various fixes to threads, downloader, and support for VirtualFileMountHTTP

panda/src/downloader/Sources.pp

@@ -36,7 +36,11 @@
     patcher.h patcher.I \
     socketStream.h socketStream.I \
     ssl_utils.h \
-    urlSpec.I urlSpec.h
+    stringStreamBuf.I stringStreamBuf.h \
+    stringStream.I stringStream.h \
+    urlSpec.I urlSpec.h \
+    virtualFileHTTP.I virtualFileHTTP.h \
+    virtualFileMountHTTP.I virtualFileMountHTTP.h
     
   #define INCLUDED_SOURCES                 \
     config_downloader.cxx \
@@ -63,7 +67,11 @@
     patcher.cxx \
     socketStream.cxx \
     ssl_utils.cxx \
-    urlSpec.cxx
+    stringStreamBuf.cxx \
+    stringStream.cxx \
+    urlSpec.cxx \
+    virtualFileHTTP.cxx \
+    virtualFileMountHTTP.cxx
 
   #define INSTALL_HEADERS \
     bioPtr.I bioPtr.h \
@@ -92,7 +100,11 @@
     patcher.h patcher.I \
     socketStream.h socketStream.I \
     ssl_utils.h \
-    urlSpec.h urlSpec.I
+    stringStreamBuf.I stringStreamBuf.h \
+    stringStream.I stringStream.h \
+    urlSpec.h urlSpec.I \
+    virtualFileHTTP.I virtualFileHTTP.h \
+    virtualFileMountHTTP.I virtualFileMountHTTP.h
     
   #define IGATESCAN all
 

panda/src/downloader/bioStreamBuf.cxx

@@ -184,11 +184,11 @@ underflow() {
         int os_error = errno;
 #endif  // WIN32_VC
 
-        // if not EOF, there may be more data on the way even if BIO
-        // indicates that no retry is necessary
-        //_is_closed = !BIO_should_retry(*_source);
-        //_is_closed = !BIO_should_read(*_source);
-        _is_closed = (BIO_eof(*_source) != 0);
+        // Though BIO_eof() is tempting, it appears there are cases in
+        // which that never returns true, if the socket is closed by
+        // the server.  But BIO_should_retry() *appears* to be
+        // reliable.
+        _is_closed = !BIO_should_retry(*_source);
         if (_is_closed) {
           downloader_cat.info()
             << "Lost connection to "
@@ -214,20 +214,19 @@ underflow() {
         }
         gbump(num_bytes);
         return EOF;
-
       } 
         
-      if (downloader_cat.is_spam()) {
-        downloader_cat.spam()
-          << "read " << read_count << " bytes from " << _source << "\n";
-      }
-
       // Slide what we did read to the top of the buffer.
       nassertr(read_count < (int)num_bytes, EOF);
       size_t delta = (int)num_bytes - read_count;
       memmove(gptr() + delta, gptr(), read_count);
       gbump(delta);
     }
+
+    if (downloader_cat.is_spam()) {
+      downloader_cat.spam()
+        << "read " << read_count << " bytes from " << _source << "\n";
+    }
   }
 
   return (unsigned char)*gptr();
@@ -246,10 +245,6 @@ write_chars(const char *start, size_t length) {
     size_t wrote_so_far = 0;
 
     int write_count = BIO_write(*_source, start, length);
-    if (downloader_cat.is_spam()) {
-      downloader_cat.spam()
-        << "wrote " << write_count << " bytes to " << _source << "\n";
-    }
     thread_consider_yield();
     while (write_count != (int)(length - wrote_so_far)) {
       if (write_count <= 0) {
@@ -295,6 +290,10 @@ write_chars(const char *start, size_t length) {
       } else {
         // wrote some characters.
         wrote_so_far += write_count;
+        if (downloader_cat.is_spam()) {
+          downloader_cat.spam()
+            << "wrote " << write_count << " bytes to " << _source << "\n";
+        }
       }
       
       // Try to write some more.
@@ -305,6 +304,10 @@ write_chars(const char *start, size_t length) {
       }
       thread_consider_yield();
     }
+    if (downloader_cat.is_spam()) {
+      downloader_cat.spam()
+        << "wrote " << write_count << " bytes to " << _source << "\n";
+    }
   }
 
   return length;

panda/src/downloader/chunkedStream.cxx

@@ -17,6 +17,19 @@
 // This module is not compiled if OpenSSL is not available.
 #ifdef HAVE_OPENSSL
 
+////////////////////////////////////////////////////////////////////
+//     Function: IChunkedStream::Destructor
+//       Access: Published, Virtual
+//  Description: 
+////////////////////////////////////////////////////////////////////
+IChunkedStream::
+~IChunkedStream() {
+  if (_channel != (HTTPChannel *)NULL) {
+    _channel->body_stream_destructs(this);
+    _channel = NULL;
+  }
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: IChunkedStream::is_closed
 //       Access: Public, Virtual

panda/src/downloader/chunkedStream.h

@@ -41,6 +41,7 @@ public:
   INLINE IChunkedStream(BioStreamPtr *source, HTTPChannel *doc);
 
   INLINE IChunkedStream &open(BioStreamPtr *source, HTTPChannel *doc);
+  virtual ~IChunkedStream();
 
   virtual bool is_closed();
   virtual void close();

panda/src/downloader/chunkedStreamBuf.cxx

@@ -33,6 +33,7 @@ ChunkedStreamBuf::
 ChunkedStreamBuf() {
   _chunk_remaining = 0;
   _done = true;
+  _wanted_nonblocking = false;
   _read_state = ISocketStream::RS_initial;
 
 #ifdef HAVE_IOSTREAM
@@ -74,6 +75,7 @@ open_read(BioStreamPtr *source, HTTPChannel *doc) {
   nassertv(!_source.is_null());
   _chunk_remaining = 0;
   _done = false;
+  _wanted_nonblocking = doc->_wanted_nonblocking;
   _read_state = ISocketStream::RS_reading;
   _doc = doc;
 
@@ -141,64 +143,87 @@ underflow() {
 ////////////////////////////////////////////////////////////////////
 size_t ChunkedStreamBuf::
 read_chars(char *start, size_t length) {
-  nassertr(!_source.is_null(), 0);
-  if (_done) {
-    return 0;
-  }
-
-  if (_chunk_remaining != 0) {
-    // Extract some of the bytes remaining in the chunk.
-    length = min(length, _chunk_remaining);
-    (*_source)->read(start, length);
-    size_t read_count = (*_source)->gcount();
-    _chunk_remaining -= read_count;
-
-    if (read_count == 0 && (*_source)->is_closed()) {
-      // Whoops, the socket closed while we were downloading.
-      _read_state = ISocketStream::RS_error;
+  while (true) {
+    nassertr(!_source.is_null(), 0);
+    if (_done) {
+      return 0;
     }
-
-    return read_count;
-  }
-
-  // Read the next chunk.
-  string line;
-  bool got_line = http_getline(line);
-  while (got_line && line.empty()) {
-    // Skip blank lines.  There really should be exactly one blank
-    // line, but who's counting?  It's tricky to count and maintain
-    // reentry for nonblocking I/O.
-    got_line = http_getline(line);
-  }
-  if (!got_line) {
-    // EOF (or data unavailable) while trying to read the chunk size.
-    if ((*_source)->is_closed()) {
-      // Whoops, the socket closed while we were downloading.
-      _read_state = ISocketStream::RS_error;
+    
+    if (_chunk_remaining != 0) {
+      // Extract some of the bytes remaining in the chunk.
+      length = min(length, _chunk_remaining);
+      (*_source)->read(start, length);
+      size_t read_count = (*_source)->gcount();
+      if (!_wanted_nonblocking) {
+        while (read_count == 0 && !(*_source)->is_closed()) {
+          // Simulate blocking.
+          thread_yield();
+          (*_source)->read(start, length);
+          read_count = (*_source)->gcount();
+        }
+      }
+      _chunk_remaining -= read_count;
+      
+      if (read_count == 0 && (*_source)->is_closed()) {
+        // Whoops, the socket closed while we were downloading.
+        _read_state = ISocketStream::RS_error;
+      }
+      
+      return read_count;
     }
-    return 0;
-  }
-  size_t chunk_size = (size_t)strtol(line.c_str(), NULL, 16);
-  if (downloader_cat.is_spam()) {
-    downloader_cat.spam()
-      << "Got chunk of size " << chunk_size << " bytes.\n";
-  }
+    
+    // Read the next chunk.
+    string line;
+    bool got_line = http_getline(line);
+    while (got_line && line.empty()) {
+      // Skip blank lines.  There really should be exactly one blank
+      // line, but who's counting?  It's tricky to count and maintain
+      // reentry for nonblocking I/O.
+      got_line = http_getline(line);
+    }
+    if (!got_line) {
+      // EOF (or data unavailable) while trying to read the chunk size.
+      if ((*_source)->is_closed()) {
+        // Whoops, the socket closed while we were downloading.
+        _read_state = ISocketStream::RS_error;
+      }
+      
+      if (!_wanted_nonblocking) {
+        // Simulate blocking.
+        thread_yield();
+        continue;  // back to the top.
+      }
 
-  if (chunk_size == 0) {
-    // Last chunk; we're done.
-    _done = true;
-    _doc->_file_size = _doc->_transfer_file_size;
-    _doc->_got_file_size = true;
-    _read_state = ISocketStream::RS_complete;
-    return 0;
-  }
+      return 0;
+    }
+    size_t chunk_size = (size_t)strtol(line.c_str(), NULL, 16);
+    if (downloader_cat.is_spam()) {
+      downloader_cat.spam()
+        << "Got chunk of size " << chunk_size << " bytes.\n";
+    }
+    
+    if (chunk_size == 0) {
+      // Last chunk; we're done.
+      _done = true;
+      if (_doc != (HTTPChannel *)NULL && _read_index == _doc->_read_index) {
+        _doc->_file_size = _doc->_transfer_file_size;
+        _doc->_got_file_size = true;
+      }
+      _read_state = ISocketStream::RS_complete;
+      return 0;
+    }
+    
+    if (_doc != (HTTPChannel *)NULL && _read_index == _doc->_read_index) {
+      _doc->_transfer_file_size += chunk_size;
+    }
+    
+    _chunk_remaining = chunk_size;
 
-  if (_doc != (HTTPChannel *)NULL && _read_index == _doc->_read_index) {
-    _doc->_transfer_file_size += chunk_size;
+    // Back to the top.
   }
 
-  _chunk_remaining = chunk_size;
-  return read_chars(start, length);
+  // Never gets here.
+  return 0;
 }
 
 ////////////////////////////////////////////////////////////////////

panda/src/downloader/chunkedStreamBuf.h

@@ -51,6 +51,7 @@ private:
   PT(BioStreamPtr) _source;
   size_t _chunk_remaining;
   bool _done;
+  bool _wanted_nonblocking;
   string _working_getline;
   ISocketStream::ReadState _read_state;
 

panda/src/downloader/config_downloader.cxx

@@ -15,6 +15,8 @@
 #include "dconfig.h"
 #include "config_downloader.h"
 #include "httpChannel.h"
+#include "virtualFileHTTP.h"
+#include "virtualFileMountHTTP.h"
 #include "pandaSystem.h"
 
 
@@ -114,6 +116,23 @@ ConfigVariableDouble http_timeout
           "It starts counting after the TCP connection has been established "
           "(http_connect_timeout, above) and the request has been sent."));
 
+ConfigVariableInt http_skip_body_size
+("http-skip-body-size", 8192,
+ PRC_DESC("This is the maximum number of bytes in a received "
+          "(but unwanted) body that will be skipped past, in "
+          "order to reset to a new request.  "
+          "See HTTPChannel::set_skip_body_size()."));
+
+ConfigVariableDouble http_idle_timeout
+("http-idle-timeout", 5.0,
+ PRC_DESC("This the amount of time, in seconds, in which a "
+          "previously-established connection is allowed to remain open "
+          "and unused.  If a previous connection has remained unused for "
+          "at least this number of seconds, it will be closed and a new "
+          "connection will be opened; otherwise, the same connection "
+          "will be reused for the next request (for a particular "
+          "HTTPChannel)."));
+
 ConfigVariableInt http_max_connect_count
 ("http-max-connect-count", 10,
  PRC_DESC("This is the maximum number of times to try reconnecting to the "
@@ -166,6 +185,8 @@ init_libdownloader() {
 
 #ifdef HAVE_OPENSSL
   HTTPChannel::init_type();
+  VirtualFileHTTP::init_type();
+  VirtualFileMountHTTP::init_type();
 
   // We need to define this here, rather than above, to guarantee that
   // it has been initialized by the time we check it.

panda/src/downloader/config_downloader.h

@@ -50,6 +50,8 @@ extern ConfigVariableString http_proxy_username;
 extern ConfigVariableBool http_proxy_tunnel;
 extern ConfigVariableDouble http_connect_timeout;
 extern ConfigVariableDouble http_timeout;
+extern ConfigVariableInt http_skip_body_size;
+extern ConfigVariableDouble http_idle_timeout;
 extern ConfigVariableInt http_max_connect_count;
 extern ConfigVariableFilename http_client_certificate_filename;
 extern ConfigVariableString http_client_certificate_passphrase;

panda/src/downloader/downloader_composite2.cxx

@@ -14,4 +14,8 @@
 #include "patcher.cxx"
 #include "socketStream.cxx"
 #include "ssl_utils.cxx"
+#include "stringStreamBuf.cxx"
+#include "stringStream.cxx"
 #include "urlSpec.cxx"
+#include "virtualFileHTTP.cxx"
+#include "virtualFileMountHTTP.cxx"

panda/src/downloader/httpChannel.I

@@ -391,6 +391,71 @@ get_http_timeout() const {
   return _http_timeout;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPChannel::set_skip_body_size
+//       Access: Published
+//  Description: Specifies the maximum number of bytes in a received
+//               (but unwanted) body that will be skipped past, in
+//               order to reset to a new request.
+//
+//               That is, if this HTTPChannel requests a file via
+//               get_document(), but does not call download_to_ram(),
+//               download_to_file(), or open_read_body(), and instead
+//               immediately requests a new file, then the HTTPChannel
+//               has a choice whether to skip past the unwanted
+//               document, or to close the connection and open a new
+//               one.  If the number of bytes to skip is more than
+//               this threshold, the connection will be closed;
+//               otherwise, the data will simply be read and
+//               discarded.
+////////////////////////////////////////////////////////////////////
+INLINE void HTTPChannel::
+set_skip_body_size(size_t skip_body_size) {
+  _skip_body_size = skip_body_size;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPChannel::get_skip_body_size
+//       Access: Published
+//  Description: Returns the maximum number of bytes in a received
+//               (but unwanted) body that will be skipped past, in
+//               order to reset to a new request.  See
+//               set_skip_body_size().
+////////////////////////////////////////////////////////////////////
+INLINE size_t HTTPChannel::
+get_skip_body_size() const {
+  return _skip_body_size;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPChannel::set_idle_timeout
+//       Access: Published
+//  Description: Specifies the amount of time, in seconds, in which a
+//               previously-established connection is allowed to
+//               remain open and unused.  If a previous connection has
+//               remained unused for at least this number of seconds,
+//               it will be closed and a new connection will be
+//               opened; otherwise, the same connection will be reused
+//               for the next request (for this particular
+//               HTTPChannel).
+////////////////////////////////////////////////////////////////////
+INLINE void HTTPChannel::
+set_idle_timeout(double idle_timeout) {
+  _idle_timeout = idle_timeout;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPChannel::get_idle_timeout
+//       Access: Published
+//  Description: Returns the amount of time, in seconds, in which an
+//               previously-established connection is allowed to
+//               remain open and unused.  See set_idle_timeout().
+////////////////////////////////////////////////////////////////////
+INLINE double HTTPChannel::
+get_idle_timeout() const {
+  return _idle_timeout;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: HTTPChannel::set_download_throttle
 //       Access: Published

panda/src/downloader/httpChannel.cxx

@@ -20,6 +20,7 @@
 #include "chunkedStream.h"
 #include "identityStream.h"
 #include "config_downloader.h"
+#include "virtualFileMountHTTP.h"
 #include "ramfile.h"
 
 #ifdef HAVE_OPENSSL
@@ -48,6 +49,8 @@ HTTPChannel(HTTPClient *client) :
   _proxy_tunnel = http_proxy_tunnel;
   _connect_timeout = http_connect_timeout;
   _http_timeout = http_timeout;
+  _skip_body_size = http_skip_body_size;
+  _idle_timeout = http_idle_timeout;
   _blocking_connect = false;
   _download_throttle = false;
   _max_bytes_per_second = downloader_byte_rate;
@@ -89,6 +92,7 @@ HTTPChannel(HTTPClient *client) :
   _started_download = false;
   _sent_so_far = 0;
   _body_stream = NULL;
+  _owns_body_stream = false;
   _sbio = NULL;
   _last_status_code = 0;
   _last_run_time = 0.0f;
@@ -106,108 +110,6 @@ HTTPChannel::
   reset_download_to();
 }
 
-////////////////////////////////////////////////////////////////////
-//     Function: HTTPChannel::get_file_system
-//       Access: Published, Virtual
-//  Description: Returns the VirtualFileSystem this file is associated
-//               with.
-////////////////////////////////////////////////////////////////////
-VirtualFileSystem *HTTPChannel::
-get_file_system() const {
-  return NULL;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: HTTPChannel::get_filename
-//       Access: Published, Virtual
-//  Description: Returns the full pathname to this file within the
-//               virtual file system.
-////////////////////////////////////////////////////////////////////
-Filename HTTPChannel::
-get_filename() const {
-  return Filename();
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: HTTPChannel::is_regular_file
-//       Access: Published, Virtual
-//  Description: Returns true if this file represents a regular file
-//               (and read_file() may be called), false otherwise.
-////////////////////////////////////////////////////////////////////
-bool HTTPChannel::
-is_regular_file() const {
-  return is_valid();
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: HTTPChannel::will_close_connection
-//       Access: Public
-//  Description: Returns true if the server has indicated it will
-//               close the connection after this document has been
-//               read, or false if it will remain open (and future
-//               documents may be requested on the same connection).
-////////////////////////////////////////////////////////////////////
-bool HTTPChannel::
-will_close_connection() const {
-  if (get_http_version() < HTTPEnum::HV_11) {
-    // pre-HTTP 1.1 always closes.
-    return true;
-  }
-
-  string connection = get_header_value("Connection");
-  if (downcase(connection) == "close") {
-    // The server says it will close.
-    return true;
-  }
-
-  if (connection.empty() && !get_persistent_connection()) {
-    // The server didn't say, but we asked it to close.
-    return true;
-  }
-
-  // Assume the server will keep it open.
-  return false;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: HTTPChannel::open_read_file
-//       Access: Public, Virtual
-//  Description: Opens the document for reading.  Returns a newly
-//               allocated istream on success (which you should
-//               pass to close_read_file() when you are done reading).
-//               Returns NULL on failure.  This may only be called
-//               once for a particular HTTPChannel.
-////////////////////////////////////////////////////////////////////
-istream *HTTPChannel::
-open_read_file(bool auto_unwrap) const {
-  return ((HTTPChannel *)this)->read_body();
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: HTTPChannel::close_read_file
-//       Access: Public
-//  Description: Closes a file opened by a previous call to
-//               open_read_file().  This really just deletes the
-//               istream pointer, but it is recommended to use this
-//               interface instead of deleting it explicitly, to help
-//               work around compiler issues.
-////////////////////////////////////////////////////////////////////
-void HTTPChannel::
-close_read_file(istream *stream) const {
-  if (stream != (istream *)NULL) {
-    // For some reason--compiler bug in gcc 3.2?--explicitly deleting
-    // the stream pointer does not call the appropriate global delete
-    // function; instead apparently calling the system delete
-    // function.  So we call the delete function by hand instead.
-#if !defined(USE_MEMORY_NOWRAPPERS) && defined(REDEFINE_GLOBAL_OPERATOR_NEW)
-    stream->~istream();
-    (*global_operator_delete)(stream);
-#else
-    delete stream;
-#endif
-  }
-}
-
 ////////////////////////////////////////////////////////////////////
 //     Function: HTTPChannel::get_status_string
 //       Access: Published
@@ -296,6 +198,36 @@ get_header_value(const string &key) const {
   return string();
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPChannel::will_close_connection
+//       Access: Published
+//  Description: Returns true if the server has indicated it will
+//               close the connection after this document has been
+//               read, or false if it will remain open (and future
+//               documents may be requested on the same connection).
+////////////////////////////////////////////////////////////////////
+bool HTTPChannel::
+will_close_connection() const {
+  if (get_http_version() < HTTPEnum::HV_11) {
+    // pre-HTTP 1.1 always closes.
+    return true;
+  }
+
+  string connection = get_header_value("Connection");
+  if (downcase(connection) == "close") {
+    // The server says it will close.
+    return true;
+  }
+
+  if (connection.empty() && !get_persistent_connection()) {
+    // The server didn't say, but we asked it to close.
+    return true;
+  }
+
+  // Assume the server will keep it open.
+  return false;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: HTTPChannel::get_file_size
 //       Access: Published, Virtual
@@ -306,7 +238,7 @@ get_header_value(const string &key) const {
 //
 //               If the file is dynamically generated, the size may
 //               not be available until a read has started
-//               (e.g. open_read_file() has been called); and even
+//               (e.g. open_read_body() has been called); and even
 //               then it may increase as more of the file is read due
 //               to the nature of HTTP/1.1 requests which can change
 //               their minds midstream about how much data they're
@@ -394,6 +326,10 @@ run() {
     case DD_ram:
       repeat_later = run_download_to_ram();
       break;
+
+    case DD_stream:
+      repeat_later = run_download_to_stream();
+      break;
     }
     if (repeat_later) {
       thread_yield();
@@ -573,7 +509,7 @@ run() {
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: HTTPChannel::read_body
+//     Function: HTTPChannel::open_read_body
 //       Access: Published
 //  Description: Returns a newly-allocated istream suitable for
 //               reading the body of the document.  This may only be
@@ -581,11 +517,19 @@ run() {
 //               post_form(), or after a call to run() has returned
 //               false.
 //
-//               The user is responsible for deleting the returned
-//               istream later.
+//               Note that, in nonblocking mode, the returned stream
+//               may report an early EOF, even before the actual end
+//               of file.  When this happens, you should call
+//               stream->is_closed() to determine whether you should
+//               attempt to read some more later.
+//
+//               The user is responsible for passing the returned
+//               istream to close_read_body() later.
 ////////////////////////////////////////////////////////////////////
 ISocketStream *HTTPChannel::
-read_body() {
+open_read_body() {
+  reset_body_stream();
+
   if ((_state != S_read_header && _state != S_begin_body) || _source.is_null()) {
     return NULL;
   }
@@ -611,9 +555,38 @@ read_body() {
     result = new IIdentityStream(_source, this, _got_file_size, _file_size);
   }
 
+  result->_channel = this;
+  _body_stream = result;
+  _owns_body_stream = false;
+
   return result;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPChannel::close_read_body
+//       Access: Public
+//  Description: Closes a file opened by a previous call to
+//               open_read_body().  This really just deletes the
+//               istream pointer, but it is recommended to use this
+//               interface instead of deleting it explicitly, to help
+//               work around compiler issues.
+////////////////////////////////////////////////////////////////////
+void HTTPChannel::
+close_read_body(istream *stream) const {
+  if (stream != (istream *)NULL) {
+    // For some reason--compiler bug in gcc 3.2?--explicitly deleting
+    // the stream pointer does not call the appropriate global delete
+    // function; instead apparently calling the system delete
+    // function.  So we call the delete function by hand instead.
+#if !defined(USE_MEMORY_NOWRAPPERS) && defined(REDEFINE_GLOBAL_OPERATOR_NEW)
+    stream->~istream();
+    (*global_operator_delete)(stream);
+#else
+    delete stream;
+#endif
+  }
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: HTTPChannel::download_to_file
 //       Access: Published
@@ -733,6 +706,66 @@ download_to_ram(Ramfile *ramfile, bool subdocument_resumes) {
   return is_download_complete();
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPChannel::download_to_stream
+//       Access: Published
+//  Description: Specifies the name of an ostream to download the
+//               resulting document to.  This should be called
+//               immediately after get_document() or
+//               begin_get_document() or related functions.
+//
+//               In the case of the blocking I/O methods like
+//               get_document(), this function will download the
+//               entire document to the file and return true if it was
+//               successfully downloaded, false otherwise.
+//
+//               In the case of non-blocking I/O methods like
+//               begin_get_document(), this function simply indicates an
+//               intention to download to the indicated file.  It
+//               returns true if the file can be opened for writing,
+//               false otherwise, but the contents will not be
+//               completely downloaded until run() has returned false.
+//               At this time, it is possible that a communications
+//               error will have left a partial file, so
+//               is_download_complete() may be called to test this.
+//
+//               If subdocument_resumes is true and the document in
+//               question was previously requested as a subdocument
+//               (i.e. get_subdocument() with a first_byte value
+//               greater than zero), this will automatically seek to
+//               the appropriate byte within the file for writing the
+//               output.  In this case, the file must already exist
+//               and must have at least first_byte bytes in it.  If
+//               subdocument_resumes is false, a subdocument will
+//               always be downloaded beginning at the first byte of
+//               the file.
+////////////////////////////////////////////////////////////////////
+bool HTTPChannel::
+download_to_stream(ostream *strm, bool subdocument_resumes) {
+  reset_download_to();
+  _download_to_stream = strm;
+  _download_to_stream->clear();
+  _subdocument_resumes = subdocument_resumes;
+
+  _download_dest = DD_stream;
+
+  if (_wanted_nonblocking) {
+    // In nonblocking mode, we can't start the download yet; that will
+    // be done later as run() is called.
+    return true;
+  }
+
+  // In normal, blocking mode, go ahead and do the download.
+  if (!open_download_file()) {
+    reset_download_to();
+    return false;
+  }
+
+  while (run()) {
+  }
+  return is_download_complete();
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: HTTPChannel::get_connection
 //       Access: Published
@@ -754,7 +787,11 @@ get_connection() {
   BioStream *stream = _source->get_stream();
   _source->set_stream(NULL);
 
-  // We're now passing ownership of the connection to the user.
+  // We're now passing ownership of the connection to the caller.
+  if (downloader_cat.is_debug()) {
+    downloader_cat.debug()
+      << "passing ownership of connection to caller.\n";
+  }
   reset_to_new();
 
   return stream;
@@ -777,6 +814,35 @@ downcase(const string &s) {
   return result;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPChannel::body_stream_destructs
+//       Access: Public
+//  Description: Called by ISocketStream destructor when _body_stream
+//               is destructing.
+////////////////////////////////////////////////////////////////////
+void HTTPChannel::
+body_stream_destructs(ISocketStream *stream) {
+  if (stream == _body_stream) {
+    if (_state == S_reading_body) {
+      switch (_body_stream->get_read_state()) {
+      case ISocketStream::RS_complete:
+        finished_body(false);
+        break;
+        
+      case ISocketStream::RS_error:
+        _state = HTTPChannel::S_failure;
+        _status_entry._status_code = HTTPChannel::SC_lost_connection;
+        break;
+    
+      default:
+        break;
+      }
+    }
+    _body_stream = NULL;
+    _owns_body_stream = false;
+  }
+}
+
 
 ////////////////////////////////////////////////////////////////////
 //     Function: HTTPChannel::reached_done_state
@@ -842,13 +908,7 @@ reached_done_state() {
     
   } else {
     // Oops, we have to download the body now.
-    // We shouldn't already be in the middle of reading some other
-    // body when we come here.
-    if (_body_stream != NULL) {
-      delete _body_stream;
-      _body_stream = (ISocketStream *)NULL;
-    }
-    _body_stream = read_body();
+    open_read_body();
     if (_body_stream == (ISocketStream *)NULL) {
       if (downloader_cat.is_debug()) {
         downloader_cat.debug()
@@ -857,8 +917,9 @@ reached_done_state() {
       return false;
 
     } else {
+      _owns_body_stream = true;
       if (_state != S_reading_body) {
-        _body_stream = NULL;
+        reset_body_stream();
       }
       _started_download = true;
 
@@ -940,7 +1001,11 @@ run_connecting() {
     }
 
   } else {
-    _state = _want_ssl ? S_setup_ssl : S_ready;
+    if (_want_ssl) {
+      _state = S_setup_ssl;
+    } else {
+      _state = S_ready;
+    }
   }
   return false;
 }
@@ -1124,7 +1189,12 @@ run_http_proxy_reading_header() {
   // Now we have a tunnel opened through the proxy.
   make_request_text();
 
-  _state = _want_ssl ? S_setup_ssl : S_ready;
+  if (_want_ssl) {
+    _state = S_setup_ssl;
+  } else {
+    _state = S_ready;
+  }
+
   return false;
 }
 
@@ -1370,7 +1440,12 @@ run_socks_proxy_connect_reply() {
       << connect_port << "\n";
   }
 
-  _state = _want_ssl ? S_setup_ssl : S_ready;
+  if (_want_ssl) {
+    _state = S_setup_ssl;
+  } else {
+    _state = S_ready;
+  }
+
   return false;
 }
 
@@ -1927,6 +2002,10 @@ run_begin_body() {
   if (will_close_connection()) {
     // If the socket will close anyway, no point in skipping past the
     // previous body; just reset.
+    if (downloader_cat.is_debug()) {
+      downloader_cat.debug()
+        << "resetting to begin body; server would close anyway.\n";
+    }
     reset_to_new();
     return false;
   }
@@ -1935,11 +2014,10 @@ run_begin_body() {
     // We have already "read" the nonexistent body.
     _state = S_read_trailer;
 
-  } else if (get_file_size() > 8192) {
+  } else if (get_file_size() > _skip_body_size) {
     // If we know the size of the body we are about to skip and it's
-    // too large (and here we arbitrarily say 8KB is too large), then
-    // don't bother skipping it--just drop the connection and get a
-    // new one.
+    // too large, then don't bother skipping it--just drop the
+    // connection and get a new one.
     if (downloader_cat.is_debug()) {
       downloader_cat.debug()
         << "Dropping connection rather than skipping past " 
@@ -1948,13 +2026,7 @@ run_begin_body() {
     reset_to_new();
 
   } else {
-    // We shouldn't already be in the middle of reading some other
-    // body when we come here.
-    if (_body_stream != NULL) {
-      delete _body_stream;
-      _body_stream = (ISocketStream *)NULL;
-    }
-    _body_stream = read_body();
+    open_read_body();
     if (_body_stream == (ISocketStream *)NULL) {
       if (downloader_cat.is_debug()) {
         downloader_cat.debug()
@@ -1963,8 +2035,9 @@ run_begin_body() {
       reset_to_new();
       
     } else {
+      _owns_body_stream = true;
       if (_state != S_reading_body) {
-        _body_stream = NULL;
+        reset_body_stream();
       }
     }
   }
@@ -1978,8 +2051,7 @@ run_begin_body() {
 //  Description: In this state we are in the process of reading the
 //               response's body.  We will only come to this function
 //               if the user did not choose to read the entire body
-//               himself (by calling read_body(), for instance, or
-//               open_read_file()).
+//               himself (by calling open_read_body()).
 //
 //               In this case we should skip past the body to reset
 //               the connection for making a new request.
@@ -1989,13 +2061,21 @@ run_reading_body() {
   if (will_close_connection()) {
     // If the socket will close anyway, no point in skipping past the
     // previous body; just reset.
+    if (downloader_cat.is_debug()) {
+      downloader_cat.debug()
+        << "resetting to read body; server would close anyway.\n";
+    }
     reset_to_new();
     return false;
   }
 
   // Skip the body we've already started.
-  if (_body_stream == NULL) {
+  if (_body_stream == NULL || !_owns_body_stream) {
     // Whoops, we're not in skip-body mode.  Better reset.
+    if (downloader_cat.is_debug()) {
+      downloader_cat.debug()
+        << "resetting, not in skip-body mode.\n";
+    }
     reset_to_new();
     return false;
   }
@@ -2009,29 +2089,14 @@ run_reading_body() {
     getline(*_body_stream, line);
   }
 
-  switch (_body_stream->get_read_state()) {
-  case ISocketStream::RS_complete:
-    finished_body(false);
-    break;
-
-  case ISocketStream::RS_error:
-    _state = HTTPChannel::S_failure;
-    _status_entry._status_code = HTTPChannel::SC_lost_connection;
-    break;
-
-  default:
-    break;
-  }
-
   if (!_body_stream->is_closed()) {
     // There's more to come later.
     return true;
   }
 
-  delete _body_stream;
-  _body_stream = NULL;
+  reset_body_stream();
 
-  // This should have been set by the _body_stream finishing.
+  // This should have been set by the call to finished_body(), above.
   nassertr(_state != S_reading_body, false);
   return false;
 }
@@ -2055,6 +2120,10 @@ run_read_body() {
   if (will_close_connection()) {
     // If the socket will close anyway, no point in skipping past the
     // previous body; just reset.
+    if (downloader_cat.is_debug()) {
+      downloader_cat.debug()
+        << "resetting to read body; server would close anyway.\n";
+    }
     reset_to_new();
     return false;
   }
@@ -2086,6 +2155,10 @@ run_read_trailer() {
   if (will_close_connection()) {
     // If the socket will close anyway, no point in skipping past the
     // previous body; just reset.
+    if (downloader_cat.is_debug()) {
+      downloader_cat.debug()
+        << "resetting to read trailer; server would close anyway.\n";
+    }
     reset_to_new();
     return false;
   }
@@ -2102,7 +2175,7 @@ run_read_trailer() {
 ////////////////////////////////////////////////////////////////////
 bool HTTPChannel::
 run_download_to_file() {
-  nassertr(_body_stream != (ISocketStream *)NULL, false);
+  nassertr(_body_stream != (ISocketStream *)NULL && _owns_body_stream, false);
 
   bool do_throttle = _wanted_nonblocking && _download_throttle;
 
@@ -2143,22 +2216,9 @@ run_download_to_file() {
 
   _download_to_file.flush();
 
-  switch (_body_stream->get_read_state()) {
-  case ISocketStream::RS_complete:
-    finished_body(false);
-    break;
-
-  case ISocketStream::RS_error:
-    _state = HTTPChannel::S_failure;
-    _status_entry._status_code = HTTPChannel::SC_lost_connection;
-    break;
-
-  default:
-    break;
-  }
-
   if (_body_stream->is_closed()) {
     // Done.
+    reset_body_stream();
     _download_to_file.close();
     _started_download = false;
     return false;
@@ -2176,7 +2236,7 @@ run_download_to_file() {
 ////////////////////////////////////////////////////////////////////
 bool HTTPChannel::
 run_download_to_ram() {
-  nassertr(_body_stream != (ISocketStream *)NULL, false);
+  nassertr(_body_stream != (ISocketStream *)NULL && _owns_body_stream, false);
   nassertr(_download_to_ramfile != (Ramfile *)NULL, false);
 
   bool do_throttle = _wanted_nonblocking && _download_throttle;
@@ -2207,22 +2267,70 @@ run_download_to_ram() {
     count = _body_stream->gcount();
   }
 
-  switch (_body_stream->get_read_state()) {
-  case ISocketStream::RS_complete:
-    finished_body(false);
-    break;
+  if (_body_stream->is_closed()) {
+    // Done.
+    reset_body_stream();
+    _started_download = false;
+    return false;
+  } else {
+    // More to come.
+    return true;
+  }
+}
 
-  case ISocketStream::RS_error:
-    _state = HTTPChannel::S_failure;
-    _status_entry._status_code = HTTPChannel::SC_lost_connection;
-    break;
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPChannel::run_download_to_stream
+//       Access: Private
+//  Description: After the headers, etc. have been read, this streams
+//               the download to the named file.
+////////////////////////////////////////////////////////////////////
+bool HTTPChannel::
+run_download_to_stream() {
+  nassertr(_body_stream != (ISocketStream *)NULL && _owns_body_stream, false);
 
-  default:
-    break;
+  bool do_throttle = _wanted_nonblocking && _download_throttle;
+
+  static const size_t buffer_size = 1024;
+  char buffer[buffer_size];
+
+  size_t remaining_this_pass = buffer_size;
+  if (do_throttle) {
+    remaining_this_pass = _bytes_per_update;
+  }
+
+  _body_stream->read(buffer, min(buffer_size, remaining_this_pass));
+  size_t count = _body_stream->gcount();
+  while (count != 0) {
+    _download_to_stream->write(buffer, count);
+    _bytes_downloaded += count;
+    if (do_throttle) {
+      nassertr(count <= remaining_this_pass, false);
+      remaining_this_pass -= count;
+      if (remaining_this_pass == 0) {
+        // That's enough for now.
+        return true;
+      }
+    }
+
+    _body_stream->read(buffer, min(buffer_size, remaining_this_pass));
+    count = _body_stream->gcount();
+  }
+
+  if (_download_to_stream->fail()) {
+    downloader_cat.warning()
+      << "Error writing to stream\n";
+    _status_entry._status_code = SC_download_write_error;
+    _state = S_failure;
+    reset_download_to();
+    return false;
   }
 
+  _download_to_stream->flush();
+
   if (_body_stream->is_closed()) {
     // Done.
+    reset_body_stream();
+    _download_to_stream = NULL;
     _started_download = false;
     return false;
   } else {
@@ -2281,12 +2389,20 @@ begin_request(HTTPEnum::Method method, const DocumentSpec &url,
   if (_proxy != new_proxy) {
     _proxy = new_proxy;
     _proxy_auth = (HTTPAuthorization *)NULL;
+    if (downloader_cat.is_debug()) {
+      downloader_cat.debug()
+        << "resetting to change proxy to " << _proxy << "\n";
+    }
     reset_to_new();
   }
 
   // Ditto with changing the nonblocking state.
   if (_nonblocking != nonblocking) {
     _nonblocking = nonblocking;
+    if (downloader_cat.is_debug()) {
+      downloader_cat.debug()
+        << "resetting to change nonblocking state to " << _nonblocking << ".\n";
+    }
     reset_to_new();
   }
 
@@ -2308,6 +2424,19 @@ begin_request(HTTPEnum::Method method, const DocumentSpec &url,
 
   // Also, reset from whatever previous request might still be pending.
   if (_state == S_failure || (_state < S_read_header && _state != S_ready)) {
+    if (downloader_cat.is_debug()) {
+      downloader_cat.debug()
+        << "resetting to clear previous request.\n";
+    }
+    reset_to_new();
+
+  } else if (TrueClock::get_global_ptr()->get_short_time() - _last_run_time >= _idle_timeout) {
+    if (downloader_cat.is_debug()) {
+      downloader_cat.debug()
+        << "resetting old connection: " 
+        << TrueClock::get_global_ptr()->get_short_time() - _last_run_time
+        << " s old.\n";
+    }
     reset_to_new();
 
   } else if (_state == S_read_header) {
@@ -2316,7 +2445,11 @@ begin_request(HTTPEnum::Method method, const DocumentSpec &url,
     _state = S_begin_body;
   }
 
-  _done_state = (_method == HTTPEnum::M_connect) ? S_ready : S_read_header;
+  if (_method == HTTPEnum::M_connect) {
+    _done_state = S_ready;
+  } else {
+    _done_state = S_read_header;
+  }
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -2381,6 +2514,7 @@ reconsider_proxy() {
 void HTTPChannel::
 reset_for_new_request() {
   reset_download_to();
+  reset_body_stream();
 
   _last_status_code = 0;
   _status_entry = StatusEntry();
@@ -2406,6 +2540,10 @@ reset_for_new_request() {
 void HTTPChannel::
 finished_body(bool has_trailer) {
   if (will_close_connection() && _download_dest == DD_none) {
+    if (downloader_cat.is_debug()) {
+      downloader_cat.debug()
+        << "resetting to finish body; server would close anyway.\n";
+    }
     reset_to_new();
 
   } else {
@@ -2421,12 +2559,12 @@ finished_body(bool has_trailer) {
 //     Function: HTTPChannel::open_download_file
 //       Access: Private
 //  Description: If a download has been requested, opens the file on
-//               disk (or prepares the RamFile) and seeks within it to
-//               the appropriate _first_byte_delivered position, so
-//               that downloaded bytes will be written to the
-//               appropriate point within the file.  Returns true if
-//               the starting position is valid, false otherwise (in
-//               which case the state is set to S_failure).
+//               disk (or prepares the RamFile or stream) and seeks
+//               within it to the appropriate _first_byte_delivered
+//               position, so that downloaded bytes will be written to
+//               the appropriate point within the file.  Returns true
+//               if the starting position is valid, false otherwise
+//               (in which case the state is set to S_failure).
 ////////////////////////////////////////////////////////////////////
 bool HTTPChannel::
 open_download_file() {
@@ -2479,6 +2617,24 @@ open_download_file() {
         _download_to_ramfile->_data = 
           _download_to_ramfile->_data.substr(0, _first_byte_delivered);
       }
+    } else if (_download_dest == DD_stream) {
+      // Windows doesn't complain if you try to seek past the end of
+      // file--it happily appends enough zero bytes to make the
+      // difference.  Blecch.  That means we need to get the file size
+      // first to check it ourselves.
+      _download_to_stream->seekp(0, ios::end);
+      if (_first_byte_delivered > (size_t)_download_to_stream->tellp()) {
+        downloader_cat.info()
+          << "Invalid starting position of byte " << _first_byte_delivered
+          << " within stream (which has " 
+          << _download_to_stream->tellp() << " bytes)\n";
+        _download_to_stream = NULL;
+        _status_entry._status_code = SC_download_invalid_range;
+        _state = S_failure;
+        return false;
+      }
+      
+      _download_to_stream->seekp(_first_byte_delivered);
     }
 
   } else {
@@ -2489,6 +2645,8 @@ open_download_file() {
       _download_to_file.seekp(0);
     } else if (_download_dest == DD_ram) {
       _download_to_ramfile->_data = string();
+    } else if (_download_dest == DD_stream) {
+      _download_to_stream->seekp(0);
     }
   }
 
@@ -2523,8 +2681,8 @@ server_getline(string &str) {
         }
         str = str.substr(0, p);
       }
-      if (downloader_cat.is_spam()) {
-        downloader_cat.spam() << "recv: " << str << "\n";
+      if (downloader_cat.is_debug()) {
+        downloader_cat.debug() << "recv: " << str << "\n";
       }
       return true;
 
@@ -2697,9 +2855,14 @@ server_send(const string &str, bool secret) {
     reset_to_new();
     return false;
   }
+
+  if (downloader_cat.is_spam()) {
+    downloader_cat.spam()
+      << "wrote " << write_count << " bytes to " << _bio << "\n";
+  }
   
 #ifndef NDEBUG
-  if (!secret && downloader_cat.is_spam()) {
+  if (!secret && downloader_cat.is_debug()) {
     show_send(str.substr(0, write_count));
   }
 #endif
@@ -2736,6 +2899,10 @@ parse_http_response(const string &line) {
     } else {
       // Maybe we were just in some bad state.  Drop the connection
       // and try again, once.
+      if (downloader_cat.is_debug()) {
+        downloader_cat.debug()
+          << "got non-HTTP response, resetting.\n";
+      }
       reset_to_new();
       _response_type = RT_non_http;
     }
@@ -3416,6 +3583,11 @@ reset_url(const URLSpec &old_url, const URLSpec &new_url) {
   if (new_url.get_scheme() != old_url.get_scheme() ||
       (_proxy.empty() && (new_url.get_server() != old_url.get_server() || 
                           new_url.get_port() != old_url.get_port()))) {
+    if (downloader_cat.is_debug()) {
+      downloader_cat.debug()
+        << "resetting for new server " 
+        << new_url.get_server_and_port() << "\n";
+    }
     reset_to_new();
   }
 }
@@ -3458,14 +3630,14 @@ show_send(const string &message) {
   size_t newline = message.find('\n', start);
   while (newline != string::npos) {
     // Assume every \n is preceded by a \r.
-    downloader_cat.spam()
+    downloader_cat.debug()
       << "send: " << message.substr(start, newline - start - 1) << "\n";
     start = newline + 1;
     newline = message.find('\n', start);
   }
 
   if (start < message.length()) {
-    downloader_cat.spam()
+    downloader_cat.debug()
       << "send: " << message.substr(start) << " (no newline)\n";
   }
 }
@@ -3483,6 +3655,7 @@ reset_download_to() {
   _started_download = false;
   _download_to_file.close();
   _download_to_ramfile = (Ramfile *)NULL;
+  _download_to_stream = NULL;
   _download_dest = DD_none;
 }
 
@@ -3497,6 +3670,24 @@ reset_to_new() {
   _state = S_new;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPChannel::reset_body_stream
+//       Access: Private
+//  Description: Clears the _body_stream pointer, if it is set.
+////////////////////////////////////////////////////////////////////
+void HTTPChannel::
+reset_body_stream() {
+  if (_owns_body_stream) {
+    if (_body_stream != (ISocketStream *)NULL) {
+      close_read_body(_body_stream);
+      nassertv(_body_stream == (ISocketStream *)NULL && !_owns_body_stream);
+    }
+  } else {
+    _body_stream = NULL;
+  }
+}
+
+
 ////////////////////////////////////////////////////////////////////
 //     Function: HTTPChannel::close_connection
 //       Access: Private
@@ -3505,10 +3696,7 @@ reset_to_new() {
 ////////////////////////////////////////////////////////////////////
 void HTTPChannel::
 close_connection() {
-  if (_body_stream != (ISocketStream *)NULL) {
-    delete _body_stream;
-    _body_stream = (ISocketStream *)NULL;
-  }
+  reset_body_stream();
   _source.clear();
   _bio.clear();
   _working_get = string();

panda/src/downloader/httpChannel.h

@@ -29,7 +29,6 @@
 #include "httpEnum.h"
 #include "urlSpec.h"
 #include "documentSpec.h"
-#include "virtualFile.h"
 #include "bioPtr.h"
 #include "bioStreamPtr.h"
 #include "pmap.h"
@@ -38,6 +37,7 @@
 #include "config_downloader.h"
 #include "filename.h"
 #include "openssl/ssl.h"
+#include "typedReferenceCount.h"
 
 class Ramfile;
 class HTTPClient;
@@ -57,22 +57,13 @@ class HTTPClient;
 //               requested from the same HTTPChannel until the first
 //               document has been fully retrieved.
 ////////////////////////////////////////////////////////////////////
-class EXPCL_PANDAEXPRESS HTTPChannel : public VirtualFile {
+class EXPCL_PANDAEXPRESS HTTPChannel : public TypedReferenceCount {
 private:
   HTTPChannel(HTTPClient *client);
 
 public:
   virtual ~HTTPChannel();
 
-  virtual VirtualFileSystem *get_file_system() const;
-  virtual Filename get_filename() const;
-
-  virtual bool is_regular_file() const;
-  virtual istream *open_read_file(bool auto_unwrap) const;
-  void close_read_file(istream *stream) const;
-
-  bool will_close_connection() const;
-
 PUBLISHED:
   // get_status_code() will either return an HTTP-style status code >=
   // 100 (e.g. 404), or one of the following values.  In general,
@@ -99,8 +90,8 @@ PUBLISHED:
     SC_ssl_invalid_server_certificate,
     SC_ssl_unexpected_server,
     
-    // These errors are only generated after a download_to_ram() or
-    // download_to_file() call has been issued.
+    // These errors are only generated after a download_to_*() call
+    // been issued.
     SC_download_open_error,
     SC_download_write_error,
     SC_download_invalid_range,
@@ -108,6 +99,7 @@ PUBLISHED:
 
   INLINE bool is_valid() const;
   INLINE bool is_connection_ready() const;
+
   INLINE const URLSpec &get_url() const;
   INLINE const DocumentSpec &get_document_spec() const;
   INLINE HTTPEnum::HTTPVersion get_http_version() const;
@@ -124,6 +116,7 @@ PUBLISHED:
 
   INLINE void set_persistent_connection(bool persistent_connection);
   INLINE bool get_persistent_connection() const;
+  bool will_close_connection() const;
 
   INLINE void set_allow_proxy(bool allow_proxy);
   INLINE bool get_allow_proxy() const;
@@ -138,6 +131,11 @@ PUBLISHED:
   INLINE void set_http_timeout(double timeout_seconds);
   INLINE double get_http_timeout() const;
 
+  INLINE void set_skip_body_size(size_t skip_body_size);
+  INLINE size_t get_skip_body_size() const;
+  INLINE void set_idle_timeout(double idle_timeout);
+  INLINE double get_idle_timeout() const;
+
   INLINE void set_download_throttle(bool download_throttle);
   INLINE bool get_download_throttle() const;
 
@@ -148,7 +146,7 @@ PUBLISHED:
   INLINE double get_max_updates_per_second() const;
 
   INLINE void set_expected_file_size(size_t file_size);
-  virtual off_t get_file_size() const;
+  off_t get_file_size() const;
   INLINE bool is_file_size_known() const;
 
   INLINE size_t get_first_byte_requested() const;
@@ -183,9 +181,12 @@ PUBLISHED:
   bool run();
   INLINE void begin_connect_to(const DocumentSpec &url);
 
-  ISocketStream *read_body();
+  ISocketStream *open_read_body();
+  void close_read_body(istream *stream) const;
+
   BLOCKING bool download_to_file(const Filename &filename, bool subdocument_resumes = true);
   BLOCKING bool download_to_ram(Ramfile *ramfile, bool subdocument_resumes = true);
+  BLOCKING bool download_to_stream(ostream *strm, bool subdocument_resumes = true);
   SocketStream *get_connection();
 
   INLINE size_t get_bytes_downloaded() const;
@@ -194,6 +195,7 @@ PUBLISHED:
 
 public:
   static string downcase(const string &s);
+  void body_stream_destructs(ISocketStream *stream);
 
 private:
   bool reached_done_state();
@@ -220,6 +222,7 @@ private:
 
   bool run_download_to_file();
   bool run_download_to_ram();
+  bool run_download_to_stream();
 
   void begin_request(HTTPEnum::Method method, const DocumentSpec &url, 
                      const string &body, bool nonblocking,
@@ -258,6 +261,7 @@ private:
 
   void reset_download_to();
   void reset_to_new();
+  void reset_body_stream();
   void close_connection();
 
   static bool more_useful_status_code(int a, int b);
@@ -312,6 +316,8 @@ private:
   bool _proxy_tunnel;
   double _connect_timeout;
   double _http_timeout;
+  size_t _skip_body_size;
+  double _idle_timeout;
   bool _blocking_connect;
   bool _download_throttle;
   double _max_bytes_per_second;
@@ -342,12 +348,14 @@ private:
     DD_none,
     DD_file,
     DD_ram,
+    DD_stream,
   };
   DownloadDest _download_dest;
   bool _subdocument_resumes;
   Filename _download_to_filename;
   pofstream _download_to_file;
   Ramfile *_download_to_ramfile;
+  ostream *_download_to_stream;
 
   int _read_index;
 
@@ -406,6 +414,7 @@ private:
   string _current_field_name;
   string _current_field_value;
   ISocketStream *_body_stream;
+  bool _owns_body_stream;
   BIO *_sbio;
   pvector<URLSpec> _redirect_trail;
   int _last_status_code;
@@ -420,9 +429,9 @@ public:
     return _type_handle;
   }
   static void init_type() {
-    VirtualFile::init_type();
+    TypedReferenceCount::init_type();
     register_type(_type_handle, "HTTPChannel",
-                  VirtualFile::get_class_type());
+                  TypedReferenceCount::get_class_type());
   }
 
 private:

panda/src/downloader/httpClient.cxx

@@ -39,6 +39,7 @@ bool HTTPClient::_ssl_initialized = false;
 // This is created once and never freed.
 X509_STORE *HTTPClient::_x509_store = NULL;
 
+PT(HTTPClient) HTTPClient::_global_ptr;
 
 ////////////////////////////////////////////////////////////////////
 //     Function: trim_blanks
@@ -1080,6 +1081,19 @@ get_header(const URLSpec &url) {
   return doc;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: HTTPClient::get_global_ptr
+//       Access: Published, Static
+//  Description: Returns the default global HTTPClient.
+////////////////////////////////////////////////////////////////////
+HTTPClient *HTTPClient::
+get_global_ptr() {
+  if (_global_ptr == NULL) {
+    _global_ptr = new HTTPClient;
+  }
+  return _global_ptr;
+}
+
 
 ////////////////////////////////////////////////////////////////////
 //     Function: HTTPClient::get_ssl_ctx

panda/src/downloader/httpClient.h

@@ -34,6 +34,7 @@
 #include "pvector.h"
 #include "pmap.h"
 #include "pset.h"
+#include "referenceCount.h"
 
 #include "openssl/ssl.h"
 
@@ -49,12 +50,18 @@ class HTTPChannel;
 //       Class : HTTPClient
 // Description : Handles contacting an HTTP server and retrieving a
 //               document.  Each HTTPClient object represents a
-//               separate context; it is up to the programmer whether
-//               one HTTPClient should be used to retrieve all
-//               documents, or a separate one should be created each
-//               time.
+//               separate context, and stores its own list of cookies,
+//               passwords, and certificates; however, a given
+//               HTTPClient is capable of making multiple simultaneous
+//               requests to the same or different servers.
+//
+//               It is up to the programmer whether one HTTPClient
+//               should be used to retrieve all documents, or a
+//               separate one should be created each time.  There is a
+//               default, global HTTPClient available in
+//               HTTPClient::get_global_ptr().
 ////////////////////////////////////////////////////////////////////
-class EXPCL_PANDAEXPRESS HTTPClient {
+class EXPCL_PANDAEXPRESS HTTPClient : public ReferenceCount {
 PUBLISHED:
   HTTPClient();
   HTTPClient(const HTTPClient &copy);
@@ -128,6 +135,8 @@ PUBLISHED:
   INLINE static string base64_encode(const string &s);
   INLINE static string base64_decode(const string &s);
 
+  static HTTPClient *get_global_ptr();
+
 public:
   SSL_CTX *get_ssl_ctx();
 
@@ -199,6 +208,9 @@ private:
 
   static bool _ssl_initialized;
   static X509_STORE *_x509_store;
+
+  static PT(HTTPClient) _global_ptr;
+
   friend class HTTPChannel;
 };
 

panda/src/downloader/identityStream.I

@@ -44,6 +44,6 @@ INLINE IIdentityStream &IIdentityStream::
 open(BioStreamPtr *source, HTTPChannel *doc, 
      bool has_content_length, size_t content_length) {
   clear((ios_iostate)0);
-  _buf.open_read(source, has_content_length, content_length);
+  _buf.open_read(source, doc, has_content_length, content_length);
   return *this;
 }

panda/src/downloader/identityStream.cxx

@@ -17,6 +17,19 @@
 // This module is not compiled if OpenSSL is not available.
 #ifdef HAVE_OPENSSL
 
+////////////////////////////////////////////////////////////////////
+//     Function: IIdentityStream::Destructor
+//       Access: Published, Virtual
+//  Description: 
+////////////////////////////////////////////////////////////////////
+IIdentityStream::
+~IIdentityStream() {
+  if (_channel != (HTTPChannel *)NULL) {
+    _channel->body_stream_destructs(this);
+    _channel = NULL;
+  }
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: IIdentityStream::is_closed
 //       Access: Public, Virtual

panda/src/downloader/identityStream.h

@@ -48,6 +48,7 @@ public:
 
   INLINE IIdentityStream &open(BioStreamPtr *source, HTTPChannel *doc,
                                bool has_content_length, size_t content_length);
+  virtual ~IIdentityStream();
 
   virtual bool is_closed();
   virtual void close();

panda/src/downloader/identityStreamBuf.cxx

@@ -31,6 +31,7 @@ IdentityStreamBuf::
 IdentityStreamBuf() {
   _has_content_length = true;
   _bytes_remaining = 0;
+  _wanted_nonblocking = false;
   _read_state = ISocketStream::RS_initial;
 
 #ifdef HAVE_IOSTREAM
@@ -67,10 +68,11 @@ IdentityStreamBuf::
 //               from the identity encoding.
 ////////////////////////////////////////////////////////////////////
 void IdentityStreamBuf::
-open_read(BioStreamPtr *source, 
+open_read(BioStreamPtr *source, HTTPChannel *doc,
           bool has_content_length, size_t content_length) {
   _source = source;
   _has_content_length = has_content_length;
+  _wanted_nonblocking = doc->_wanted_nonblocking;
   _bytes_remaining = content_length;
   _read_state = ISocketStream::RS_reading;
 }
@@ -134,6 +136,15 @@ read_chars(char *start, size_t length) {
     // file.
     (*_source)->read(start, length);
     read_count = (*_source)->gcount();
+
+    if (!_wanted_nonblocking) {
+      while (read_count == 0 && !(*_source)->is_closed()) {
+        // Simulate blocking.
+        thread_yield();
+        (*_source)->read(start, length);
+        read_count = (*_source)->gcount();
+      }
+    }
   
     if (read_count == 0) {
       if ((*_source)->is_closed()) {
@@ -144,12 +155,21 @@ read_chars(char *start, size_t length) {
     }
 
   } else {
-    // Extract some of the bytes remaining in the chunk.
+    // Extract some of the remaining bytes, but do not read past the
+    // content_length restriction.
 
     if (_bytes_remaining != 0) {
       length = min(length, _bytes_remaining);
       (*_source)->read(start, length);
       read_count = (*_source)->gcount();
+      if (!_wanted_nonblocking) {
+        while (read_count == 0 && !(*_source)->is_closed()) {
+          // Simulate blocking.
+          thread_yield();
+          (*_source)->read(start, length);
+          read_count = (*_source)->gcount();
+        }
+      }
       nassertr(read_count <= _bytes_remaining, 0);
       _bytes_remaining -= read_count;
   

panda/src/downloader/identityStreamBuf.h

@@ -24,6 +24,8 @@
 #include "pointerTo.h"
 #include "socketStream.h"
 
+class HTTPChannel;
+
 ////////////////////////////////////////////////////////////////////
 //       Class : IdentityStreamBuf
 // Description : The streambuf object that implements
@@ -34,7 +36,7 @@ public:
   IdentityStreamBuf();
   virtual ~IdentityStreamBuf();
 
-  void open_read(BioStreamPtr *source,
+  void open_read(BioStreamPtr *source, HTTPChannel *doc,
                  bool has_content_length, size_t content_length);
   void close_read();
 
@@ -50,6 +52,7 @@ private:
   PT(BioStreamPtr) _source;
   bool _has_content_length;
   size_t _bytes_remaining;
+  bool _wanted_nonblocking;
   ISocketStream::ReadState _read_state;
   char *_buffer;
 

panda/src/downloader/socketStream.I

@@ -181,6 +181,7 @@ flush() {
 ////////////////////////////////////////////////////////////////////
 INLINE ISocketStream::
 ISocketStream(streambuf *buf) : istream(buf), SSReader(this) {
+  _channel = NULL;
 }
 
 ////////////////////////////////////////////////////////////////////

panda/src/downloader/socketStream.cxx

@@ -15,6 +15,7 @@
 #include "socketStream.h"
 #include "datagram.h"
 #include "datagramIterator.h"
+#include "httpChannel.h"
 
 #ifdef HAVE_OPENSSL
 
@@ -220,4 +221,31 @@ send_datagram(const Datagram &dg) {
   return !is_closed();
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: ISocketStream::Destructor
+//       Access: Published, Virtual
+//  Description: 
+////////////////////////////////////////////////////////////////////
+ISocketStream::
+~ISocketStream() {
+  // This should already have been cleared by the subclass destructor.
+  nassertv(_channel == NULL);
+}
+
+
+////////////////////////////////////////////////////////////////////
+//     Function: ISocketStream::set_hold_ptr
+//       Access: Published
+//  Description: Associates a reference-counting pointer with the
+//               stream.  The stream does nothing with the pointer
+//               other than hold its reference count for its own
+//               lifetime (or until set_hold_ptr() is called again).
+//
+//               At the moment, this is used only by VirtualFileHTTP.
+////////////////////////////////////////////////////////////////////
+void ISocketStream::
+set_hold_ptr(const TypedReferenceCount *ptr) {
+  _hold_ptr = ptr;
+}
+
 #endif  // HAVE_OPENSSL

panda/src/downloader/socketStream.h

@@ -20,6 +20,8 @@
 #include "config_express.h" // for collect_tcp
 #include "datagram.h"
 #include "pdeque.h"
+#include "typedReferenceCount.h"
+#include "pointerTo.h"
 
 // At the present, this module is not compiled if OpenSSL is not
 // available, since the only current use for it is to implement
@@ -27,6 +29,8 @@
 
 #ifdef HAVE_OPENSSL
 
+class HTTPChannel;
+
 ////////////////////////////////////////////////////////////////////
 //       Class : SSReader
 // Description : An internal class for reading from a socket stream.
@@ -120,6 +124,7 @@ private:
 class EXPCL_PANDAEXPRESS ISocketStream : public istream, public SSReader {
 public:
   INLINE ISocketStream(streambuf *buf);
+  virtual ~ISocketStream();
 
 PUBLISHED:
   enum ReadState {
@@ -132,6 +137,16 @@ PUBLISHED:
   virtual bool is_closed() = 0;
   virtual void close() = 0;
   virtual ReadState get_read_state() = 0;
+
+  void set_hold_ptr(const TypedReferenceCount *ptr);
+
+protected:
+  HTTPChannel *_channel;
+
+private:
+  CPT(TypedReferenceCount) _hold_ptr;
+
+  friend class HTTPChannel;
 };
 
 ////////////////////////////////////////////////////////////////////

panda/src/putil/stringStream.h → panda/src/downloader/stringStream.h

@@ -24,7 +24,7 @@
 //               data to an internal buffer, which can be retrieved
 //               and/or set as a string.
 ////////////////////////////////////////////////////////////////////
-class EXPCL_PANDA_PUTIL StringStream : public iostream {
+class EXPCL_PANDAEXPRESS StringStream : public iostream {
 PUBLISHED:
   INLINE StringStream();
   INLINE StringStream(const string &source);

panda/src/putil/stringStreamBuf.h → panda/src/downloader/stringStreamBuf.h

@@ -25,7 +25,7 @@
 //               contents can be appended to or extracted at any time
 //               by application code.
 ////////////////////////////////////////////////////////////////////
-class EXPCL_PANDA_PUTIL StringStreamBuf : public streambuf {
+class EXPCL_PANDAEXPRESS StringStreamBuf : public streambuf {
 public:
   StringStreamBuf();
   virtual ~StringStreamBuf();

panda/src/downloader/virtualFileHTTP.I

@@ -0,0 +1,26 @@
+// Filename: virtualFileHTTP.I
+// Created by:  drose (31Oct08)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) Carnegie Mellon University.  All rights reserved.
+//
+// All use of this software is subject to the terms of the revised BSD
+// license.  You should have received a copy of this license along
+// with this source code in a file named "LICENSE."
+//
+////////////////////////////////////////////////////////////////////
+
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileHTTP::is_implicit_pz_file
+//       Access: Published
+//  Description: Returns true if this file is a .pz file that should
+//               be implicitly decompressed on load, or false if it is
+//               not a .pz file or if it should not be decompressed.
+////////////////////////////////////////////////////////////////////
+INLINE bool VirtualFileHTTP::
+is_implicit_pz_file() const {
+  return _implicit_pz_file;
+}

panda/src/downloader/virtualFileHTTP.cxx

@@ -0,0 +1,278 @@
+// Filename: virtualFileHTTP.cxx
+// Created by:  drose (31Oct08)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) Carnegie Mellon University.  All rights reserved.
+//
+// All use of this software is subject to the terms of the revised BSD
+// license.  You should have received a copy of this license along
+// with this source code in a file named "LICENSE."
+//
+////////////////////////////////////////////////////////////////////
+
+#include "virtualFileHTTP.h"
+#include "virtualFileMountHTTP.h"
+#include "stringStream.h"
+#include "zStream.h"
+
+TypeHandle VirtualFileHTTP::_type_handle;
+
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileHTTP::Constructor
+//       Access: Public
+//  Description: 
+////////////////////////////////////////////////////////////////////
+VirtualFileHTTP::
+VirtualFileHTTP(VirtualFileMountHTTP *mount, const Filename &local_filename,
+                bool implicit_pz_file, bool status_only) :
+  _mount(mount),
+  _local_filename(local_filename),
+  _implicit_pz_file(implicit_pz_file),
+  _status_only(status_only)
+{
+  URLSpec url(_mount->get_root());
+  url.set_path(_mount->get_root().get_path() + _local_filename.c_str());
+  _channel = _mount->get_channel();
+  if (_status_only) {
+    _channel->get_header(url);
+  } else {
+    _channel->get_document(url);
+  }
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileHTTP::Destructor
+//       Access: Published, Virtual
+//  Description: 
+////////////////////////////////////////////////////////////////////
+VirtualFileHTTP::
+~VirtualFileHTTP() {
+  // Recycle the associated HTTPChannel, so we can use it again later
+  // without having to close the connection to the server.
+  _mount->recycle_channel(_channel);
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileHTTP::get_file_system
+//       Access: Published, Virtual
+//  Description: Returns the VirtualFileSystem this file is associated
+//               with.
+////////////////////////////////////////////////////////////////////
+VirtualFileSystem *VirtualFileHTTP::
+get_file_system() const {
+  return _mount->get_file_system();
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileHTTP::get_filename
+//       Access: Published, Virtual
+//  Description: Returns the full pathname to this file within the
+//               virtual file system.
+////////////////////////////////////////////////////////////////////
+Filename VirtualFileHTTP::
+get_filename() const {
+  string mount_point = _mount->get_mount_point();
+  if (_local_filename.empty()) {
+    if (mount_point.empty()) {
+      return "/";
+    } else {
+      return string("/") + mount_point;
+    }
+
+  } else {
+    if (mount_point.empty()) {
+      return string("/") + _local_filename.get_fullpath();
+    } else {
+      return string("/") + mount_point + string("/") + _local_filename.get_fullpath();
+    }
+  }
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileHTTP::has_file
+//       Access: Published, Virtual
+//  Description: Returns true if this file exists, false otherwise.
+////////////////////////////////////////////////////////////////////
+bool VirtualFileHTTP::
+has_file() const {
+  return _channel->is_valid();
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileHTTP::is_directory
+//       Access: Published, Virtual
+//  Description: Returns true if this file represents a directory (and
+//               scan_directory() may be called), false otherwise.
+////////////////////////////////////////////////////////////////////
+bool VirtualFileHTTP::
+is_directory() const {
+  return false;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileHTTP::is_regular_file
+//       Access: Published, Virtual
+//  Description: Returns true if this file represents a regular file
+//               (and read_file() may be called), false otherwise.
+////////////////////////////////////////////////////////////////////
+bool VirtualFileHTTP::
+is_regular_file() const {
+  return _channel->is_valid();
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileHTTP::open_read_file
+//       Access: Published, Virtual
+//  Description: Opens the file for reading.  Returns a newly
+//               allocated istream on success (which you should
+//               eventually delete when you are done reading).
+//               Returns NULL on failure.
+//
+//               If auto_unwrap is true, an explicitly-named .pz file
+//               is automatically decompressed and the decompressed
+//               contents are returned.  This is different than
+//               vfs-implicit-pz, which will automatically decompress
+//               a file if the extension .pz is *not* given.
+////////////////////////////////////////////////////////////////////
+istream *VirtualFileHTTP::
+open_read_file(bool auto_unwrap) const {
+  if (_status_only) {
+    return NULL;
+  }
+
+  // We pre-download the file into a StringStream, then return a
+  // buffer to that.  It seems safer, since we can guarantee the file
+  // comes all at once without timeouts along the way.
+  StringStream *strstream = new StringStream;
+  if (!fetch_file(strstream)) {
+    delete strstream;
+    return NULL;
+  }
+  
+  return return_file(strstream, auto_unwrap);
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileHTTP::fetch_file
+//       Access: Private
+//  Description: Downloads the entire file from the web server into
+//               the indicated iostream.  Returns true on success,
+//               false on failure.
+//
+//               This seems to be safer than returning the socket
+//               stream directly, since this way we can better control
+//               timeouts and other internet hiccups.  We can also
+//               offer seeking on the resulting stream.
+////////////////////////////////////////////////////////////////////
+bool VirtualFileHTTP::
+fetch_file(ostream *buffer_stream) const {
+  _channel->download_to_stream(buffer_stream, false);
+  if (!_channel->is_download_complete()) {
+    // Failure to download fully.  Try again to download more.
+
+    URLSpec url(_mount->get_root());
+    url.set_path(_mount->get_root().get_path() + _local_filename.c_str());
+
+    size_t bytes_downloaded = _channel->get_bytes_downloaded();
+    size_t last_byte = bytes_downloaded;
+
+    while (bytes_downloaded > 0 && !_channel->is_download_complete()) {
+      _channel->get_subdocument(url, last_byte, 0);
+      _channel->download_to_stream(buffer_stream, true);
+      bytes_downloaded = _channel->get_bytes_downloaded();
+      last_byte = _channel->get_last_byte_delivered();
+    }
+  }
+
+  return _channel->is_download_complete() && _channel->is_valid();
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileHTTP::return_file
+//       Access: Private
+//  Description: After downloading the entire file via fetch_file(),
+//               rewinds the file stream and returns it as its own
+//               readable stream.
+////////////////////////////////////////////////////////////////////
+istream *VirtualFileHTTP::
+return_file(istream *buffer_stream, bool auto_unwrap) const {
+  // Will we be automatically unwrapping a .pz file?
+  bool do_unwrap = (_implicit_pz_file || (auto_unwrap && _local_filename.get_extension() == "pz"));
+
+  istream *result = buffer_stream;
+#ifdef HAVE_ZLIB
+  if (result != (istream *)NULL && do_unwrap) {
+    // We have to slip in a layer to decompress the file on the fly.
+    IDecompressStream *wrapper = new IDecompressStream(result, true);
+    result = wrapper;
+  }
+#endif  // HAVE_ZLIB
+
+  return result;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileHTTP::was_read_successful
+//       Access: Public
+//  Description: Call this method after a reading the istream returned
+//               by open_read_file() to completion.  If it returns
+//               true, the file was read completely and without error;
+//               if it returns false, there may have been some errors
+//               or a truncated file read.  This is particularly
+//               likely if the stream is a VirtualFileHTTP.
+////////////////////////////////////////////////////////////////////
+bool VirtualFileHTTP::
+was_read_successful() const {
+  return _channel->is_valid() && _channel->is_download_complete();
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileHTTP::get_file_size
+//       Access: Published, Virtual
+//  Description: Returns the current size on disk (or wherever it is)
+//               of the already-open file.  Pass in the stream that
+//               was returned by open_read_file(); some
+//               implementations may require this stream to determine
+//               the size.
+////////////////////////////////////////////////////////////////////
+off_t VirtualFileHTTP::
+get_file_size(istream *stream) const {
+  return _channel->get_file_size();
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileHTTP::get_file_size
+//       Access: Published, Virtual
+//  Description: Returns the current size on disk (or wherever it is)
+//               of the file before it has been opened.
+////////////////////////////////////////////////////////////////////
+off_t VirtualFileHTTP::
+get_file_size() const {
+  return _channel->get_file_size();
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileHTTP::get_timestamp
+//       Access: Published, Virtual
+//  Description: Returns a time_t value that represents the time the
+//               file was last modified, to within whatever precision
+//               the operating system records this information (on a
+//               Windows95 system, for instance, this may only be
+//               accurate to within 2 seconds).
+//
+//               If the timestamp cannot be determined, either because
+//               it is not supported by the operating system or
+//               because there is some error (such as file not found),
+//               returns 0.
+////////////////////////////////////////////////////////////////////
+time_t VirtualFileHTTP::
+get_timestamp() const {
+  const DocumentSpec &spec = _channel->get_document_spec();
+  if (spec.has_date()) {
+    return spec.get_date().get_time();
+  }
+  return 0;
+}

panda/src/downloader/virtualFileHTTP.h

@@ -0,0 +1,89 @@
+// Filename: virtualFileHTTP.h
+// Created by:  drose (31Oct08)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) Carnegie Mellon University.  All rights reserved.
+//
+// All use of this software is subject to the terms of the revised BSD
+// license.  You should have received a copy of this license along
+// with this source code in a file named "LICENSE."
+//
+////////////////////////////////////////////////////////////////////
+
+#ifndef VIRTUALFILEHTTP_H
+#define VIRTUALFILEHTTP_H
+
+#include "pandabase.h"
+
+#include "virtualFile.h"
+#include "httpChannel.h"
+#include "urlSpec.h"
+
+class VirtualFileMountHTTP;
+
+////////////////////////////////////////////////////////////////////
+//       Class : VirtualFileHTTP
+// Description : This maps a document retrieved from an HTTPClient
+//               into the VirtualFileSystem, allowing models etc. to
+//               be loaded directly from a web page.
+////////////////////////////////////////////////////////////////////
+class EXPCL_PANDAEXPRESS VirtualFileHTTP : public VirtualFile {
+public:
+  VirtualFileHTTP(VirtualFileMountHTTP *mount,
+                  const Filename &local_filename,
+                  bool implicit_pz_file,
+                  bool status_only);
+  virtual ~VirtualFileHTTP();
+
+  virtual VirtualFileSystem *get_file_system() const;
+  virtual Filename get_filename() const;
+
+  virtual bool has_file() const;
+  virtual bool is_directory() const;
+  virtual bool is_regular_file() const;
+  INLINE bool is_implicit_pz_file() const;
+
+  virtual istream *open_read_file(bool auto_unwrap) const;
+  virtual bool was_read_successful() const;
+  virtual off_t get_file_size(istream *stream) const;
+  virtual off_t get_file_size() const;
+  virtual time_t get_timestamp() const;
+
+private:
+  bool fetch_file(ostream *buffer_stream) const;
+  istream *return_file(istream *buffer_stream, bool auto_unwrap) const;
+
+  VirtualFileMountHTTP *_mount;
+  Filename _local_filename;
+  bool _implicit_pz_file;
+  bool _status_only;
+  URLSpec _url;
+  PT(HTTPChannel) _channel;
+
+public:
+  virtual TypeHandle get_type() const {
+    return get_class_type();
+  }
+  virtual TypeHandle force_init_type() {init_type(); return get_class_type();}
+
+PUBLISHED:
+  static TypeHandle get_class_type() {
+    return _type_handle;
+  }
+
+public:
+  static void init_type() {
+    VirtualFile::init_type();
+    register_type(_type_handle, "VirtualFileHTTP",
+                  VirtualFile::get_class_type());
+  }
+
+private:
+  static TypeHandle _type_handle;
+};
+
+#include "virtualFileHTTP.I"
+
+#endif

panda/src/downloader/virtualFileMountHTTP.I

@@ -0,0 +1,36 @@
+// Filename: virtualFileMountHTTP.I
+// Created by:  drose (30Oct08)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) Carnegie Mellon University.  All rights reserved.
+//
+// All use of this software is subject to the terms of the revised BSD
+// license.  You should have received a copy of this license along
+// with this source code in a file named "LICENSE."
+//
+////////////////////////////////////////////////////////////////////
+
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileMountHTTP::get_http_client
+//       Access: Published
+//  Description: Returns the HTTPClient object that services this
+//               mount point.
+////////////////////////////////////////////////////////////////////
+INLINE HTTPClient *VirtualFileMountHTTP::
+get_http_client() const {
+  return _http;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileMountHTTP::get_root
+//       Access: Published
+//  Description: Returns the URL that represents the root of this
+//               mount point.
+////////////////////////////////////////////////////////////////////
+INLINE const URLSpec &VirtualFileMountHTTP::
+get_root() const {
+  return _root;
+}

panda/src/downloader/virtualFileMountHTTP.cxx

@@ -0,0 +1,227 @@
+// Filename: virtualFileMountHTTP.cxx
+// Created by:  drose (30Oct08)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) Carnegie Mellon University.  All rights reserved.
+//
+// All use of this software is subject to the terms of the revised BSD
+// license.  You should have received a copy of this license along
+// with this source code in a file named "LICENSE."
+//
+////////////////////////////////////////////////////////////////////
+
+#include "virtualFileMountHTTP.h"
+#include "virtualFileHTTP.h"
+#include "virtualFileSystem.h"
+
+#ifdef HAVE_OPENSSL
+
+TypeHandle VirtualFileMountHTTP::_type_handle;
+
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileMountHTTP::Constructor
+//       Access: Published
+//  Description: 
+////////////////////////////////////////////////////////////////////
+VirtualFileMountHTTP::
+VirtualFileMountHTTP(const URLSpec &root, HTTPClient *http) :
+  _http(http),
+  _root(root)
+{
+  // Make sure the root ends on a slash.  The implicit trailing slash
+  // is a semi-standard internet convention.
+  string path = _root.get_path();
+  if (!path.empty() && path[path.length() - 1] != '/') {
+    path += '/';
+    _root.set_path(path);
+  }
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileMountHTTP::Destructor
+//       Access: Public, Virtual
+//  Description: 
+////////////////////////////////////////////////////////////////////
+VirtualFileMountHTTP::
+~VirtualFileMountHTTP() {
+}
+
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileMountHTTP::has_file
+//       Access: Public, Virtual
+//  Description: Returns true if the indicated file exists within the
+//               mount system.
+////////////////////////////////////////////////////////////////////
+bool VirtualFileMountHTTP::
+has_file(const Filename &) const {
+  return false;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileMountHTTP::is_directory
+//       Access: Public, Virtual
+//  Description: Returns true if the indicated file exists within the
+//               mount system and is a directory.
+////////////////////////////////////////////////////////////////////
+bool VirtualFileMountHTTP::
+is_directory(const Filename &) const {
+  return false;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileMountHTTP::is_regular_file
+//       Access: Public, Virtual
+//  Description: Returns true if the indicated file exists within the
+//               mount system and is a regular file.
+////////////////////////////////////////////////////////////////////
+bool VirtualFileMountHTTP::
+is_regular_file(const Filename &) const {
+  return false;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileMountHTTP::make_virtual_file
+//       Access: Public, Virtual
+//  Description: Constructs and returns a new VirtualFile instance
+//               that corresponds to the indicated filename within
+//               this mount point.  The returned VirtualFile object
+//               does not imply that the given file actually exists;
+//               but if the file does exist, then the handle can be
+//               used to read it.
+////////////////////////////////////////////////////////////////////
+PT(VirtualFile) VirtualFileMountHTTP::
+make_virtual_file(const string &local_filename,
+                  const Filename &original_filename, bool implicit_pz_file,
+                  bool status_only) {
+  PT(VirtualFileHTTP) vfile = 
+    new VirtualFileHTTP(this, local_filename, implicit_pz_file, status_only);
+  vfile->set_original_filename(original_filename);
+
+  return vfile.p();
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileMountHTTP::open_read_file
+//       Access: Public, Virtual
+//  Description: Opens the file for reading, if it exists.  Returns a
+//               newly allocated istream on success (which you should
+//               eventually delete when you are done reading).
+//               Returns NULL on failure.
+////////////////////////////////////////////////////////////////////
+istream *VirtualFileMountHTTP::
+open_read_file(const Filename &) const {
+  return NULL;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileMountHTTP::get_file_size
+//       Access: Published, Virtual
+//  Description: Returns the current size on disk (or wherever it is)
+//               of the already-open file.  Pass in the stream that
+//               was returned by open_read_file(); some
+//               implementations may require this stream to determine
+//               the size.
+////////////////////////////////////////////////////////////////////
+off_t VirtualFileMountHTTP::
+get_file_size(const Filename &, istream *) const {
+  return 0;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileMountHTTP::get_file_size
+//       Access: Published, Virtual
+//  Description: Returns the current size on disk (or wherever it is)
+//               of the file before it has been opened.
+////////////////////////////////////////////////////////////////////
+off_t VirtualFileMountHTTP::
+get_file_size(const Filename &) const {
+  return 0;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileMountHTTP::get_timestamp
+//       Access: Published, Virtual
+//  Description: Returns a time_t value that represents the time the
+//               file was last modified, to within whatever precision
+//               the operating system records this information (on a
+//               Windows95 system, for instance, this may only be
+//               accurate to within 2 seconds).
+//
+//               If the timestamp cannot be determined, either because
+//               it is not supported by the operating system or
+//               because there is some error (such as file not found),
+//               returns 0.
+////////////////////////////////////////////////////////////////////
+time_t VirtualFileMountHTTP::
+get_timestamp(const Filename &) const {
+  return 0;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileMountHTTP::scan_directory
+//       Access: Public, Virtual
+//  Description: Fills the given vector up with the list of filenames
+//               that are local to this directory, if the filename is
+//               a directory.  Returns true if successful, or false if
+//               the file is not a directory or cannot be read.
+////////////////////////////////////////////////////////////////////
+bool VirtualFileMountHTTP::
+scan_directory(vector_string &, const Filename &) const {
+  return false;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileMountHTTP::output
+//       Access: Public, Virtual
+//  Description: 
+////////////////////////////////////////////////////////////////////
+void VirtualFileMountHTTP::
+output(ostream &out) const {
+  out << _root;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileMountHTTP::get_channel
+//       Access: Public
+//  Description: Returns an HTTPChannel object suitable for use for
+//               extracting a document from the current URL root.
+////////////////////////////////////////////////////////////////////
+PT(HTTPChannel) VirtualFileMountHTTP::
+get_channel() {
+  PT(HTTPChannel) channel;
+  _channels_lock.acquire();
+
+  if (!_channels.empty()) {
+    // If we have some channels sitting around, grab one.  Grab the
+    // one on the end; it was most recently pushed, and therefore most
+    // likely to be still alive.
+    channel = _channels.back();
+    _channels.pop_back();
+  } else {
+    // If we don't have any channels standing by, make a new one.
+    channel = _http->make_channel(true);
+  }
+
+  _channels_lock.release();
+  return channel;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileMountHTTP::recycle_channel
+//       Access: Public
+//  Description: Accepts an HTTPChannel that is no longer being used,
+//               and restores it to standby duty, so that it will be
+//               returned by a future call to get_channel().
+////////////////////////////////////////////////////////////////////
+void VirtualFileMountHTTP::
+recycle_channel(HTTPChannel *channel) {
+  _channels_lock.acquire();
+  _channels.push_back(channel);
+  _channels_lock.release();
+}
+
+#endif  // HAVE_OPENSSL

panda/src/downloader/virtualFileMountHTTP.h

@@ -0,0 +1,95 @@
+// Filename: virtualFileMountHTTP.h
+// Created by:  drose (30Oct08)
+//
+////////////////////////////////////////////////////////////////////
+//
+// PANDA 3D SOFTWARE
+// Copyright (c) Carnegie Mellon University.  All rights reserved.
+//
+// All use of this software is subject to the terms of the revised BSD
+// license.  You should have received a copy of this license along
+// with this source code in a file named "LICENSE."
+//
+////////////////////////////////////////////////////////////////////
+
+#ifndef VIRTUALFILEMOUNTHTTP_H
+#define VIRTUALFILEMOUNTHTTP_H
+
+#include "pandabase.h"
+
+#ifdef HAVE_OPENSSL
+
+#include "virtualFileMount.h"
+#include "httpClient.h"
+#include "httpChannel.h"
+#include "urlSpec.h"
+#include "pointerTo.h"
+#include "mutexImpl.h"
+
+////////////////////////////////////////////////////////////////////
+//       Class : VirtualFileMountHTTP
+// Description : Maps a web page (URL root) into the
+//               VirtualFileSystem.
+////////////////////////////////////////////////////////////////////
+class EXPCL_PANDAEXPRESS VirtualFileMountHTTP : public VirtualFileMount {
+PUBLISHED:
+  VirtualFileMountHTTP(const URLSpec &root, HTTPClient *http = HTTPClient::get_global_ptr());
+  virtual ~VirtualFileMountHTTP();
+
+  INLINE HTTPClient *get_http_client() const;
+  INLINE const URLSpec &get_root() const;
+
+public:
+  virtual PT(VirtualFile) make_virtual_file(const string &local_filename,
+                                            const Filename &original_filename, 
+                                            bool implicit_pz_file,
+                                            bool status_only);
+
+  virtual bool has_file(const Filename &file) const;
+  virtual bool is_directory(const Filename &file) const;
+  virtual bool is_regular_file(const Filename &file) const;
+
+  virtual istream *open_read_file(const Filename &file) const;
+  virtual off_t get_file_size(const Filename &file, istream *stream) const;
+  virtual off_t get_file_size(const Filename &file) const;
+  virtual time_t get_timestamp(const Filename &file) const;
+
+  virtual bool scan_directory(vector_string &contents, 
+                              const Filename &dir) const;
+
+  virtual void output(ostream &out) const;
+
+  PT(HTTPChannel) get_channel();
+  void recycle_channel(HTTPChannel *channel);
+
+private:
+  PT(HTTPClient) _http;
+  URLSpec _root;
+
+  MutexImpl _channels_lock;
+  typedef pvector< PT(HTTPChannel) > Channels;
+  Channels _channels;
+
+public:
+  virtual TypeHandle get_type() const {
+    return get_class_type();
+  }
+  virtual TypeHandle force_init_type() {init_type(); return get_class_type();}
+  static TypeHandle get_class_type() {
+    return _type_handle;
+  }
+  static void init_type() {
+    VirtualFileMount::init_type();
+    register_type(_type_handle, "VirtualFileMountHTTP",
+                  VirtualFileMount::get_class_type());
+  }
+
+private:
+  static TypeHandle _type_handle;
+};
+
+#include "virtualFileMountHTTP.I"
+
+#endif  // HAVE_OPENSSL
+
+#endif

panda/src/express/virtualFile.cxx

@@ -20,6 +20,16 @@
 
 TypeHandle VirtualFile::_type_handle;
 
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFile::has_file
+//       Access: Published, Virtual
+//  Description: Returns true if this file exists, false otherwise.
+////////////////////////////////////////////////////////////////////
+bool VirtualFile::
+has_file() const {
+  return false;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: VirtualFile::is_directory
 //       Access: Published, Virtual
@@ -170,7 +180,7 @@ open_read_file(bool auto_unwrap) const {
 ////////////////////////////////////////////////////////////////////
 off_t VirtualFile::
 get_file_size(istream *stream) const {
-  return 0;
+  return get_file_size();
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -228,6 +238,21 @@ close_read_file(istream *stream) const {
   }
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFile::was_read_successful
+//       Access: Public
+//  Description: Call this method after a reading the istream returned
+//               by open_read_file() to completion.  If it returns
+//               true, the file was read completely and without error;
+//               if it returns false, there may have been some errors
+//               or a truncated file read.  This is particularly
+//               likely if the stream is a VirtualFileHTTP.
+////////////////////////////////////////////////////////////////////
+bool VirtualFile::
+was_read_successful() const {
+  return true;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: VirtualFile::read_file
 //       Access: Public

panda/src/express/virtualFile.h

@@ -42,6 +42,7 @@ PUBLISHED:
   virtual Filename get_filename() const=0;
   INLINE const Filename &get_original_filename() const;
 
+  virtual bool has_file() const;
   virtual bool is_directory() const;
   virtual bool is_regular_file() const;
 
@@ -54,6 +55,7 @@ PUBLISHED:
   BLOCKING INLINE string read_file(bool auto_unwrap) const;
   BLOCKING virtual istream *open_read_file(bool auto_unwrap) const;
   BLOCKING void close_read_file(istream *stream) const;
+  virtual bool was_read_successful() const;
   BLOCKING virtual off_t get_file_size(istream *stream) const;
   BLOCKING virtual off_t get_file_size() const;
   BLOCKING virtual time_t get_timestamp() const;

panda/src/express/virtualFileComposite.cxx

@@ -39,6 +39,16 @@ get_filename() const {
   return _filename;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileComposite::has_file
+//       Access: Published, Virtual
+//  Description: Returns true if this file exists, false otherwise.
+////////////////////////////////////////////////////////////////////
+bool VirtualFileComposite::
+has_file() const {
+  return true;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: VirtualFileComposite::is_directory
 //       Access: Published, Virtual

panda/src/express/virtualFileComposite.h

@@ -36,6 +36,7 @@ public:
   virtual VirtualFileSystem *get_file_system() const;
   virtual Filename get_filename() const;
 
+  virtual bool has_file() const;
   virtual bool is_directory() const;
 
 protected:

panda/src/express/virtualFileMount.I

@@ -19,14 +19,9 @@
 //  Description: 
 ////////////////////////////////////////////////////////////////////
 INLINE VirtualFileMount::
-VirtualFileMount(VirtualFileSystem *file_system,
-                 const Filename &physical_filename,
-                 const Filename &mount_point,
-                 int mount_flags) :
-  _file_system(file_system),
-  _physical_filename(physical_filename),
-  _mount_point(mount_point),
-  _mount_flags(mount_flags)
+VirtualFileMount() :
+  _file_system(NULL),
+  _mount_flags(0)
 {
 }
 
@@ -42,17 +37,6 @@ get_file_system() const {
 }
 
 
-////////////////////////////////////////////////////////////////////
-//     Function: VirtualFileMount::get_physical_filename
-//       Access: Public
-//  Description: Returns the name of the source file on the OS
-//               filesystem of the directory or file that is mounted.
-////////////////////////////////////////////////////////////////////
-INLINE const Filename &VirtualFileMount::
-get_physical_filename() const {
-  return _physical_filename;
-}
-
 ////////////////////////////////////////////////////////////////////
 //     Function: VirtualFileMount::get_mount_point
 //       Access: Public

panda/src/express/virtualFileMount.cxx

@@ -13,6 +13,7 @@
 ////////////////////////////////////////////////////////////////////
 
 #include "virtualFileMount.h"
+#include "virtualFileSimple.h"
 
 TypeHandle VirtualFileMount::_type_handle;
 
@@ -24,6 +25,32 @@ TypeHandle VirtualFileMount::_type_handle;
 ////////////////////////////////////////////////////////////////////
 VirtualFileMount::
 ~VirtualFileMount() {
+  nassertv(_file_system == NULL);
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileMount::make_virtual_file
+//       Access: Public, Virtual
+//  Description: Constructs and returns a new VirtualFile instance
+//               that corresponds to the indicated filename within
+//               this mount point.  The returned VirtualFile object
+//               does not imply that the given file actually exists;
+//               but if the file does exist, then the handle can be
+//               used to read it.
+////////////////////////////////////////////////////////////////////
+PT(VirtualFile) VirtualFileMount::
+make_virtual_file(const string &local_filename,
+                  const Filename &original_filename, bool implicit_pz_file,
+                  bool) {
+  Filename local(local_filename);
+  if (original_filename.is_text()) {
+    local.set_text();
+  }
+  PT(VirtualFileSimple) file =
+    new VirtualFileSimple(this, local, implicit_pz_file);
+  file->set_original_filename(original_filename);
+
+  return file.p();
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -58,7 +85,7 @@ close_read_file(istream *stream) const {
 ////////////////////////////////////////////////////////////////////
 void VirtualFileMount::
 output(ostream &out) const {
-  out << get_physical_filename();
+  out << get_type();
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -68,5 +95,5 @@ output(ostream &out) const {
 ////////////////////////////////////////////////////////////////////
 void VirtualFileMount::
 write(ostream &out) const {
-  out << get_physical_filename() << " on /" << get_mount_point() << "\n";
+  out << *this << " on /" << get_mount_point() << "\n";
 }

panda/src/express/virtualFileMount.h

@@ -17,9 +17,10 @@
 
 #include "pandabase.h"
 
+#include "virtualFile.h"
 #include "filename.h"
 #include "pointerTo.h"
-#include "typedObject.h"
+#include "typedReferenceCount.h"
 
 class VirtualFileSystem;
 
@@ -29,19 +30,21 @@ class VirtualFileSystem;
 //               within a VirtualFileSystem.  Normally users don't
 //               need to monkey with this class directly.
 ////////////////////////////////////////////////////////////////////
-class EXPCL_PANDAEXPRESS VirtualFileMount : public TypedObject {
-public:
-  INLINE VirtualFileMount(VirtualFileSystem *file_system,
-                          const Filename &physical_filename,
-                          const Filename &mount_point,
-                          int mount_flags);
+class EXPCL_PANDAEXPRESS VirtualFileMount : public TypedReferenceCount {
+PUBLISHED:
+  INLINE VirtualFileMount();
   virtual ~VirtualFileMount();
 
   INLINE VirtualFileSystem *get_file_system() const;
-  INLINE const Filename &get_physical_filename() const;
   INLINE const Filename &get_mount_point() const;
   INLINE int get_mount_flags() const;
 
+public:
+  virtual PT(VirtualFile) make_virtual_file(const string &local_filename,
+                                            const Filename &original_filename,
+                                            bool implicit_pz_file,
+                                            bool status_only);
+
   virtual bool has_file(const Filename &file) const=0;
   virtual bool is_directory(const Filename &file) const=0;
   virtual bool is_regular_file(const Filename &file) const=0;
@@ -55,13 +58,12 @@ public:
   virtual bool scan_directory(vector_string &contents, 
                               const Filename &dir) const=0;
 
-
+PUBLISHED:
   virtual void output(ostream &out) const;
   virtual void write(ostream &out) const;
 
 protected:
   VirtualFileSystem *_file_system;
-  Filename _physical_filename;
   Filename _mount_point;
   int _mount_flags;
 
@@ -75,13 +77,15 @@ public:
     return _type_handle;
   }
   static void init_type() {
-    TypedObject::init_type();
+    TypedReferenceCount::init_type();
     register_type(_type_handle, "VirtualFileMount",
-                  TypedObject::get_class_type());
+                  TypedReferenceCount::get_class_type());
   }
 
 private:
   static TypeHandle _type_handle;
+
+  friend class VirtualFileSystem;
 };
 
 INLINE ostream &operator << (ostream &out, const VirtualFileMount &mount);

panda/src/express/virtualFileMountMultifile.I

@@ -19,12 +19,7 @@
 //  Description: 
 ////////////////////////////////////////////////////////////////////
 INLINE VirtualFileMountMultifile::
-VirtualFileMountMultifile(VirtualFileSystem *file_system,
-                          Multifile *multifile,
-                          const Filename &mount_point,
-                          int mount_flags) :
-  VirtualFileMount(file_system, multifile->get_multifile_name(),
-                   mount_point, mount_flags),
+VirtualFileMountMultifile(Multifile *multifile) :
   _multifile(multifile)
 {
 }

panda/src/express/virtualFileMountMultifile.cxx

@@ -149,3 +149,13 @@ scan_directory(vector_string &contents, const Filename &dir) const {
   return _multifile->scan_directory(contents, dir);
 }
 
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileMountMultifile::output
+//       Access: Public, Virtual
+//  Description: 
+////////////////////////////////////////////////////////////////////
+void VirtualFileMountMultifile::
+output(ostream &out) const {
+  out << _multifile->get_multifile_name();
+}

panda/src/express/virtualFileMountMultifile.h

@@ -27,15 +27,13 @@
 //               VirtualFileSystem.
 ////////////////////////////////////////////////////////////////////
 class EXPCL_PANDAEXPRESS VirtualFileMountMultifile : public VirtualFileMount {
-public:
-  INLINE VirtualFileMountMultifile(VirtualFileSystem *file_system,
-                                   Multifile *multifile, 
-                                   const Filename &mount_point,
-                                   int mount_flags);
+PUBLISHED:
+  INLINE VirtualFileMountMultifile(Multifile *multifile);
   virtual ~VirtualFileMountMultifile();
 
   INLINE Multifile *get_multifile() const;
 
+public:
   virtual bool has_file(const Filename &file) const;
   virtual bool is_directory(const Filename &file) const;
   virtual bool is_regular_file(const Filename &file) const;
@@ -48,6 +46,7 @@ public:
   virtual bool scan_directory(vector_string &contents, 
                               const Filename &dir) const;
 
+  virtual void output(ostream &out) const;
 
 private:
   PT(Multifile) _multifile;

panda/src/express/virtualFileMountSystem.I

@@ -19,10 +19,18 @@
 //  Description: 
 ////////////////////////////////////////////////////////////////////
 INLINE VirtualFileMountSystem::
-VirtualFileMountSystem(VirtualFileSystem *file_system,
-                       const Filename &physical_filename,
-                       const Filename &mount_point,
-                       int mount_flags) :
-  VirtualFileMount(file_system, physical_filename, mount_point, mount_flags)
+VirtualFileMountSystem(const Filename &physical_filename) :
+  _physical_filename(physical_filename)
 {
 }
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileMountSystem::get_physical_filename
+//       Access: Public
+//  Description: Returns the name of the source file on the OS
+//               filesystem of the directory or file that is mounted.
+////////////////////////////////////////////////////////////////////
+INLINE const Filename &VirtualFileMountSystem::
+get_physical_filename() const {
+  return _physical_filename;
+}

panda/src/express/virtualFileMountSystem.cxx

@@ -195,3 +195,13 @@ scan_directory(vector_string &contents, const Filename &dir) const {
   return pathname.scan_directory(contents);
 }
 
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileMountSystem::output
+//       Access: Public, Virtual
+//  Description: 
+////////////////////////////////////////////////////////////////////
+void VirtualFileMountSystem::
+output(ostream &out) const {
+  out << get_physical_filename();
+}

panda/src/express/virtualFileMountSystem.h

@@ -25,13 +25,12 @@
 //               VirtualFileSystem.
 ////////////////////////////////////////////////////////////////////
 class EXPCL_PANDAEXPRESS VirtualFileMountSystem : public VirtualFileMount {
-public:
-  INLINE VirtualFileMountSystem(VirtualFileSystem *file_system,
-                                const Filename &physical_filename,
-                                const Filename &mount_point,
-                                int mount_flags);
+PUBLISHED:
+  INLINE VirtualFileMountSystem(const Filename &physical_filename);
 
+  INLINE const Filename &get_physical_filename() const;
 
+public:
   virtual bool has_file(const Filename &file) const;
   virtual bool is_directory(const Filename &file) const;
   virtual bool is_regular_file(const Filename &file) const;
@@ -44,6 +43,11 @@ public:
   virtual bool scan_directory(vector_string &contents, 
                               const Filename &dir) const;
 
+  virtual void output(ostream &out) const;
+
+private:
+  Filename _physical_filename;
+
 public:
   virtual TypeHandle get_type() const {
     return get_class_type();

panda/src/express/virtualFileSimple.cxx

@@ -56,6 +56,16 @@ get_filename() const {
   }
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileSimple::has_file
+//       Access: Published, Virtual
+//  Description: Returns true if this file exists, false otherwise.
+////////////////////////////////////////////////////////////////////
+bool VirtualFileSimple::
+has_file() const {
+  return _mount->has_file(_local_filename);
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: VirtualFileSimple::is_directory
 //       Access: Published, Virtual

panda/src/express/virtualFileSimple.h

@@ -35,6 +35,7 @@ public:
   virtual VirtualFileSystem *get_file_system() const;
   virtual Filename get_filename() const;
 
+  virtual bool has_file() const;
   virtual bool is_directory() const;
   virtual bool is_regular_file() const;
   INLINE bool is_implicit_pz_file() const;

panda/src/express/virtualFileSystem.I

@@ -21,7 +21,8 @@
 ////////////////////////////////////////////////////////////////////
 INLINE bool VirtualFileSystem::
 exists(const Filename &filename) const {
-  return get_file(filename) != (VirtualFile *)NULL;
+  cerr << "exists: " << filename << "\n";
+  return get_file(filename, true) != (VirtualFile *)NULL;
 }
 
 ////////////////////////////////////////////////////////////////////

panda/src/express/virtualFileSystem.cxx

@@ -55,12 +55,9 @@ VirtualFileSystem::
 ////////////////////////////////////////////////////////////////////
 bool VirtualFileSystem::
 mount(Multifile *multifile, const string &mount_point, int flags) {
-  VirtualFileMountMultifile *mount = 
-    new VirtualFileMountMultifile(this, multifile, 
-                                  normalize_mount_point(mount_point),
-                                  flags);
-  _mounts.push_back(mount);
-  return true;
+  VirtualFileMountMultifile *new_mount = 
+    new VirtualFileMountMultifile(multifile);
+  return mount(new_mount, mount_point, flags);
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -94,16 +91,12 @@ mount(const Filename &physical_filename, const string &mount_point,
   }
 
   if (physical_filename.is_directory()) {
-    VirtualFileMountSystem *mount =
-      new VirtualFileMountSystem(this, physical_filename, 
-                                 normalize_mount_point(mount_point),
-                                 flags);
-    _mounts.push_back(mount);
-    return true;
+    VirtualFileMountSystem *new_mount =
+      new VirtualFileMountSystem(physical_filename);
+    return mount(new_mount, mount_point, flags);
   } else {
     // It's not a directory; it must be a Multifile.
     PT(Multifile) multifile = new Multifile;
-
     multifile->set_encryption_password(password);
 
     // For now these are always opened read only.  Maybe later we'll
@@ -117,6 +110,24 @@ mount(const Filename &physical_filename, const string &mount_point,
   }
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileSystem::mount
+//       Access: Published
+//  Description: Adds the given VirtualFileMount object to the mount
+//               list.  This is a lower-level function that the other
+//               flavors of mount(); it requires you to create a
+//               VirtualFileMount object specifically.
+////////////////////////////////////////////////////////////////////
+bool VirtualFileSystem::
+mount(VirtualFileMount *mount, const string &mount_point, int flags) {
+  nassertr(mount->_file_system == NULL, false);
+  mount->_file_system = this;
+  mount->_mount_point = normalize_mount_point(mount_point);
+  mount->_mount_flags = flags;
+  _mounts.push_back(mount);
+  return true;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: VirtualFileSystem::unmount
 //       Access: Published
@@ -137,7 +148,8 @@ unmount(Multifile *multifile) {
         DCAST(VirtualFileMountMultifile, mount);
       if (mmount->get_multifile() == multifile) {
         // Remove this one.  Don't increment wi.
-        delete mount;
+        mount->_file_system = NULL;
+
       } else {
         // Don't remove this one.
         ++wi;
@@ -157,10 +169,9 @@ unmount(Multifile *multifile) {
 ////////////////////////////////////////////////////////////////////
 //     Function: VirtualFileSystem::unmount
 //       Access: Published
-//  Description: Unmounts all appearances of the indicated physical
-//               filename (either a directory name or a Multifile
-//               name) from the file system.  Returns the number of
-//               appearances unmounted.
+//  Description: Unmounts all appearances of the indicated directory
+//               name or multifile name from the file system.  Returns
+//               the number of appearances unmounted.
 ////////////////////////////////////////////////////////////////////
 int VirtualFileSystem::
 unmount(const Filename &physical_filename) {
@@ -170,9 +181,59 @@ unmount(const Filename &physical_filename) {
     VirtualFileMount *mount = (*ri);
     (*wi) = mount;
 
-    if (mount->get_physical_filename() == physical_filename) {
+    if (mount->is_exact_type(VirtualFileMountSystem::get_class_type())) {
+      VirtualFileMountSystem *smount = 
+        DCAST(VirtualFileMountSystem, mount);
+      if (smount->get_physical_filename() == physical_filename) {
+        // Remove this one.  Don't increment wi.
+        mount->_file_system = NULL;
+        
+      } else {
+        // Don't remove this one.
+        ++wi;
+      }
+
+    } else if (mount->is_exact_type(VirtualFileMountMultifile::get_class_type())) {
+      VirtualFileMountMultifile *mmount = 
+        DCAST(VirtualFileMountMultifile, mount);
+      if (mmount->get_multifile()->get_multifile_name() == physical_filename) {
+        // Remove this one.  Don't increment wi.
+        mount->_file_system = NULL;
+
+      } else {
+        // Don't remove this one.
+        ++wi;
+      }
+
+    } else {
+      // Don't remove this one.
+      ++wi;
+    }
+    ++ri;
+  }
+
+  int num_removed = _mounts.end() - wi;
+  _mounts.erase(wi, _mounts.end());
+  return num_removed;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileSystem::unmount
+//       Access: Published
+//  Description: Unmounts the indicated VirtualFileMount object
+//               from the file system.  Returns the number of
+//               appearances unmounted.
+////////////////////////////////////////////////////////////////////
+int VirtualFileSystem::
+unmount(VirtualFileMount *mount) {
+  Mounts::iterator ri, wi;
+  wi = ri = _mounts.begin();
+  while (ri != _mounts.end()) {
+    (*wi) = (*ri);
+    if ((*ri) == mount) {
       // Remove this one.  Don't increment wi.
-      delete mount;
+      (*ri)->_file_system = NULL;
+
     } else {
       // Don't remove this one.
       ++wi;
@@ -203,7 +264,8 @@ unmount_point(const string &mount_point) {
 
     if (mount->get_mount_point() == nmp) {
       // Remove this one.  Don't increment wi.
-      delete mount;
+      mount->_file_system = NULL;
+
     } else {
       // Don't remove this one.
       ++wi;
@@ -224,10 +286,9 @@ unmount_point(const string &mount_point) {
 ////////////////////////////////////////////////////////////////////
 int VirtualFileSystem::
 unmount_all() {
-  Mounts::iterator mi;
-  for (mi = _mounts.begin(); mi != _mounts.end(); ++mi) {
-    VirtualFileMount *mount = (*mi);
-    delete mount;
+  Mounts::iterator ri;
+  for (ri = _mounts.begin(); ri != _mounts.end(); ++ri) {
+    (*ri)->_file_system = NULL;
   }
 
   int num_removed = _mounts.size();
@@ -235,6 +296,28 @@ unmount_all() {
   return num_removed;
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileSystem::get_num_mounts
+//       Access: Published
+//  Description: Returns the number of individual mounts in the
+//               system.
+////////////////////////////////////////////////////////////////////
+int VirtualFileSystem::
+get_num_mounts() const {
+  return _mounts.size();
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: VirtualFileSystem::get_mount
+//       Access: Published
+//  Description: Returns the nth mount in the system.
+////////////////////////////////////////////////////////////////////
+VirtualFileMount *VirtualFileSystem::
+get_mount(int n) const {
+  nassertr(n >= 0 && n < (int)_mounts.size(), NULL);
+  return _mounts[n];
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: VirtualFileSystem::chdir
 //       Access: Published
@@ -254,7 +337,7 @@ chdir(const string &new_directory) {
     return true;
   }
 
-  PT(VirtualFile) file = get_file(new_directory);
+  PT(VirtualFile) file = get_file(new_directory, true);
   if (file != (VirtualFile *)NULL && file->is_directory()) {
     _cwd = file->get_filename();
     return true;
@@ -278,9 +361,18 @@ get_cwd() const {
 //  Description: Looks up the file by the indicated name in the file
 //               system.  Returns a VirtualFile pointer representing
 //               the file if it is found, or NULL if it is not.
+//
+//               If status_only is true, the file will be checked for
+//               existence and length and so on, but the returned
+//               file's contents cannot be read.  This is an
+//               optimization which is especially important for
+//               certain mount types, for instance HTTP, for which
+//               opening a file to determine its status is
+//               substantially less expensive than opening it to read
+//               its contents.
 ////////////////////////////////////////////////////////////////////
 PT(VirtualFile) VirtualFileSystem::
-get_file(const Filename &filename) const {
+get_file(const Filename &filename, bool status_only) const {
   nassertr(!filename.empty(), NULL);
   Filename pathname(filename);
   if (pathname.is_local()) {
@@ -306,26 +398,24 @@ get_file(const Filename &filename) const {
     if (strpath == mount_point) {
       // Here's an exact match on the mount point.  This filename is
       // the root directory of this mount object.
-      if (found_match(found_file, composite_file, mount, "", pathname,
-                      false)) {
+      if (consider_match(found_file, composite_file, mount, "", pathname,
+                         false, status_only)) {
         return found_file;
       }
     } else if (mount_point.empty()) {
       // This is the root mount point; all files are in here.
-      if (mount->has_file(strpath)) {
-        // Bingo!
-        if (found_match(found_file, composite_file, mount, strpath, 
-                        pathname, false)) {
-          return found_file;
-        }
+      if (consider_match(found_file, composite_file, mount, strpath, 
+                         pathname, false, status_only)) {
+        return found_file;
+      }
 #ifdef HAVE_ZLIB
-      } else if (vfs_implicit_pz && mount->has_file(strpath_pz)) {
-        if (found_match(found_file, composite_file, mount, strpath_pz, 
-                        pathname, true)) {
+      if (vfs_implicit_pz) {
+        if (consider_match(found_file, composite_file, mount, strpath_pz, 
+                           pathname, true, status_only)) {
           return found_file;
         }
-#endif  // HAVE_ZLIB
       }
+#endif  // HAVE_ZLIB
 
     } else if (strpath.length() > mount_point.length() &&
                strpath.substr(0, mount_point.length()) == mount_point &&
@@ -333,21 +423,19 @@ get_file(const Filename &filename) const {
       // This pathname falls within this mount system.
       Filename local_filename = strpath.substr(mount_point.length() + 1);
       Filename local_filename_pz = strpath_pz.substr(mount_point.length() + 1);
-      if (mount->has_file(local_filename)) {
-        // Bingo!
-        if (found_match(found_file, composite_file, mount, local_filename, 
-                        pathname, false)) {
-          return found_file;
-        }
+      if (consider_match(found_file, composite_file, mount, local_filename, 
+                         pathname, false, status_only)) {
+        return found_file;
+      }
 #ifdef HAVE_ZLIB
-      } else if (vfs_implicit_pz && mount->has_file(local_filename_pz)) {
+      if (vfs_implicit_pz) {
         // Bingo!
-        if (found_match(found_file, composite_file, mount, local_filename_pz,
-                        pathname, true)) {
+        if (consider_match(found_file, composite_file, mount, local_filename_pz,
+                           pathname, true, status_only)) {
           return found_file;
         }
+      }
 #endif  // HAVE_ZLIB
-      }            
     }
   }
   return found_file;
@@ -362,9 +450,10 @@ get_file(const Filename &filename) const {
 //               found.
 ////////////////////////////////////////////////////////////////////
 PT(VirtualFile) VirtualFileSystem::
-find_file(const Filename &filename, const DSearchPath &searchpath) const {
+find_file(const Filename &filename, const DSearchPath &searchpath,
+          bool status_only) const {
   if (!filename.is_local()) {
-    return get_file(filename);
+    return get_file(filename, status_only);
   }
 
   int num_directories = searchpath.get_num_directories();
@@ -378,7 +467,7 @@ find_file(const Filename &filename, const DSearchPath &searchpath) const {
       // true), we don't prefix another one.
       match = filename;
     }
-    PT(VirtualFile) found_file = get_file(match);
+    PT(VirtualFile) found_file = get_file(match, status_only);
     if (found_file != (VirtualFile *)NULL) {
       return found_file;
     }
@@ -403,7 +492,7 @@ resolve_filename(Filename &filename,
   PT(VirtualFile) found;
 
   if (filename.is_local()) {
-    found = find_file(filename, searchpath);
+    found = find_file(filename, searchpath, true);
 
     if (found.is_null()) {
       // We didn't find it with the given extension; can we try the
@@ -411,7 +500,7 @@ resolve_filename(Filename &filename,
       if (filename.get_extension().empty() && !default_extension.empty()) {
         Filename try_ext = filename;
         try_ext.set_extension(default_extension);
-        found = find_file(try_ext, searchpath);
+        found = find_file(try_ext, searchpath, true);
       }
     }
   } else {
@@ -424,7 +513,7 @@ resolve_filename(Filename &filename,
       if (filename.get_extension().empty() && !default_extension.empty()) {
         Filename try_ext = filename;
         try_ext.set_extension(default_extension);
-        found = get_file(try_ext);
+        found = get_file(try_ext, true);
       }
     }
   }
@@ -605,7 +694,7 @@ get_global_ptr() {
 ////////////////////////////////////////////////////////////////////
 istream *VirtualFileSystem::
 open_read_file(const Filename &filename, bool auto_unwrap) const {
-  PT(VirtualFile) file = get_file(filename);
+  PT(VirtualFile) file = get_file(filename, false);
   if (file == (VirtualFile *)NULL) {
     return NULL;
   }
@@ -706,30 +795,34 @@ normalize_mount_point(const string &mount_point) const {
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: VirtualFileSystem::found_match
+//     Function: VirtualFileSystem::consider_match
 //       Access: Private
-//  Description: Evaluates one match found during a get_file()
-//               operation.  There may be multiple matches for a
-//               particular filename due to the ambiguities introduced
-//               by allowing multiple mount points, so we may have to
-//               keep searching even after the first match is found.
+//  Description: Evaluates one possible filename match found during a
+//               get_file() operation.  There may be multiple matches
+//               for a particular filename due to the ambiguities
+//               introduced by allowing multiple mount points, so we
+//               may have to keep searching even after the first match
+//               is found.
 //
 //               Returns true if the search should terminate now, or
 //               false if it should keep iterating.
 ////////////////////////////////////////////////////////////////////
 bool VirtualFileSystem::
-found_match(PT(VirtualFile) &found_file, VirtualFileComposite *&composite_file,
-            VirtualFileMount *mount, const string &local_filename,
-            const Filename &original_filename, bool implicit_pz_file) const {
+consider_match(PT(VirtualFile) &found_file, VirtualFileComposite *&composite_file,
+               VirtualFileMount *mount, const string &local_filename,
+               const Filename &original_filename, bool implicit_pz_file,
+               bool status_only) const {
+  PT(VirtualFile) vfile = 
+    mount->make_virtual_file(local_filename, original_filename, false, status_only);
+  if (!vfile->has_file()) {
+    // Keep looking.
+    return false;
+  }
+
   if (found_file == (VirtualFile *)NULL) {
     // This was our first match.  Save it.
-    Filename local(local_filename);
-    if (original_filename.is_text()) {
-      local.set_text();
-    }
-    found_file = new VirtualFileSimple(mount, local, implicit_pz_file);
-    found_file->set_original_filename(original_filename);
-    if (!mount->is_directory(local_filename)) {
+    found_file = vfile;
+    if (!found_file->is_directory()) {
       // If it's not a directory, we're done.
       return true;
     }
@@ -742,10 +835,11 @@ found_match(PT(VirtualFile) &found_file, VirtualFileComposite *&composite_file,
   } else {
     // This was our second match.  The previous match(es) must
     // have been directories.
-    if (!mount->is_directory(local_filename)) {
+    if (!vfile->is_directory()) {
       // However, this one isn't a directory.  We're done.
       return true;
     }
+
     if (!implicit_pz_file) {
       // At least two directories matched to the same path.  We
       // need a composite directory.
@@ -756,8 +850,8 @@ found_match(PT(VirtualFile) &found_file, VirtualFileComposite *&composite_file,
         composite_file->add_component(found_file);
         found_file = composite_file;
       }
-      composite_file->add_component
-        (new VirtualFileSimple(mount, local_filename, false));
+
+      composite_file->add_component(vfile);
     }
   }
 

panda/src/express/virtualFileSystem.h

@@ -18,6 +18,7 @@
 #include "pandabase.h"
 
 #include "virtualFile.h"
+#include "virtualFileMount.h"
 #include "filename.h"
 #include "dSearchPath.h"
 #include "pointerTo.h"
@@ -25,7 +26,6 @@
 #include "pvector.h"
 
 class Multifile;
-class VirtualFileMount;
 class VirtualFileComposite;
 
 ////////////////////////////////////////////////////////////////////
@@ -45,24 +45,29 @@ PUBLISHED:
   ~VirtualFileSystem();
 
   enum MountFlags {
-    MF_owns_pointer   = 0x0001,    // This flag is no longer used.
     MF_read_only      = 0x0002,
   };
 
   BLOCKING bool mount(Multifile *multifile, const string &mount_point, int flags);
   BLOCKING bool mount(const Filename &physical_filename, const string &mount_point, 
                       int flags, const string &password = "");
+  bool mount(VirtualFileMount *mount, const string &mount_point, int flags);
   BLOCKING int unmount(Multifile *multifile);
   BLOCKING int unmount(const Filename &physical_filename);
+  int unmount(VirtualFileMount *mount);
   BLOCKING int unmount_point(const string &mount_point);
   BLOCKING int unmount_all();
 
+  int get_num_mounts() const;
+  VirtualFileMount *get_mount(int n) const;
+
   BLOCKING bool chdir(const string &new_directory);
   BLOCKING const Filename &get_cwd() const;
 
-  BLOCKING PT(VirtualFile) get_file(const Filename &filename) const;
+  BLOCKING PT(VirtualFile) get_file(const Filename &filename, bool status_only = false) const;
   BLOCKING PT(VirtualFile) find_file(const Filename &filename, 
-                                     const DSearchPath &searchpath) const;
+                                     const DSearchPath &searchpath,
+                                     bool status_only = false) const;
   BLOCKING bool resolve_filename(Filename &filename, const DSearchPath &searchpath,
                                  const string &default_extension = string()) const;
   BLOCKING int find_all_files(const Filename &filename, const DSearchPath &searchpath,
@@ -91,13 +96,14 @@ public:
 
 private:
   Filename normalize_mount_point(const string &mount_point) const;
-  bool found_match(PT(VirtualFile) &found_file, VirtualFileComposite *&composite_file,
-                   VirtualFileMount *mount, const string &local_filename,
-                   const Filename &original_filename, bool implicit_pz_file) const;
+  bool consider_match(PT(VirtualFile) &found_file, VirtualFileComposite *&composite_file,
+                      VirtualFileMount *mount, const string &local_filename,
+                      const Filename &original_filename, bool implicit_pz_file,
+                      bool status_only) const;
   static void parse_option(const string &option,
                            int &flags, string &password);
 
-  typedef pvector<VirtualFileMount *> Mounts;
+  typedef pvector<PT(VirtualFileMount) > Mounts;
   Mounts _mounts;
   Filename _cwd;
 

panda/src/gobj/texture.h

@@ -618,7 +618,7 @@ protected:
   Filename _alpha_filename;
   Filename _fullpath;
   Filename _alpha_fullpath;
-  string _texture_pool_key;
+  Filename _texture_pool_key;
 
   // The number of channels of the primary file we use.  1, 2, 3, or 4.
   int _primary_file_num_channels;

panda/src/gobj/texturePool.I

@@ -202,6 +202,18 @@ release_all_textures() {
   get_global_ptr()->ns_release_all_textures();
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: TexturePool::rehash
+//       Access: Published, Static
+//  Description: Should be called when the model-path changes, to blow
+//               away the cache of texture pathnames found along the
+//               model-path.
+////////////////////////////////////////////////////////////////////
+INLINE void TexturePool::
+rehash() {
+  get_global_ptr()->_relpath_lookup.clear();
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: TexturePool::garbage_collect
 //       Access: Published, Static

panda/src/gobj/texturePool.cxx

@@ -24,7 +24,7 @@
 #include "texturePoolFilter.h"
 #include "configVariableList.h"
 #include "load_dso.h"
-#include "lightMutexHolder.h"
+#include "mutexHolder.h"
 
 TexturePool *TexturePool::_global_ptr;
 
@@ -51,7 +51,7 @@ write(ostream &out) {
 ////////////////////////////////////////////////////////////////////
 void TexturePool::
 register_texture_type(MakeTextureFunc *func, const string &extensions) {
-  LightMutexHolder holder(_lock);
+  MutexHolder holder(_lock);
 
   vector_string words;
   extract_words(downcase(extensions), words);
@@ -70,7 +70,7 @@ register_texture_type(MakeTextureFunc *func, const string &extensions) {
 ////////////////////////////////////////////////////////////////////
 void TexturePool::
 register_filter(TexturePoolFilter *filter) {
-  LightMutexHolder holder(_lock);
+  MutexHolder holder(_lock);
 
   gobj_cat.info()
     << "Registering Texture filter " << *filter << "\n";
@@ -87,7 +87,7 @@ register_filter(TexturePoolFilter *filter) {
 ////////////////////////////////////////////////////////////////////
 TexturePool::MakeTextureFunc *TexturePool::
 get_texture_type(const string &extension) const {
-  LightMutexHolder holder(_lock);
+  MutexHolder holder(_lock);
 
   string c = downcase(extension);
   TypeRegistry::const_iterator ti;
@@ -139,7 +139,7 @@ make_texture(const string &extension) const {
 ////////////////////////////////////////////////////////////////////
 void TexturePool::
 write_texture_types(ostream &out, int indent_level) const {
-  LightMutexHolder holder(_lock);
+  MutexHolder holder(_lock);
 
   PNMFileTypeRegistry *pnm_reg = PNMFileTypeRegistry::get_global_ptr();
   pnm_reg->write(out, indent_level);
@@ -206,17 +206,10 @@ TexturePool() {
 ////////////////////////////////////////////////////////////////////
 bool TexturePool::
 ns_has_texture(const Filename &orig_filename) {
-  LightMutexHolder holder(_lock);
+  MutexHolder holder(_lock);
 
-  Filename filename(orig_filename);
-
-  if (!_fake_texture_image.empty()) {
-    filename = _fake_texture_image;
-  }
-
-  VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
-  vfs->resolve_filename(filename, get_texture_path());
-  vfs->resolve_filename(filename, get_model_path());
+  Filename filename;
+  resolve_filename(filename, orig_filename);
 
   Textures::const_iterator ti;
   ti = _textures.find(filename);
@@ -236,18 +229,11 @@ ns_has_texture(const Filename &orig_filename) {
 Texture *TexturePool::
 ns_load_texture(const Filename &orig_filename, int primary_file_num_channels,
                 bool read_mipmaps, const LoaderOptions &options) {
-  Filename filename(orig_filename);
-
-  if (!_fake_texture_image.empty()) {
-    filename = _fake_texture_image;
-  }
-
-  VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
-  vfs->resolve_filename(filename, get_texture_path()) ||
-    vfs->resolve_filename(filename, get_model_path());
+  Filename filename;
 
   {
-    LightMutexHolder holder(_lock);
+    MutexHolder holder(_lock);
+    resolve_filename(filename, orig_filename);
     Textures::const_iterator ti;
     ti = _textures.find(filename);
     if (ti != _textures.end()) {
@@ -312,7 +298,7 @@ ns_load_texture(const Filename &orig_filename, int primary_file_num_channels,
   tex->_texture_pool_key = filename;
 
   {
-    LightMutexHolder holder(_lock);
+    MutexHolder holder(_lock);
 
     // Now look again--someone may have just loaded this texture in
     // another thread.
@@ -359,23 +345,18 @@ ns_load_texture(const Filename &orig_filename,
                 int primary_file_num_channels,
                 int alpha_file_channel,
                 bool read_mipmaps, const LoaderOptions &options) {
-  Filename filename(orig_filename);
-  Filename alpha_filename(orig_alpha_filename);
-
   if (!_fake_texture_image.empty()) {
     return ns_load_texture(_fake_texture_image, primary_file_num_channels,
                            read_mipmaps, options);
   }
 
-  VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
-  vfs->resolve_filename(filename, get_texture_path()) ||
-    vfs->resolve_filename(filename, get_model_path());
-  
-  vfs->resolve_filename(alpha_filename, get_texture_path()) ||
-    vfs->resolve_filename(alpha_filename, get_model_path());
+  Filename filename;
+  Filename alpha_filename;
 
   {
-    LightMutexHolder holder(_lock);
+    MutexHolder holder(_lock);
+    resolve_filename(filename, orig_filename);
+    resolve_filename(alpha_filename, orig_alpha_filename);
 
     Textures::const_iterator ti;
     ti = _textures.find(filename);
@@ -444,7 +425,7 @@ ns_load_texture(const Filename &orig_filename,
   tex->_texture_pool_key = filename;
 
   {
-    LightMutexHolder holder(_lock);
+    MutexHolder holder(_lock);
 
     // Now look again.
     Textures::const_iterator ti;
@@ -486,15 +467,13 @@ ns_load_texture(const Filename &orig_filename,
 Texture *TexturePool::
 ns_load_3d_texture(const Filename &filename_pattern,
                    bool read_mipmaps, const LoaderOptions &options) {
-  Filename filename(filename_pattern);
-  filename.set_pattern(true);
-
-  VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
-  vfs->resolve_filename(filename, get_texture_path()) ||
-    vfs->resolve_filename(filename, get_model_path());
+  Filename orig_filename(filename_pattern);
+  orig_filename.set_pattern(true);
 
+  Filename filename;
   {
-    LightMutexHolder holder(_lock);
+    MutexHolder holder(_lock);
+    resolve_filename(filename, orig_filename);
 
     Textures::const_iterator ti;
     ti = _textures.find(filename);
@@ -549,7 +528,7 @@ ns_load_3d_texture(const Filename &filename_pattern,
   tex->_texture_pool_key = filename;
 
   {
-    LightMutexHolder holder(_lock);
+    MutexHolder holder(_lock);
 
     // Now look again.
     Textures::const_iterator ti;
@@ -580,15 +559,13 @@ ns_load_3d_texture(const Filename &filename_pattern,
 Texture *TexturePool::
 ns_load_cube_map(const Filename &filename_pattern, bool read_mipmaps, 
                  const LoaderOptions &options) {
-  Filename filename(filename_pattern);
-  filename.set_pattern(true);
-
-  VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
-  vfs->resolve_filename(filename, get_texture_path()) ||
-    vfs->resolve_filename(filename, get_model_path());
+  Filename orig_filename(filename_pattern);
+  orig_filename.set_pattern(true);
 
+  Filename filename;
   {
-    LightMutexHolder holder(_lock);
+    MutexHolder holder(_lock);
+    resolve_filename(filename, orig_filename);
 
     Textures::const_iterator ti;
     ti = _textures.find(filename);
@@ -643,7 +620,7 @@ ns_load_cube_map(const Filename &filename_pattern, bool read_mipmaps,
   tex->_texture_pool_key = filename;
 
   {
-    LightMutexHolder holder(_lock);
+    MutexHolder holder(_lock);
 
     // Now look again.
     Textures::const_iterator ti;
@@ -673,7 +650,7 @@ ns_load_cube_map(const Filename &filename_pattern, bool read_mipmaps,
 ////////////////////////////////////////////////////////////////////
 Texture *TexturePool::
 ns_get_normalization_cube_map(int size) {
-  LightMutexHolder holder(_lock);
+  MutexHolder holder(_lock);
 
   if (_normalization_cube_map == (Texture *)NULL) {
     _normalization_cube_map = new Texture("normalization_cube_map");
@@ -693,7 +670,7 @@ ns_get_normalization_cube_map(int size) {
 ////////////////////////////////////////////////////////////////////
 Texture *TexturePool::
 ns_get_alpha_scale_map() {
-  LightMutexHolder holder(_lock);
+  MutexHolder holder(_lock);
 
   if (_alpha_scale_map == (Texture *)NULL) {
     _alpha_scale_map = new Texture("alpha_scale_map");
@@ -711,7 +688,7 @@ ns_get_alpha_scale_map() {
 void TexturePool::
 ns_add_texture(Texture *tex) {
   PT(Texture) keep = tex;
-  LightMutexHolder holder(_lock);
+  MutexHolder holder(_lock);
 
   if (!tex->_texture_pool_key.empty()) {
     ns_release_texture(tex);
@@ -734,7 +711,7 @@ ns_add_texture(Texture *tex) {
 ////////////////////////////////////////////////////////////////////
 void TexturePool::
 ns_release_texture(Texture *tex) {
-  LightMutexHolder holder(_lock);
+  MutexHolder holder(_lock);
 
   if (!tex->_texture_pool_key.empty()) {
     Textures::iterator ti;
@@ -744,6 +721,9 @@ ns_release_texture(Texture *tex) {
     }
     tex->_texture_pool_key = string();
   }
+
+  // Blow away the cache of resolved relative filenames.
+  _relpath_lookup.clear();
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -753,7 +733,7 @@ ns_release_texture(Texture *tex) {
 ////////////////////////////////////////////////////////////////////
 void TexturePool::
 ns_release_all_textures() {
-  LightMutexHolder holder(_lock);
+  MutexHolder holder(_lock);
 
   Textures::iterator ti;
   for (ti = _textures.begin(); ti != _textures.end(); ++ti) {
@@ -763,6 +743,9 @@ ns_release_all_textures() {
 
   _textures.clear();
   _normalization_cube_map = NULL;
+
+  // Blow away the cache of resolved relative filenames.
+  _relpath_lookup.clear();
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -772,7 +755,7 @@ ns_release_all_textures() {
 ////////////////////////////////////////////////////////////////////
 int TexturePool::
 ns_garbage_collect() {
-  LightMutexHolder holder(_lock);
+  MutexHolder holder(_lock);
 
   int num_released = 0;
   Textures new_set;
@@ -814,7 +797,7 @@ ns_garbage_collect() {
 ////////////////////////////////////////////////////////////////////
 void TexturePool::
 ns_list_contents(ostream &out) const {
-  LightMutexHolder holder(_lock);
+  MutexHolder holder(_lock);
 
   int total_size;
   int total_ram_size;
@@ -844,6 +827,35 @@ ns_list_contents(ostream &out) const {
   out << "texture pool size - texture pool ram: " << total_size - total_ram_size << "\n";
 }
 
+////////////////////////////////////////////////////////////////////
+//     Function: TexturePool::resolve_filename
+//       Access: Private
+//  Description: Searches for the indicated filename along the texture
+//               and model path.  If the filename was previously
+//               searched for, doesn't search again, as an
+//               optimization.  Assumes _lock is held.
+////////////////////////////////////////////////////////////////////
+void TexturePool::
+resolve_filename(Filename &new_filename, const Filename &orig_filename) {
+  if (!_fake_texture_image.empty()) {
+    new_filename = _fake_texture_image;
+    return;
+  }
+
+  RelpathLookup::iterator rpi = _relpath_lookup.find(orig_filename);
+  if (rpi != _relpath_lookup.end()) {
+    new_filename = (*rpi).second;
+    return;
+  }
+
+  new_filename = orig_filename;
+  VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr();
+  vfs->resolve_filename(new_filename, get_texture_path()) ||
+    vfs->resolve_filename(new_filename, get_model_path());
+
+  _relpath_lookup[orig_filename] = new_filename;
+}
+
 ////////////////////////////////////////////////////////////////////
 //     Function: TexturePool::try_load_cache
 //       Access: Private
@@ -985,7 +997,7 @@ pre_load(const Filename &orig_filename, const Filename &orig_alpha_filename,
          bool read_mipmaps, const LoaderOptions &options) {
   PT(Texture) tex;
 
-  LightMutexHolder holder(_lock);
+  MutexHolder holder(_lock);
 
   FilterRegistry::iterator fi;
   for (fi = _filter_registry.begin();
@@ -1011,7 +1023,7 @@ PT(Texture) TexturePool::
 post_load(Texture *tex) {
   PT(Texture) result = tex;
 
-  LightMutexHolder holder(_lock);
+  MutexHolder holder(_lock);
 
   FilterRegistry::iterator fi;
   for (fi = _filter_registry.begin();

panda/src/gobj/texturePool.h

@@ -20,7 +20,7 @@
 #include "filename.h"
 #include "config_gobj.h"
 #include "loaderOptions.h"
-#include "lightMutex.h"
+#include "pmutex.h"
 #include "pmap.h"
 
 class TexturePoolFilter;
@@ -65,6 +65,7 @@ PUBLISHED:
   INLINE static void add_texture(Texture *texture);
   INLINE static void release_texture(Texture *texture);
   INLINE static void release_all_textures();
+  INLINE static void rehash();
 
   INLINE static int garbage_collect();
 
@@ -118,6 +119,8 @@ private:
   int ns_garbage_collect();
   void ns_list_contents(ostream &out) const;
 
+  void resolve_filename(Filename &new_filename, const Filename &orig_filename);
+
   void try_load_cache(PT(Texture) &tex, BamCache *cache, 
                       const Filename &filename, PT(BamCacheRecord) &record, 
                       bool &compressed_cache_record,
@@ -136,9 +139,12 @@ private:
 
   static TexturePool *_global_ptr;
 
-  LightMutex _lock;
-  typedef phash_map<string,  PT(Texture), string_hash> Textures;
-  Textures _textures;
+  Mutex _lock;
+  typedef pmap<Filename, PT(Texture)> Textures;
+  Textures _textures;  // indexed by fullpath
+  typedef pmap<Filename, Filename> RelpathLookup;
+  RelpathLookup _relpath_lookup;
+
   string _fake_texture_image;
 
   PT(Texture) _normalization_cube_map;

panda/src/pipeline/conditionVarSimpleImpl.cxx

@@ -51,7 +51,7 @@ wait(double timeout) {
   // variable semantics, after all).
   ThreadSimpleManager *manager = ThreadSimpleManager::get_global_ptr();
   ThreadSimpleImpl *thread = manager->get_current_thread();
-  manager->enqueue_ready(thread);
+  manager->enqueue_ready(thread, true);
   manager->next_context();
 
   _mutex.acquire();
@@ -69,7 +69,7 @@ do_notify() {
     // There had been a thread waiting on this condition variable.
     // Switch contexts immediately, to make fairness more likely.
     ThreadSimpleImpl *thread = manager->get_current_thread();
-    manager->enqueue_ready(thread);
+    manager->enqueue_ready(thread, false);
     manager->next_context();
   }
 }
@@ -86,7 +86,7 @@ do_notify_all() {
     // There had been a thread waiting on this condition variable.
     // Switch contexts immediately, to make fairness more likely.
     ThreadSimpleImpl *thread = manager->get_current_thread();
-    manager->enqueue_ready(thread);
+    manager->enqueue_ready(thread, false);
     manager->next_context();
   }
 }

panda/src/pipeline/mutexDebug.cxx

@@ -356,11 +356,13 @@ do_release() {
         nassertv(_lock_count > 0);
       }
     } else {
-      
+
+      /*
       if (thread_cat.is_debug()) {
         thread_cat.debug()
           << *current_thread << " releasing " << *this << "\n";
       }
+      */
       _cvar_impl.notify();
     }
   }

panda/src/pipeline/mutexSimpleImpl.cxx

@@ -54,7 +54,7 @@ do_release() {
     // There had been a thread waiting on this mutex.  Switch contexts
     // immediately, to make fairness more likely.
     ThreadSimpleImpl *thread = manager->get_current_thread();
-    manager->enqueue_ready(thread);
+    manager->enqueue_ready(thread, false);
     manager->next_context();
   }
 }

panda/src/pipeline/threadSimpleImpl.I

@@ -80,7 +80,7 @@ yield() {
   ThreadSimpleManager *manager = ThreadSimpleManager::get_global_ptr();
   if (manager->is_same_system_thread()) {
     ThreadSimpleImpl *thread = manager->get_current_thread();
-    thread->yield_this();
+    thread->yield_this(true);
   } else {
     manager->system_yield();
   }
@@ -109,7 +109,7 @@ INLINE void ThreadSimpleImpl::
 consider_yield_this() {
   double now = _manager->get_current_time();
   if (now >= _stop_time) {
-    yield_this();
+    yield_this(false);
   }
 }
 

panda/src/pipeline/threadSimpleImpl.cxx

@@ -135,7 +135,7 @@ start(ThreadPriority priority, bool joinable) {
 
   init_thread_context(&_context, _stack, _stack_size, st_begin_thread, this);
 
-  _manager->enqueue_ready(this);
+  _manager->enqueue_ready(this, false);
   return true;
 }
 
@@ -215,8 +215,8 @@ sleep_this(double seconds) {
 //  Description: 
 ////////////////////////////////////////////////////////////////////
 void ThreadSimpleImpl::
-yield_this() {
-  _manager->enqueue_ready(this);
+yield_this(bool volunteer) {
+  _manager->enqueue_ready(this, true);
   _manager->next_context();
 }
 
@@ -254,7 +254,7 @@ begin_thread() {
   // Any threads that were waiting to join with this thread now become ready.
   JoiningThreads::iterator jti;
   for (jti = _joining_threads.begin(); jti != _joining_threads.end(); ++jti) {
-    _manager->enqueue_ready(*jti);
+    _manager->enqueue_ready(*jti, false);
   }
   _joining_threads.clear();
 

panda/src/pipeline/threadSimpleImpl.h

@@ -81,7 +81,7 @@ public:
   INLINE static void consider_yield();
 
   void sleep_this(double seconds);
-  void yield_this();
+  void yield_this(bool volunteer);
   INLINE void consider_yield_this();
 
   INLINE double get_wake_time() const;

panda/src/pipeline/threadSimpleManager.cxx

@@ -88,12 +88,20 @@ ThreadSimpleManager() :
 //               thread will be executed when its turn comes.  If the
 //               thread is not the currently executing thread, its
 //               _jmp_context should be filled appropriately.
+//
+//               If volunteer is true, the thread is volunteering to
+//               sleep before its timeslice has been used up.  If
+//               volunteer is false, the thread would still be running
+//               if it could.
 ////////////////////////////////////////////////////////////////////
 void ThreadSimpleManager::
-enqueue_ready(ThreadSimpleImpl *thread) {
+enqueue_ready(ThreadSimpleImpl *thread, bool volunteer) {
   // We actually add it to _next_ready, so that we can tell when we
   // have processed every thread in a given epoch.
   _next_ready.push_back(thread);
+  if (volunteer) {
+    ++_num_next_ready_volunteers;
+  }
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -501,7 +509,9 @@ choose_next_context() {
     while (_ready.empty()) {
       if (!_next_ready.empty()) {
         // We've finished an epoch.
+        bool all_volunteers = (_num_next_ready_volunteers == (int)_next_ready.size());
         _ready.swap(_next_ready);
+        _num_next_ready_volunteers = 0;
 
         if (new_epoch && !_tick_records.empty()) {
           // Pop the oldest timeslice record off when we finish an
@@ -519,10 +529,12 @@ choose_next_context() {
 
         } else {
           // Otherwise, we're legitimately at the end of an epoch.
-          // Yield, to give some time back to the system.
-
-          // On second thought, this might yield too much.
-          //system_yield();
+          if (all_volunteers) {
+            // All of our non-blocked, non-sleeping threads have
+            // voluntarily yielded.  Therefore, yield the whole
+            // process.
+            system_yield();
+          }
         }
         new_epoch = true;
         
@@ -577,6 +589,10 @@ choose_next_context() {
       _current_thread = chosen_thread;
       break;
     }
+
+    // This thread is not ready to wake up yet.  Put it back for next
+    // epoch.  It doesn't count as a volunteer, though--its timeslice
+    // was used up.
     _next_ready.push_back(chosen_thread);
   }
 

panda/src/pipeline/threadSimpleManager.h

@@ -58,7 +58,7 @@ private:
   ThreadSimpleManager();
 
 public:
-  void enqueue_ready(ThreadSimpleImpl *thread);
+  void enqueue_ready(ThreadSimpleImpl *thread, bool volunteer);
   void enqueue_sleep(ThreadSimpleImpl *thread, double seconds);
   void enqueue_block(ThreadSimpleImpl *thread, BlockerSimple *blocker);
   bool unblock_one(BlockerSimple *blocker);
@@ -118,6 +118,7 @@ private:
   // FIFO list of ready threads.
   FifoThreads _ready;
   FifoThreads _next_ready;
+  int _num_next_ready_volunteers;
 
   typedef pmap<BlockerSimple *, FifoThreads> Blocked;
   Blocked _blocked;

panda/src/putil/Sources.pp

@@ -59,8 +59,6 @@
     simpleHashMap.I simpleHashMap.h \
     sparseArray.I sparseArray.h \
     string_utils.I string_utils.N string_utils.h \
-    stringStreamBuf.I stringStreamBuf.h \
-    stringStream.I stringStream.h \
     timedCycle.I timedCycle.h typedWritable.I \
     typedWritable.h typedWritableReferenceCount.I \
     typedWritableReferenceCount.h updateSeq.I updateSeq.h \
@@ -107,8 +105,6 @@
     simpleHashMap.cxx \
     sparseArray.cxx \
     string_utils.cxx \
-    stringStreamBuf.cxx \
-    stringStream.cxx \
     timedCycle.cxx typedWritable.cxx \
     typedWritableReferenceCount.cxx updateSeq.cxx \
     uniqueIdAllocator.cxx \
@@ -167,8 +163,6 @@
     simpleHashMap.I simpleHashMap.h \
     sparseArray.I sparseArray.h \
     string_utils.I string_utils.h \
-    stringStreamBuf.I stringStreamBuf.h \
-    stringStream.I stringStream.h \
     timedCycle.I timedCycle.h typedWritable.I \
     typedWritable.h typedWritableReferenceCount.I \
     typedWritableReferenceCount.h updateSeq.I updateSeq.h \

panda/src/putil/putil_composite2.cxx

@@ -17,8 +17,6 @@
 #include "simpleHashMap.cxx"
 #include "sparseArray.cxx"
 #include "string_utils.cxx"
-#include "stringStreamBuf.cxx"
-#include "stringStream.cxx"
 #include "timedCycle.cxx"
 #include "typedWritable.cxx"
 #include "typedWritableReferenceCount.cxx"