Browse Source

new multifile format for upcoming support of direct loading from multifiles

David Rose 23 years ago
parent
commit
b1dc8b2fa6

+ 0 - 347
panda/src/downloader/asyncDecompressor.cxx

@@ -1,347 +0,0 @@
-// Filename: asyncDecompressor.cxx
-// Created by:  mike (09Jan97)
-//
-////////////////////////////////////////////////////////////////////
-//
-// PANDA 3D SOFTWARE
-// Copyright (c) 2001, Disney Enterprises, Inc.  All rights reserved
-//
-// All use of this software is subject to the terms of the Panda 3d
-// Software license.  You should have received a copy of this license
-// along with this source code; you will also find a current copy of
-// the license at http://www.panda3d.org/license.txt .
-//
-// To contact the maintainers of this program write to
-// [email protected] .
-//
-////////////////////////////////////////////////////////////////////
-
-// This file is compiled only if we have zlib installed.
-
-////////////////////////////////////////////////////////////////////
-// Includes
-////////////////////////////////////////////////////////////////////
-
-#include "config_downloader.h"
-
-#include <event.h>
-#include <pt_Event.h>
-#include <throw_event.h>
-#include <eventParameter.h>
-#include <filename.h>
-#include <stdio.h>
-
-#include "asyncDecompressor.h"
-
-////////////////////////////////////////////////////////////////////
-// Defines
-////////////////////////////////////////////////////////////////////
-
-////////////////////////////////////////////////////////////////////
-//       Class : DecompressorToken
-// Description : Holds a request for the decompressor.
-////////////////////////////////////////////////////////////////////
-class DecompressorToken : public ReferenceCount {
-public:
-  INLINE DecompressorToken(uint id, const Filename &source_file,
-                    const Filename &dest_file, const string &event_name) {
-    _id = id;
-    _source_file = source_file;
-    _dest_file = dest_file;
-    _event_name = event_name;
-  }
-  int _id;
-  Filename _source_file;
-  Filename _dest_file;
-  string _event_name;
-};
-
-////////////////////////////////////////////////////////////////////
-//     Function: Decompressor::Constructor
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-Decompressor::
-Decompressor(void) : AsyncUtility() {
-  PT(Buffer) buffer = new Buffer(decompressor_buffer_size);
-  init(buffer);
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Decompressor::Constructor
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-Decompressor::
-Decompressor(PT(Buffer) buffer) : AsyncUtility() {
-  init(buffer);
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Decompressor::Constructor
-//       Access: Private
-//  Description:
-////////////////////////////////////////////////////////////////////
-void Decompressor::
-init(PT(Buffer) buffer) {
-  nassertv(!buffer.is_null());
-  _frequency = decompressor_frequency;
-  _token_board = new DecompressorTokenBoard;
-  _half_buffer_length = buffer->get_length()/2;
-  _buffer = buffer;
-  char *temp_name = tempnam(NULL, "dc");
-  _temp_file_name = temp_name;
-  _temp_file_name.set_binary();
-  delete temp_name;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Decompressor::Destructor
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-Decompressor::
-~Decompressor(void) {
-  destroy_thread();
-
-  delete _token_board;
-  _temp_file_name.unlink();
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Decompressor::request_decompress
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-int Decompressor::
-request_decompress(const Filename &source_file, const string &event_name) {
-  Filename dest_file = source_file;
-  string extension = source_file.get_extension();
-  if (extension == "pz")
-    dest_file = source_file.get_fullpath_wo_extension();
-  else {
-    if (downloader_cat.is_debug())
-      downloader_cat.debug()
-        << "Decompressor::request_decompress() - Unknown file extension: ."
-        << extension << endl;
-  }
-  return request_decompress(source_file, dest_file, event_name);
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Decompressor::request_decompress
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-int Decompressor::
-request_decompress(const Filename &source_file, const Filename &dest_file,
-                   const string &event_name) {
-
-  PT(DecompressorToken) tok;
-  if (_threads_enabled) {
-
-    // Make sure we actually are threaded
-    if (!_threaded) {
-      downloader_cat.info()
-        << "Decompressor::request_decompress() - create_thread() was "
-        << "never called!  Calling it now..." << endl;
-      create_thread();
-    }
-
-    // We need to grab the lock in order to signal the condition variable
-#ifdef HAVE_IPC
-    _lock.lock();
-#endif
-
-      if (_token_board->_waiting.is_full()) {
-        downloader_cat.error()
-          << "Downloader::request_download() - Too many pending requests\n";
-        return 0;
-      }
-
-      if (downloader_cat.is_debug()) {
-        downloader_cat.debug()
-          << "Decompress requested for file: " << source_file << endl;
-      }
-
-      tok = new DecompressorToken(_next_token++, source_file, dest_file,
-                                        event_name);
-      _token_board->_waiting.insert(tok);
-
-#ifdef HAVE_IPC
-      _request_cond->signal();
-    _lock.unlock();
-#endif
-
-  } else {
-    // If we're not running asynchronously, process the load request
-    // directly now.
-    if (_token_board->_waiting.is_full()) {
-      downloader_cat.error()
-        << "Downloader::request_download() - Too many pending requests\n";
-      return 0;
-    }
-    if (downloader_cat.is_debug()) {
-      downloader_cat.debug()
-        << "Decompress requested for file: " << source_file << endl;
-    }
-
-    tok = new DecompressorToken(_next_token++, source_file, dest_file,
-                                        event_name);
-    _token_board->_waiting.insert(tok);
-    process_request();
-  }
-
-  return tok->_id;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Decompressor::process_request
-//       Access: Private
-//  Description: Serves any requests on the token board, moving them
-//               to the done queue.
-////////////////////////////////////////////////////////////////////
-bool Decompressor::
-process_request() {
-  if (_shutdown) {
-    if (downloader_cat.is_debug())
-      downloader_cat.debug()
-        << "Decompressor shutting down...\n";
-    return false;
-  }
-
-  // If there is actually a request token - process it
-  while (!_token_board->_waiting.is_empty()) {
-    PT(DecompressorToken) tok = _token_board->_waiting.extract();
-    if (decompress(tok->_source_file, tok->_dest_file)) {
-      _token_board->_done.insert(tok);
-
-      // Throw a "done" event now.
-      if (!tok->_event_name.empty()) {
-        PT_Event done = new Event(tok->_event_name);
-        done->add_parameter(EventParameter((int)tok->_id));
-        throw_event(done);
-      }
-
-      if (downloader_cat.is_debug()) {
-        downloader_cat.debug()
-          << "Decompressor::process_request() - decompress complete for "
-          << tok->_source_file << "\n";
-      }
-    }
-  }
-
-  return true;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Decompressor::decompress
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-bool Decompressor::
-decompress(Filename &source_file) {
-  Filename dest_file = source_file;
-  string extension = source_file.get_extension();
-  if (extension == "pz")
-    dest_file = source_file.get_fullpath_wo_extension();
-  else {
-    if (downloader_cat.is_debug())
-      downloader_cat.debug()
-        << "Decompressor::request_decompress() - Unknown file extension: ."
-        << extension << endl;
-    return false;
-  }
-  return decompress(source_file, dest_file);
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Decompressor::decompress
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-bool Decompressor::
-decompress(Filename &source_file, Filename &dest_file) {
-
-  // Open source file
-  ifstream read_stream;
-  source_file.set_binary();
-  if (!source_file.open_read(read_stream)) {
-    downloader_cat.error()
-      << "Decompressor::decompress() - Error opening source file: "
-      << source_file << endl;
-    return false;
-  }
-
-  // Determine source file length
-  read_stream.seekg(0, ios::end);
-  int source_file_length = read_stream.tellg();
-  if (source_file_length == 0) {
-    downloader_cat.warning()
-      << "Decompressor::decompress() - Zero length file: "
-      << source_file << endl;
-    return true;
-  }
-  read_stream.seekg(0, ios::beg);
-
-  // Open destination file
-  ofstream write_stream;
-  dest_file.set_binary();
-  if (!dest_file.open_write(write_stream)) {
-    downloader_cat.error()
-      << "Decompressor::decompress() - Error opening dest file: "
-      << source_file << endl;
-    return false;
-  }
-
-  // Read from the source file into the first half of the buffer,
-  // decompress into the second half of the buffer, write the second
-  // half of the buffer to disk, and repeat.
-  int total_bytes_read = 0;
-  bool read_all_input = false;
-  bool handled_all_input = false;
-  int source_buffer_length;
-  ZDecompressor decompressor;
-  while (handled_all_input == false) {
-
-    // See if there is anything left in the source file
-    if (read_all_input == false) {
-      read_stream.read(_buffer->_buffer, _half_buffer_length);
-      source_buffer_length = read_stream.gcount();
-      total_bytes_read += source_buffer_length;
-      if (read_stream.eof()) {
-        nassertr(total_bytes_read == source_file_length, false);
-        read_all_input = true;
-      }
-    }
-
-    char *next_in = _buffer->_buffer;
-    int avail_in = source_buffer_length;
-    char *dest_buffer = _buffer->_buffer + source_buffer_length;
-    char *next_out = dest_buffer;
-    int dest_buffer_length = _buffer->get_length() - source_buffer_length;
-    int avail_out = dest_buffer_length;
-    nassertr(avail_out > 0 && avail_in > 0, false);
-
-    while (avail_in > 0) {
-      int ret = decompressor.decompress_to_stream(next_in, avail_in,
-                        next_out, avail_out, dest_buffer,
-                        dest_buffer_length, write_stream);
-      if (ret == ZCompressorBase::S_error)
-        return false;
-      if ((int)decompressor.get_total_in() == source_file_length &&
-          avail_out == dest_buffer_length)
-        handled_all_input = true;
-    }
-
-    nap();
-
-  }
-
-  read_stream.close();
-  write_stream.close();
-
-  source_file.unlink();
-
-  return true;
-}

+ 0 - 64
panda/src/downloader/asyncDecompressor.h

@@ -1,64 +0,0 @@
-// Filename: asyncDecompressor.h
-// Created by:  mike (09Jan97)
-//
-////////////////////////////////////////////////////////////////////
-//
-// PANDA 3D SOFTWARE
-// Copyright (c) 2001, Disney Enterprises, Inc.  All rights reserved
-//
-// All use of this software is subject to the terms of the Panda 3d
-// Software license.  You should have received a copy of this license
-// along with this source code; you will also find a current copy of
-// the license at http://www.panda3d.org/license.txt .
-//
-// To contact the maintainers of this program write to
-// [email protected] .
-//
-////////////////////////////////////////////////////////////////////
-#ifndef ASYNCDECOMPRESSOR_H
-#define ASYNCDECOMPRESSOR_H
-//
-////////////////////////////////////////////////////////////////////
-// Includes
-////////////////////////////////////////////////////////////////////
-#include <pandabase.h>
-#include <filename.h>
-#include <tokenBoard.h>
-#include <buffer.h>
-#include "zcompressor.h"
-#include "asyncUtility.h"
-
-class DecompressorToken;
-
-////////////////////////////////////////////////////////////////////
-//       Class : Decompressor
-// Description :
-////////////////////////////////////////////////////////////////////
-class EXPCL_PANDAEXPRESS Decompressor : public AsyncUtility {
-PUBLISHED:
-  Decompressor(void);
-  Decompressor(PT(Buffer) buffer);
-  virtual ~Decompressor(void);
-
-  int request_decompress(const Filename &source_file,
-                         const string &event_name);
-  int request_decompress(const Filename &source_file,
-                         const Filename &dest_file,
-                         const string &event_name);
-
-  bool decompress(Filename &source_file);
-  bool decompress(Filename &source_file, Filename &dest_file);
-
-private:
-  void init(PT(Buffer) buffer);
-  virtual bool process_request(void);
-
-  typedef TokenBoard<DecompressorToken> DecompressorTokenBoard;
-  DecompressorTokenBoard *_token_board;
-
-  PT(Buffer) _buffer;
-  int _half_buffer_length;
-  Filename _temp_file_name;
-};
-
-#endif

+ 0 - 110
panda/src/downloader/asyncDownloader.I

@@ -1,110 +0,0 @@
-// Filename: asyncDownloader.I
-// Created by:  mike (09Jan97)
-//
-////////////////////////////////////////////////////////////////////
-//
-// PANDA 3D SOFTWARE
-// Copyright (c) 2001, Disney Enterprises, Inc.  All rights reserved
-//
-// All use of this software is subject to the terms of the Panda 3d
-// Software license.  You should have received a copy of this license
-// along with this source code; you will also find a current copy of
-// the license at http://www.panda3d.org/license.txt .
-//
-// To contact the maintainers of this program write to
-// [email protected] .
-//
-////////////////////////////////////////////////////////////////////
-
-#include "config_downloader.h"
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::set_byte_rate
-//       Access: Public
-//  Description: Note: modem speeds are reported in bits, so you
-//               need to convert!
-////////////////////////////////////////////////////////////////////
-INLINE void Downloader::
-set_byte_rate(float bytes) {
-  nassertv(bytes > 0.0);
-  if (bytes == _byte_rate)
-    return;
-#ifdef HAVE_IPC
-  _buffer_lock.lock();
-#endif
-
-  _new_byte_rate = bytes;
-
-#ifdef HAVE_IPC
-  _buffer_lock.unlock();
-#endif
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::get_byte_rate
-//       Access: Public
-//  Description: Returns byte rate in bytes.
-////////////////////////////////////////////////////////////////////
-INLINE float Downloader::
-get_byte_rate(void) const {
-  return _byte_rate;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::enable_download
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-INLINE void Downloader::
-enable_download(bool val) {
-  _download_enabled = val;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::is_download_enabled
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-INLINE bool Downloader::
-is_download_enabled(void) const {
-  return _download_enabled;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::set_disk_write_frequency
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-INLINE void Downloader::
-set_disk_write_frequency(int frequency) {
-  nassertv(frequency > 0);
-#ifdef HAVE_IPC
-    _buffer_lock.lock();
-#endif
-
-  _new_disk_write_frequency = frequency;
-
-#ifdef HAVE_IPC
-    _buffer_lock.unlock();
-#endif
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::get_disk_write_frequency
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-INLINE int Downloader::
-get_disk_write_frequency(void) const {
-  return _disk_write_frequency;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::get_last_attempt_stalled
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-INLINE bool Downloader::
-get_last_attempt_stalled(void) const {
-  return _last_attempt_stalled;
-}

+ 0 - 1045
panda/src/downloader/asyncDownloader.cxx

@@ -1,1045 +0,0 @@
-// Filename: asyncDownloader.cxx
-// Created by:  mike (09Jan97)
-//
-////////////////////////////////////////////////////////////////////
-//
-// PANDA 3D SOFTWARE
-// Copyright (c) 2001, Disney Enterprises, Inc.  All rights reserved
-//
-// All use of this software is subject to the terms of the Panda 3d
-// Software license.  You should have received a copy of this license
-// along with this source code; you will also find a current copy of
-// the license at http://www.panda3d.org/license.txt .
-//
-// To contact the maintainers of this program write to
-// [email protected] .
-//
-////////////////////////////////////////////////////////////////////
-
-#include "config_downloader.h"
-
-#include <event.h>
-#include <pt_Event.h>
-#include <throw_event.h>
-#include <eventParameter.h>
-#include <filename.h>
-#include <errno.h>
-#include <math.h>
-
-#include "asyncDownloader.h"
-#include <circBuffer.h>
-#include "plist.h"
-
-#if !defined(WIN32_VC)
-// #define errno wsaGetLastError()
-  #include <sys/time.h>
-  #include <netinet/in.h>
-  #include <arpa/inet.h>
-  #include <netdb.h>
-#endif
-
-////////////////////////////////////////////////////////////////////
-// Defines
-////////////////////////////////////////////////////////////////////
-enum send_status {
-  SS_error,
-  SS_timeout,
-  SS_success
-};
-
-enum receive_status {
-  RS_error,
-  RS_timeout,
-  RS_success,
-  RS_eof
-};
-
-////////////////////////////////////////////////////////////////////
-//       Class : DownloaderToken
-// Description : Holds a request for the downloader.
-////////////////////////////////////////////////////////////////////
-class DownloaderToken : public ReferenceCount {
-public:
-  INLINE DownloaderToken(uint id, const string &file_name,
-        const Filename &file_dest, const string &event_name,
-        int first_byte, int last_byte, int total_bytes,
-        bool partial_content, bool sync) : _id(id), _first_byte(first_byte),
-                _last_byte(last_byte), _total_bytes(total_bytes) {
-    _file_name = file_name;
-    _event_name = event_name;
-    _file_dest = file_dest;
-    _partial_content = partial_content;
-    _sync = sync;
-  }
-  uint _id;
-  string _file_name;
-  Filename _file_dest;
-  string _event_name;
-  int _first_byte;
-  int _last_byte;
-  int _total_bytes;
-  bool _partial_content;
-  bool _sync;
-};
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::Constructor
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-Downloader::
-Downloader(void) : AsyncUtility() {
-  init();
-}
-
-#if 0
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::Constructor
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-Downloader::
-Downloader(PT(Buffer) buffer) : AsyncUtility() {
-  init(buffer);
-}
-#endif
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::init
-//       Access: Private
-//  Description:
-////////////////////////////////////////////////////////////////////
-void Downloader::
-init(void) {
-  _disk_write_frequency = downloader_disk_write_frequency;
-  _new_disk_write_frequency = 0;
-  _byte_rate = downloader_byte_rate;
-  _new_byte_rate = 0;
-  _frequency = downloader_frequency;
-  nassertv(_frequency > 0 && _byte_rate > 0);
-  _read_size = _byte_rate * _frequency;
-  _disk_buffer_size = _disk_write_frequency * _read_size;
-  _buffer = new Buffer(_disk_buffer_size);
-  _connected = false;
-  _token_board = new DownloaderTokenBoard;
-  _download_enabled = true;
-  _last_attempt_stalled = true;
-  // We need to flush after every write in case we're interrupted
-  _dest_stream.setf(ios::unitbuf, 0);
-  _last_attempt_stalled = false;
-  _current_attempt_stalled = false;
-
-#if defined(WIN32)
-  WSAData mydata;
-  int answer1 = WSAStartup(0x0101, &mydata);
-  if(answer1 != 0) {
-    downloader_cat.error()
-      << "Downloader::Downloader() - Error initializing TCP stack!"
-      << endl;
-  }
-#endif
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::Destructor
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-Downloader::
-~Downloader() {
-  if (_connected)
-    disconnect_from_server();
-
-  destroy_thread();
-
-  delete _token_board;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::connect_to_server
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-bool Downloader::
-connect_to_server(const string &name, uint port) {
-  if (downloader_cat.is_debug())
-    downloader_cat.debug()
-      << "Downloader connecting to server: " << name
-      << " on port: " << port << endl;
-
-  _server_name = name;
-
-  _sin.sin_family = PF_INET;
-  _sin.sin_port = htons(port);
-  ulong addr = (ulong)inet_addr(name.c_str());
-  struct hostent *hp = NULL;
-
-  if (addr == INADDR_NONE) {
-    hp = gethostbyname(name.c_str());
-    if (hp != NULL)
-      (void)memcpy(&_sin.sin_addr, hp->h_addr, (uint)hp->h_length);
-    else {
-      downloader_cat.error()
-        << "Downloader::connect_to_server() - gethostbyname() failed: "
-        << strerror(errno) << endl;
-      return false;
-    }
-  } else
-    (void)memcpy(&_sin.sin_addr, &addr, sizeof(addr));
-
-  return connect_to_server();
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::connect_to_server
-//       Access: Private
-//  Description:
-////////////////////////////////////////////////////////////////////
-bool Downloader::
-connect_to_server(void) {
-  if (_connected == true)
-    return true;
-
-  _socket = 0xffffffff;
-  _socket = socket(PF_INET, SOCK_STREAM, 0);
-  if (_socket == (int)0xffffffff) {
-    downloader_cat.error()
-      << "Downloader::connect_to_server() - socket failed: "
-      << strerror(errno) << endl;
-    return false;
-  }
-
-  _connected = true;
-
-  if (connect(_socket, (struct sockaddr *)&_sin, sizeof(_sin)) < 0) {
-    downloader_cat.error()
-      << "Downloader::connect_to_server() - connect() failed: "
-      << strerror(errno) << endl;
-    disconnect_from_server();
-    _connected = false;
-  }
-
-  return _connected;
-}
-
-///////////////////////////////////////////////////////////////////
-//     Function: Downloader::disconnect_from_server
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-void Downloader::
-disconnect_from_server(void) {
-  if (downloader_cat.is_debug())
-    downloader_cat.debug()
-      << "Downloader disconnecting from server..." << endl;
-
-#if defined(WIN32)
-  (void)closesocket(_socket);
-#else
-  (void)close(_socket);
-#endif
-
-  _connected = false;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::request_sync_download
-//       Access: Public
-//  Description: Requests the synchronous download of a complete file.
-////////////////////////////////////////////////////////////////////
-int Downloader::
-request_sync_download(const string &file_name, const Filename &file_dest,
-                const string &event_name) {
-  return request_download(file_name, file_dest, event_name, true);
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::request_sync_download
-//       Access: Public
-//  Description: Requests the synchronous download of a complete file.
-////////////////////////////////////////////////////////////////////
-int Downloader::
-request_sync_download(const string &file_name, const Filename &file_dest,
-                const string &event_name, int first_byte,
-                int last_byte, int total_bytes, bool partial_content) {
-  return request_download(file_name, file_dest, event_name, first_byte,
-                        last_byte, total_bytes, partial_content, true);
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::request_download
-//       Access: Public
-//  Description: Requests the download of a complete file.
-////////////////////////////////////////////////////////////////////
-int Downloader::
-request_download(const string &file_name, const Filename &file_dest,
-                const string &event_name, bool sync) {
-  return request_download(file_name, file_dest, event_name, 0, 0, 0,
-                                false, sync);
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::request_download
-//       Access: Public
-//  Description: Requests an asynchronous load of a file.  The request
-//               will be queued and served by the asynchronous thread.
-//               If event_name is nonempty, it is the name of the
-//               event that will be thrown (with the uint id as its
-//               single parameter) when the loading is completed later.
-//
-//               The return value is an integer which can be used to
-//               identify this particular request later to
-//               fetch_load(), or 0 if there has been an error.
-//
-//               Can be used to request a partial download of a file.
-////////////////////////////////////////////////////////////////////
-int Downloader::
-request_download(const string &file_name, const Filename &file_dest,
-                        const string &event_name, int first_byte,
-                        int last_byte, int total_bytes,
-                        bool partial_content, bool sync) {
-
-  nassertr(first_byte <= last_byte && last_byte <= total_bytes, 0);
-
-  PT(DownloaderToken) tok;
-  if (_threads_enabled) {
-
-    // Make sure we actually are threaded
-    if (!_threaded) {
-      downloader_cat.info()
-        << "Downloader::request_download() - create_thread() was "
-        << "never called!  Calling it now..." << endl;
-      create_thread();
-    }
-
-    // We need to grab the lock in order to signal the condition variable
-#ifdef HAVE_IPC
-    _lock.lock();
-#endif
-
-      if (_token_board->_waiting.full()) {
-        downloader_cat.error()
-          << "Downloader::request_download() - Too many pending requests\n";
-        return 0;
-      }
-
-      if (downloader_cat.is_debug()) {
-        downloader_cat.debug()
-          << "Download requested for file: " << file_name << "\n";
-      }
-
-      tok = new DownloaderToken(_next_token++, file_name, file_dest,
-                event_name, first_byte, last_byte, total_bytes,
-                                        partial_content, sync);
-      _token_board->_waiting.push_back(tok);
-
-#ifdef HAVE_IPC
-      _request_cond->signal();
-    _lock.unlock();
-#endif
-
-  } else {
-    // If we're not running asynchronously, process the load request
-    // directly now.
-    if (_token_board->_waiting.full()) {
-      downloader_cat.error()
-        << "Downloader::request_download() - Too many pending requests\n";
-      return 0;
-    }
-    if (downloader_cat.is_debug()) {
-      downloader_cat.debug()
-        << "Load requested for file: " << file_name << "\n";
-    }
-
-    tok = new DownloaderToken(_next_token++, file_name, file_dest,
-                event_name, first_byte, last_byte, total_bytes,
-                                        partial_content, sync);
-    _token_board->_waiting.push_back(tok);
-    process_request();
-  }
-
-  return tok->_id;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::process_request
-//       Access: Private
-//  Description: Serves any requests on the token board, moving them
-//               to the done queue.
-////////////////////////////////////////////////////////////////////
-bool Downloader::
-process_request() {
-  if (_shutdown) {
-    if (downloader_cat.is_debug())
-      downloader_cat.debug()
-        << "Downloader shutting down...\n";
-    return false;
-  }
-
-  // If there is actually a request token - process it
-  while (!_token_board->_waiting.empty()) {
-    PT(DownloaderToken) tok = _token_board->_waiting.front();
-    _token_board->_waiting.pop_front();
-    int ret = download(tok->_file_name, tok->_file_dest, tok->_event_name,
-                 tok->_first_byte, tok->_last_byte, tok->_total_bytes,
-                 tok->_partial_content, tok->_sync, tok->_id);
-    nassertr(tok->_event_name.empty() == false, false);
-    PT_Event return_event = new Event(tok->_event_name);
-    return_event->add_parameter(EventParameter((int)tok->_id));
-    if (ret == DS_success) {
-      _token_board->_done.push_back(tok);
-      return_event->add_parameter(EventParameter(DS_success));
-
-      // Throw a "done" event now.
-      if (!tok->_event_name.empty()) {
-        PT_Event done = new Event(tok->_event_name);
-        done->add_parameter(EventParameter((int)tok->_id));
-        throw_event(done);
-      }
-
-      if (downloader_cat.is_debug()) {
-        downloader_cat.debug()
-          << "Downloader::process_request() - downloading complete for "
-          << tok->_file_name << "\n";
-      }
-    } else {
-      return_event->add_parameter(EventParameter(ret));
-    }
-    throw_event(return_event);
-  }
-
-  return true;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::safe_send
-//       Access: Private
-//  Description:
-////////////////////////////////////////////////////////////////////
-int Downloader::
-safe_send(int socket, const char *data, int length, long timeout) {
-  if (length == 0) {
-    downloader_cat.error()
-      << "Downloader::safe_send() - requested 0 length send!" << endl;
-    return SS_error;
-  }
-  int bytes = 0;
-  struct timeval tv;
-  tv.tv_sec = timeout;
-  tv.tv_usec = 0;
-  fd_set wset;
-  FD_ZERO(&wset);
-  while (bytes < length) {
-    FD_SET(socket, &wset);
-    int sret = select(socket + 1, NULL, &wset, NULL, &tv);
-    if (sret == 0) {
-      downloader_cat.error()
-        << "Downloader::safe_send() - select timed out after: "
-        << timeout << " seconds" << endl;
-      return SS_timeout;
-    } else if (sret == -1) {
-      downloader_cat.error()
-        << "Downloader::safe_send() - error: " << strerror(errno) << endl;
-      return SS_error;
-    }
-    int ret = send(socket, data, length, 0);
-    if (ret > 0)
-      bytes += ret;
-    else {
-      downloader_cat.error()
-        << "Downloader::safe_send() - error: " << strerror(errno) << endl;
-      return SS_error;
-    }
-  }
-  return SS_success;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::safe_receive
-//       Access: Private
-//  Description:
-////////////////////////////////////////////////////////////////////
-int Downloader::
-safe_receive(int socket, DownloadStatus &status, int length,
-                                long timeout, int &bytes) {
-  bytes = 0;
-  if (length == 0) {
-    downloader_cat.error()
-      << "Downloader::safe_receive() - requested 0 length receive!" << endl;
-    return RS_error;
-  }
-  struct timeval tv;
-  tv.tv_sec = timeout;
-  tv.tv_usec = 0;
-  fd_set rset;
-  FD_ZERO(&rset);
-  while (bytes < length) {
-    FD_SET(socket, &rset);
-    int sret = select(socket + 1, &rset, NULL, NULL, &tv);
-    if (sret == 0) {
-      downloader_cat.warning()
-        << "Downloader::safe_receive() - select timed out after: "
-        << timeout << " seconds" << endl;
-      return RS_timeout;
-    } else if (sret == -1) {
-      downloader_cat.error()
-        << "Downloader::safe_receive() - error: " << strerror(errno) << endl;
-      return RS_error;
-    }
-    int ret = recv(socket, status._next_in, length - bytes, 0);
-    if (ret > 0) {
-      if (downloader_cat.is_debug())
-        downloader_cat.debug()
-          << "Downloader::safe_receive() - recv() got: " << ret << " bytes"
-          << endl;
-      bytes += ret;
-      status._next_in += ret;
-      status._bytes_in_buffer += ret;
-      if (bytes < length) {
-        if (downloader_cat.is_debug())
-          downloader_cat.debug()
-            << "Downloader::safe_receive() - Download stalled" << endl;
-        _current_attempt_stalled = true;
-      }
-    } else if (ret == 0) {
-      if (downloader_cat.is_debug())
-        downloader_cat.debug()
-          << "Downloader::safe_receive() - End of file" << endl;
-      return RS_eof;
-    } else {
-      downloader_cat.error()
-        << "Downloader::safe_receive() - error: " << strerror(errno) << endl;
-      return RS_error;
-    }
-  }
-  nassertr(bytes == length, RS_error);
-  return RS_success;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::attempt_read
-//       Access: Private
-//  Description:
-////////////////////////////////////////////////////////////////////
-int Downloader::
-attempt_read(int length, DownloadStatus &status, int &bytes_read) {
-
-  bytes_read = 0;
-  for (int i = 0; i < downloader_timeout_retries; i++) {
-
-    // Ensure we have enough room in the buffer to download length bytes
-    // If we don't have enough room, write the buffer to disk
-    if (status._bytes_in_buffer + length > _disk_buffer_size) {
-      if (downloader_cat.is_debug())
-        downloader_cat.debug()
-          << "Downloader::attempt_read() - Flushing buffer" << endl;
-
-      if (write_to_disk(status) == false)
-        return RS_error;
-    }
-
-    // Make the request for length bytes
-    int bytes;
-    int ans = safe_receive(_socket, status, length,
-                                (long)downloader_timeout, bytes);
-    bytes_read += bytes;
-
-    switch (ans) {
-      case RS_error:
-      case RS_eof:
-        return ans;
-      case RS_timeout:
-        // Try again
-        break;
-      case RS_success:
-        nassertr(bytes == length, RS_error);
-        return RS_success;
-      default:
-        downloader_cat.error()
-          << "Downloader::attempt_read() - unknown return condition "
-          << "from safe_receive() : " << ans << endl;
-        return RS_error;
-    }
-  }
-
-  // We timed out on retries consecutive attempts - this is considered
-  // a true timeout
-  return RS_timeout;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::download
-//       Access: Private
-//  Description:
-////////////////////////////////////////////////////////////////////
-int Downloader::
-download(const string &file_name, Filename file_dest,
-                const string &event_name, int first_byte, int last_byte,
-                int total_bytes, bool partial_content, bool sync, uint id) {
-
-  if (_download_enabled == false) {
-    if (downloader_cat.is_debug())
-      downloader_cat.debug()
-        << "Downloader::download() - downloading is disabled" << endl;
-    return DS_abort;
-  }
-
-  // Make sure we are still connected to the server
-  if (connect_to_server() == false)
-    return DS_abort;
-
-  // Attempt to open the destination file
-  file_dest.set_binary();
-  bool result;
-  if (partial_content == true && first_byte > 0)
-    result = file_dest.open_append(_dest_stream);
-  else
-    result = file_dest.open_write(_dest_stream);
-  if (result == false) {
-    downloader_cat.error()
-      << "Downloader::download() - Error opening file: " << file_dest
-      << " for writing" << endl;
-    return DS_abort;
-  }
-
-  // Send an HTTP request for the file to the server
-  string request = "GET ";
-  request += file_name;
-  request += " HTTP/1.1\012Host: ";
-  request += _server_name;
-  request += "\012Connection: close";
-  if (partial_content == true) {
-    if (downloader_cat.is_debug())
-      downloader_cat.debug()
-        << "Downloader::download() - Requesting byte range: " << first_byte
-        << "-" << last_byte << endl;
-    request += "\012Range: bytes=";
-    stringstream start_stream;
-    start_stream << first_byte << "-" << last_byte;
-    request += start_stream.str();
-  }
-  request += "\012\012";
-  int outlen = request.size();
-  if (downloader_cat.is_debug())
-    downloader_cat.debug()
-      << "Downloader::download() - Sending request:\n" << request << endl;
-  int send_ret = safe_send(_socket, request.c_str(), outlen,
-                        (long)downloader_timeout);
-
-  // Handle timeouts on the send
-  if (send_ret == SS_timeout) {
-    for (int sr = 0; sr < downloader_timeout_retries; sr++) {
-      send_ret = safe_send(_socket, request.c_str(), outlen,
-                                (long)downloader_timeout);
-      if (send_ret != SS_timeout)
-        break;
-    }
-    if (send_ret == SS_timeout) {
-      // We've really timed out - throw an event
-      downloader_cat.error()
-        << "Downloader::download() - send timed out after: "
-        << downloader_timeout_retries << " retries" << endl;
-      return DS_timeout;
-    }
-  }
-
-  if (send_ret == SS_error)
-    return DS_abort;
-
-  // Create a download status to maintain download progress information
-  DownloadStatus status(_buffer->_buffer, event_name, first_byte, last_byte,
-                        total_bytes, partial_content, id);
-  bool got_any_data = false;
-
-  // Loop at the requested frequency until the download completes
-  for (;;) {
-    bool resize_buffer = false;
-
-    // Ensure that these don't change while we're computing read_size
-#ifdef HAVE_IPC
-    _buffer_lock.lock();
-#endif
-
-    nassertr(_frequency > 0, DS_abort);
-    // If byte rate has changed, recompute read size and write buffer size
-    if (_new_byte_rate > 0) {
-      _read_size = (int)ceil(_new_byte_rate * _frequency);
-      _byte_rate = _new_byte_rate;
-      _new_byte_rate = 0;
-      resize_buffer = true;
-    }
-
-    // If the disk write frequency has changed, compute a new buffer size
-    if (_new_disk_write_frequency > 0) {
-      _disk_write_frequency = _new_disk_write_frequency;
-      _new_disk_write_frequency = 0;
-      resize_buffer = true;
-    }
-
-    if (resize_buffer == true) {
-      // Flush the write buffer before resizing it
-      if (status._bytes_in_buffer > 0) {
-        if (downloader_cat.is_debug())
-          downloader_cat.debug()
-            << "Downloader::download() - Flushing buffer" << endl;
-
-        if (write_to_disk(status) == false) {
-          downloader_cat.error()
-            << "Downloader::download() - failed to flush buffer during "
-            << "resize" << endl;
-          return DS_abort;
-        }
-      }
-
-      // Resize the buffer
-      _disk_buffer_size = (_disk_write_frequency * _read_size);
-
-      if (downloader_cat.is_debug())
-        downloader_cat.debug()
-          << "Downloader::download() - resizing disk buffer to: "
-          << _disk_buffer_size << endl;
-      _buffer.clear();
-      downloader_cat.debug()
-        << "Downloader::download() - buffer cleared" << endl;
-      _buffer = new Buffer(_disk_buffer_size);
-      // Update the status with the new buffer
-      status._buffer = _buffer->_buffer;
-      status.reset();
-      downloader_cat.debug()
-        << "Downloader::download() - new buffer created" << endl;
-    }
-
-#ifdef HAVE_IPC
-    _buffer_lock.unlock();
-#endif
-
-    // Attempt to read
-    int bytes_read;
-
-    int ret = attempt_read(_read_size, status, bytes_read);
-    if (downloader_cat.is_debug())
-      downloader_cat.debug()
-        << "Downloader::download() - stalled status: " << _current_attempt_stalled
-        << endl;
-
-    _last_attempt_stalled = _current_attempt_stalled;
-    _current_attempt_stalled = false;
-
-    if (bytes_read > 0)
-      got_any_data = true;
-
-    switch (ret) {
-      case RS_error:
-
-        downloader_cat.error()
-          << "Downloader::download() - Error reading from socket: "
-          << strerror(errno) << endl;
-        return DS_abort;
-
-      case RS_timeout:
-
-        {
-          // We've really timed out - throw an event
-          downloader_cat.error()
-            << "Downloader::download() - receive timed out after: "
-            << downloader_timeout_retries << " retries" << endl;
-          if (bytes_read > 0) {
-            if (write_to_disk(status) == false) {
-              downloader_cat.error()
-                << "Downloader::download() - write to disk failed after "
-                << "timeout!" << endl;
-              return DS_abort;
-            }
-          }
-          return DS_timeout;
-        }
-
-      case RS_success:
-
-        if (downloader_cat.is_debug())
-          downloader_cat.debug()
-            << "Downloader::download() - Got: " << bytes_read << " bytes"
-            << endl;
-        break;
-
-      case RS_eof:
-
-        {
-          // We occasionally will get 0 bytes on the first attempt - we
-          // don't want to treat this as end of file in any case
-          if (got_any_data == true) {
-            if (downloader_cat.is_debug())
-              downloader_cat.debug()
-                << "Download for: " << file_name << " completed" << endl;
-            bool ret = true;
-            if (bytes_read > 0)
-              ret = write_to_disk(status);
-            _dest_stream.close();
-
-            // The "Connection: close" line tells server to close connection
-            // when the download is complete
-            _connected = false;
-            if (ret == false)
-              return DS_abort;
-            return DS_success;
-          } else {
-            if (downloader_cat.is_debug())
-              downloader_cat.debug()
-                << "Downloader::download() - Received 0 bytes" << endl;
-          }
-        }
-        break;
-
-      default:
-
-        downloader_cat.error()
-          << "Downloader::download() - Unknown return value from "
-          << "attempt_read() : " << ret << endl;
-        return DS_abort;
-
-    } // switch(ret)
-
-    // Sleep for the requested frequency
-    nap();
-
-  } // for (;;)
-
-  downloader_cat.error()
-    << "Downloader::download() - Dropped out of for loop without returning!"
-    << endl;
-  return DS_abort;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::parse_http_response
-//       Access: Private
-//  Description: Check the HTTP response from the server
-////////////////////////////////////////////////////////////////////
-bool Downloader::
-parse_http_response(const string &resp) {
-  size_t ws = resp.find(" ", 0);
-  string httpstr = resp.substr(0, ws);
-  if (!(httpstr == "HTTP/1.1")) {
-    downloader_cat.error()
-      << "Downloader::parse_http_response() - not HTTP/1.1 - got: "
-      << httpstr << endl;
-    return false;
-  }
-  size_t ws2 = resp.find(" ", ws);
-  string numstr = resp.substr(ws, ws2);
-  nassertr(numstr.length() > 0, false);
-  int num = atoi(numstr.c_str());
-  switch (num) {
-    case 200:
-    case 206:
-      return true;
-    case 202:
-      // Accepted - server may not honor request, though
-      if (downloader_cat.is_debug())
-        downloader_cat.debug()
-          << "Downloader::parse_http_response() - got a 202 Accepted - "
-          << "server does not guarantee to honor this request" << endl;
-      return true;
-    case 201:
-    case 203:
-    case 204:
-    case 205:
-    default:
-      break;
-  }
-
-  downloader_cat.error()
-    << "Downloader::parse_http_response() - Invalid response: "
-    << resp << endl;
-  return false;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::parse_header
-//       Access: Private
-//  Description: Looks for a valid header.  If it finds one, it
-//               calculates the header length and strips it from
-//               the download status structure.  Function returns false
-//               on an error condition, otherwise true.
-////////////////////////////////////////////////////////////////////
-bool Downloader::
-parse_header(DownloadStatus &status) {
-
-  if (status._header_is_complete == true)
-    return true;
-
-  if (status._bytes_in_buffer == 0) {
-    downloader_cat.error()
-      << "Downloader::parse_header() - Empty buffer!" << endl;
-    return false;
-  }
-
-  string bufstr((char *)status._start, status._bytes_in_buffer);
-  size_t p  = 0;
-  while (p < bufstr.length()) {
-    // Server sends out CR LF (\r\n) as newline delimiter
-    size_t nl = bufstr.find("\015\012", p);
-    if (nl == string::npos) {
-      downloader_cat.error()
-        << "Downloader::parse_header() - No newlines in buffer of "
-        << "length: " << status._bytes_in_buffer << endl;
-      return false;
-    } else if (p == 0 && nl == p) {
-      downloader_cat.error()
-        << "Downloader::parse_header() - Buffer begins with newline!"
-        << endl;
-        return false;
-    }
-
-    string component = bufstr.substr(p, nl - p);
-
-    // The first line of the response should say whether
-    // got an error or not
-    if (status._first_line_complete == false) {
-      status._first_line_complete = true;
-      if (parse_http_response(component) == true) {
-        if (downloader_cat.is_debug())
-          downloader_cat.debug()
-            << "Downloader::parse_header() - Header is valid: "
-            << component << endl;
-        status._header_is_valid = true;
-      } else {
-        return false;
-      }
-    }
-
-    // Look for content length
-    size_t cpos = component.find(":");
-    string tline = component.substr(0, cpos);
-    if (status._partial_content == true && tline == "Content-Length") {
-      tline = component.substr(cpos + 2, string::npos);
-      int server_download_bytes = atoi(tline.c_str());
-      int client_download_bytes = status._last_byte - status._first_byte;
-      if (status._first_byte == 0)
-        client_download_bytes += 1;
-      if (client_download_bytes != server_download_bytes) {
-        downloader_cat.error()
-          << "Downloader::parse_header() - server size = "
-          << server_download_bytes << ", client size = "
-          << client_download_bytes << " ("
-          << status._last_byte << "-" << status._first_byte << ")" << endl;
-        return false;
-      }
-    }
-
-    // Two consecutive (CR LF)s indicates end of HTTP header
-    if (nl == p) {
-      if (downloader_cat.is_debug())
-        downloader_cat.debug()
-          << "Downloader::parse_header() - Header is complete" << endl;
-      status._header_is_complete = true;
-
-      // Strip the header out of the status buffer
-      int header_length = nl + 2;
-      status._start += header_length;
-      status._bytes_in_buffer -= header_length;
-
-      if (downloader_cat.is_debug())
-        downloader_cat.debug()
-          << "Downloader::parse_header() - Stripping out header of size: "
-          << header_length << endl;
-
-      return true;
-    }
-
-    p = nl + 2;
-  }
-
-  if (status._header_is_complete == false) {
-    if (downloader_cat.is_debug())
-      downloader_cat.debug()
-        << "Downloader::parse_header() - Reached end of buffer without "
-        << "successfully parsing the header - buffer size: "
-        << status._bytes_in_buffer << endl;
-  }
-
-  return true;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::write_to_disk
-//       Access: Private
-//  Description: Writes a download to disk.  If there is a header,
-//               the pointer and size are adjusted so the header
-//               is excluded.  Function returns false on error
-//               condition.
-////////////////////////////////////////////////////////////////////
-bool Downloader::
-write_to_disk(DownloadStatus &status) {
-
-  // Ensure the header has been parsed successfully first
-  if (parse_header(status) == false)
-    return false;
-
-  if (status._header_is_complete == false) {
-    downloader_cat.error()
-      << "Downloader::write_to_disk() - Incomplete HTTP header - "
-      << "(or header was larger than download buffer) - "
-      << "try increasing download-buffer-size" << endl;
-    return false;
-  }
-
-  // Write what we have so far to disk
-  if (status._bytes_in_buffer > 0) {
-    if (downloader_cat.is_debug())
-      downloader_cat.debug()
-        << "Downloader::write_to_disk() - Writing "
-        << status._bytes_in_buffer << " to disk" << endl;
-
-    _dest_stream.write(status._start, status._bytes_in_buffer);
-    status._total_bytes_written += status._bytes_in_buffer;
-
-    // Throw an event to indicate how many bytes have been written so far
-    if (!status._event_name.empty()) {
-      PT_Event write_event = new Event(status._event_name);
-      write_event->add_parameter(EventParameter((int)status._id));
-      write_event->add_parameter(EventParameter(DS_write));
-      write_event->add_parameter(EventParameter(status._total_bytes_written));
-      throw_event(write_event);
-    }
-  }
-
-  status.reset();
-
-  return true;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::DownloadStatus::constructor
-//       Access: Private
-//  Description:
-////////////////////////////////////////////////////////////////////
-Downloader::DownloadStatus::
-DownloadStatus(char *buffer, const string &event_name, int first_byte,
-        int last_byte, int total_bytes, bool partial_content, uint id) {
-  _first_line_complete = false;
-  _header_is_complete = false;
-  _header_is_valid = false;
-  _buffer = buffer;
-  _event_name = event_name;
-  _first_byte = first_byte;
-  _last_byte = last_byte;
-  _total_bytes = total_bytes;
-  _partial_content = partial_content;
-  _id = id;
-  reset();
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Downloader::DownloadStatus::reset
-//       Access: Public
-//  Description: Resets the status buffer for more downloading after
-//               a write.
-////////////////////////////////////////////////////////////////////
-void Downloader::DownloadStatus::
-reset(void) {
-  _start = _buffer;
-  _next_in = _start;
-  _bytes_in_buffer = 0;
-  _total_bytes_written = 0;
-}

+ 0 - 150
panda/src/downloader/asyncDownloader.h

@@ -1,150 +0,0 @@
-// Filename: asyncDownloader.h
-// Created by:  mike (09Jan97)
-//
-////////////////////////////////////////////////////////////////////
-//
-// PANDA 3D SOFTWARE
-// Copyright (c) 2001, Disney Enterprises, Inc.  All rights reserved
-//
-// All use of this software is subject to the terms of the Panda 3d
-// Software license.  You should have received a copy of this license
-// along with this source code; you will also find a current copy of
-// the license at http://www.panda3d.org/license.txt .
-//
-// To contact the maintainers of this program write to
-// [email protected] .
-//
-////////////////////////////////////////////////////////////////////
-#ifndef ASYNCDOWNLOADER_H
-#define ASYNCDOWNLOADER_H
-//
-////////////////////////////////////////////////////////////////////
-// Includes
-////////////////////////////////////////////////////////////////////
-#include <pandabase.h>
-#include <notify.h>
-#include <filename.h>
-#include <tokenBoard.h>
-#include <buffer.h>
-#include "asyncUtility.h"
-
-#if defined(WIN32_VC)
-  #include <winsock.h>
-#else
-  #include <netinet/in.h>  // Irix seems to require this one for resolv.h.
-  #include <sys/types.h>
-  #include <sys/socket.h>
-  #include <resolv.h>
-#endif
-
-class DownloaderToken;
-
-////////////////////////////////////////////////////////////////////
-//       Class : Downloader
-// Description :
-////////////////////////////////////////////////////////////////////
-class EXPCL_PANDAEXPRESS Downloader : public AsyncUtility {
-PUBLISHED:
-  enum DownloadCode {
-    DS_write = 2,
-    DS_success = 1,
-    DS_abort = -1,
-    DS_timeout = -2
-  };
-  Downloader(void);
-  //Downloader(PT(Buffer) buffer);
-  virtual ~Downloader(void);
-
-  bool connect_to_server(const string &name, uint port=80);
-  void disconnect_from_server(void);
-
-  int request_sync_download(const string &file_name, const Filename &file_dest,
-                        const string &event_name);
-  int request_sync_download(const string &file_name, const Filename &file_dest,
-                        const string &event_name, int first_byte,
-                        int last_byte, int total_bytes,
-                        bool partial_content = true);
-  int request_download(const string &file_name, const Filename &file_dest,
-                        const string &event_name, bool sync = false);
-  int request_download(const string &file_name, const Filename &file_dest,
-                        const string &event_name, int first_byte,
-                        int last_byte, int total_bytes,
-                        bool partial_content = true, bool sync = false);
-
-  INLINE void set_byte_rate(float bytes);
-  INLINE float get_byte_rate(void) const;
-  INLINE void set_disk_write_frequency(int frequency);
-  INLINE int get_disk_write_frequency(void) const;
-  INLINE void enable_download(bool val);
-  INLINE bool is_download_enabled(void) const;
-  INLINE bool get_last_attempt_stalled(void) const;
-
-private:
-  class DownloadStatus {
-  public:
-    DownloadStatus(char *buffer, const string &event_name, int first_byte,
-                        int last_byte, int total_bytes, bool partial_content,
-                        uint id);
-    void reset(void);
-
-  public:
-    bool _first_line_complete;
-    bool _header_is_complete;
-    bool _header_is_valid;
-    char *_start;
-    char *_next_in;
-    int _bytes_in_buffer;
-    string _event_name;
-    int _total_bytes_written;
-    int _first_byte;
-    int _last_byte;
-    int _total_bytes;
-    bool _partial_content;
-    uint _id;
-    char *_buffer;
-  };
-
-  void init();
-  int download(const string &file_name, Filename file_dest,
-                        const string &event_name, int first_byte,
-                        int last_byte, int total_bytes, bool partial_content,
-                        bool sync, uint id);
-  virtual bool process_request(void);
-  bool parse_header(DownloadStatus &status);
-  bool write_to_disk(DownloadStatus &status);
-  bool connect_to_server(void);
-  int safe_send(int socket, const char *data, int length, long timeout);
-  int safe_receive(int socket, DownloadStatus &status, int length,
-                                long timeout, int &bytes);
-  bool parse_http_response(const string &resp);
-  int attempt_read(int length, DownloadStatus &status, int &bytes_read);
-
-  typedef TokenBoard<DownloaderToken> DownloaderTokenBoard;
-  DownloaderTokenBoard *_token_board;
-
-  bool _connected;
-
-#ifdef HAVE_IPC
-  mutex _buffer_lock;
-#endif
-
-  int _socket;
-  PT(Buffer) _buffer;
-  int _disk_write_frequency;
-  int _new_disk_write_frequency;
-  float _byte_rate;
-  float _new_byte_rate;
-  int _read_size;
-  bool _download_enabled;
-  ofstream _dest_stream;
-  int _disk_buffer_size;
-  bool _last_attempt_stalled;
-  bool _current_attempt_stalled;
-
-  string _server_name;
-  struct sockaddr_in _sin;
-};
-
-#include "asyncDownloader.I"
-
-#endif

+ 0 - 262
panda/src/downloader/asyncExtractor.cxx

@@ -1,262 +0,0 @@
-// Filename: asyncExtractor.cxx
-// Created by:  mike (09Jan97)
-//
-////////////////////////////////////////////////////////////////////
-//
-// PANDA 3D SOFTWARE
-// Copyright (c) 2001, Disney Enterprises, Inc.  All rights reserved
-//
-// All use of this software is subject to the terms of the Panda 3d
-// Software license.  You should have received a copy of this license
-// along with this source code; you will also find a current copy of
-// the license at http://www.panda3d.org/license.txt .
-//
-// To contact the maintainers of this program write to
-// [email protected] .
-//
-////////////////////////////////////////////////////////////////////
-
-#include "config_downloader.h"
-
-#include <event.h>
-#include <pt_Event.h>
-#include <throw_event.h>
-#include <eventParameter.h>
-#include <filename.h>
-
-#include "asyncExtractor.h"
-
-////////////////////////////////////////////////////////////////////
-// Defines
-////////////////////////////////////////////////////////////////////
-
-////////////////////////////////////////////////////////////////////
-//       Class : ExtractorToken
-// Description : Holds a request for the extractor.
-////////////////////////////////////////////////////////////////////
-class ExtractorToken : public ReferenceCount {
-public:
-  INLINE ExtractorToken(uint id, const Filename &source_file,
-                        const string &event_name,
-                        const Filename &rel_path) {
-    _id = id;
-    _source_file = source_file;
-    _event_name = event_name;
-    _rel_path = rel_path;
-  }
-  int _id;
-  Filename _source_file;
-  string _event_name;
-  Filename _rel_path;
-};
-
-////////////////////////////////////////////////////////////////////
-//     Function: Extractor::Constructor
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-Extractor::
-Extractor(void) : AsyncUtility() {
-  PT(Buffer) buffer = new Buffer(extractor_buffer_size);
-  init(buffer);
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Extractor::Constructor
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-Extractor::
-Extractor(PT(Buffer) buffer) : AsyncUtility() {
-  init(buffer);
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Extractor::Constructor
-//       Access: Private
-//  Description:
-////////////////////////////////////////////////////////////////////
-void Extractor::
-init(PT(Buffer) buffer) {
-  nassertv(!buffer.is_null());
-  _frequency = extractor_frequency;
-  _token_board = new ExtractorTokenBoard;
-  _buffer = buffer;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Extractor::Destructor
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-Extractor::
-~Extractor(void) {
-  destroy_thread();
-
-  delete _token_board;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Extractor::request_extract
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-int Extractor::
-request_extract(const Filename &source_file, const string &event_name,
-                const Filename &rel_path) {
-
-  PT(ExtractorToken) tok;
-  if (_threads_enabled) {
-
-    // Make sure we actually are threaded
-    if (!_threaded) {
-      downloader_cat.info()
-        << "Extractor::request_extract() - create_thread() was "
-        << "never called!  Calling it now..." << endl;
-      create_thread();
-    }
-
-    // We need to grab the lock in order to signal the condition variable
-#ifdef HAVE_IPC
-    _lock.lock();
-#endif
-
-      if (_token_board->_waiting.is_full()) {
-        downloader_cat.error()
-          << "Extractor::request_extract() - Too many pending requests\n";
-        return 0;
-      }
-
-      if (downloader_cat.is_debug()) {
-        downloader_cat.debug()
-          << "Extract requested for file: " << source_file << endl;
-      }
-
-      tok = new ExtractorToken(_next_token++, source_file, event_name,
-                                                rel_path);
-      _token_board->_waiting.insert(tok);
-
-#ifdef HAVE_IPC
-      _request_cond->signal();
-    _lock.unlock();
-#endif
-
-  } else {
-    // If we're not running asynchronously, process the load request
-    // directly now.
-    if (_token_board->_waiting.is_full()) {
-      downloader_cat.error()
-        << "Extractor::request_extract() - Too many pending requests\n";
-      return 0;
-    }
-    if (downloader_cat.is_debug()) {
-      downloader_cat.debug()
-        << "Extract requested for file: " << source_file << endl;
-    }
-
-    tok = new ExtractorToken(_next_token++, source_file, event_name,
-                                                rel_path);
-    _token_board->_waiting.insert(tok);
-    process_request();
-  }
-
-  return tok->_id;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Extractor::process_request
-//       Access: Private
-//  Description: Serves any requests on the token board, moving them
-//               to the done queue.
-////////////////////////////////////////////////////////////////////
-bool Extractor::
-process_request() {
-  if (_shutdown) {
-    if (downloader_cat.is_debug())
-      downloader_cat.debug()
-        << "Extractor shutting down...\n";
-    return false;
-  }
-
-  // If there is actually a request token - process it
-  while (!_token_board->_waiting.is_empty()) {
-    PT(ExtractorToken) tok = _token_board->_waiting.extract();
-    if (extract(tok->_source_file, tok->_rel_path)) {
-      _token_board->_done.insert(tok);
-
-      // Throw a "done" event now.
-      if (!tok->_event_name.empty()) {
-        PT_Event done = new Event(tok->_event_name);
-        done->add_parameter(EventParameter((int)tok->_id));
-        throw_event(done);
-      }
-
-      if (downloader_cat.is_debug()) {
-        downloader_cat.debug()
-          << "Extractor::process_request() - extract complete for "
-          << tok->_source_file << "\n";
-      }
-    }
-  }
-
-  return true;
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Extractor::extract
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-bool Extractor::
-extract(Filename &source_file, const Filename &rel_path) {
-
-  // Open source file
-  ifstream read_stream;
-  source_file.set_binary();
-  if (!source_file.open_read(read_stream)) {
-    downloader_cat.error()
-      << "Extractor::extract() - Error opening source file: "
-      << source_file << endl;
-    return false;
-  }
-
-  // Determine source file length
-  read_stream.seekg(0, ios::end);
-  int source_file_length = read_stream.tellg();
-  read_stream.seekg(0, ios::beg);
-
-  // Read the multifile header
-  Multifile mfile;
-
-  // Read from the source file and write to the appropriate extracted file
-  int total_bytes_read = 0;
-  bool read_all_input = false;
-  bool handled_all_input = false;
-  int source_buffer_length;
-  while (handled_all_input == false) {
-
-    // See if there is anything left in the source file
-    if (read_all_input == false) {
-      read_stream.read(_buffer->_buffer, _buffer->get_length());
-      source_buffer_length = read_stream.gcount();
-      total_bytes_read += source_buffer_length;
-      if (read_stream.eof()) {
-        nassertr(total_bytes_read == source_file_length, false);
-        read_all_input = true;
-      }
-    }
-
-    // Write to the out file
-    char *start = _buffer->_buffer;
-    int size = source_buffer_length;
-    if (mfile.write_extract(start, size, rel_path) == true)
-      handled_all_input = true;
-
-    nap();
-  }
-
-  read_stream.close();
-  source_file.unlink();
-
-  return true;
-}

+ 0 - 58
panda/src/downloader/asyncExtractor.h

@@ -1,58 +0,0 @@
-// Filename: asyncExtractor.h
-// Created by:  mike (09Jan97)
-//
-////////////////////////////////////////////////////////////////////
-//
-// PANDA 3D SOFTWARE
-// Copyright (c) 2001, Disney Enterprises, Inc.  All rights reserved
-//
-// All use of this software is subject to the terms of the Panda 3d
-// Software license.  You should have received a copy of this license
-// along with this source code; you will also find a current copy of
-// the license at http://www.panda3d.org/license.txt .
-//
-// To contact the maintainers of this program write to
-// [email protected] .
-//
-////////////////////////////////////////////////////////////////////
-#ifndef ASYNCEXTRACTOR_H
-#define ASYNCEXTRACTOR_H
-//
-////////////////////////////////////////////////////////////////////
-// Includes
-////////////////////////////////////////////////////////////////////
-#include <pandabase.h>
-#include <filename.h>
-#include <tokenBoard.h>
-#include <buffer.h>
-#include <multifile.h>
-#include "asyncUtility.h"
-
-class ExtractorToken;
-
-////////////////////////////////////////////////////////////////////
-//       Class : Extractor
-// Description :
-////////////////////////////////////////////////////////////////////
-class EXPCL_PANDAEXPRESS Extractor : public AsyncUtility {
-PUBLISHED:
-  Extractor(void);
-  Extractor(PT(Buffer) buffer);
-  virtual ~Extractor(void);
-
-  int request_extract(const Filename &source_file,
-                      const string &event_name, const Filename &rel_path = "");
-
-  bool extract(Filename &source_file, const Filename &rel_path);
-
-private:
-  void init(PT(Buffer) buffer);
-  virtual bool process_request(void);
-
-  typedef TokenBoard<ExtractorToken> ExtractorTokenBoard;
-  ExtractorTokenBoard *_token_board;
-
-  PT(Buffer) _buffer;
-};
-
-#endif

+ 5 - 10
panda/src/downloader/extractor.I

@@ -21,16 +21,11 @@
 ////////////////////////////////////////////////////////////////////
 //     Function: Extractor::get_progress
 //       Access: Public
-//  Description:
+//  Description: Returns the fraction of the Multifile extracted so
+//               far.
 ////////////////////////////////////////////////////////////////////
 INLINE float Extractor::
-get_progress(void) const {
-  if (_initiated == false) {
-    downloader_cat.warning()
-      << "Extractor::get_progress() - Extraction has not been initiated"
-      << endl;
-    return 0.0;
-  }
-  nassertr(_source_file_length > 0, 0.0);
-  return ((float)_total_bytes_read / (float)_source_file_length);
+get_progress() const {
+  nassertr(_initiated, 0.0f);
+  return ((float)_subfile_index / (float)(_multifile.get_num_subfiles() + 1));
 }

+ 108 - 113
panda/src/downloader/extractor.cxx

@@ -16,28 +16,12 @@
 //
 ////////////////////////////////////////////////////////////////////
 
-#include "config_downloader.h"
-
-#include <filename.h>
-#include <error_utils.h>
-
-#include <errno.h>
 #include "extractor.h"
+#include "config_downloader.h"
 
-////////////////////////////////////////////////////////////////////
-// Defines
-////////////////////////////////////////////////////////////////////
+#include "filename.h"
+#include "error_utils.h"
 
-////////////////////////////////////////////////////////////////////
-//     Function: Extractor::Constructor
-//       Access: Public
-//  Description:
-////////////////////////////////////////////////////////////////////
-Extractor::
-Extractor(void) {
-  PT(Buffer) buffer = new Buffer(extractor_buffer_size);
-  init(buffer);
-}
 
 ////////////////////////////////////////////////////////////////////
 //     Function: Extractor::Constructor
@@ -45,24 +29,8 @@ Extractor(void) {
 //  Description:
 ////////////////////////////////////////////////////////////////////
 Extractor::
-Extractor(PT(Buffer) buffer) {
-  init(buffer);
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Extractor::Constructor
-//       Access: Private
-//  Description:
-////////////////////////////////////////////////////////////////////
-void Extractor::
-init(PT(Buffer) buffer) {
-  if (downloader_cat.is_debug())
-    downloader_cat.debug()
-      << "Extractor constructor called" << endl;
+Extractor() {
   _initiated = false;
-  nassertv(!buffer.is_null());
-  _buffer = buffer;
-  _mfile = NULL;
 }
 
 ////////////////////////////////////////////////////////////////////
@@ -71,136 +39,163 @@ init(PT(Buffer) buffer) {
 //  Description:
 ////////////////////////////////////////////////////////////////////
 Extractor::
-~Extractor(void) {
+~Extractor() {
   if (downloader_cat.is_debug())
     downloader_cat.debug()
       << "Extractor destructor called" << endl;
-  if (_initiated == true)
+  if (_initiated) {
     cleanup();
+  }
 }
 
 ////////////////////////////////////////////////////////////////////
 //     Function: Extractor::initiate
 //       Access: Public
-//  Description:
+//  Description: Begins the extraction process.  Returns EU_success if
+//               successful, EU_error_abort otherwise.
+//
+//               After calling initiate(), you should repeatedly call
+//               run() as a background task until the file is
+//               completely extracted.
 ////////////////////////////////////////////////////////////////////
 int Extractor::
-initiate(Filename &source_file, const Filename &rel_path) {
-
-  if (_initiated == true) {
+initiate(const Filename &multifile_name, const Filename &extract_to) {
+  if (_initiated) {
     downloader_cat.error()
       << "Extractor::initiate() - Extraction has already been initiated"
       << endl;
     return EU_error_abort;
   }
 
-  // Open source file
-  _source_file = source_file;
-  _source_file.set_binary();
-  if (!_source_file.open_read(_read_stream)) {
+  _multifile_name = multifile_name;
+  _extract_to = extract_to;
+
+  if (!_multifile.open_read(_multifile_name)) {
     downloader_cat.error()
-      << "Extractor::extract() - Error opening source file: "
-      << _source_file << " : " << strerror(errno) << endl;
-    return get_write_error();
+      << "Error opening Multifile " << _multifile_name << ".\n";
+    return EU_error_abort;
   }
 
-  _rel_path = rel_path;
-
-  // Determine source file length
-  _read_stream.seekg(0, ios::end);
-  _source_file_length = _read_stream.tellg();
-  _read_stream.seekg(0, ios::beg);
-
-  _total_bytes_read = 0;
-  _read_all_input = false;
-  _handled_all_input = false;
-  _mfile = new Multifile();
+  _subfile_index = 0;
+  _subfile_pos = 0;
+  _subfile_length = 0;
+  _read = (istream *)NULL;
   _initiated = true;
-  return EU_success;
-}
 
-////////////////////////////////////////////////////////////////////
-//     Function: Extractor::cleanup
-//       Access: Private
-//  Description:
-////////////////////////////////////////////////////////////////////
-void Extractor::
-cleanup(void) {
-  if (downloader_cat.is_debug())
-    downloader_cat.debug()
-      << "Extractor cleanup called" << endl;
-  if (_initiated == false) {
-    downloader_cat.error()
-      << "Extractor::cleanup() - Extraction has not been initiated"
-      << endl;
-    return;
-  }
-
-  delete _mfile;
-  _mfile = NULL;
-  _read_stream.close();
-//  _source_file.unlink();
-  _initiated = false;
+  return EU_success;
 }
 
 ////////////////////////////////////////////////////////////////////
 //     Function: Extractor::run
 //       Access: Public
-//  Description:
+//  Description: Extracts the next small unit of data from the
+//               Multifile.  Returns EU_ok if progress is continuing,
+//               EU_error_abort if there is a problem, or EU_success
+//               when the last piece has been extracted.
 ////////////////////////////////////////////////////////////////////
 int Extractor::
-run(void) {
-  if (_initiated == false) {
+run() {
+  if (!_initiated) {
     downloader_cat.error()
       << "Extractor::run() - Extraction has not been initiated"
       << endl;
     return EU_error_abort;
   }
 
-  // See if there is anything left in the source file
-  if (_read_all_input == false) {
-    _read_stream.read(_buffer->_buffer, _buffer->get_length());
-    _source_buffer_length = _read_stream.gcount();
-    _total_bytes_read += _source_buffer_length;
-    if (_read_stream.eof()) {
-      nassertr(_total_bytes_read == _source_file_length, false);
-      _read_all_input = true;
+  if (_read == (istream *)NULL) {
+    // Time to open the next subfile.
+    if (_subfile_index >= _multifile.get_num_subfiles()) {
+      // All done!
+      cleanup();
+      return EU_success;
     }
-  }
 
-  char *buffer_start = _buffer->_buffer;
-  int buffer_size = _source_buffer_length;
+    Filename subfile_filename(_extract_to, 
+                              _multifile.get_subfile_name(_subfile_index));
+    subfile_filename.set_binary();
+    subfile_filename.make_dir();
+    if (!subfile_filename.open_write(_write)) {
+      downloader_cat.error()
+        << "Unable to write to " << subfile_filename << ".\n";
+      cleanup();
+      return EU_error_abort;
+    }
 
-  // Write to the out file
-  int write_ret = _mfile->write(buffer_start, buffer_size, _rel_path);
-  if (write_ret == EU_success) {
-    cleanup();
-    return EU_success;
-  } else if (write_ret < 0) {
-    downloader_cat.error()
-      << "Extractor::run() - got error from Multifile: " << write_ret
-      << endl;
-    return write_ret;
+    _subfile_length = _multifile.get_subfile_length(_subfile_index);
+    _subfile_pos = 0;
+    _read = &_multifile.open_read_subfile(_subfile_index);
+
+  } else if (_subfile_pos >= _subfile_length) {
+    // Time to close this subfile.
+    _multifile.close_subfile();
+    _write.close();
+    _read = (istream *)NULL;
+    _subfile_index++;
+
+  } else {
+    // Read a number of bytes from the subfile and write them to the
+    // output.
+    size_t max_bytes = min((size_t)extractor_buffer_size, 
+                           _subfile_length - _subfile_pos);
+    for (size_t p = 0; p < max_bytes; p++) {
+      int byte = _read->get();
+      if (_read->eof() || _read->fail()) {
+        downloader_cat.error()
+          << "Unexpected EOF on multifile " << _multifile_name << ".\n";
+        cleanup();
+        return EU_error_abort;
+      }
+      _write.put(byte);
+    }
+    _subfile_pos += max_bytes;
   }
+
   return EU_ok;
 }
 
 ////////////////////////////////////////////////////////////////////
 //     Function: Extractor::extract
 //       Access: Public
-//  Description:
+//  Description: A convenience function to extract the Multifile all
+//               at once, when you don't care about doing it in the
+//               background.
 ////////////////////////////////////////////////////////////////////
 bool Extractor::
-extract(Filename &source_file, const Filename &rel_path) {
+extract(const Filename &source_file, const Filename &rel_path) {
   int ret = initiate(source_file, rel_path);
-  if (ret < 0)
+  if (ret < 0) {
     return false;
+  }
   for (;;) {
     ret = run();
-    if (ret == EU_success)
+    if (ret == EU_success) {
       return true;
-    if (ret < 0)
+    }
+    if (ret < 0) {
       return false;
+    }
   }
   return false;
 }
+
+////////////////////////////////////////////////////////////////////
+//     Function: Extractor::cleanup
+//       Access: Private
+//  Description:
+////////////////////////////////////////////////////////////////////
+void Extractor::
+cleanup() {
+  if (downloader_cat.is_debug())
+    downloader_cat.debug()
+      << "Extractor cleanup called" << endl;
+  if (!_initiated) {
+    downloader_cat.error()
+      << "Extractor::cleanup() - Extraction has not been initiated"
+      << endl;
+    return;
+  }
+
+  _multifile.close();
+  _write.close();
+  _read = (istream *)NULL;
+}

+ 30 - 28
panda/src/downloader/extractor.h

@@ -17,49 +17,51 @@
 ////////////////////////////////////////////////////////////////////
 #ifndef EXTRACTOR_H
 #define EXTRACTOR_H
-//
-////////////////////////////////////////////////////////////////////
-// Includes
-////////////////////////////////////////////////////////////////////
-#include <pandabase.h>
-#include <filename.h>
-#include <buffer.h>
-#include <multifile.h>
-#include <pointerTo.h>
+
+#include "pandabase.h"
+#include "filename.h"
+#include "buffer.h"
+#include "multifile.h"
+#include "pointerTo.h"
 
 ////////////////////////////////////////////////////////////////////
 //       Class : Extractor
-// Description :
+// Description : This class automatically extracts the contents of a
+//               Multifile to the current directory (or to a specified
+//               directory) in the background.
+//
+//               It is designed to limit its use of system resources
+//               and run unobtrusively in the background.  After
+//               initiate(), each call to run() extracts another small
+//               portion of the Multifile.  Call run() repeatedly
+//               whenever you have spare cycles until run() returns
+//               EU_success.
 ////////////////////////////////////////////////////////////////////
 class EXPCL_PANDAEXPRESS Extractor {
 PUBLISHED:
-  Extractor(void);
-  Extractor(PT(Buffer) buffer);
-  virtual ~Extractor(void);
+  Extractor();
+  ~Extractor();
 
-  int initiate(Filename &source_file, const Filename &rel_path = "");
-  int run(void);
+  int initiate(const Filename &multifile_name, const Filename &extract_to = "");
+  int run();
 
-  bool extract(Filename &source_file, const Filename &rel_path = "");
+  bool extract(const Filename &multifile_name, const Filename &extract_to = "");
 
   INLINE float get_progress(void) const;
 
 private:
-  void init(PT(Buffer) buffer);
-  void cleanup(void);
+  void cleanup();
 
   bool _initiated;
-  PT(Buffer) _buffer;
+  Filename _multifile_name;
+  Filename _extract_to;
+  Multifile _multifile;
 
-  ifstream _read_stream;
-  int _source_file_length;
-  Multifile *_mfile;
-  int _total_bytes_read;
-  bool _read_all_input;
-  bool _handled_all_input;
-  int _source_buffer_length;
-  Filename _source_file;
-  Filename _rel_path;
+  int _subfile_index;
+  size_t _subfile_pos;
+  size_t _subfile_length;
+  istream *_read;
+  ofstream _write;
 };
 
 #include "extractor.I"

+ 322 - 57
panda/src/downloadertools/multify.cxx

@@ -16,89 +16,354 @@
 //
 ////////////////////////////////////////////////////////////////////
 
-#include <pandabase.h>
+#include "pandabase.h"
 #ifndef HAVE_GETOPT
-  #include <gnu_getopt.h>
+  #include "gnu_getopt.h"
 #else
   #include <getopt.h>
 #endif
-#include <multifile.h>
-#include <filename.h>
-#ifndef OLD_WAY
-  #include <extractor.h>
-#endif
+#include "multifile.h"
+#include "filename.h"
+#include <stdio.h>
+
+
+bool create = false;      // -c
+bool append = false;      // -r
+bool list = false;        // -t
+bool extract = false;     // -x
+bool verbose = false;     // -v
+Filename multifile_name;  // -f
+bool got_multifile_name = false;
+bool to_stdout = false;   // -O
+Filename chdir_to;        // -C
+bool got_chdir_to = false;
+size_t scale_factor = 0;  // -F
+
+void 
+usage() {
+  cerr << "Usage: multify -[c|r|t|x] -f <multifile_name> [options] <subfile_name> ...\n";
+}
+
+void 
+help() {
+  cerr << "multify is used to store and extract files from a Panda Multifile.\n"
+       << "This is similar to a tar or zip file in that it is an archive file that\n"
+       << "contains a number of subfiles that may later be extracted.\n\n"
+
+       << "The command-line options for multify are designed to be similar to those\n"
+       << "for tar, the traditional Unix archiver utility.\n\n";
+  usage();
+}
+
+bool
+is_named(const string &subfile_name, int argc, char *argv[]) {
+  // Returns true if the indicated subfile appears on the list of
+  // files named on the command line.
+  if (argc < 2) {
+    // No named files; everything is listed.
+    return true;
+  }
+
+  for (int i = 1; i < argc; i++) {
+    if (subfile_name == argv[i]) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+bool
+add_directory(Multifile &multifile, const Filename &directory_name) {
+  vector_string files;
+  if (!directory_name.scan_directory(files)) {
+    cerr << "Unable to scan directory " << directory_name << "\n";
+    return false;
+  }
+
+  bool okflag = true;
+
+  vector_string::const_iterator fi;
+  for (fi = files.begin(); fi != files.end(); ++fi) {
+    Filename subfile_name(directory_name, (*fi));
+    if (subfile_name.is_directory()) {
+      okflag = add_directory(multifile, subfile_name);
+
+    } else if (!subfile_name.exists()) {
+      cerr << "Not found: " << subfile_name << "\n";
+      okflag = false;
+
+    } else {
+      if (verbose) {
+        cout << subfile_name << "\n";
+      }
+      if (!multifile.add_subfile(subfile_name, subfile_name)) {
+        cerr << "Unable to add " << subfile_name << ".\n";
+        okflag = false;
+      }
+    }
+  }
+
+  return okflag;
+}
+
+bool
+add_files(int argc, char *argv[]) {
+  Multifile multifile;
+  if (append) {
+    if (!multifile.open_read_write(multifile_name)) {
+      cerr << "Unable to open " << multifile_name << " for updating.\n";
+      return false;
+    }
+  } else {
+    if (!multifile.open_write(multifile_name)) {
+      cerr << "Unable to open " << multifile_name << " for writing.\n";
+      return false;
+    }
+  }
+
+  if (scale_factor != 0 && scale_factor != multifile.get_scale_factor()) {
+    cerr << "Setting scale factor to " << scale_factor << "\n";
+    multifile.set_scale_factor(scale_factor);
+  }
+
+  bool okflag = true;
+  for (int i = 1; i < argc; i++) {
+    Filename subfile_name = Filename::from_os_specific(argv[i]);
+    if (subfile_name.is_directory()) {
+      if (!add_directory(multifile, subfile_name)) {
+        okflag = false;
+      }
+
+    } else if (!subfile_name.exists()) {
+      cerr << "Not found: " << subfile_name << "\n";
+      okflag = false;
+
+    } else {
+      if (verbose) {
+        cout << subfile_name << "\n";
+      }
+      if (!multifile.add_subfile(subfile_name, subfile_name)) {
+        cerr << "Unable to add " << subfile_name << ".\n";
+        okflag = false;
+      }
+    }
+  }
+
+  if (multifile.needs_repack()) {
+    if (!multifile.repack()) {
+      cerr << "Failed to write " << multifile_name << ".\n";
+      okflag = false;
+    }
+  } else {
+    if (!multifile.flush()) {
+      cerr << "Failed to write " << multifile_name << ".\n";
+      okflag = false;
+    }
+  }
+
+  return okflag;
+}
+
+bool
+extract_files(int argc, char *argv[]) {
+  if (!multifile_name.exists()) {
+    cerr << multifile_name << " not found.\n";
+    return false;
+  }
+  Multifile multifile;
+  if (!multifile.open_read(multifile_name)) {
+    cerr << "Unable to open " << multifile_name << " for reading.\n";
+    return false;
+  }
+
+  int num_subfiles = multifile.get_num_subfiles();
+
+  for (int i = 0; i < num_subfiles; i++) {
+    string subfile_name = multifile.get_subfile_name(i);
+    if (is_named(subfile_name, argc, argv)) {
+      Filename filename = subfile_name;
+      if (got_chdir_to) {
+        filename = Filename(chdir_to, subfile_name);
+      }
+      if (to_stdout) {
+        if (verbose) {
+          cerr << filename << "\n";
+        }
+        multifile.extract_subfile_to(i, cout);
+      } else {
+        if (verbose) {
+          cout << filename << "\n";
+        }
+        multifile.extract_subfile(i, filename);
+      }
+    }
+  }
+
+  return true;
+}
+
+bool
+list_files(int argc, char *argv[]) {
+  if (!multifile_name.exists()) {
+    cerr << multifile_name << " not found.\n";
+    return false;
+  }
+  Multifile multifile;
+  if (!multifile.open_read(multifile_name)) {
+    cerr << "Unable to open " << multifile_name << " for reading.\n";
+    return false;
+  }
+
+  int num_subfiles = multifile.get_num_subfiles();
+  
+  if (verbose) {
+    cout << num_subfiles << " subfiles:\n" << flush;
+    for (int i = 0; i < num_subfiles; i++) {
+      string subfile_name = multifile.get_subfile_name(i);
+      if (is_named(subfile_name, argc, argv)) {
+        printf("%12d  %s\n", 
+               multifile.get_subfile_length(i),
+               subfile_name.c_str());
+      }
+    }
+    fflush(stdout);
+    if (multifile.get_scale_factor() != 1) {
+      cout << "Scale factor is " << multifile.get_scale_factor() << "\n";
+    }
+    if (multifile.needs_repack()) {
+      cout << "Multifile needs to be repacked.\n";
+    }
+  } else {
+    for (int i = 0; i < num_subfiles; i++) {
+      string subfile_name = multifile.get_subfile_name(i);
+      if (is_named(subfile_name, argc, argv)) {
+        cout << subfile_name << "\n";
+      }
+    }
+  }
+
+  return true;
+}
 
 int
 main(int argc, char *argv[]) {
-  if (argc < 3) {
-    cerr << "Usage: multify -[x,c|vr] <dest_file> <src_file> ..." << endl;
-    return 0;
+  if (argc < 2) {
+    usage();
+    return 1;
   }
 
-  bool verbose = true;
-  bool extract = false;
+  // To emulate tar, we assume an implicit hyphen in front of the
+  // first argument if there is not one already.
+  if (argc >= 2) {
+    if (*argv[1] != '-' && *argv[1] != '\0') {
+      char *new_arg = new char[strlen(argv[1]) + 2];
+      new_arg[0] = '-';
+      strcpy(new_arg + 1, argv[1]);
+      argv[1] = new_arg;
+    }
+  }
 
   extern char *optarg;
   extern int optind;
-  int flag = getopt(argc, argv, "xcvr:");
+  static const char *optflags = "crtxvf:OC:F:h";
+  int flag = getopt(argc, argv, optflags);
   Filename rel_path;
   while (flag != EOF) {
     switch (flag) {
-      case 'x':
-        extract = true;
-        break;
-      case 'c':
-        break;
-      case 'v':
-        verbose = true;
-        break;
-      case 'r':
-        rel_path = optarg;
-        break;
-      default:
-        cerr << "Unhandled switch: " << flag << endl;
-        break;
+    case 'c':
+      create = true;
+      break;
+    case 'r':
+      append = true;
+      break;
+    case 't':
+      list = true;
+      break;
+    case 'x':
+      extract = true;
+      break;
+    case 'v':
+      verbose = true;
+      break;
+    case 'f':
+      multifile_name = Filename::from_os_specific(optarg);
+      got_multifile_name = true;
+      break;
+    case 'C':
+      chdir_to = Filename::from_os_specific(optarg);
+      got_chdir_to = true;
+      break;
+    case 'O':
+      to_stdout = true;
+      break;
+    case 'F':
+      {
+        char *endptr;
+        scale_factor = strtol(optarg, &endptr, 10);
+        if (*endptr != '\0') {
+          cerr << "Invalid integer: " << optarg << "\n";
+          usage();
+          return 1;
+        }
+        if (scale_factor == 0) {
+          cerr << "Scale factor must be nonzero.\n";
+          usage();
+          return 1;
+        }
+      }
+      break;
+
+    case 'h':
+      help();
+      return 1;
+    case '?':
+      usage();
+      return 1;
+    default:
+      cerr << "Unhandled switch: " << flag << endl;
+      break;
     }
-    flag = getopt(argc, argv, "xcvr:");
+    flag = getopt(argc, argv, optflags);
   }
   argc -= (optind - 1);
   argv += (optind - 1);
 
-  if (argc <= 1) {
-    cerr << "Usage: multify -[x,c|v] <dest_file> <src_file> ..." << endl;
-    return 0;
+  // We should have exactly one of these options.
+  if ((create?1:0) + (append?1:0) + (list?1:0) + (extract?1:0) != 1) {
+    cerr << "Exactly one of -c, -r, -t, -x must be specified.\n";
+    usage();
+    return 1;
   }
 
-  Filename dest_file = argv[1];
-  dest_file.set_binary();
+  if (!got_multifile_name) {
+    if (argc <= 1) {
+      usage();
+      return 1;
+    }
 
-  if (verbose == true) {
-    if (extract == true)
-      cerr << "Extracting from: " << dest_file << endl;
-    else
-      cerr << "Appending to: " << dest_file << endl;
+    // For now, we allow -f to be omitted, and use the first argument
+    // as the archive name, for backward compatibility.  Later we will
+    // remove this.
+    multifile_name = Filename::from_os_specific(argv[1]);
+    cerr << "Warning: using " << multifile_name
+         << " as archive name.  Use -f in the future to specify this.\n";
+    argc--;
+    argv++;
   }
 
-  Multifile mfile;
-
-  if (extract == false) {
-    for (int i = 2; i < argc; i++) {
-      Filename in_file = argv[i];
-      in_file.set_binary();
-      mfile.add(in_file);
-    }
+  bool okflag = true;
+  if (create || append) {
+    okflag = add_files(argc, argv);
+  } else if (extract) {
+    okflag = extract_files(argc, argv);
+  } else { // list
+    okflag = list_files(argc, argv);
+  }
 
-    if (mfile.write(dest_file) == false)
-      cerr << "Failed to write: " << dest_file << endl;
+  if (okflag) {
+    return 0;
   } else {
-#ifdef OLD_WAY
-    mfile.read(dest_file);
-    mfile.extract_all(rel_path);
-#else
-    Extractor extor;
-    extor.extract(dest_file, rel_path);
-#endif
+    return 1;
   }
-
-  return 1;
 }

+ 67 - 47
panda/src/express/Sources.pp

@@ -9,55 +9,61 @@
   #define COMBINED_SOURCES $[TARGET]_composite1.cxx $[TARGET]_composite2.cxx
 
   #define SOURCES \
-     bigEndian.h buffer.I buffer.h \
-     checksumHashGenerator.I checksumHashGenerator.h circBuffer.I \
-     circBuffer.h clockObject.I clockObject.h config_express.h \
-     datagram.I datagram.h datagramGenerator.I \
-     datagramGenerator.h datagramInputFile.I datagramInputFile.h \
-     datagramIterator.I datagramIterator.h datagramOutputFile.I \
-     datagramOutputFile.h datagramSink.I datagramSink.h \
-     dcast.T dcast.h \
-     error_utils.h \
-     get_config_path.h hashGeneratorBase.I hashGeneratorBase.h \
-     hashVal.I hashVal.h indent.I indent.h littleEndian.h \
-     memoryInfo.I memoryInfo.h \
-     memoryUsage.I memoryUsage.h \
-     memoryUsagePointerCounts.I memoryUsagePointerCounts.h \
-     memoryUsagePointers.I memoryUsagePointers.h \
-     multifile.I multifile.h namable.I \
-     namable.h nativeNumericData.I nativeNumericData.h \
-     numeric_types.h pointerTo.I pointerTo.h \
-     pointerToArray.I pointerToArray.h \
-     profileTimer.I profileTimer.h \
-     pta_uchar.h referenceCount.I referenceCount.h \
-     register_type.I register_type.h \
-     reversedNumericData.I reversedNumericData.h tokenBoard.I \
-     tokenBoard.h trueClock.I trueClock.h typeHandle.I \
-     typeHandle.h typedObject.I typedObject.h \
-     typedReferenceCount.I typedReferenceCount.h typedef.h \
-     typeRegistry.I typeRegistry.h \
-     typeRegistryNode.I typeRegistryNode.h \
-     vector_uchar.h \
-     $[if $[HAVE_CRYPTO], \
-        crypto_utils.cxx crypto_utils.h patchfile.I \
-        patchfile.cxx patchfile.h ]
+    bigEndian.h buffer.I buffer.h \
+    checksumHashGenerator.I checksumHashGenerator.h circBuffer.I \
+    circBuffer.h clockObject.I clockObject.h config_express.h \
+    datagram.I datagram.h datagramGenerator.I \
+    datagramGenerator.h datagramInputFile.I datagramInputFile.h \
+    datagramIterator.I datagramIterator.h datagramOutputFile.I \
+    datagramOutputFile.h datagramSink.I datagramSink.h \
+    dcast.T dcast.h \
+    error_utils.h \
+    get_config_path.h hashGeneratorBase.I hashGeneratorBase.h \
+    hashVal.I hashVal.h indent.I indent.h \
+    indirectLess.I indirectLess.h \
+    littleEndian.h \
+    memoryInfo.I memoryInfo.h \
+    memoryUsage.I memoryUsage.h \
+    memoryUsagePointerCounts.I memoryUsagePointerCounts.h \
+    memoryUsagePointers.I memoryUsagePointers.h \
+    multifile.I multifile.h namable.I \
+    namable.h nativeNumericData.I nativeNumericData.h \
+    numeric_types.h \
+    ordered_vector.h ordered_vector.I ordered_vector.T \
+    pointerTo.I pointerTo.h \
+    pointerToArray.I pointerToArray.h \
+    profileTimer.I profileTimer.h \
+    pta_uchar.h referenceCount.I referenceCount.h \
+    register_type.I register_type.h \
+    reversedNumericData.I reversedNumericData.h tokenBoard.I \
+    tokenBoard.h trueClock.I trueClock.h typeHandle.I \
+    typeHandle.h typedObject.I typedObject.h \
+    typedReferenceCount.I typedReferenceCount.h typedef.h \
+    typeRegistry.I typeRegistry.h \
+    typeRegistryNode.I typeRegistryNode.h \
+    vector_uchar.h \
+    $[if $[HAVE_CRYPTO], \
+       crypto_utils.cxx crypto_utils.h patchfile.I \
+       patchfile.cxx patchfile.h ]
 
   #define INCLUDED_SOURCES  \
-     buffer.cxx checksumHashGenerator.cxx clockObject.cxx \
-     config_express.cxx datagram.cxx datagramGenerator.cxx \
-     datagramInputFile.cxx datagramIterator.cxx \
-     datagramOutputFile.cxx datagramSink.cxx dcast.cxx error_utils.cxx \
-     get_config_path.cxx \
-     hashGeneratorBase.cxx hashVal.cxx indent.cxx \
-     memoryInfo.cxx memoryUsage.cxx memoryUsagePointerCounts.cxx \
-     memoryUsagePointers.cxx multifile.cxx namable.cxx \
-     nativeNumericData.cxx profileTimer.cxx \
-     pta_uchar.cxx referenceCount.cxx register_type.cxx \
-     reversedNumericData.cxx trueClock.cxx typeHandle.cxx \
-     typedObject.cxx typedReferenceCount.cxx \
-     typeRegistry.cxx typeRegistryNode.cxx vector_uchar.cxx
+    buffer.cxx checksumHashGenerator.cxx clockObject.cxx \
+    config_express.cxx datagram.cxx datagramGenerator.cxx \
+    datagramInputFile.cxx datagramIterator.cxx \
+    datagramOutputFile.cxx datagramSink.cxx dcast.cxx error_utils.cxx \
+    get_config_path.cxx \
+    hashGeneratorBase.cxx hashVal.cxx indent.cxx \
+    memoryInfo.cxx memoryUsage.cxx memoryUsagePointerCounts.cxx \
+    memoryUsagePointers.cxx multifile.cxx namable.cxx \
+    nativeNumericData.cxx \
+    ordered_vector.cxx \
+    profileTimer.cxx \
+    pta_uchar.cxx referenceCount.cxx register_type.cxx \
+    reversedNumericData.cxx trueClock.cxx typeHandle.cxx \
+    typedObject.cxx typedReferenceCount.cxx \
+    typeRegistry.cxx typeRegistryNode.cxx vector_uchar.cxx
 
-  #define INSTALL_HEADERS                       \
+  #define INSTALL_HEADERS  \
     bigEndian.h buffer.I buffer.h checksumHashGenerator.I  \
     checksumHashGenerator.h circBuffer.I circBuffer.h clockObject.I \
     clockObject.h config_express.h datagram.I datagram.h \
@@ -66,12 +72,15 @@
     datagramOutputFile.I datagramOutputFile.h datagramSink.I \
     datagramSink.h dcast.T dcast.h \
     error_utils.h get_config_path.h hashGeneratorBase.I \
-    hashGeneratorBase.h hashVal.I hashVal.h indent.I indent.h \
+    hashGeneratorBase.h hashVal.I hashVal.h \
+    indent.I indent.h \
+    indirectLess.I indirectLess.h \
     littleEndian.h memoryInfo.I memoryInfo.h memoryUsage.I \
     memoryUsage.h memoryUsagePointerCounts.I \
     memoryUsagePointerCounts.h memoryUsagePointers.I \
     memoryUsagePointers.h multifile.I multifile.h namable.I namable.h \
     nativeNumericData.I nativeNumericData.h numeric_types.h \
+    ordered_vector.h ordered_vector.I ordered_vector.T \
     patchfile.I patchfile.h pointerTo.I pointerTo.h \
     pointerToArray.I pointerToArray.h profileTimer.I \
     profileTimer.h pta_uchar.h referenceCount.I referenceCount.h \
@@ -98,3 +107,14 @@
 
 #end test_bin_target
 
+
+#begin test_bin_target
+  #define TARGET test_ordered_vector
+
+  #define SOURCES \
+    test_ordered_vector.cxx
+
+  #define LOCAL_LIBS $[LOCAL_LIBS] putil
+  #define OTHER_LIBS $[OTHER_LIBS] pystub
+
+#end test_bin_target

+ 1 - 0
panda/src/express/express_composite2.cxx

@@ -2,6 +2,7 @@
 #include "multifile.cxx"
 #include "namable.cxx"
 #include "nativeNumericData.cxx"
+#include "ordered_vector.cxx"
 #include "profileTimer.cxx"
 #include "pta_uchar.cxx"
 #include "referenceCount.cxx"

+ 0 - 0
panda/src/putil/indirectLess.I → panda/src/express/indirectLess.I


+ 1 - 1
panda/src/putil/indirectLess.h → panda/src/express/indirectLess.h

@@ -19,7 +19,7 @@
 #ifndef INDIRECTLESS_H
 #define INDIRECTLESS_H
 
-#include <pandabase.h>
+#include "pandabase.h"
 
 ////////////////////////////////////////////////////////////////////
 //       Class : IndirectLess

+ 168 - 15
panda/src/express/multifile.I

@@ -16,27 +16,180 @@
 //
 ////////////////////////////////////////////////////////////////////
 
+
 ////////////////////////////////////////////////////////////////////
-//     Function: Multifile::write_header
-//       Access: Private
-//  Description: Writes the header uncompressed with platform-
-//               independent byte ordering
+//     Function: Multifile::is_read_valid
+//       Access: Published
+//  Description: Returns true if the Multifile has been opened for
+//               read mode and there have been no errors, and
+//               individual Subfile contents may be extracted.
+////////////////////////////////////////////////////////////////////
+INLINE bool Multifile::
+is_read_valid() const {
+  return (_read != (istream *)NULL && !_read->fail() && 
+          _open_subfile == (Subfile *)NULL);
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Multifile::is_write_valid
+//       Access: Published
+//  Description: Returns true if the Multifile has been opened for
+//               write mode and there have been no errors, and
+//               Subfiles may be added or removed from the Multifile.
+////////////////////////////////////////////////////////////////////
+INLINE bool Multifile::
+is_write_valid() const {
+  return (_write != (ostream *)NULL && !_write->fail() && 
+          _open_subfile == (Subfile *)NULL);
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Multifile::needs_repack
+//       Access: Published
+//  Description: Returns true if the Multifile index is suboptimal and
+//               should be repacked.  Call repack() to achieve this.
+////////////////////////////////////////////////////////////////////
+INLINE bool Multifile::
+needs_repack() const {
+  return _needs_repack;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Multifile::set_scale_factor
+//       Access: Published
+//  Description: Changes the internal scale factor for this Multifile.
+//
+//               This is normally 1, but it may be set to any
+//               arbitrary value (greater than zero) to support
+//               Multifile archives that exceed 4GB, if necessary.
+//               (Individual subfiles may still not exceed 4GB.)
+//
+//               All addresses within the file are rounded up to the
+//               next multiple of _scale_factor, and zeros are written
+//               to the file to fill the resulting gaps.  Then the
+//               address is divided by _scale_factor and written out
+//               as a 32-bit integer.  Thus, setting a scale factor of
+//               2 supports up to 8GB files, 3 supports 12GB files,
+//               etc.
+//
+//               Calling this function on an already-existing
+//               Multifile forces an immediate repack() operation.
 ////////////////////////////////////////////////////////////////////
 INLINE void Multifile::
-write_header(ofstream &write_stream) {
-  _datagram.clear();
-  _datagram.add_uint32(_magic_number);
-  _datagram.add_int32(_num_mfiles);
-  string msg = _datagram.get_message();
-  write_stream.write((char *)msg.data(), msg.length());
+set_scale_factor(size_t scale_factor) {
+  nassertv(scale_factor != 0);
+  if (_scale_factor != scale_factor) {
+    _scale_factor = scale_factor;
+    if (_next_index != 0) {
+      repack();
+    }
+  }
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Multifile::get_scale_factor
+//       Access: Published
+//  Description: Returns the internal scale factor for this Multifile.
+//               See set_scale_factor().
+////////////////////////////////////////////////////////////////////
+INLINE size_t Multifile::
+get_scale_factor() const {
+  return _scale_factor;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Multifile::word_to_streampos
+//       Access: Private
+//  Description: Converts a size_t address read from the file to
+//               a streampos byte address within the file.
+////////////////////////////////////////////////////////////////////
+INLINE streampos Multifile::
+word_to_streampos(size_t word) const {
+  return (streampos)word * (streampos)_scale_factor;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Multifile::streampos_to_word
+//       Access: Private
+//  Description: Converts a streampos byte address within the file to
+//               a size_t value suitable for writing to the file.
+////////////////////////////////////////////////////////////////////
+INLINE size_t Multifile::
+streampos_to_word(streampos fpos) const {
+  return (size_t)((fpos + _scale_factor - 1) / _scale_factor); 
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Multifile::normalize_streampos
+//       Access: Private
+//  Description: Rounds the streampos byte address up to the next
+//               multiple of _scale_factor.  Only multiples of
+//               _scale_factor may be written to the file.
+////////////////////////////////////////////////////////////////////
+INLINE streampos Multifile::
+normalize_streampos(streampos fpos) const {
+  return word_to_streampos(streampos_to_word(fpos));
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Multifile::Subfile::Constructor
+//       Access: Public
+//  Description: 
+////////////////////////////////////////////////////////////////////
+INLINE Multifile::Subfile::
+Subfile(const string &name) :
+  _name(name)
+{
+  _index_start = 0;
+  _data_start = 0;
+  _data_length = 0;
+  _source = (istream *)NULL;
+  _flags = 0;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Multifile::Subfile::operator <
+//       Access: Public
+//  Description: 
+////////////////////////////////////////////////////////////////////
+INLINE bool Multifile::Subfile::
+operator < (const Multifile::Subfile &other) const {
+  return _name < other._name;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Multifile::Subfile::is_deleted
+//       Access: Public
+//  Description: Returns true if the Subfile indicates it has been
+//               deleted (removed from the index), false otherwise.
+//               This should never be true of Subfiles that currently
+//               appear in either the _subfiles or _new_subfiles
+//               lists.
+////////////////////////////////////////////////////////////////////
+INLINE bool Multifile::Subfile::
+is_deleted() const {
+  return (_flags & SF_deleted) != 0;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Multifile::Subfile::is_index_invalid
+//       Access: Public
+//  Description: Returns true if there was some problem reading the
+//               index record for this Subfile from the Multifile.
+////////////////////////////////////////////////////////////////////
+INLINE bool Multifile::Subfile::
+is_index_invalid() const {
+  return (_flags & SF_index_invalid) != 0;
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: Multifile::get_header_length
+//     Function: Multifile::Subfile::is_data_invalid
 //       Access: Public
-//  Description:
+//  Description: Returns true if there was some problem reading the
+//               data contents of this Subfile, particularly when
+//               copying into the Multifile.
 ////////////////////////////////////////////////////////////////////
-INLINE int Multifile::
-get_header_length(void) const {
-  return _header_length;
+INLINE bool Multifile::Subfile::
+is_data_invalid() const {
+  return (_flags & SF_data_invalid) != 0;
 }

+ 967 - 522
panda/src/express/multifile.cxx

@@ -16,696 +16,1141 @@
 //
 ////////////////////////////////////////////////////////////////////
 
-#include <pandabase.h>
-#include "config_express.h"
-#include "error_utils.h"
+#include "multifile.h"
 
+#include "config_express.h"
 #include <algorithm>
-#include <errno.h>
 
-#include "multifile.h"
+// This sequence of bytes begins each Multifile to identify it as a
+// Multifile.
+const char Multifile::_header[] = "pmf\0\n\r";
+const size_t Multifile::_header_size = 6;
 
-////////////////////////////////////////////////////////////////////
-// Defines
-////////////////////////////////////////////////////////////////////
-PN_uint32 Multifile::_magic_number = 0xbeeffeeb;
+// These numbers identify the version of the Multifile.  Generally, a
+// change in the major version is intolerable; while a Multifile with
+// an older minor version may still be read.
+const int Multifile::_current_major_ver = 1;
+const int Multifile::_current_minor_ver = 0;
+
+//
+// A Multifile consists of the following elements:
+//
+// (1) A header.  This is always the first n bytes of the Multifile,
+// and contains a magic number to identify the file, as well as
+// version numbers and any file-specific parameters.
+//
+//   char[6]    The string Multifile::_header, a magic number.
+//   int16      The file's major version number
+//   int16      The file's minor version number
+//   uint32     Scale factor.  This scales all address references within
+//              the file.  Normally 1, this may be set larger to
+//              support Multifiles larger than 4GB.
+
+//
+// (2) Zero or more index entries, one for each subfile within the
+// Multifile.  These entries are of variable length.  The first one of
+// these immediately follows the header, and the first word of each
+// index entry contains the address of the next index entry.  A zero
+// "next" address marks the end of the chain.  These may appear at any
+// point within the Multifile; they do not necessarily appear in
+// sequential order at the beginning of the file (although they will
+// after the file has been "packed").
+//
+//   uint32     The address of the next entry.
+//   uint32     The address of this subfile's data record.
+//   uint32     The length in bytes of this subfile's data record.
+//   uint16     The Subfile::_flags member.
+//   uint16     The length in bytes of the subfile's name.
+//   char[n]    The subfile's name.
+//
+// (3) Zero or more data entries, one for each subfile.  These may
+// appear at any point within the Multifile; they do not necessarily
+// follow each index entry, nor are they necessarily all grouped
+// together at the end (although they will be all grouped together at
+// the end after the file has been "packed").  These are just blocks
+// of literal data.
+//
 
 ////////////////////////////////////////////////////////////////////
-//     Function: Multifile::Memfile::Constructor
-//       Access: Public
+//     Function: Multifile::Constructor
+//       Access: Published
 //  Description:
 ////////////////////////////////////////////////////////////////////
-Multifile::Memfile::
-Memfile(void) {
-  if (express_cat.is_debug())
-    express_cat.debug()
-      << "Multifile::Memfile constructor called" << endl;
-  reset();
-  _header_length_buf_length = sizeof(_header_length);
-  _header_length_buf = new char[_header_length_buf_length];
+Multifile::
+Multifile() {
+  _read = (istream *)NULL;
+  _write = (ostream *)NULL;
+  _next_index = 0;
+  _last_index = 0;
+  _needs_repack = false;
+  _scale_factor = 1;
+  _file_major_ver = 0;
+  _file_minor_ver = 0;
+  _open_subfile = (Subfile *)NULL;
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: Multifile::Memfile::Destructor
-//       Access: Public
+//     Function: Multifile::Destructor
+//       Access: Published
 //  Description:
 ////////////////////////////////////////////////////////////////////
-Multifile::Memfile::
-~Memfile(void) {
-  if (express_cat.is_debug())
-    express_cat.debug()
-      << "Multifile::Memfile destructor called" << endl;
-  if (_buffer != (char *)0L)
-    delete _buffer;
-  _buffer = (char *)0L;
-  delete _header_length_buf;
+Multifile::
+~Multifile() {
+  close();
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: Multifile::Memfile::reset
-//       Access: Public
-//  Description:
+//     Function: Multifile::open_read
+//       Access: Published
+//  Description: Opens the named Multifile on disk for reading.  The
+//               Multifile index is read in, and the list of subfiles
+//               becomes available; individual subfiles may then be
+//               extracted or read, but the list of subfiles may not
+//               be modified.
+//
+//               Also see the version of open_read() which accepts an
+//               istream.  Returns true on success, false on failure.
 ////////////////////////////////////////////////////////////////////
-void Multifile::Memfile::
-reset(void) {
-  if (express_cat.is_debug())
-    express_cat.debug()
-      << "Multifile::Memfile reset called" << endl;
-  _header_length_parsed = false;
-  _header_parsed = false;
-  _header_length = 0;
-  _buffer_length = 0;
-  //if (_buffer != (char *)0L)
-  //  delete _buffer;
-  _buffer = (char *)0L;
-  _file_open = false;
-  _bytes_written = 0;
-  //_datagram.clear();
-}
-
-////////////////////////////////////////////////////////////////////
-//     Function: Multifile::Memfile::parse_header_length
-//       Access: Public
-//  Description: Fills up _datagram until it has sizeof(_header_length)
-//               bytes and then extracts _header_length from _datagram.
-//               Returns true when this has been accomplished.
-//               Advances start to end of header_length.
-////////////////////////////////////////////////////////////////////
-bool Multifile::Memfile::
-parse_header_length(char *&start, int &size) {
-  if (_header_length_parsed == true)
-    return true;
-
-  int bytes_so_far = _datagram.get_length();
-  if (bytes_so_far + size < _header_length_buf_length) {
-    _datagram.append_data(start, size);
-    start += size;
-    size = 0;
+bool Multifile::
+open_read(const Filename &multifile_name) {
+  close();
+  Filename fname = multifile_name;
+  fname.set_binary();
+  if (!fname.open_read(_read_file)) {
     return false;
   }
+  _read = &_read_file;
+  _multifile_name = multifile_name;
+  return read_index();
+}
 
-  _datagram.append_data(start, _header_length_buf_length - bytes_so_far);
-  nassertr((int)_datagram.get_length() == _header_length_buf_length, false);
-
-  // Advance start and adjust size
-  nassertr(_header_length_buf_length >= bytes_so_far, false);
-  start += (_header_length_buf_length - bytes_so_far);
-  nassertr(size >= _header_length_buf_length, false);
-  size -= (_header_length_buf_length - bytes_so_far);
-
-  DatagramIterator di(_datagram);
-  _header_length = di.get_int32();
-
-  if (express_cat.is_debug())
-    express_cat.debug()
-      << "Multifile::Memfile::parse_header_length() - header length: "
-      << _header_length << endl;
-
-  nassertr(_header_length > _header_length_buf_length + (int)sizeof(_buffer_length), false);
-
-  _header_length_parsed = true;
-
+////////////////////////////////////////////////////////////////////
+//     Function: Multifile::open_write
+//       Access: Published
+//  Description: Opens the named Multifile on disk for writing.  If
+//               there already exists a file by that name, it is
+//               deleted.  The Multifile is then prepared for
+//               accepting a brand new set of subfiles, which will be
+//               written to the indicated filename.  Individual
+//               subfiles may not be extracted or read.
+//
+//               Also see the version of open_write() which accepts an
+//               ostream.  Returns true on success, false on failure.
+////////////////////////////////////////////////////////////////////
+bool Multifile::
+open_write(const Filename &multifile_name) {
+  close();
+  Filename fname = multifile_name;
+  fname.set_binary();
+  if (!fname.open_write(_write_file)) {
+    return false;
+  }
+  _write = &_write_file;
+  _multifile_name = multifile_name;
   return true;
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: Multifile::Memfile::parse_header
-//       Access: Public
-//  Description: Returns true when a complete header has been parsed.
-//               Advances the start pointer to the end of the header.
+//     Function: Multifile::open_read_write
+//       Access: Published
+//  Description: Opens the named Multifile on disk for reading and
+//               writing.  If there already exists a file by that
+//               name, its index is read.  Subfiles may be added or
+//               removed, and the resulting changes will be written to
+//               the named file.
+//
+//               Also see the version of open_read_write() which
+//               accepts an iostream.  Returns true on success, false
+//               on failure.
 ////////////////////////////////////////////////////////////////////
-bool Multifile::Memfile::
-parse_header(char *&start, int& size) {
-  if (_header_parsed == true)
-    return true;
-
-  // Determine header length
-  if (parse_header_length(start, size) == false)
+bool Multifile::
+open_read_write(const Filename &multifile_name) {
+  close();
+  Filename fname = multifile_name;
+  fname.set_binary();
+  bool exists = fname.exists();
+  if (!fname.open_read_write(_read_write_file)) {
     return false;
-
-  int bytes_so_far = _datagram.get_length();
-
-  // Make sure we don't exceed the length of the header
-  int tsize = size;
-  if ((size + bytes_so_far) >= _header_length) {
-    nassertr(bytes_so_far <= _header_length, false);
-    tsize = _header_length - bytes_so_far;
-    _header_parsed = true;
   }
+  _read = &_read_write_file;
+  _write = &_read_write_file;
+  _multifile_name = multifile_name;
 
-  // Accumulate bytes collected so far
-  _datagram.append_data(start, tsize);
+  if (exists) {
+    return read_index();
+  } else {
+    return true;
+  }
+}
 
-  // Fill in data for the memfile
-  if (_header_parsed == true) {
-    nassertr((int)_datagram.get_length() == _header_length, false);
-    DatagramIterator di(_datagram);
-    PN_int32 header_length = di.get_int32();
-    nassertr(header_length == _header_length, false);
-    _name = di.extract_bytes(_header_length - _header_length_buf_length -
-                             sizeof(_buffer_length));
-    _buffer_length = di.get_int32();
-    nassertr(_buffer_length >= 0, false);
+////////////////////////////////////////////////////////////////////
+//     Function: Multifile::close
+//       Access: Published
+//  Description: Closes the Multifile if it is open.  All changes are
+//               flushed to disk, and the file becomes invalid for
+//               further operations until the next call to open().
+////////////////////////////////////////////////////////////////////
+void Multifile::
+close() {
+  close_subfile();
+  flush();
+  _read = (istream *)NULL;
+  _write = (ostream *)NULL;
+  _next_index = 0;
+  _last_index = 0;
+  _needs_repack = false;
+  _scale_factor = 1;
+  _file_major_ver = 0;
+  _file_minor_ver = 0;
+
+  _read_file.close();
+  _write_file.close();
+  _read_write_file.close();
+  _multifile_name = Filename();
+
+  clear_subfiles();
+}
 
-    express_cat.debug()
-      << "Multifile::Memfile::parse_header() - Got a header for mem "
-      << "file: " << _name << " header length: " << _header_length
-      << " buffer length: " << _buffer_length << endl;
+////////////////////////////////////////////////////////////////////
+//     Function: Multifile::add_subfile
+//       Access: Published
+//  Description: Adds a file on disk as a subfile to the Multifile.
+//               The file named by filename will be read and added to
+//               the Multifile at the next call to flush().
+//
+//               Returns true on success, false on failure.
+////////////////////////////////////////////////////////////////////
+bool Multifile::
+add_subfile(const string &subfile_name, const Filename &filename) {
+  nassertr(is_write_valid(), false);
 
-    // Advance start pointer to the end of the header
-    start += tsize;
-    nassertr(size >= tsize, false);
-    size -= tsize;
-    _datagram.clear();
+  if (!filename.exists()) {
+    return false;
   }
+  Subfile *subfile = new Subfile(subfile_name);
+
+  subfile->_source_filename = filename;
+  subfile->_source_filename.set_binary();
 
-  return _header_parsed;
+  return add_new_subfile(subfile);
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: Multifile::Memfile::read
-//       Access: Public
-//  Description: Reads from an individual file
-////////////////////////////////////////////////////////////////////
-bool Multifile::Memfile::
-read(const Filename &name) {
-  ifstream read_stream;
-  if (!name.open_read(read_stream)) {
-    express_cat.error()
-      << "Multifile::Memfile() - Failed to open input file: "
-      << name << endl;
+//     Function: Multifile::flush
+//       Access: Published
+//  Description: Writes all contents of the Multifile to disk.  Until
+//               flush() is called, add_subfile() and remove_subfile()
+//               do not actually do anything to disk.  At this point,
+//               all of the recently-added subfiles are read and their
+//               contents are added to the end of the Multifile, and
+//               the recently-removed subfiles are marked gone from
+//               the Multifile.
+//
+//               This may result in a suboptimal index.  To guarantee
+//               that the index is written at the beginning of the
+//               file, call repack() instead of flush().
+//
+//               It is not necessary to call flush() explicitly unless
+//               you are concerned about reading the recently-added
+//               subfiles immediately.
+//
+//               Returns true on success, false on failure.
+////////////////////////////////////////////////////////////////////
+bool Multifile::
+flush() {
+  if (!is_write_valid()) {
     return false;
   }
 
-  if (express_cat.is_debug())
-    express_cat.debug()
-      << "Multifile::Memfile::read() - Reading file: " << name << endl;
+  if (_next_index == 0) {
+    // If we don't have an index yet, we don't have a header.  Write
+    // the header.
+    if (!write_header()) {
+      return false;
+    }
+  }
 
-  _header_length = name.length() + sizeof(_header_length) +
-        sizeof(_buffer_length);
-  _name = name;
+  nassertr(_write != (ostream *)NULL, false);
 
-  // Determine the length of the file
-  read_stream.seekg(0, ios::end);
-  _buffer_length = read_stream.tellg();
-  _buffer = new char[_buffer_length];
+  // First, mark out all of the removed subfiles.
+  PendingSubfiles::iterator pi;
+  for (pi = _removed_subfiles.begin(); pi != _removed_subfiles.end(); ++pi) {
+    Subfile *subfile = (*pi);
+    subfile->rewrite_index_flags(*_write);
+    delete subfile;
+  }
+  _removed_subfiles.clear();
+
+  bool wrote_ok = true;
+
+  if (!_new_subfiles.empty()) {
+    // Add a few more files to the end.  We always add subfiles at the
+    // end of the multifile, so go there first.
+    if (_last_index != 0) {
+      _write->seekp(0, ios::end);
+      if (_write->fail()) {
+        express_cat.info()
+          << "Unable to seek Multifile " << _multifile_name << ".\n";
+        return false;
+      }
+      _next_index = _write->tellp();
+      _next_index = pad_to_streampos(_next_index);
+
+      // And update the forward link from the last_index to point to
+      // this new index location.
+      _write->seekp(_last_index);
+      Datagram dg;
+      dg.add_uint32(streampos_to_word(_next_index));
+      _write->write(dg.get_data(), dg.get_length());
+    }
 
-  // Read the file
-  read_stream.seekg(0, ios::beg);
-  read_stream.read(_buffer, _buffer_length);
+    _write->seekp(_next_index);
+    nassertr(_next_index == _write->tellp(), false);
+    
+    // Ok, here we are at the end of the file.  Write out the
+    // recently-added subfiles here.  First, count up the index size.
+    for (pi = _new_subfiles.begin(); pi != _new_subfiles.end(); ++pi) {
+      Subfile *subfile = (*pi);
+      _last_index = _next_index;
+      _next_index = subfile->write_index(*_write, _next_index, this);
+      _next_index = pad_to_streampos(_next_index);
+      nassertr(_next_index == _write->tellp(), false);
+    }
+    
+    // Now we're at the end of the index.  Write a 0 here to mark the
+    // end.
+    Datagram dg;
+    dg.add_uint32(0);
+    _write->write(dg.get_data(), dg.get_length());
+    _next_index += 4;
+    _next_index = pad_to_streampos(_next_index);
+
+    // All right, now write out each subfile's data.
+    for (pi = _new_subfiles.begin(); pi != _new_subfiles.end(); ++pi) {
+      Subfile *subfile = (*pi);
+      _next_index = subfile->write_data(*_write, _read, _next_index);
+      _next_index = pad_to_streampos(_next_index);
+      if (subfile->is_data_invalid()) {
+        wrote_ok = false;
+      }
+      nassertr(_next_index == _write->tellp(), false);
+    }
+    
+    // Now go back and fill in the proper addresses for the data start.
+    // We didn't do it in the first pass, because we don't really want
+    // to keep all those pile handles open, and so we didn't have to
+    // determine each pile's length ahead of time.
+    for (pi = _new_subfiles.begin(); pi != _new_subfiles.end(); ++pi) {
+      Subfile *subfile = (*pi);
+      subfile->rewrite_index_data_start(*_write, this);
+    }
 
-  return (_buffer_length > 0);
+    _new_subfiles.clear();
+  }
+
+  _write->flush();
+  if (!wrote_ok || _write->fail()) {
+    express_cat.info()
+      << "Unable to update Multifile " << _multifile_name << ".\n";
+    close();
+    return false;
+  }
+  return true;
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: Multifile::Memfile::read_from_multifile
-//       Access: Public
-//  Description: Reads a Memfile from an open Multifile ifstream
-////////////////////////////////////////////////////////////////////
-bool Multifile::Memfile::
-read_from_multifile(ifstream &read_stream) {
-  read_stream.read(_header_length_buf, _header_length_buf_length);
-  char *start = _header_length_buf;
-  int size = _header_length_buf_length;
-  if (parse_header_length(start, size) == false) {
-    express_cat.error()
-      << "Multifile::Memfile::read_from_multifile() - invalid header "
-      << "length" << endl;
+//     Function: Multifile::repack
+//       Access: Published
+//  Description: Forces a complete rewrite of the Multifile and all of
+//               its contents, so that its index will appear at the
+//               beginning of the file with all of the subfiles listed
+//               in alphabetical order.  This is considered optimal
+//               for reading, and is the standard configuration; but
+//               it is not essential to do this.
+//
+//               It is only valid to call this if the Multifile was
+//               opened using open_read_write() and an explicit
+//               filename, rather than an iostream.  Also, we must
+//               have write permission to the directory containing the
+//               Multifile.
+//
+//               Returns true on success, false on failure.
+////////////////////////////////////////////////////////////////////
+bool Multifile::
+repack() {
+  nassertr(is_write_valid() && is_read_valid(), false);
+  nassertr(!_multifile_name.empty(), false);
+
+  // First, we open a temporary filename to copy the Multifile to.
+  Filename temp_filename =
+    Filename::temporary(_multifile_name.get_dirname(), "mftemp");
+  temp_filename.set_binary();
+  ofstream temp;
+  if (!temp_filename.open_write(temp)) {
+    express_cat.info()
+      << "Unable to open temporary file " << temp_filename << "\n";
     return false;
   }
 
-  // Read the rest of the header
-  char *header_buf = new char[_header_length - _header_length_buf_length];
-  read_stream.read(header_buf, _header_length - _header_length_buf_length);
-  start = header_buf;
-  size = _header_length;
-  if (parse_header(start, size) == false) {
-    delete header_buf;
-    express_cat.error()
-      << "Multifile::Memfile::read_from_multifile() - Invalid header"
-      << endl;
+  // Now we scrub our internal structures so it looks like we're a
+  // brand new Multifile.
+  PendingSubfiles::iterator pi;
+  for (pi = _removed_subfiles.begin(); pi != _removed_subfiles.end(); ++pi) {
+    Subfile *subfile = (*pi);
+    delete subfile;
+  }
+  _removed_subfiles.clear();
+  _new_subfiles.clear();
+  copy(_subfiles.begin(), _subfiles.end(), back_inserter(_new_subfiles));
+  _next_index = 0;
+  _last_index = 0;
+
+  // And we write our contents to our new temporary file.
+  _write = &temp;
+  if (!flush()) {
+    temp.close();
+    temp_filename.unlink();
     return false;
   }
 
-  delete header_buf;
-
-  express_cat.debug()
-    << "Multifile::Memfile::read_from_multifile() - Got a valid Memfile "
-    << "header: name: " << _name << " length: " << _buffer_length << endl;
+  // Now close everything, and move the temporary file back over our
+  // original file.
+  Filename orig_name = _multifile_name;
+  temp.close();
+  close();
+  if (!temp_filename.rename_to(orig_name)) {
+    express_cat.info()
+      << "Unable to rename temporary file " << temp_filename << " to "
+      << orig_name.get_basename() << ".\n";
+    return false;
+  }
 
-  _buffer = new char[_buffer_length];
-  read_stream.read(_buffer, _buffer_length);
+  if (!open_read_write(orig_name)) {
+    express_cat.info()
+      << "Unable to read newly repacked " << _multifile_name 
+      << ".\n";
+    return false;
+  }
 
   return true;
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: Multifile::Memfile::write
-//       Access: Public
-//  Description: Writes to a individual file
-////////////////////////////////////////////////////////////////////
-bool Multifile::Memfile::
-write(const Filename &rel_path) {
-  ofstream write_stream;
-  Filename name = rel_path.get_fullpath() + _name.get_fullpath();
-  name.set_binary();
-  name.make_dir();
-  if (!name.open_write(write_stream)) {
-    express_cat.error()
-      << "Multifile::Memfile::write() - Failed to open output file: "
-      << name << endl;
-    return false;
-  }
-
-  express_cat.debug()
-    << "Writing to file: " << name << endl;
-
-  write_stream.write(_buffer, _buffer_length);
-  return true;
+//     Function: Multifile::get_num_subfiles
+//       Access: Published
+//  Description: Returns the number of subfiles within the Multifile.
+//               The subfiles may be accessed in alphabetical order by
+//               iterating through [0 .. get_num_subfiles()).
+////////////////////////////////////////////////////////////////////
+int Multifile::
+get_num_subfiles() const {
+  return _subfiles.size();
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: Multifile::Memfile::write
-//       Access: Public
-//  Description: Writes a Memfile to an open Multifile ofstream
+//     Function: Multifile::find_subfile
+//       Access: Published
+//  Description: Returns the index of the subfile with the indicated
+//               name, or -1 if the named subfile is not within the
+//               Multifile.
 ////////////////////////////////////////////////////////////////////
-void Multifile::Memfile::
-write_to_multifile(ofstream &write_stream) {
-  express_cat.debug()
-    << "Writing: " << _name << " to multifile" << endl;
+int Multifile::
+find_subfile(const string &subfile_name) const {
+  Subfile find_subfile(subfile_name);
+  Subfiles::const_iterator fi;
+  fi = _subfiles.find(&find_subfile);
+  if (fi == _subfiles.end()) {
+    // Not present.
+    return -1;
+  }
+  return (fi - _subfiles.begin());
+}
 
-  _datagram.clear();
-  _datagram.add_int32(_header_length);
-  string path_name = _name.get_fullpath();
-  _datagram.append_data(path_name.c_str(), path_name.length());
-  _datagram.add_int32(_buffer_length);
+////////////////////////////////////////////////////////////////////
+//     Function: Multifile::remove_subfile
+//       Access: Published
+//  Description: Removes the nth subfile from the Multifile.  This
+//               will cause all subsequent index numbers to decrease
+//               by one.  The file will not actually be removed from
+//               the disk until the next call to flush().
+//
+//               Note that this does not actually remove the data from
+//               the indicated subfile; it simply removes it from the
+//               index.  The Multifile will not be reduced in size
+//               after this operation, until the next call to
+//               repack().
+////////////////////////////////////////////////////////////////////
+void Multifile::
+remove_subfile(int index) {
+  nassertv(is_write_valid());
+  nassertv(index >= 0 && index < (int)_subfiles.size());
+  Subfile *subfile = _subfiles[index];
+  subfile->_flags |= SF_deleted;
+  _removed_subfiles.push_back(subfile);
+  _subfiles.erase(_subfiles.begin() + index);
+
+  _needs_repack = true;
+}
 
-  string msg = _datagram.get_message();
-  write_stream.write((char *)msg.data(), msg.length());
-  write_stream.write(_buffer, _buffer_length);
+////////////////////////////////////////////////////////////////////
+//     Function: Multifile::get_subfile_name
+//       Access: Published
+//  Description: Returns the name of the nth subfile.
+////////////////////////////////////////////////////////////////////
+const string &Multifile::
+get_subfile_name(int index) const {
+#ifndef NDEBUG
+  static string empty_string;
+  nassertr(index >= 0 && index < (int)_subfiles.size(), empty_string);
+#endif
+  return _subfiles[index]->_name;
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: Multifile::Memfile::write
-//       Access: Public
-//  Description: Returns true when the memfile has been parsed and
-//               written to disk.
-//               Advances the start pointer as it writes.
-////////////////////////////////////////////////////////////////////
-int Multifile::Memfile::
-write(char *&start, int &size, const Filename &rel_path) {
-  // Make sure we've got a complete header first
-  if (parse_header(start, size) == false) {
-    return EU_ok;
-  }
-
-  // Try to open the file for writing
-  if (_file_open == false) {
-    Filename name = rel_path.get_fullpath() + _name.get_fullpath();
-    name.set_binary();
-    name.make_dir();
-    if (express_cat.is_debug())
-      express_cat.debug()
-        << "Multifile::Memfile::write() - Opening mem file: " << name
-        << " for writing" << endl;
-    if ((_file_open = name.open_write(_write_stream)) == false) {
-      express_cat.error()
-        << "Multfile::Memfile::write() - Couldn't open file: "
-        << name << " : " << strerror(errno) << endl;
-      return get_write_error();
-    }
-    _file_open = true;
-  }
+//     Function: Multifile::get_subfile_length
+//       Access: Published
+//  Description: Returns the data length of the nth subfile.  This
+//               might return 0 if the subfile has recently been added
+//               and flush() has not yet been called.
+////////////////////////////////////////////////////////////////////
+size_t Multifile::
+get_subfile_length(int index) const {
+  nassertr(index >= 0 && index < (int)_subfiles.size(), 0);
+  return _subfiles[index]->_data_length;
+}
 
-  // Don't write more than the buffer length
-  int done = EU_ok;
-  int tsize = size;
-  nassertr(_buffer_length >= _bytes_written, false);
-  int missing_bytes = _buffer_length - _bytes_written;
-  if (size >= missing_bytes) {
-    tsize = missing_bytes;
-    done = EU_success;
+////////////////////////////////////////////////////////////////////
+//     Function: Multifile::read_subfile
+//       Access: Published
+//  Description: Fills the indicated Datagram with the data from the
+//               nth subfile.
+////////////////////////////////////////////////////////////////////
+void Multifile::
+read_subfile(int index, Datagram &data) {
+  nassertv(is_read_valid());
+  nassertv(index >= 0 && index < (int)_subfiles.size());
+  data.clear();
+
+  istream &in = open_read_subfile(index);
+  size_t length = _subfiles[index]->_data_length;
+  for (size_t p = 0; p < length; p++) {
+    int byte = in.get();
+    data.add_int8(byte);
   }
+  bool failed = in.fail();
+  close_subfile();
+  nassertv(!failed);
+}
 
-  _write_stream.write(start, tsize);
-  start += tsize;
-  _bytes_written += tsize;
-  nassertr(size >= tsize, false);
-  size -= tsize;
-
-  if (done == EU_success) {
-    _write_stream.close();
-    express_cat.debug()
-      << "Multifile::Memfile::write() - Closing mem file" << endl;
+////////////////////////////////////////////////////////////////////
+//     Function: Multifile::extract_subfile
+//       Access: Published
+//  Description: Extracts the nth subfile into a file with the given
+//               name.
+////////////////////////////////////////////////////////////////////
+bool Multifile::
+extract_subfile(int index, const Filename &filename) {
+  nassertr(is_read_valid(), false);
+  nassertr(index >= 0 && index < (int)_subfiles.size(), false);
+  
+  Filename fname = filename;
+  fname.set_binary();
+  fname.make_dir();
+  ofstream out;
+  if (!fname.open_write(out)) {
+    express_cat.info()
+      << "Unable to write to file " << filename << "\n";
+    return false;
   }
 
-  return done;
+  return extract_subfile_to(index, out);
 }
 
-
-
 ////////////////////////////////////////////////////////////////////
-//     Function: Multifile::Constructor
+//     Function: Multifile::open_read
 //       Access: Public
-//  Description:
+//  Description: Opens an anonymous Multifile for reading using an
+//               istream.  There must be seek functionality via
+//               seekg() and tellg() on the istream.
+//
+//               This version of open_read() does not close the
+//               istream when Multifile.close() is called.
 ////////////////////////////////////////////////////////////////////
-Multifile::
-Multifile(void) {
-  if (express_cat.is_debug())
-    express_cat.debug()
-      << "Multifile constructor called" << endl;
-  reset();
-  _header_length = sizeof(_magic_number) + sizeof(_num_mfiles);
+bool Multifile::
+open_read(istream *multifile_stream) {
+  close();
+  _read = multifile_stream;
+  return read_index();
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: Multifile::Destructor
+//     Function: Multifile::open_write
 //       Access: Public
-//  Description:
+//  Description: Opens an anonymous Multifile for writing using an
+//               ostream.  There must be seek functionality via
+//               seekp() and tellp() on the pstream.
+//
+//               This version of open_write() does not close the
+//               ostream when Multifile.close() is called.
 ////////////////////////////////////////////////////////////////////
-Multifile::
-~Multifile(void) {
-  if (express_cat.is_debug())
-    express_cat.debug()
-      << "Multifile destructor called" << endl;
-  _files.erase(_files.begin(), _files.end());
-  if (_current_mfile != NULL)
-    delete _current_mfile;
+bool Multifile::
+open_write(ostream *multifile_stream) {
+  close();
+  _write = multifile_stream;
+  return true;
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: Multifile::evaluate
+//     Function: Multifile::open_read_write
 //       Access: Public
-//  Description: Checks for a valid compressed or uncompressed
-//               Multifile.
+//  Description: Opens an anonymous Multifile for reading and writing
+//               using an iostream.  There must be seek functionality
+//               via seekg()/seekp() and tellg()/tellp() on the
+//               iostream.
+//
+//               This version of open_read_write() does not close the
+//               iostream when Multifile.close() is called.
 ////////////////////////////////////////////////////////////////////
-int Multifile::
-evaluate(const char *start, int size) {
-  if (size < (int)sizeof(_magic_number)) {
-    express_cat.debug()
-      << "Multifile::evaluate() - not enough bytes to determine" << endl;
-    return T_unknown;
+bool Multifile::
+open_read_write(iostream *multifile_stream) {
+  close();
+  _read = multifile_stream;
+  _write = multifile_stream;
+
+  // Check whether the read stream is empty.
+  _read->seekg(0, ios::end);
+  if (_read->tellg() == 0) {
+    // The read stream is empty, which is always valid.
+    return true;
   }
 
-  Datagram dgram;
-  dgram.append_data(start, sizeof(_magic_number));
-  DatagramIterator di(dgram);
-  PN_uint32 magic_number = di.get_uint32();
-  if (magic_number == _magic_number)
-    return T_valid;
-
-  express_cat.debug()
-    << "Invalid magic number: " << magic_number << " ("
-    << _magic_number << ")" << endl;
-
-  return T_invalid;
+  // The read stream is not empty, so we'd better have a valid
+  // Multifile.
+  _read->seekg(0);
+  return read_index();
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: Multifile::parse_header
-//       Access: Public
-//  Description: Returns true when a complete header has been parsed.
+//     Function: Multifile::add_subfile
+//       Access: Published
+//  Description: Adds a file on disk as a subfile to the Multifile.
+//               The indicated istream will be read and its contents
+//               added to the Multifile at the next call to flush().
 ////////////////////////////////////////////////////////////////////
-int Multifile::
-parse_header(char *&start, int &size) {
-  if (_header_parsed == true)
-    return EU_success;
-
-  int dgramsize = _datagram.get_length();
-  int tsize = size;
-
-  // Make sure we don't exceed the length of the header
-  nassertr(_header_length >= dgramsize, EU_error_abort);
-  int missing_bytes = _header_length - dgramsize;
-  if (size >= missing_bytes) {
-    tsize = missing_bytes;
-    _header_parsed = true;
-  }
-
-  // Accumulate bytes collected so far
-  _datagram.append_data(start, tsize);
-
-  // Verify magic number
-  if (_header_parsed == true) {
-    DatagramIterator di(_datagram);
-    PN_uint32 magic_number = di.get_uint32();
-    if (magic_number != _magic_number) {
-      express_cat.error()
-        << "Multifile::parse_header() - Invalid magic number: "
-        << magic_number << " (" << _magic_number << ")" << endl;
-      return EU_error_abort;
-    }
-    _num_mfiles = di.get_int32();
-    if (_num_mfiles <= 0) {
-      express_cat.debug()
-        << "Multifile::parse_header() - No memfiles in multifile"
-        << endl;
-      return EU_error_file_empty;
-    }
+bool Multifile::
+add_subfile(const string &subfile_name, istream *subfile_data) {
+  nassertr(is_write_valid(), false);
 
-    // Advance start pointer to the end of the header
-    start += tsize;
-    nassertr(size >= tsize, false);
-    size -= tsize;
-    _datagram.clear();
-    return EU_success;
-  }
+  Subfile *subfile = new Subfile(subfile_name);
 
-  return EU_ok;
+  subfile->_source = subfile_data;
+
+  return add_new_subfile(subfile);
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: Multifile::add
+//     Function: Multifile::extract_subfile_to
 //       Access: Public
-//  Description:
+//  Description: Extracts the nth subfile to the indicated ostream.
 ////////////////////////////////////////////////////////////////////
 bool Multifile::
-add(const Filename &name) {
-  Memfile *mfile = new Memfile;
-  if (mfile->read(name)) {
-    _files.push_back(mfile);
-    return true;
+extract_subfile_to(int index, ostream &out) {
+  nassertr(is_read_valid(), false);
+  nassertr(index >= 0 && index < (int)_subfiles.size(), false);
+
+  istream &in = open_read_subfile(index);
+  size_t length = _subfiles[index]->_data_length;
+  for (size_t p = 0; p < length; p++) {
+    int byte = in.get();
+    out.put(byte);
   }
-  return false;
+  bool failed = in.fail();
+  close_subfile();
+  nassertr(!failed, false);
+
+  return (!out.fail());
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: Multifile::remove
+//     Function: Multifile::open_read_subfile
 //       Access: Public
-//  Description:
+//  Description: Returns an istream that may be used to read the
+//               indicated subfile.  The istream may or may not be a
+//               reference within the Multifile itself, so relative
+//               seeks are acceptable but global seeks will fail.
+//               Also, checking for eof() may fail; you must use
+//               get_subfile_length() instead.
+//
+//               It is not valid to perform any other operations on
+//               this Multifile until the indicated subfile has been
+//               read to completion and close_subfile() has been
+//               called.
 ////////////////////////////////////////////////////////////////////
-bool Multifile::
-remove(const Filename &name) {
-  MemfileList::iterator found;
-  found = find_if(_files.begin(), _files.end(), MemfileMatch(name));
-  if (found != _files.end()) {
-    _files.erase(found);
-    return true;
+istream &Multifile::
+open_read_subfile(int index) {
+#ifndef NDEBUG
+  ifstream empty_stream;
+  nassertr(_open_subfile == (Subfile *)NULL, empty_stream);
+  nassertr(is_read_valid(), empty_stream);
+  nassertr(index >= 0 && index < (int)_subfiles.size(), empty_stream);
+#endif
+  _open_subfile = _subfiles[index];
+  if (_open_subfile->_source != (istream *)NULL) {
+    _open_subfile->_source->seekg(0);
+    return *_open_subfile->_source;
   }
-  return false;
+  if (!_open_subfile->_source_filename.empty()) {
+    _open_subfile->_source_filename.open_read(_subfile_read);
+    return _subfile_read;
+  }
+  nassertr(_open_subfile->_data_start != 0, empty_stream);
+  _read->seekg(_open_subfile->_data_start);
+  return *_read;
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: Multifile::has_file
+//     Function: Multifile::close_subfile
 //       Access: Public
-//  Description:
+//  Description: "Closes" the istream that was returned via a previous
+//               call to open_read_subfile(), and makes other
+//               operations on the Multifile valid once more.
 ////////////////////////////////////////////////////////////////////
-bool Multifile::
-has_file(const Filename &name) {
-  MemfileList::iterator found;
-  found = find_if(_files.begin(), _files.end(), MemfileMatch(name));
-  return (found != _files.end());
+void Multifile::
+close_subfile() {
+  _open_subfile = (Subfile *)NULL;
+  _subfile_read.close();
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: Multifile::read
-//       Access: Public
-//  Description: Reads a multifile from disk
+//     Function: Multifile::pad_to_streampos
+//       Access: Private
+//  Description: Assumes the _write pointer is at the indicated fpos,
+//               rounds the fpos up to the next legitimate address
+//               (using normalize_streampos()), and writes enough
+//               zeros to the stream to fill the gap.  Returns the new
+//               fpos.
 ////////////////////////////////////////////////////////////////////
-bool Multifile::
-read(Filename &name) {
-
-  // Open the multifile for reading
-  ifstream read_stream;
-  name.set_binary();
-  if (!name.open_read(read_stream)) {
-    express_cat.error()
-      << "Multifile::read() - Failed to open input file: "
-      << name << endl;
-    return false;
+streampos Multifile::
+pad_to_streampos(streampos fpos) {
+  nassertr(_write != (ostream *)NULL, fpos);
+  nassertr(_write->tellp() == fpos, fpos);
+  streampos new_fpos = normalize_streampos(fpos);
+  while (fpos < new_fpos) {
+    _write->put(0);
+    fpos++;
   }
+  return fpos;
+}
 
-  // Check for a valid header
-  char *buffer = new char[_header_length];
-  read_stream.read(buffer, _header_length);
-  char *start = buffer;
-  int len = _header_length;
-  int ret = parse_header(start, len);
-  delete buffer;
-  if (ret < 0) {
-    express_cat.error()
-      << "Multifile::read() - invalid header" << endl;
-    return false;
+////////////////////////////////////////////////////////////////////
+//     Function: Multifile::add_new_subfile
+//       Access: Private
+//  Description: Adds a newly-allocated Subfile pointer to the
+//               Multifile.
+////////////////////////////////////////////////////////////////////
+bool Multifile::
+add_new_subfile(Subfile *subfile) {
+  if (_next_index != 0) {
+    // If we're adding a Subfile to an already-existing Multifile, we
+    // will eventually need to repack the file.
+    _needs_repack = true;
   }
 
-  if (_num_mfiles <= 0) {
-    express_cat.error()
-      << "Multfile::read() - No files in Multfile: " << name << endl;
-    return false;
+  pair<Subfiles::iterator, bool> insert_result = _subfiles.insert(subfile);
+  if (!insert_result.second) {
+    // Hmm, unable to insert.  There must already be a subfile by that
+    // name.  Remove the old one.
+    Subfile *old_subfile = (*insert_result.first);
+    old_subfile->_flags |= SF_deleted;
+    _removed_subfiles.push_back(old_subfile);
+    (*insert_result.first) = subfile;
   }
 
-  // Read all the Memfiles
-  for (int i = 0; i < _num_mfiles; i++) {
-    Memfile *mfile = new Memfile;
-    mfile->read_from_multifile(read_stream);
-    _files.push_back(mfile);
+  _new_subfiles.push_back(subfile);
+  return true;
+}
+
+////////////////////////////////////////////////////////////////////
+//     Function: Multifile::clear_subfiles
+//       Access: Private
+//  Description: Removes the set of subfiles from the tables and frees
+//               their associated memory.
+////////////////////////////////////////////////////////////////////
+void Multifile::
+clear_subfiles() {
+  PendingSubfiles::iterator pi;
+  for (pi = _removed_subfiles.begin(); pi != _removed_subfiles.end(); ++pi) {
+    Subfile *subfile = (*pi);
+    subfile->rewrite_index_flags(*_write);
+    delete subfile;
   }
+  _removed_subfiles.clear();
 
-  return true;
+  // We don't have to delete the ones in _new_subfiles, because these
+  // also appear in _subfiles.
+  _new_subfiles.clear();
+
+  Subfiles::iterator fi;
+  for (fi = _subfiles.begin(); fi != _subfiles.end(); ++fi) {
+    Subfile *subfile = (*fi);
+    delete subfile;
+  }
+  _subfiles.clear();
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: Multifile::write
-//       Access: Public
-//  Description:
+//     Function: Multifile::read_index
+//       Access: Private
+//  Description: Reads the Multifile header and index.  Returns true
+//               if successful, false if the Multifile is not valid.
 ////////////////////////////////////////////////////////////////////
 bool Multifile::
-write(Filename name) {
+read_index() {
+  nassertr(_read != (istream *)NULL, false);
+  static const size_t header_followup_size = 2 + 2 + 4;
+  static const size_t total_header_size = _header_size + header_followup_size;
+  char this_header[total_header_size];
+  _read->read(this_header, total_header_size);
+  if (_read->fail() || _read->gcount() != total_header_size) {
+    express_cat.info()
+      << "Unable to read Multifile header " << _multifile_name << ".\n";
+    close();
+    return false;
+  }
+  if (memcmp(this_header, _header, _header_size) != 0) {
+    express_cat.info()
+      << _multifile_name << " is not a Multifile.\n";
+    return false;
+  }
 
-  ofstream write_stream;
-  name.set_binary();
-  if (!name.open_write(write_stream)) {
-    express_cat.error()
-      << "Multifile::write() - Error opening file: " << name << endl;
+  // Now get the version numbers out.
+  Datagram dg(this_header + _header_size, header_followup_size);
+  DatagramIterator dgi(dg);
+  _file_major_ver = dgi.get_int16();
+  _file_minor_ver = dgi.get_int16();
+  _scale_factor = dgi.get_uint32();
+
+  if (_file_major_ver != _current_major_ver ||
+      (_file_major_ver == _current_major_ver && 
+       _file_minor_ver > _current_minor_ver)) {
+    express_cat.info()
+      << _multifile_name << " has version " << _file_major_ver << "."
+      << _file_minor_ver << ", expecting version " 
+      << _current_major_ver << "." << _current_minor_ver << ".\n";
     return false;
   }
 
-  _num_mfiles = _files.size();
-  if (_num_mfiles == 0) {
-    express_cat.debug()
-      << "No files in Multifile to write" << endl;
+
+  // Now read the index out.
+  _next_index = _read->tellg();
+  _next_index = normalize_streampos(_next_index);  
+  _read->seekg(_next_index);
+  _last_index = 0;
+  streampos index_forward;
+
+  Subfile *subfile = new Subfile("");
+  index_forward = subfile->read_index(*_read, _next_index, this);
+  while (index_forward != 0) {
+    _last_index = _next_index;
+    if (subfile->is_deleted()) {
+      // Ignore deleted Subfiles in the index.
+      _needs_repack = true;
+      delete subfile;
+    } else {
+      _subfiles.push_back(subfile);
+    }
+    if (index_forward != normalize_streampos(_read->tellg())) {
+      // If the index entries don't follow exactly sequentially, the
+      // file ought to be repacked.
+      _needs_repack = true;
+    }
+    _read->seekg(index_forward);
+    _next_index = index_forward;
+    subfile = new Subfile("");
+    index_forward = subfile->read_index(*_read, _next_index, this);
+  }
+  if (subfile->is_index_invalid()) {
+    express_cat.info()
+      << "Error reading index for " << _multifile_name << ".\n";
+    close();
+    delete subfile;
     return false;
   }
 
-  express_cat.debug()
-    << "Multifile::write() - Writing multifile: " << name << endl;
+  size_t before_size = _subfiles.size();
+  _subfiles.sort();
+  size_t after_size = _subfiles.size();
 
-  write_header(write_stream);
+  // If these don't match, the same filename appeared twice in the
+  // index, which shouldn't be possible.
+  nassertr(before_size == after_size, true);
 
-  MemfileList::iterator i;
-  for (i = _files.begin(); i != _files.end(); ++i)
-    (*i)->write_to_multifile(write_stream);
+  delete subfile;
+  return true;
+}  
 
-  write_stream.close();
+////////////////////////////////////////////////////////////////////
+//     Function: Multifile::write_header
+//       Access: Private
+//  Description: Writes just the header part of the Multifile, not the
+//               index.
+////////////////////////////////////////////////////////////////////
+bool Multifile::
+write_header() {
+  nassertr(_write != (ostream *)NULL, false);
+  nassertr(_write->tellp() == 0, false);
+  _write->write(_header, _header_size);
+  Datagram dg;
+  dg.add_int16(_current_major_ver);
+  dg.add_int16(_current_minor_ver);
+  dg.add_uint32(_scale_factor);
+  _write->write(dg.get_data(), dg.get_length());
+
+  _next_index = _write->tellp();
+  _next_index = pad_to_streampos(_next_index);
+  _last_index = 0;
+
+  if (_write->fail()) {
+    express_cat.info()
+      << "Unable to write header for " << _multifile_name << ".\n";
+    close();
+    return false;
+  }
 
   return true;
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: Multifile::write
+//     Function: Multifile::Subfile::read_index
 //       Access: Public
-//  Description: Returns true when all the memfiles have been
-//               written.
-//               Advances the start pointer as it writes.
+//  Description: Reads the index record for the Subfile from the
+//               indicated istream.  Assumes the istream has already
+//               been positioned to the indicated stream position,
+//               fpos, the start of the index record.  Returns the
+//               position within the file of the next index record.
 ////////////////////////////////////////////////////////////////////
-int Multifile::
-write(char *&start, int &size, const Filename &rel_path) {
-  // Make sure we have a complete header first
-  int parse_ret = parse_header(start, size);
-  if (parse_ret < 0) {
-    express_cat.error()
-     << "Multifile::write() - bad header" << endl;
-    return parse_ret;
-  }
-
-  while (_num_mfiles > 0) {
-    if (_current_mfile == NULL) {
-      _current_mfile = new Memfile;
-    }
-    int write_ret = _current_mfile->write(start, size, rel_path);
-    if (write_ret == EU_success) {
-      _num_mfiles--;
-      delete _current_mfile;
-      _current_mfile = NULL;
-    } else {
-      if (write_ret < 0) {
-        express_cat.error()
-          << "Multifile::write() - bad write: " << write_ret << endl;
-      }
-      return write_ret;
-    }
+streampos Multifile::Subfile::
+read_index(istream &read, streampos fpos, Multifile *multifile) {
+  nassertr(read.tellg() == fpos, fpos);
+
+  // First, get the next stream position.  We do this separately,
+  // because if it is zero, we don't get anything else.
+
+  static const size_t next_index_size = 4;
+  char next_index_buffer[next_index_size];
+  read.read(next_index_buffer, next_index_size);
+  if (read.fail() || read.gcount() != next_index_size) {
+    _flags |= SF_index_invalid;
+    return 0;
+  }
+
+  Datagram idg(next_index_buffer, next_index_size);
+  DatagramIterator idgi(idg);
+  streampos next_index = multifile->word_to_streampos(idgi.get_uint32());
+
+  if (next_index == 0) {
+    return 0;
+  }
+
+  // Now get the rest of the index (except the name, which is variable
+  // length).
+
+  _index_start = fpos;
+
+  static const size_t index_size = 4 + 4 + 2 + 2;
+  char index_buffer[index_size];
+  read.read(index_buffer, index_size);
+  if (read.fail() || read.gcount() != index_size) {
+    _flags |= SF_index_invalid;
+    return 0;
+  }
+
+  Datagram dg(index_buffer, index_size);
+  DatagramIterator dgi(dg);
+  _data_start = multifile->word_to_streampos(dgi.get_uint32());
+  _data_length = dgi.get_uint32();
+  _flags = dgi.get_uint16();
+  size_t name_length = dgi.get_uint16();
+
+  // And finally, get the rest of the name.
+  char *name_buffer = new char[name_length];
+  nassertr(name_buffer != (char *)NULL, next_index);
+  for (size_t ni = 0; ni < name_length; ni++) {
+    name_buffer[ni] = read.get() ^ 0xff;
   }
+  _name = string(name_buffer, name_length);
+  delete[] name_buffer;
 
-  return EU_success;
+  return next_index;
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: Multifile::write_extract
+//     Function: Multifile::Subfile::write_index
 //       Access: Public
-//  Description: Returns true when entire Multifile has been
-//               extracted to disk files.
+//  Description: Writes the index record for the Subfile to the
+//               indicated ostream.  Assumes the istream has already
+//               been positioned to the indicated stream position,
+//               fpos, the start of the index record, and that this is
+//               the effective end of the file.  Returns the position
+//               within the file of the next index record.
+//
+//               The _index_start member is updated by this operation.
 ////////////////////////////////////////////////////////////////////
-bool Multifile::
-write_extract(char *&start, int &size, const Filename &rel_path) {
-  int parse_ret = parse_header(start, size);
-  if (parse_ret < 0)
-    return false;
-  if (_current_mfile == (Memfile *)0L)
-    _current_mfile = new Memfile;
-  for (;;) {
-    if (_current_mfile->write(start, size, rel_path) == false)
-      return false;
-    if (++_mfiles_written == _num_mfiles)
-      return true;
-    _current_mfile->reset();
+streampos Multifile::Subfile::
+write_index(ostream &write, streampos fpos, Multifile *multifile) {
+  nassertr(write.tellp() == fpos, fpos);
+
+  _index_start = fpos;
+
+  // This will be the contents of this particular index record.
+  Datagram dg;
+  dg.add_uint32(multifile->streampos_to_word(_data_start));
+  dg.add_uint32(_data_length);
+  dg.add_uint16(_flags);
+  dg.add_uint16(_name.length());
+
+  // For no real good reason, we'll invert all the bits in the name.
+  // The only reason we do this is to make it inconvenient for a
+  // casual browser of the Multifile to discover the names of the
+  // files stored within it.  Naturally, this isn't real obfuscation
+  // or security.
+  string::iterator ni;
+  for (ni = _name.begin(); ni != _name.end(); ++ni) {
+    dg.add_int8((*ni) ^ 0xff);
   }
+
+  size_t this_index_size = 4 + dg.get_length();
+
+  // Plus, we will write out the next index address first.
+  streampos next_index = fpos + this_index_size;
+  Datagram idg;
+  idg.add_uint32(multifile->streampos_to_word(next_index));
+
+  write.write(idg.get_data(), idg.get_length());
+  write.write(dg.get_data(), dg.get_length());
+
+  return next_index;
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: Multifile::reset
+//     Function: Multifile::Subfile::write_index
 //       Access: Public
-//  Description:
+//  Description: Writes the data record for the Subfile to the
+//               indicated ostream: the actual contents of the
+//               Subfile.  Assumes the istream has already been
+//               positioned to the indicated stream position, fpos,
+//               the start of the data record, and that this is the
+//               effective end of the file.  Returns the position
+//               within the file of the next data record.
+//
+//               The _data_start and _data_length members are updated
+//               by this operation.
+//
+//               If the "read" pointer is non-NULL, it is the readable
+//               istream of a Multifile in which the Subfile might
+//               already be packed.  This is used for reading the
+//               contents of the Subfile during a repack() operation.
 ////////////////////////////////////////////////////////////////////
-void Multifile::
-reset(void) {
-  if (express_cat.is_debug())
-    express_cat.debug()
-      << "Multifile reset called" << endl;
-  _header_parsed = false;
-  _num_mfiles = 0;
-  _current_mfile = (Memfile *)0L;
-  _datagram.clear();
-  _files.erase(_files.begin(), _files.end());
-  _mfiles_written = 0;
+streampos Multifile::Subfile::
+write_data(ostream &write, istream *read, streampos fpos) {
+  nassertr(write.tellp() == fpos, fpos);
+
+  _data_start = fpos;
+
+  istream *source = _source;
+  ifstream source_file;
+  if (source == (istream *)NULL && !_source_filename.empty()) {
+    // If we have a filename, open it up and read that.
+    if (!_source_filename.open_read(source_file)) {
+      // Unable to open the source file.
+      express_cat.info()
+        << "Unable to read " << _source_filename << ".\n";
+      _flags |= SF_data_invalid;
+      _data_length = 0;
+    } else {
+      source = &source_file;
+    }
+  }
+
+  if (source == (istream *)NULL) {
+    // We don't have any source data.  Perhaps we're reading from an
+    // already-packed Subfile (e.g. during repack()).
+    if (read == (istream *)NULL) {
+      // No, we're just screwed.
+      express_cat.info()
+        << "No source for subfile " << _name << ".\n";
+      _flags |= SF_data_invalid;
+    } else {
+      // Read the data from the original Multifile.
+      for (size_t p = 0; p < _data_length; p++) {
+        int byte = read->get();
+        if (read->eof() || read->fail()) {
+          // Unexpected EOF or other failure on the source file.
+          express_cat.info()
+            << "Unexpected EOF for subfile " << _name << ".\n";
+          _flags |= SF_data_invalid;
+        }
+        write.put(byte);
+      }
+    }
+  } else {
+    // We do have source data.  Copy it in, and also measure its
+    // length.
+    _data_length = 0;
+    int byte = source->get();
+    while (!source->eof() && !source->fail()) {
+      _data_length++;
+      write.put(byte);
+      byte = source->get();
+    }
+  }
+
+  _source = (istream *)NULL;
+  _source_filename = Filename();
+  source_file.close();
+
+  return fpos + _data_length;
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: Multifile::extract
+//     Function: Multifile::Subfile::rewrite_index_data_start
 //       Access: Public
-//  Description:
+//  Description: Seeks within the indicate fstream back to the index
+//               record and rewrites just the _data_start and
+//               _data_length part of the index record.
 ////////////////////////////////////////////////////////////////////
-bool Multifile::
-extract(const Filename &name, const Filename &rel_path) {
-  MemfileList::iterator found;
-  found = find_if(_files.begin(), _files.end(), MemfileMatch(name));
-  if (found != _files.end()) {
-    (*found)->write(rel_path);
-    return true;
-  }
-  return false;
+void Multifile::Subfile::
+rewrite_index_data_start(ostream &write, Multifile *multifile) {
+  nassertv(_index_start != 0);
+
+  static const size_t data_start_offset = 4;
+  size_t data_start_pos = _index_start + data_start_offset;
+  write.seekp(data_start_pos);
+  nassertv(!write.fail());
+
+  Datagram dg;
+  dg.add_uint32(multifile->streampos_to_word(_data_start));
+  dg.add_uint32(_data_length);
+  write.write(dg.get_data(), dg.get_length());
 }
 
 ////////////////////////////////////////////////////////////////////
-//     Function: Multifile::extract_all
+//     Function: Multifile::Subfile::rewrite_index_flags
 //       Access: Public
-//  Description:
+//  Description: Seeks within the indicate fstream back to the index
+//               record and rewrites just the _flags part of the
+//               index record.
 ////////////////////////////////////////////////////////////////////
-void Multifile::
-extract_all(const Filename &rel_path) {
-  express_cat.debug()
-    << "Multifile::extract_all() - Extracting all files" << endl;
-
-  MemfileList::iterator i;
-  for (i = _files.begin(); i != _files.end(); ++i)
-    (*i)->write(rel_path);
+void Multifile::Subfile::
+rewrite_index_flags(ostream &write) {
+  nassertv(_index_start != 0);
+
+  static const size_t flags_offset = 4 + 4 + 4;
+  size_t flags_pos = _index_start + flags_offset;
+  write.seekp(flags_pos);
+  nassertv(!write.fail());
+
+  Datagram dg;
+  dg.add_uint16(_flags);
+  write.write(dg.get_data(), dg.get_length());
 }

+ 106 - 83
panda/src/express/multifile.h

@@ -15,21 +15,19 @@
 // [email protected] .
 //
 ////////////////////////////////////////////////////////////////////
+
 #ifndef MULTIFILE_H
 #define MULTIFILE_H
-//
-////////////////////////////////////////////////////////////////////
-// Includes
-////////////////////////////////////////////////////////////////////
-#include <pandabase.h>
 
-#include "typedef.h"
+#include "pandabase.h"
+
 #include "datagram.h"
 #include "datagramIterator.h"
 
-#include <notify.h>
-#include <filename.h>
-#include "plist.h"
+#include "filename.h"
+#include "ordered_vector.h"
+#include "indirectLess.h"
+#include "pvector.h"
 
 ////////////////////////////////////////////////////////////////////
 //       Class : Multifile
@@ -37,95 +35,120 @@
 ////////////////////////////////////////////////////////////////////
 class EXPCL_PANDAEXPRESS Multifile {
 PUBLISHED:
-  enum Type {
-    T_unknown,
-    T_valid,
-    T_invalid,
-  };
+  Multifile();
+  ~Multifile();
 
-  Multifile(void);
-  ~Multifile(void);
+  bool open_read(const Filename &multifile_name);
+  bool open_write(const Filename &multifile_name);
+  bool open_read_write(const Filename &multifile_name);
+  void close();
 
-  INLINE int get_header_length(void) const;
+  INLINE bool is_read_valid() const;
+  INLINE bool is_write_valid() const;
+  INLINE bool needs_repack() const;
 
-  static int evaluate(const char *start, int size);
+  INLINE void set_scale_factor(size_t scale_factor);
+  INLINE size_t get_scale_factor() const;
 
-  bool add(const Filename &name);
-  bool remove(const Filename &name);
-  bool has_file(const Filename &name);
+  bool add_subfile(const string &subfile_name, const Filename &filename);
+  bool flush();
+  bool repack();
 
-  bool read(Filename &name);
-  bool write(Filename name);
-  int write(char *&start, int &size, const Filename &rel_path = "");
-  bool write_extract(char *&start, int &size, const Filename &rel_path = "");
-  bool extract(const Filename &name, const Filename &rel_path = "");
-  void extract_all(const Filename &rel_path = "");
+  int get_num_subfiles() const;
+  int find_subfile(const string &subfile_name) const;
+  void remove_subfile(int index);
+  const string &get_subfile_name(int index) const;
+  size_t get_subfile_length(int index) const;
 
-  void reset(void);
-  int parse_header(char *&start, int &size);
-
-private:
-
-  INLINE void write_header(ofstream &write_stream);
+  void read_subfile(int index, Datagram &datagram);
+  bool extract_subfile(int index, const Filename &filename);
 
 public:
-  // This nested class must be public so we can make a list of its
-  // pointers.  Weird.
-  class Memfile {
-  public:
-    Memfile(void);
-    ~Memfile(void);
-    bool parse_header_length(char *&start, int &size);
-    bool parse_header(char *&start, int &size);
-
-    bool read(const Filename &name);
-    bool read_from_multifile(ifstream &read_stream);
-    bool write(const Filename &rel_path);
-    void write_to_multifile(ofstream &write_stream);
-    int write(char *&start, int &size, const Filename &rel_path = "");
-    void reset(void);
+  // Special interfaces to work with iostreams, not necessarily files.
+  bool open_read(istream *multifile_stream);
+  bool open_write(ostream *multifile_stream);
+  bool open_read_write(iostream *multifile_stream);
+  bool add_subfile(const string &subfile_name, istream *subfile_data);
 
-  public:
-    bool _header_length_parsed;
-    bool _header_parsed;
-    Datagram _datagram;
-    char *_header_length_buf;
-    int _header_length_buf_length;
-
-    PN_int32 _header_length;
-    Filename _name;
-    PN_int32 _buffer_length;
-    char* _buffer;
-
-    bool _file_open;
-    ofstream _write_stream;
-    int _bytes_written;
-  };
+  bool extract_subfile_to(int index, ostream &out);
+  istream &open_read_subfile(int index);
+  void close_subfile();
 
 private:
-  typedef plist<Memfile *> MemfileList;
+  enum SubfileFlags {
+    SF_deleted        = 0x0001,
+    SF_index_invalid  = 0x0002,
+    SF_data_invalid   = 0x0004,
+  };
 
-  class MemfileMatch {
+  class Subfile {
   public:
-    MemfileMatch(const Filename &name) {
-      _want_name = name;
-    }
-    bool operator()(Memfile *mfile) const {
-      return mfile->_name == _want_name;
-    }
-    Filename _want_name;
+    INLINE Subfile(const string &name);
+    INLINE bool operator < (const Subfile &other) const;
+    streampos read_index(istream &read, streampos fpos,
+                         Multifile *multfile);
+    streampos write_index(ostream &write, streampos fpos,
+                          Multifile *multifile);
+    streampos write_data(ostream &write, istream *read, streampos fpos);
+    void rewrite_index_data_start(ostream &write, Multifile *multifile);
+    void rewrite_index_flags(ostream &write);
+    INLINE bool is_deleted() const;
+    INLINE bool is_index_invalid() const;
+    INLINE bool is_data_invalid() const;
+
+    string _name;
+    streampos _index_start;
+    streampos _data_start;
+    size_t _data_length;
+    istream *_source;
+    Filename _source_filename;
+    int _flags;
   };
 
-private:
-  MemfileList _files;
-  PN_int32 _num_mfiles;
-  bool _header_parsed;
-  Memfile *_current_mfile;
-  Datagram _datagram;
-  int _header_length;
-  int _mfiles_written;
-
-  static PN_uint32 _magic_number;
+  INLINE streampos word_to_streampos(size_t word) const;
+  INLINE size_t streampos_to_word(streampos fpos) const;
+  INLINE streampos normalize_streampos(streampos fpos) const;
+  streampos pad_to_streampos(streampos fpos);
+
+  bool add_new_subfile(Subfile *subfile);
+  void clear_subfiles();
+  bool read_index();
+  bool write_header();
+
+
+  typedef ov_set<Subfile *, IndirectLess<Subfile> > Subfiles;
+  Subfiles _subfiles;
+  typedef pvector<Subfile *> PendingSubfiles;
+  PendingSubfiles _new_subfiles;
+  PendingSubfiles _removed_subfiles;
+
+  istream *_read;
+  ostream *_write;
+  streampos _next_index;
+  streampos _last_index;
+
+  bool _needs_repack;
+  size_t _scale_factor;
+
+  ifstream _read_file;
+  ofstream _write_file;
+  fstream _read_write_file;
+  Filename _multifile_name;
+
+  int _file_major_ver;
+  int _file_minor_ver;
+
+  Subfile *_open_subfile;
+  // This is just to support open_read_subfile() for those subfiles
+  // that have recently been added but not flushed.
+  ifstream _subfile_read;
+
+  static const char _header[];
+  static const size_t _header_size;
+  static const int _current_major_ver;
+  static const int _current_minor_ver;
+
+  friend class Subfile;
 };
 
 #include "multifile.I"

+ 0 - 0
panda/src/putil/ordered_vector.I → panda/src/express/ordered_vector.I


+ 0 - 0
panda/src/putil/ordered_vector.T → panda/src/express/ordered_vector.T


+ 0 - 0
panda/src/putil/ordered_vector.cxx → panda/src/express/ordered_vector.cxx


+ 0 - 1
panda/src/putil/ordered_vector.h → panda/src/express/ordered_vector.h

@@ -20,7 +20,6 @@
 #define ORDERED_VECTOR_H
 
 #include "pandabase.h"
-#include "config_util.h"
 
 #include "pvector.h"
 #include "pset.h"

+ 0 - 0
panda/src/putil/test_ordered_vector.cxx → panda/src/express/test_ordered_vector.cxx


+ 1 - 16
panda/src/putil/Sources.pp

@@ -30,13 +30,11 @@
     globalPointerRegistry.I globalPointerRegistry.h \
     indirectCompareNames.I indirectCompareNames.h \
     indirectCompareTo.I indirectCompareTo.h \
-    indirectLess.I indirectLess.h \
     ioPtaDatagramFloat.h ioPtaDatagramInt.h \
     ioPtaDatagramShort.h keyboardButton.h lineStream.I \
     lineStream.h lineStreamBuf.I lineStreamBuf.h \
     modifierButtons.I modifierButtons.h mouseButton.h \
     mouseData.I mouseData.h nameUniquifier.I nameUniquifier.h \
-    ordered_vector.h ordered_vector.I ordered_vector.T \
     pipeline.h pipeline.I \
     pipelineCycler.h pipelineCycler.I \
     pipelineCyclerBase.h pipelineCyclerBase.I \
@@ -67,7 +65,6 @@
     keyboardButton.cxx lineStream.cxx lineStreamBuf.cxx \
     modifierButtons.cxx mouseButton.cxx mouseData.cxx \
     nameUniquifier.cxx \
-    ordered_vector.cxx \
     pipeline.cxx \
     pipelineCycler.cxx \
     pipelineCyclerBase.cxx \
@@ -100,13 +97,12 @@
     globPattern.I globPattern.h \
     globalPointerRegistry.I globalPointerRegistry.h \
     indirectCompareNames.I indirectCompareNames.h indirectCompareTo.I \
-    indirectCompareTo.h indirectLess.I indirectLess.h \
+    indirectCompareTo.h \
     ioPtaDatagramFloat.h ioPtaDatagramInt.h \
     ioPtaDatagramShort.h iterator_types.h keyboardButton.h lineStream.I \
     lineStream.h lineStreamBuf.I lineStreamBuf.h modifierButtons.I \
     modifierButtons.h mouseButton.h mouseData.I mouseData.h \
     nameUniquifier.I nameUniquifier.h \
-    ordered_vector.h ordered_vector.I ordered_vector.T \
     pipeline.h pipeline.I \
     pipelineCycler.h pipelineCycler.I \
     pipelineCyclerBase.h pipelineCyclerBase.I \
@@ -170,14 +166,3 @@
     test_linestream.cxx
 
 #end test_bin_target
-
-#begin test_bin_target
-  #define TARGET test_ordered_vector
-
-  #define SOURCES \
-    test_ordered_vector.cxx
-
-  #define LOCAL_LIBS $[LOCAL_LIBS] putil
-  #define OTHER_LIBS $[OTHER_LIBS] pystub
-
-#end test_bin_target

+ 0 - 1
panda/src/putil/putil_composite2.cxx

@@ -1,5 +1,4 @@
 #include "nameUniquifier.cxx"
-#include "ordered_vector.cxx"
 #include "pipeline.cxx"
 #include "pipelineCycler.cxx"
 #include "pipelineCyclerBase.cxx"