Buffered stream API for big files: add next try of a prototype.^

code/CMakeLists.txt

@@ -171,6 +171,7 @@ SET( Common_SRCS
   Bitmap.h
   XMLTools.h
   Version.cpp
+  IOStreamBuffer.h
 )
 SOURCE_GROUP(Common FILES ${Common_SRCS})
 

code/ObjFileImporter.cpp

@@ -45,6 +45,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include "ObjFileImporter.h"
 #include "ObjFileParser.h"
 #include "ObjFileData.h"
+#include "IOStreamBuffer.h"
 #include <memory>
 #include <assimp/Importer.hpp>
 #include <assimp/scene.h>
@@ -70,78 +71,6 @@ namespace Assimp {
 
 using namespace std;
 
-template<class T>
-struct IOStreamBuffer {
-    BaseImporter *m_importer;
-    IOStream *m_stream;
-    size_t m_cacheSize;
-    std::vector<T> m_buffer;
-    size_t m_filesize;
-    size_t c;
-	size_t m_numBlocks;
-	size_t m_sizeLastBlock;
-
-    IOStreamBuffer( BaseImporter *imp, IOStream *stream, size_t cache = 4096 )
-    : m_importer( imp )
-    , m_stream( stream )
-    , m_cacheSize( cache )
-    , m_buffer()
-    , m_filesize( 0 )
-    , m_blockIndex( 0 )
-	, m_numBlocks( 0 ) 
-	, m_sizeLastBlock( 0 ) {
-        m_buffer.resize( m_cacheSize );
-        m_filesize = m_stream->FileSize();
-        m_numBlocks = m_filesize / m_cacheSize;
-        m_sizeLastBlock = m_filesize % m_cacheSize;
-    }
-
-    ~IOStreamBuffer() {
-        clear();
-    }
-
-    void clear() {
-        m_cacheSize = 4096;
-        m_buffer.resize( 0 );
-    }
-
-    bool mapPosToBlock( size_t pos, size_t &blockIdx, size_t &blockPos ) {
-        if ( pos > m_filesize ) {
-            return false;
-        }
-        blockIdx = pos / m_cacheSize;
-        blockPos = pos % m_cacheSize;
-
-        return true;
-    }
-
-    void loadBlock( size_t blockIdx ) {
-        size_t pos = blockIdx * m_cacheSize;
-        size_t sizeToRead( m_cacheSize );
-        if ( m_blockIdx == ( m_numBlocks - 1 ) ) {
-            sizeToRead = m_sizeLastBlock;
-        }
-
-        m_stream->Seek( pos <, aiOrigin_SET );
-        m_stream->Read( &m_buffer[ 0 ], sizeof( T ), sizeToRead );
-        m_blockIndex = blockIdx;
-
-        BaseImporter::ConvertToUTF8( m_buffer );
-    }
-
-    T &operator [] (size_t pos ) {
-        size_t blockIdx( 0 ), blockPos( 0 );
-        if ( !mapPosToBlock( pos, blockIdx, blockPos ) ) {
-            throw DeadlyImportError( "OBJ-file-access out of bounds." );
-        }
-        if ( m_blockIndex != blockIdx ) {
-            loadBlock( blockIdx );
-        }
-
-        return m_buffer[ blockPos ];
-	}
-};
-
 // ------------------------------------------------------------------------------------------------
 //  Default constructor
 ObjFileImporter::ObjFileImporter() :
@@ -198,8 +127,11 @@ void ObjFileImporter::InternReadFile( const std::string &file, aiScene* pScene,
         throw DeadlyImportError( "OBJ-file is too small.");
     }
 
+    IOStreamBuffer<char> streamedBuffer;
+    streamedBuffer.open( fileStream.get() );
+
     // Allocate buffer and read file into it
-    TextFileToBuffer( fileStream.get(),m_Buffer);
+    //TextFileToBuffer( fileStream.get(),m_Buffer);
 
     // Get the model name
     std::string  modelName, folderName;
@@ -222,7 +154,7 @@ void ObjFileImporter::InternReadFile( const std::string &file, aiScene* pScene,
     const unsigned int updateProgressEveryBytes = 100 * 1024;
     const unsigned int progressTotal = (3*m_Buffer.size()/updateProgressEveryBytes);
     // process all '\'
-    std::vector<char> ::iterator iter = m_Buffer.begin();
+    /*std::vector<char> ::iterator iter = m_Buffer.begin();
     while (iter != m_Buffer.end())
     {
         if (*iter == '\\')
@@ -241,17 +173,19 @@ void ObjFileImporter::InternReadFile( const std::string &file, aiScene* pScene,
             m_progress->UpdateFileRead(++progress, progressTotal);
             progressCounter = 0;
         }
-    }
+    }*/
 
     // 1/3rd progress
     m_progress->UpdateFileRead(1, 3);
 
     // parse the file into a temporary representation
-    ObjFileParser parser(m_Buffer, modelName, pIOHandler, m_progress, file);
+    ObjFileParser parser( streamedBuffer, modelName, pIOHandler, m_progress, file);
 
     // And create the proper return structures out of it
     CreateDataFromImport(parser.GetModel(), pScene);
 
+    streamedBuffer.close();
+
     // Clean up allocated storage for the next import
     m_Buffer.clear();
 

code/ObjFileParser.cpp

@@ -61,9 +61,9 @@ const std::string ObjFileParser::DEFAULT_MATERIAL = AI_DEFAULT_MATERIAL_NAME;
 
 // -------------------------------------------------------------------
 //  Constructor with loaded data and directories.
-ObjFileParser::ObjFileParser(std::vector<char> &data, const std::string &modelName, IOSystem *io, ProgressHandler* progress, const std::string &originalObjFileName) :
-    m_DataIt(data.begin()),
-    m_DataItEnd(data.end()),
+ObjFileParser::ObjFileParser( IOStreamBuffer<char> &streamBuffer, const std::string &modelName, IOSystem *io, ProgressHandler* progress, const std::string &originalObjFileName) :
+    m_DataIt(),
+    m_DataItEnd(),
     m_pModel(NULL),
     m_uiLine(0),
     m_pIO( io ),
@@ -83,7 +83,7 @@ ObjFileParser::ObjFileParser(std::vector<char> &data, const std::string &modelNa
     m_pModel->m_MaterialMap[ DEFAULT_MATERIAL ] = m_pModel->m_pDefaultMaterial;
 
     // Start parsing the file
-    parseFile();
+    parseFile( streamBuffer );
 }
 
 // -------------------------------------------------------------------
@@ -103,31 +103,33 @@ ObjFile::Model *ObjFileParser::GetModel() const
 
 // -------------------------------------------------------------------
 //  File parsing method.
-void ObjFileParser::parseFile()
-{
-    if (m_DataIt == m_DataItEnd)
-        return;
-
+void ObjFileParser::parseFile( IOStreamBuffer<char> &streamBuffer ) {
     // only update every 100KB or it'll be too slow
     const unsigned int updateProgressEveryBytes = 100 * 1024;
     unsigned int progressCounter = 0;
-    const unsigned int bytesToProcess = std::distance(m_DataIt, m_DataItEnd);
+    const unsigned int bytesToProcess = streamBuffer.size();
     const unsigned int progressTotal = 3 * bytesToProcess;
     const unsigned int progressOffset = bytesToProcess;
     unsigned int processed = 0;
 
-    DataArrayIt lastDataIt = m_DataIt;
+    //DataArrayIt lastDataIt = m_DataIt;
+
+    bool endOfFile( false );
+    std::vector<char> buffer;
+
+    //while ( m_DataIt != m_DataItEnd )
+    while ( streamBuffer.getNextLine( buffer ) ) {
+        m_DataIt = buffer.begin();
+        m_DataItEnd = buffer.end();
 
-    while (m_DataIt != m_DataItEnd)
-    {
         // Handle progress reporting
-        processed += std::distance(lastDataIt, m_DataIt);
+        /*processed += std::distance(lastDataIt, m_DataIt);
         lastDataIt = m_DataIt;
         if (processed > (progressCounter * updateProgressEveryBytes))
         {
             progressCounter++;
             m_progress->UpdateFileRead(progressOffset + processed*2, progressTotal);
-        }
+        }*/
 
         // parse line
         switch (*m_DataIt)
@@ -149,8 +151,8 @@ void ObjFileParser::parseFile()
                     }
                 } else if (*m_DataIt == 't') {
                     // read in texture coordinate ( 2D or 3D )
-                                        ++m_DataIt;
-                                        getVector( m_pModel->m_TextureCoord );
+                    ++m_DataIt;
+                    getVector( m_pModel->m_TextureCoord );
                 } else if (*m_DataIt == 'n') {
                     // Read in normal vector definition
                     ++m_DataIt;

code/ObjFileParser.h

@@ -46,6 +46,7 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #include <assimp/vector2.h>
 #include <assimp/vector3.h>
 #include <assimp/mesh.h>
+#include "IOStreamBuffer.h"
 
 namespace Assimp {
 
@@ -72,7 +73,7 @@ public:
 
 public:
     /// \brief  Constructor with data array.
-    ObjFileParser(std::vector<char> &Data, const std::string &strModelName, IOSystem* io, ProgressHandler* progress, const std::string &originalObjFileName);
+    ObjFileParser( IOStreamBuffer<char> &streamBuffer, const std::string &strModelName, IOSystem* io, ProgressHandler* progress, const std::string &originalObjFileName);
     /// \brief  Destructor
     ~ObjFileParser();
     /// \brief  Model getter.
@@ -80,7 +81,7 @@ public:
 
 private:
     /// Parse the loaded file
-    void parseFile();
+    void parseFile( IOStreamBuffer<char> &streamBuffer );
     /// Method to copy the new delimited word in the current line.
     void copyNextWord(char *pBuffer, size_t length);
     /// Method to copy the new line.

code/ObjTools.h

@@ -246,6 +246,20 @@ string_type trim_whitespaces(string_type str)
     return str;
 }
 
+template<class T>
+bool hasLineEnd( T it, T end ) {
+    bool hasLineEnd( false );
+    while ( !isEndOfBuffer( it, end ) ) {
+        it++;
+        if ( IsLineEnd( it ) ) {
+            hasLineEnd = true;
+            break;
+        }
+    }
+
+    return hasLineEnd;
+}
+
 } // Namespace Assimp
 
 #endif // OBJ_TOOLS_H_INC