- STEP: support UTF16, UTF32 and Mac/Roman escape sequences.

code/CMakeLists.txt

@@ -384,6 +384,8 @@ SET(IFC_SRCS
 	STEPFile.h
 	STEPFileReader.h
 	STEPFileReader.cpp
+	STEPFileEncoding.cpp
+	STEPFileEncoding.h
 )
 SOURCE_GROUP( IFC FILES ${IFC_SRCS})
 

code/STEPFileEncoding.cpp

@@ -0,0 +1,433 @@
+/*
+Open Asset Import Library (assimp)
+----------------------------------------------------------------------
+
+Copyright (c) 2006-2012, assimp team
+All rights reserved.
+
+Redistribution and use of this software in source and binary forms, 
+with or without modification, are permitted provided that the 
+following conditions are met:
+
+* Redistributions of source code must retain the above
+  copyright notice, this list of conditions and the
+  following disclaimer.
+
+* Redistributions in binary form must reproduce the above
+  copyright notice, this list of conditions and the
+  following disclaimer in the documentation and/or other
+  materials provided with the distribution.
+
+* Neither the name of the assimp team, nor the names of its
+  contributors may be used to endorse or promote products
+  derived from this software without specific prior
+  written permission of the assimp team.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+----------------------------------------------------------------------
+*/
+
+/** @file  STEPFileEncoding.cpp
+ *  @brief STEP character handling, string unescaping
+ */
+#include "AssimpPCH.h"
+#include "STEPFileEncoding.h"
+#include "fast_atof.h"
+
+#include "../contrib/ConvertUTF/ConvertUTF.h"
+
+using namespace Assimp;
+
+// roman1 to utf16 table
+static const UTF16 mac_codetable[] = {
+	// 0x20 unassig./nonprint. slots
+	 0x0020	,
+	 0x0021	,
+	 0x0022	,
+	 0x0023	,
+	 0x0024	,
+	 0x0025	,
+	 0x0026	,
+	 0x0027	,
+	 0x0028	,
+	 0x0029	,
+	 0x002A	,
+	 0x002B	,
+	 0x002C	,
+	 0x002D	,
+	 0x002E	,
+	 0x002F	,
+	 0x0030	,
+	 0x0031	,
+	 0x0032	,
+	 0x0033	,
+	 0x0034	,
+	 0x0035	,
+	 0x0036	,
+	 0x0037	,
+	 0x0038	,
+	 0x0039	,
+	 0x003A	,
+	 0x003B	,
+	 0x003C	,
+	 0x003D	,
+	 0x003E	,
+	 0x003F	,
+	 0x0040	,
+	 0x0041	,
+	 0x0042	,
+	 0x0043	,
+	 0x0044	,
+	 0x0045	,
+	 0x0046	,
+	 0x0047	,
+	 0x0048	,
+	 0x0049	,
+	 0x004A	,
+	 0x004B	,
+	 0x004C	,
+	 0x004D	,
+	 0x004E	,
+	 0x004F	,
+	 0x0050	,
+	 0x0051	,
+	 0x0052	,
+	 0x0053	,
+	 0x0054	,
+	 0x0055	,
+	 0x0056	,
+	 0x0057	,
+	 0x0058	,
+	 0x0059	,
+	 0x005A	,
+	 0x005B	,
+	 0x005C	,
+	 0x005D	,
+	 0x005E	,
+	 0x005F	,
+	 0x0060	,
+	 0x0061	,
+	 0x0062	,
+	 0x0063	,
+	 0x0064	,
+	 0x0065	,
+	 0x0066	,
+	 0x0067	,
+	 0x0068	,
+	 0x0069	,
+	 0x006A	,
+	 0x006B	,
+	 0x006C	,
+	 0x006D	,
+	 0x006E	,
+	 0x006F	,
+	 0x0070	,
+	 0x0071	,
+	 0x0072	,
+	 0x0073	,
+	 0x0074	,
+	 0x0075	,
+	 0x0076	,
+	 0x0077	,
+	 0x0078	,
+	 0x0079	,
+	 0x007A	,
+	 0x007B	,
+	 0x007C	,
+	 0x007D	,
+	 0x007E	,
+	 0x0000 , // unassig.
+	 0x00C4	,
+	 0x00C5	,
+	 0x00C7	,
+	 0x00C9	,
+	 0x00D1	,
+	 0x00D6	,
+	 0x00DC	,
+	 0x00E1	,
+	 0x00E0	,
+	 0x00E2	,
+	 0x00E4	,
+	 0x00E3	,
+	 0x00E5	,
+	 0x00E7	,
+	 0x00E9	,
+	 0x00E8	,
+	 0x00EA	,
+	 0x00EB	,
+	 0x00ED	,
+	 0x00EC	,
+	 0x00EE	,
+	 0x00EF	,
+	 0x00F1	,
+	 0x00F3	,
+	 0x00F2	,
+	 0x00F4	,
+	 0x00F6	,
+	 0x00F5	,
+	 0x00FA	,
+	 0x00F9	,
+	 0x00FB	,
+	 0x00FC	,
+	 0x2020	,
+	 0x00B0	,
+	 0x00A2	,
+	 0x00A3	,
+	 0x00A7	,
+	 0x2022	,
+	 0x00B6	,
+	 0x00DF	,
+	 0x00AE	,
+	 0x00A9	,
+	 0x2122	,
+	 0x00B4	,
+	 0x00A8	,
+	 0x2260	,
+	 0x00C6	,
+	 0x00D8	,
+	 0x221E	,
+	 0x00B1	,
+	 0x2264	,
+	 0x2265	,
+	 0x00A5	,
+	 0x00B5	,
+	 0x2202	,
+	 0x2211	,
+	 0x220F	,
+	 0x03C0	,
+	 0x222B	,
+	 0x00AA	,
+	 0x00BA	,
+	 0x03A9	,
+	 0x00E6	,
+	 0x00F8	,
+	 0x00BF	,
+	 0x00A1	,
+	 0x00AC	,
+	 0x221A	,
+	 0x0192	,
+	 0x2248	,
+	 0x2206	,
+	 0x00AB	,
+	 0x00BB	,
+	 0x2026	,
+	 0x00A0	,
+	 0x00C0	,
+	 0x00C3	,
+	 0x00D5	,
+	 0x0152	,
+	 0x0153	,
+	 0x2013	,
+	 0x2014	,
+	 0x201C	,
+	 0x201D	,
+	 0x2018	,
+	 0x2019	,
+	 0x00F7	,
+	 0x25CA	,
+	 0x00FF	,
+	 0x0178	,
+	 0x2044	,
+	 0x20AC	,
+	 0x2039	,
+	 0x203A	,
+	 0xFB01	,
+	 0xFB02	,
+	 0x2021	,
+	 0x00B7	,
+	 0x201A	,
+	 0x201E	,
+	 0x2030	,
+	 0x00C2	,
+	 0x00CA	,
+	 0x00C1	,
+	 0x00CB	,
+	 0x00C8	,
+	 0x00CD	,
+	 0x00CE	,
+	 0x00CF	,
+	 0x00CC	,
+	 0x00D3	,
+	 0x00D4	,
+	 0xF8FF	,
+	 0x00D2	,
+	 0x00DA	,
+	 0x00DB	,
+	 0x00D9	,
+	 0x0131	,
+	 0x02C6	,
+	 0x02DC	,
+	 0x00AF	,
+	 0x02D8	,
+	 0x02D9	,
+	 0x02DA	,
+	 0x00B8	,
+	 0x02DD	,
+	 0x02DB	,
+	 0x02C7
+};
+
+// ------------------------------------------------------------------------------------------------
+bool STEP::StringToUTF8(std::string& s) 
+{
+	// very basic handling for escaped string sequences
+	// http://doc.spatial.com/index.php?title=InterOp:Connect/STEP&redirect=no
+	
+	for (size_t i = 0; i < s.size(); ) {
+		if (s[i] == '\\') {
+			// \S\X - cp1252 (X is the character remapped to [0,127])
+			if (i+3 < s.size() && s[i+1] == 'S' && s[i+2] == '\\') {
+				// http://stackoverflow.com/questions/5586214/how-to-convert-char-from-iso-8859-1-to-utf-8-in-c-multiplatformly
+				ai_assert((uint8_t)s[i+3] < 0x80);
+				const uint8_t ch = s[i+3] + 0x80;
+
+				s[i] = 0xc0 | (ch & 0xc0) >> 6;
+				s[i+1] =  0x80 | (ch & 0x3f);
+
+				s.erase(i + 2,2);
+				++i;
+			}
+			// \X\xx - mac/roman (xx is a hex sequence)
+			else if (i+4 < s.size() && s[i+1] == 'X' && s[i+2] == '\\') {
+				
+				const uint8_t macval = HexOctetToDecimal(s.c_str() + i + 3);
+				if(macval < 0x20) {
+					return false;
+				}
+
+				ai_assert(sizeof(mac_codetable) / sizeof(mac_codetable[0]) == 0x100-0x20);
+
+				const UTF32 unival = mac_codetable[macval - 0x20], *univalp = &unival;
+
+				UTF8 temp[5], *tempp = temp;
+				ai_assert(sizeof(UTF8) == 1);
+
+				if(ConvertUTF32toUTF8(&univalp, univalp+1, &tempp, tempp+sizeof(temp), lenientConversion) != conversionOK) {
+					return false;
+				}
+
+				const size_t outcount = static_cast<size_t>(tempp-temp);
+
+				s.erase(i,5);
+				s.insert(i, reinterpret_cast<char*>(temp), outcount);
+				i += outcount;
+			}
+			// \Xn\ .. \X0\ - various unicode encodings (n=2: utf16; n=4: utf32)
+			else if (i+3 < s.size() && s[i+1] == 'X' && s[i+2] >= '0' && s[i+2] <= '9') {
+				switch(s[i+2]) {
+					// utf16
+				case '2':
+					// utf32
+				case '4':						
+					if (s[i+3] == '\\') {
+						const size_t basei = i+4;
+						// scan for \X0\ 
+						size_t j = basei, jend = s.size()-4;
+						for (; j < jend; ++j) {
+							if (s[j] == '\\' && s[j] == 'X' && s[j] == '0' && s[j] == '\\') {
+								break;
+							}
+						}
+						if (j == jend) {						
+							return false;
+						}
+
+						if (j == basei) {
+							s.erase(i,8);
+							continue;
+						}
+						
+						if (s[i+2] == '2') {
+							if (((j - basei) % 4) != 0) {
+								return false;
+							}
+							
+							const size_t count = (j-basei)/4;
+							boost::scoped_array<UTF16> src = boost::scoped_array<UTF16>(new UTF16[count]);
+
+							const char* cur = s.c_str() + basei;
+							for (size_t k = 0; k < count; ++k, cur += 4) {
+								src[k] = (static_cast<UTF16>(HexOctetToDecimal(cur)) << 8u)  |
+									 static_cast<UTF16>(HexOctetToDecimal(cur+2));
+							}
+
+							const size_t dcount = count * 3; // this is enough to hold all possible outputs
+							boost::scoped_array<UTF8> dest = boost::scoped_array<UTF8>(new UTF8[dcount]);
+
+							const UTF16* srct = src.get();
+							UTF8* destt = dest.get();
+							if(ConvertUTF16toUTF8(&srct, srct+count, &destt, destt+dcount, lenientConversion) != conversionOK) {
+								return false;
+							}
+
+							const size_t outcount = static_cast<size_t>(destt-dest.get());
+
+							s.erase(i,(j+4-i));
+
+							ai_assert(sizeof(UTF8) == 1);
+							s.insert(i, reinterpret_cast<char*>(dest.get()), outcount);
+
+							i += outcount;
+							continue;
+						}
+						else if (s[i+2] == '4') {
+							if (((j - basei) % 8) != 0) {
+								return false;
+							}
+
+							const size_t count = (j-basei)/8;
+							boost::scoped_array<UTF32> src = boost::scoped_array<UTF32>(new UTF32[count]);
+
+							const char* cur = s.c_str() + basei;
+							for (size_t k = 0; k < count; ++k, cur += 8) {
+								src[k] = (static_cast<UTF32>(HexOctetToDecimal(cur  )) << 24u) |
+									     (static_cast<UTF32>(HexOctetToDecimal(cur+2)) << 16u) |
+										 (static_cast<UTF32>(HexOctetToDecimal(cur+4)) << 8u)  |
+									     (static_cast<UTF32>(HexOctetToDecimal(cur+6)));
+							}
+
+							const size_t dcount = count * 5; // this is enough to hold all possible outputs
+							boost::scoped_array<UTF8> dest = boost::scoped_array<UTF8>(new UTF8[dcount]);
+
+							const UTF32* srct = src.get();
+							UTF8* destt = dest.get();
+							if(ConvertUTF32toUTF8(&srct, srct+count, &destt, destt+dcount, lenientConversion) != conversionOK) {
+								return false;
+							}
+
+							const size_t outcount = static_cast<size_t>(destt-dest.get());
+
+							s.erase(i,(j+4-i));
+
+							ai_assert(sizeof(UTF8) == 1);
+							s.insert(i, reinterpret_cast<char*>(dest.get()), outcount);
+
+							i += outcount;
+							continue;
+						}
+					}				
+
+					break;
+					
+					// TODO: other encoding patterns?
+
+				default:
+					return false;
+				}
+			}
+		}
+		++i;
+	}
+	return true;
+}

code/STEPFileEncoding.h

@@ -0,0 +1,63 @@
+/*
+Open Asset Import Library (assimp)
+----------------------------------------------------------------------
+
+Copyright (c) 2006-2012, assimp team
+All rights reserved.
+
+Redistribution and use of this software in source and binary forms, 
+with or without modification, are permitted provided that the 
+following conditions are met:
+
+* Redistributions of source code must retain the above
+  copyright notice, this list of conditions and the
+  following disclaimer.
+
+* Redistributions in binary form must reproduce the above
+  copyright notice, this list of conditions and the
+  following disclaimer in the documentation and/or other
+  materials provided with the distribution.
+
+* Neither the name of the assimp team, nor the names of its
+  contributors may be used to endorse or promote products
+  derived from this software without specific prior
+  written permission of the assimp team.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+----------------------------------------------------------------------
+*/
+
+#ifndef INCLUDED_AI_STEPFILEENCODING_H
+#define INCLUDED_AI_STEPFILEENCODING_H
+
+#include <string>
+
+namespace Assimp {
+namespace STEP {
+
+
+	// --------------------------------------------------------------------------
+	// Convert an ASCII STEP identifier with possibly escaped character
+	// sequences using foreign encodings to plain UTF8.
+	// 
+	// Return false if an error occurs, s may or may not be modified in
+	// this case and could still contain escape sequences (even partly
+	// escaped ones).
+	bool StringToUTF8(std::string& s);
+	
+
+} // ! STEP
+} // ! Assimp
+
+#endif

code/STEPFileReader.cpp

@@ -44,9 +44,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  */
 #include "AssimpPCH.h"
 #include "STEPFileReader.h"
+#include "STEPFileEncoding.h"
 #include "TinyFormatter.h"
 #include "fast_atof.h"
 
+
 using namespace Assimp;
 namespace EXPRESS = STEP::EXPRESS;
 
@@ -331,7 +333,6 @@ void STEP::ReadFile(DB& db,const EXPRESS::ConversionSchema& scheme,
 	}
 }
 
-
 // ------------------------------------------------------------------------------------------------
 boost::shared_ptr<const EXPRESS::DataType> EXPRESS::DataType::Parse(const char*& inout,uint64_t line, const EXPRESS::ConversionSchema* schema /*= NULL*/)
 {
@@ -419,34 +420,12 @@ boost::shared_ptr<const EXPRESS::DataType> EXPRESS::DataType::Parse(const char*&
 
 		inout = cur + 1;
 
-		// very basic handling for escaped string sequences
-		// http://doc.spatial.com/index.php?title=InterOp:Connect/STEP&redirect=no
-		// UTF16: \X2\ ... \X0\
-		// UTF32: \X4\ ... \X0\
-		// Mac: \X8\xx (xx is a hex sequence)
-		// cp1252: \S\X (X is the character remapped to [0,127])
-		// ? more of them ?
-
-		// Note: assimp is supposed to output UTF8 strings
-
+		// assimp is supposed to output UTF8 strings, so we have to deal
+		// with foreign encodings.
 		std::string stemp = std::string(start, static_cast<size_t>(cur - start));
-		for (size_t i = 0; i < stemp.size(); ++i) {
-			if (stemp[i] == '\\') {
-				if (i+3 < stemp.size() && stemp[i+1] == 'S' && stemp[i+2] == '\\') {
-					// http://stackoverflow.com/questions/5586214/how-to-convert-char-from-iso-8859-1-to-utf-8-in-c-multiplatformly
-					ai_assert((uint8_t)stemp[i+3] < 0x80);
-					const uint8_t ch = stemp[i+3] + 0x80;
-
-					stemp[i] = 0xc0 | (ch & 0xc0) >> 6;
-					stemp[i+1] =  0x80 | (ch & 0x3f);
-
-					stemp.erase(i + 2,2);
-					++i;
-				}
-				else if (i+2 < stemp.size() && stemp[i+1] == 'X' && IsNumeric(stemp[i+2])) {
-					// TODO: warn
-				}
-			}
+		if(!StringToUTF8(stemp)) {
+			// TODO: route this to a correct logger with line numbers etc., better error messages
+			DefaultLogger::get()->error("an error occurred reading escape sequences in ASCII text");
 		}
 
 		return boost::make_shared<EXPRESS::STRING>(stemp);