Merge pull request #4375 from assimp/kimkulling/introduce_compressionv5.2.0

Introduce compression class to encapsulate compression via zlib

11 files changed, 272 insertions, 78 deletions

diff --git a/code/AssetLib/Assbin/AssbinFileWriter.cpp b/code/AssetLib/Assbin/AssbinFileWriter.cpp
index 72fbfdcb1..1d16f179e 100644
--- a/code/AssetLib/Assbin/AssbinFileWriter.cpp
+++ b/code/AssetLib/Assbin/AssbinFileWriter.cpp
@@ -182,6 +182,8 @@ inline size_t Write<aiVertexWeight>(IOStream *stream, const aiVertexWeight &v) {
     return t + Write<float>(stream, v.mWeight);
 }
 
+constexpr size_t MatrixSize = 64;
+
 // -----------------------------------------------------------------------------------
 // Serialize a mat4x4
 template <>
@@ -192,7 +194,7 @@ inline size_t Write<aiMatrix4x4>(IOStream *stream, const aiMatrix4x4 &m) {
         }
     }
 
-    return 64;
+    return MatrixSize;
 }
 
 // -----------------------------------------------------------------------------------
diff --git a/code/AssetLib/XGL/XGLLoader.cpp b/code/AssetLib/XGL/XGLLoader.cpp
index b003f852a..a711598f9 100644
--- a/code/AssetLib/XGL/XGLLoader.cpp
+++ b/code/AssetLib/XGL/XGLLoader.cpp
@@ -44,9 +44,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #ifndef ASSIMP_BUILD_NO_XGL_IMPORTER
 
 #include "XGLLoader.h"
+#include "Common/Compression.h"
+
 #include <assimp/ParsingUtils.h>
 #include <assimp/fast_atof.h>
-
 #include <assimp/MemoryIOWrapper.h>
 #include <assimp/StreamReader.h>
 #include <assimp/importerdesc.h>
@@ -57,15 +58,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 using namespace Assimp;
 
-// zlib is needed for compressed XGL files
-#ifndef ASSIMP_BUILD_NO_COMPRESSED_XGL
-#  ifdef ASSIMP_BUILD_NO_OWN_ZLIB
-#    include <zlib.h>
-#  else
-#    include <contrib/zlib/zlib.h>
-#  endif
-#endif
-
 namespace Assimp { // this has to be in here because LogFunctions is in ::Assimp
 
 template <>
@@ -73,6 +65,7 @@ const char *LogFunctions<XGLImporter>::Prefix() {
     static auto prefix = "XGL: ";
 	return prefix;
 }
+
 } // namespace Assimp
 
 static const aiImporterDesc desc = {
@@ -118,8 +111,8 @@ const aiImporterDesc *XGLImporter::GetInfo() const {
 // ------------------------------------------------------------------------------------------------
 // Imports the given file into the given scene structure.
 void XGLImporter::InternReadFile(const std::string &pFile, aiScene *pScene, IOSystem *pIOHandler) {
-#ifndef ASSIMP_BUILD_NO_COMPRESSED_XGL
-	std::vector<Bytef> uncompressed;
+ #ifndef ASSIMP_BUILD_NO_COMPRESSED_XGL
+	std::vector<unsigned char> uncompressed;
 #endif
 
 	m_scene = pScene;
@@ -137,46 +130,14 @@ void XGLImporter::InternReadFile(const std::string &pFile, aiScene *pScene, IOSy
 #else
 		std::unique_ptr<StreamReaderLE> raw_reader(new StreamReaderLE(stream));
 
-		// build a zlib stream
-		z_stream zstream;
-		zstream.opaque = Z_NULL;
-		zstream.zalloc = Z_NULL;
-		zstream.zfree = Z_NULL;
-		zstream.data_type = Z_BINARY;
-
-		// raw decompression without a zlib or gzip header
-		inflateInit2(&zstream, -MAX_WBITS);
-
-		// skip two extra bytes, zgl files do carry a crc16 upfront (I think)
-		raw_reader->IncPtr(2);
-
-		zstream.next_in = reinterpret_cast<Bytef *>(raw_reader->GetPtr());
-		zstream.avail_in = (uInt) raw_reader->GetRemainingSize();
-
-		size_t total = 0l;
-
-		// TODO: be smarter about this, decompress directly into heap buffer
-		// and decompress the data .... do 1k chunks in the hope that we won't kill the stack
-#define MYBLOCK 1024
-		Bytef block[MYBLOCK];
-		int ret;
-		do {
-			zstream.avail_out = MYBLOCK;
-			zstream.next_out = block;
-			ret = inflate(&zstream, Z_NO_FLUSH);
-
-			if (ret != Z_STREAM_END && ret != Z_OK) {
-				ThrowException("Failure decompressing this file using gzip, seemingly it is NOT a compressed .XGL file");
-			}
-			const size_t have = MYBLOCK - zstream.avail_out;
-			total += have;
-			uncompressed.resize(total);
-			memcpy(uncompressed.data() + total - have, block, have);
-		} while (ret != Z_STREAM_END);
-
-		// terminate zlib
-		inflateEnd(&zstream);
-
+        Compression c;
+        size_t total = 0l;
+        if (c.open()) {
+            // skip two extra bytes, zgl files do carry a crc16 upfront (I think)
+            raw_reader->IncPtr(2);
+            total = c.decompress((unsigned char *)raw_reader->GetPtr(), raw_reader->GetRemainingSize(), uncompressed);
+            c.close();
+        }
 		// replace the input stream with a memory stream
 		stream.reset(new MemoryIOStream(reinterpret_cast<uint8_t *>(uncompressed.data()), total));
 #endif
@@ -291,7 +252,8 @@ aiNode *XGLImporter::ReadObject(XmlNode &node, TempScope &scope) {
 			const std::string &s = ai_stdStrToLower(child.name());
 			if (s == "mesh") {
 				const size_t prev = scope.meshes_linear.size();
-				if (ReadMesh(child, scope)) {
+                bool empty;
+				if (ReadMesh(child, scope, empty)) {
 					const size_t newc = scope.meshes_linear.size();
 					for (size_t i = 0; i < newc - prev; ++i) {
 						meshes.push_back(static_cast<unsigned int>(i + prev));
@@ -475,12 +437,12 @@ aiMesh *XGLImporter::ToOutputMesh(const TempMaterialMesh &m) {
 }
 
 // ------------------------------------------------------------------------------------------------
-bool XGLImporter::ReadMesh(XmlNode &node, TempScope &scope) {
+bool XGLImporter::ReadMesh(XmlNode &node, TempScope &scope, bool &empty) {
 	TempMesh t;
 
 	std::map<unsigned int, TempMaterialMesh> bymat;
     const unsigned int mesh_id = ReadIDAttr(node);
-
+    bool empty_mesh = true;
 	for (XmlNode &child : node.children()) {
         const std::string &s = ai_stdStrToLower(child.name());
 
@@ -539,6 +501,9 @@ bool XGLImporter::ReadMesh(XmlNode &node, TempScope &scope) {
 					mid = ResolveMaterialRef(sub_child, scope);
 				}
 			}
+            if (has[0] || has[1] || has[2]) {
+                empty_mesh = false;
+            }
 
 			if (mid == ~0u) {
 				ThrowException("missing material index");
@@ -590,6 +555,11 @@ bool XGLImporter::ReadMesh(XmlNode &node, TempScope &scope) {
 			scope.meshes.insert(std::pair<unsigned int, aiMesh *>(mesh_id, m));
 		}
 	}
+    if (empty_mesh) {
+        LogWarn("Mesh is empty, skipping.");
+        empty = empty_mesh;
+        return false;
+    }
 
 	// no id == not a reference, insert this mesh right *here*
 	return mesh_id == ~0u;
@@ -759,7 +729,7 @@ aiVector2D XGLImporter::ReadVec2(XmlNode &node) {
     std::string val;
     XmlParser::getValueAsString(node, val);
     const char *s = val.c_str();
-	ai_real v[2];
+    ai_real v[2] = {};
 	for (int i = 0; i < 2; ++i) {
 		if (!SkipSpaces(&s)) {
 			LogError("unexpected EOL, failed to parse vec2");
diff --git a/code/AssetLib/XGL/XGLLoader.h b/code/AssetLib/XGL/XGLLoader.h
index 903f114cf..ae7ccddc2 100644
--- a/code/AssetLib/XGL/XGLLoader.h
+++ b/code/AssetLib/XGL/XGLLoader.h
@@ -186,7 +186,7 @@ private:
     void ReadLighting(XmlNode &node, TempScope &scope);
     aiLight *ReadDirectionalLight(XmlNode &node);
     aiNode *ReadObject(XmlNode &node, TempScope &scope);
-    bool ReadMesh(XmlNode &node, TempScope &scope);
+    bool ReadMesh(XmlNode &node, TempScope &scope, bool &empty);
     void ReadMaterial(XmlNode &node, TempScope &scope);
     aiVector2D ReadVec2(XmlNode &node);
     aiVector3D ReadVec3(XmlNode &node);
diff --git a/code/CMakeLists.txt b/code/CMakeLists.txt
index 7ce933585..5936d6d06 100644
--- a/code/CMakeLists.txt
+++ b/code/CMakeLists.txt
@@ -166,6 +166,8 @@ SET( Logging_SRCS
 SOURCE_GROUP(Logging FILES ${Logging_SRCS})
 
 SET( Common_SRCS
+  Common/Compression.cpp
+  Common/Compression.h
   Common/BaseImporter.cpp
   Common/BaseProcess.cpp
   Common/BaseProcess.h
diff --git a/code/Common/Base64.cpp b/code/Common/Base64.cpp
index 2916cae7f..3e9c47405 100644
--- a/code/Common/Base64.cpp
+++ b/code/Common/Base64.cpp
@@ -3,7 +3,7 @@
 Open Asset Import Library (assimp)
 ---------------------------------------------------------------------------
 
-Copyright (c) 2006-2021, assimp team
+Copyright (c) 2006-2022, assimp team
 
 All rights reserved.
 
@@ -57,7 +57,7 @@ static const uint8_t tableDecodeBase64[128] = {
     41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 0, 0, 0, 0, 0
 };
 
-static const char* tableEncodeBase64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=";
+static const char *tableEncodeBase64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=";
 
 static inline char EncodeChar(uint8_t b) {
     return tableEncodeBase64[size_t(b)];
@@ -104,17 +104,16 @@ void Encode(const uint8_t *in, size_t inLength, std::string &out) {
     }
 }
 
-void Encode(const std::vector<uint8_t>& in, std::string &out) {
-    Encode (in.data (), in.size (), out);
+void Encode(const std::vector<uint8_t> &in, std::string &out) {
+    Encode(in.data(), in.size(), out);
 }
 
-std::string Encode (const std::vector<uint8_t>& in) {
+std::string Encode(const std::vector<uint8_t> &in) {
     std::string encoded;
-    Encode (in, encoded);
+    Encode(in, encoded);
     return encoded;
 }
 
-
 size_t Decode(const char *in, size_t inLength, uint8_t *&out) {
     if (inLength % 4 != 0) {
         throw DeadlyImportError("Invalid base64 encoded data: \"", std::string(in, std::min(size_t(32), inLength)), "\", length:", inLength);
@@ -159,23 +158,22 @@ size_t Decode(const char *in, size_t inLength, uint8_t *&out) {
     return outLength;
 }
 
-size_t Decode(const std::string& in, std::vector<uint8_t>& out) {
-    uint8_t* outPtr = nullptr;
-    size_t decodedSize = Decode (in.data (), in.size (), outPtr);
+size_t Decode(const std::string &in, std::vector<uint8_t> &out) {
+    uint8_t *outPtr = nullptr;
+    size_t decodedSize = Decode(in.data(), in.size(), outPtr);
     if (outPtr == nullptr) {
         return 0;
     }
-    out.assign (outPtr, outPtr + decodedSize);
+    out.assign(outPtr, outPtr + decodedSize);
     delete[] outPtr;
     return decodedSize;
 }
 
-std::vector<uint8_t> Decode (const std::string& in) {
+std::vector<uint8_t> Decode(const std::string &in) {
     std::vector<uint8_t> result;
-    Decode (in, result);
+    Decode(in, result);
     return result;
 }
 
-}
-
-}
+} // namespace Base64
+} // namespace Assimp
diff --git a/code/Common/Compression.cpp b/code/Common/Compression.cpp
new file mode 100644
index 000000000..2e3defa8c
--- /dev/null
+++ b/code/Common/Compression.cpp
@@ -0,0 +1,141 @@
+/*
+Open Asset Import Library (assimp)
+----------------------------------------------------------------------
+
+Copyright (c) 2006-2022, assimp team
+
+All rights reserved.
+
+Redistribution and use of this software in source and binary forms,
+with or without modification, are permitted provided that the
+following conditions are met:
+
+* Redistributions of source code must retain the above
+  copyright notice, this list of conditions and the
+  following disclaimer.
+
+* Redistributions in binary form must reproduce the above
+  copyright notice, this list of conditions and the
+  following disclaimer in the documentation and/or other
+  materials provided with the distribution.
+
+* Neither the name of the assimp team, nor the names of its
+  contributors may be used to endorse or promote products
+  derived from this software without specific prior
+  written permission of the assimp team.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+----------------------------------------------------------------------
+*/
+
+#include "Compression.h"
+#include <assimp/ai_assert.h>
+#include <assimp/Exceptional.h>
+
+#ifdef ASSIMP_BUILD_NO_OWN_ZLIB
+#include <zlib.h>
+#else
+#include "../contrib/zlib/zlib.h"
+#endif
+
+namespace Assimp {
+
+struct Compression::impl {
+    bool mOpen;
+    z_stream mZSstream;
+
+    impl() :
+            mOpen(false) {}
+};
+
+Compression::Compression() :
+        mImpl(new impl) {
+    // empty
+}
+
+Compression::~Compression() {
+    ai_assert(mImpl != nullptr);
+
+    delete mImpl;
+}
+
+bool Compression::open() {
+    ai_assert(mImpl != nullptr);
+
+    if (mImpl->mOpen) {
+        return false;
+    }
+
+    // build a zlib stream
+    mImpl->mZSstream.opaque = Z_NULL;
+    mImpl->mZSstream.zalloc = Z_NULL;
+    mImpl->mZSstream.zfree = Z_NULL;
+    mImpl->mZSstream.data_type = Z_BINARY;
+
+    // raw decompression without a zlib or gzip header
+    inflateInit2(&mImpl->mZSstream, -MAX_WBITS);
+    mImpl->mOpen = true;
+
+    return mImpl->mOpen;
+}
+
+constexpr size_t MYBLOCK = 1024;
+
+size_t Compression::decompress(unsigned char *data, size_t in, std::vector<unsigned char> &uncompressed) {
+    ai_assert(mImpl != nullptr);
+
+    mImpl->mZSstream.next_in = reinterpret_cast<Bytef *>(data);
+    mImpl->mZSstream.avail_in = (uInt)in;
+
+    Bytef block[MYBLOCK] = {};
+    int ret = 0;
+    size_t total = 0l;
+    do {
+        mImpl->mZSstream.avail_out = MYBLOCK;
+        mImpl->mZSstream.next_out = block;
+        ret = inflate(&mImpl->mZSstream, Z_NO_FLUSH);
+
+        if (ret != Z_STREAM_END && ret != Z_OK) {
+            throw DeadlyImportError("Compression", "Failure decompressing this file using gzip.");
+
+        }
+        const size_t have = MYBLOCK - mImpl->mZSstream.avail_out;
+        total += have;
+        uncompressed.resize(total);
+        ::memcpy(uncompressed.data() + total - have, block, have);
+    } while (ret != Z_STREAM_END);
+
+    return total;
+}
+
+bool Compression::isOpen() const {
+    ai_assert(mImpl != nullptr);
+
+    return mImpl->mOpen;
+}
+
+bool Compression::close() {
+    ai_assert(mImpl != nullptr);
+
+    if (!mImpl->mOpen) {
+        return false;
+    }
+
+    inflateEnd(&mImpl->mZSstream);
+    mImpl->mOpen = false;
+
+    return true;
+}
+
+} // namespace Assimp
diff --git a/code/Common/Compression.h b/code/Common/Compression.h
new file mode 100644
index 000000000..20c92cb60
--- /dev/null
+++ b/code/Common/Compression.h
@@ -0,0 +1,80 @@
+/*
+Open Asset Import Library (assimp)
+----------------------------------------------------------------------
+
+Copyright (c) 2006-2022, assimp team
+
+All rights reserved.
+
+Redistribution and use of this software in source and binary forms,
+with or without modification, are permitted provided that the
+following conditions are met:
+
+* Redistributions of source code must retain the above
+  copyright notice, this list of conditions and the
+  following disclaimer.
+
+* Redistributions in binary form must reproduce the above
+  copyright notice, this list of conditions and the
+  following disclaimer in the documentation and/or other
+  materials provided with the distribution.
+
+* Neither the name of the assimp team, nor the names of its
+  contributors may be used to endorse or promote products
+  derived from this software without specific prior
+  written permission of the assimp team.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+----------------------------------------------------------------------
+*/
+
+#pragma once
+
+#include <vector>
+
+namespace Assimp {
+
+/// @brief This class provides the decompression of zlib-compressed data.
+class Compression {
+public:
+    /// @brief  The class constructor.
+    Compression();
+
+    ///	@brief  The class destructor.
+    ~Compression();
+
+    /// @brief  Will open the access to the compression.
+    /// @return true if close was successful, false if not.
+    bool open();
+
+    /// @brief  Will return the open state.
+    /// @return true if the access is opened, false if not.
+    bool isOpen() const;
+
+    /// @brief  Will close the decompress access.
+    /// @return true if close was successful, false if not.
+    bool close();
+
+    /// @brief Will decompress the data buffer.
+    /// @param[in] data         The data to decompress
+    /// @param[in] in           The size of the data.
+    /// @param[out uncompressed A std::vector containing the decompressed data.
+    size_t decompress(unsigned char *data, size_t in, std::vector<unsigned char> &uncompressed);
+
+private:
+    struct impl;
+    impl *mImpl;
+};
+
+} // namespace Assimp
diff --git a/code/Common/IOSystem.cpp b/code/Common/IOSystem.cpp
index 77305d193..1e63827ba 100644
--- a/code/Common/IOSystem.cpp
+++ b/code/Common/IOSystem.cpp
@@ -3,7 +3,7 @@
 Open Asset Import Library (assimp)
 ---------------------------------------------------------------------------
 
-Copyright (c) 2006-2019, assimp team
+Copyright (c) 2006-2022, assimp team
 
 All rights reserved.
 
diff --git a/code/Common/material.cpp b/code/Common/material.cpp
index 8f791a313..a7541d4f1 100644
--- a/code/Common/material.cpp
+++ b/code/Common/material.cpp
@@ -4,7 +4,6 @@ Open Asset Import Library (assimp)
 
 Copyright (c) 2006-2022, assimp team
 
-
 All rights reserved.
 
 Redistribution and use of this software in source and binary forms,
diff --git a/code/Common/simd.cpp b/code/Common/simd.cpp
index 9eb547f9a..0dd437d26 100644
--- a/code/Common/simd.cpp
+++ b/code/Common/simd.cpp
@@ -5,8 +5,6 @@ Open Asset Import Library (assimp)
 
 Copyright (c) 2006-2022, assimp team
 
-
-
 All rights reserved.
 
 Redistribution and use of this software in source and binary forms,
diff --git a/include/assimp/Base64.hpp b/include/assimp/Base64.hpp
index 894eb449b..ee319aceb 100644
--- a/include/assimp/Base64.hpp
+++ b/include/assimp/Base64.hpp
@@ -50,6 +50,10 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 namespace Assimp {
 namespace Base64 {
 
+/// @brief Will encode the given 
+/// @param in 
+/// @param inLength 
+/// @param out 
 void Encode(const uint8_t *in, size_t inLength, std::string &out);
 void Encode(const std::vector<uint8_t>& in, std::string &out);
 std::string Encode(const std::vector<uint8_t>& in);