diff options
Diffstat (limited to 'src/corelib/text/qbytearray.cpp')
| -rw-r--r-- | src/corelib/text/qbytearray.cpp | 332 |
1 files changed, 236 insertions, 96 deletions
diff --git a/src/corelib/text/qbytearray.cpp b/src/corelib/text/qbytearray.cpp index f728877e4a..ba6e79543a 100644 --- a/src/corelib/text/qbytearray.cpp +++ b/src/corelib/text/qbytearray.cpp @@ -22,6 +22,7 @@ #ifndef QT_NO_COMPRESS #include <zconf.h> #include <zlib.h> +#include <qxpfunctional.h> #endif #include <ctype.h> #include <limits.h> @@ -110,23 +111,29 @@ char *qstrcpy(char *dst, const char *src) A safe \c strncpy() function. Copies at most \a len bytes from \a src (stopping at \a len or the - terminating '\\0' whichever comes first) into \a dst and returns a - pointer to \a dst. Guarantees that \a dst is '\\0'-terminated. If - \a src or \a dst is \nullptr, returns \nullptr immediately. + terminating '\\0' whichever comes first) into \a dst. Guarantees that \a + dst is '\\0'-terminated, except when \a dst is \nullptr or \a len is 0. If + \a src is \nullptr, returns \nullptr, otherwise returns \a dst. This function assumes that \a dst is at least \a len characters long. \note If \a dst and \a src overlap, the behavior is undefined. + \note Unlike strncpy(), this function does \e not write '\\0' to all \a + len bytes of \a dst, but stops after the terminating '\\0'. In this sense, + it's similar to C11's strncpy_s(). + \sa qstrcpy() */ char *qstrncpy(char *dst, const char *src, size_t len) { - if (!src || !dst) - return nullptr; - if (len > 0) { + if (dst && len > 0) { + if (!src) { + *dst = '\0'; + return nullptr; + } #ifdef Q_CC_MSVC strncpy_s(dst, len, src, len - 1); #else @@ -134,7 +141,7 @@ char *qstrncpy(char *dst, const char *src, size_t len) #endif dst[len-1] = '\0'; } - return dst; + return src ? dst : nullptr; } /*! \fn size_t qstrlen(const char *str) @@ -508,7 +515,9 @@ quint16 qChecksum(QByteArrayView data, Qt::ChecksumType standard) \sa qUncompress(const QByteArray &data) */ -/*! \relates QByteArray +/*! + \fn QByteArray qCompress(const uchar* data, qsizetype nbytes, int compressionLevel) + \relates QByteArray \overload @@ -517,44 +526,197 @@ quint16 qChecksum(QByteArrayView data, Qt::ChecksumType standard) */ #ifndef QT_NO_COMPRESS +using CompressSizeHint_t = quint32; // 32-bit BE, historically + +enum class ZLibOp : bool { Compression, Decompression }; + +Q_DECL_COLD_FUNCTION +static const char *zlibOpAsString(ZLibOp op) +{ + switch (op) { + case ZLibOp::Compression: return "qCompress"; + case ZLibOp::Decompression: return "qUncompress"; + } + Q_UNREACHABLE(); + return nullptr; +} + +Q_DECL_COLD_FUNCTION +static QByteArray zlibError(ZLibOp op, const char *what) +{ + qWarning("%s: %s", zlibOpAsString(op), what); + return QByteArray(); +} + +Q_DECL_COLD_FUNCTION +static QByteArray dataIsNull(ZLibOp op) +{ + return zlibError(op, "Data is null"); +} + +Q_DECL_COLD_FUNCTION +static QByteArray lengthIsNegative(ZLibOp op) +{ + return zlibError(op, "Input length is negative"); +} + +Q_DECL_COLD_FUNCTION +static QByteArray tooMuchData(ZLibOp op) +{ + return zlibError(op, "Not enough memory"); +} + +Q_DECL_COLD_FUNCTION +static QByteArray invalidCompressedData() +{ + return zlibError(ZLibOp::Decompression, "Input data is corrupted"); +} + +Q_DECL_COLD_FUNCTION +static QByteArray unexpectedZlibError(ZLibOp op, int err, const char *msg) +{ + qWarning("%s unexpected zlib error: %s (%d)", + zlibOpAsString(op), + msg ? msg : "", + err); + return QByteArray(); +} + +static QByteArray xxflate(ZLibOp op, QArrayDataPointer<char> out, QByteArrayView input, + qxp::function_ref<int(z_stream *) const> init, + qxp::function_ref<int(z_stream *, size_t) const> processChunk, + qxp::function_ref<void(z_stream *) const> deinit) +{ + if (out.data() == nullptr) // allocation failed + return tooMuchData(op); + qsizetype capacity = out.allocatedCapacity(); + + const auto initalSize = out.size; + + z_stream zs = {}; + zs.next_in = reinterpret_cast<uchar *>(const_cast<char *>(input.data())); // 1980s C API... + if (const int err = init(&zs); err != Z_OK) + return unexpectedZlibError(op, err, zs.msg); + const auto sg = qScopeGuard([&] { deinit(&zs); }); + + using ZlibChunkSize_t = decltype(zs.avail_in); + static_assert(!std::is_signed_v<ZlibChunkSize_t>); + static_assert(std::is_same_v<ZlibChunkSize_t, decltype(zs.avail_out)>); + constexpr auto MaxChunkSize = std::numeric_limits<ZlibChunkSize_t>::max(); + [[maybe_unused]] + constexpr auto MaxStatisticsSize = std::numeric_limits<decltype(zs.total_out)>::max(); + + size_t inputLeft = size_t(input.size()); + + int res; + do { + Q_ASSERT(out.freeSpaceAtBegin() == 0); // ensure prepend optimization stays out of the way + Q_ASSERT(capacity == out.allocatedCapacity()); + + if (zs.avail_out == 0) { + Q_ASSERT(size_t(out.size) - initalSize > MaxStatisticsSize || // total_out overflow + size_t(out.size) - initalSize == zs.total_out); + Q_ASSERT(out.size <= capacity); + + qsizetype avail_out = capacity - out.size; + if (avail_out == 0) { + out->reallocateAndGrow(QArrayData::GrowsAtEnd, 1); // grow to next natural capacity + if (out.data() == nullptr) // reallocation failed + return tooMuchData(op); + capacity = out.allocatedCapacity(); + avail_out = capacity - out.size; + } + zs.next_out = reinterpret_cast<uchar *>(out.data()) + out.size; + zs.avail_out = size_t(avail_out) > size_t(MaxChunkSize) ? MaxChunkSize + : ZlibChunkSize_t(avail_out); + out.size += zs.avail_out; + + Q_ASSERT(zs.avail_out > 0); + } + + if (zs.avail_in == 0) { + // zs.next_in is kept up-to-date by processChunk(), so nothing to do + zs.avail_in = inputLeft > MaxChunkSize ? MaxChunkSize : ZlibChunkSize_t(inputLeft); + inputLeft -= zs.avail_in; + } + + res = processChunk(&zs, inputLeft); + } while (res == Z_OK); + + switch (res) { + case Z_STREAM_END: + out.size -= zs.avail_out; + Q_ASSERT(size_t(out.size) - initalSize > MaxStatisticsSize || // total_out overflow + size_t(out.size) - initalSize == zs.total_out); + Q_ASSERT(out.size <= out.allocatedCapacity()); + out.data()[out.size] = '\0'; + return QByteArray(std::move(out)); + + case Z_MEM_ERROR: + return tooMuchData(op); + + case Z_BUF_ERROR: + Q_UNREACHABLE(); // cannot happen - we supply a buffer that can hold the result, + // or else error out early + + case Z_DATA_ERROR: // can only happen on decompression + Q_ASSERT(op == ZLibOp::Decompression); + return invalidCompressedData(); + + default: + return unexpectedZlibError(op, res, zs.msg); + } +} + QByteArray qCompress(const uchar* data, qsizetype nbytes, int compressionLevel) { + constexpr qsizetype HeaderSize = sizeof(CompressSizeHint_t); if (nbytes == 0) { - return QByteArray(4, '\0'); - } - if (!data) { - qWarning("qCompress: Data is null"); - return QByteArray(); + return QByteArray(HeaderSize, '\0'); } + if (!data) + return dataIsNull(ZLibOp::Compression); + + if (nbytes < 0) + return lengthIsNegative(ZLibOp::Compression); + if (compressionLevel < -1 || compressionLevel > 9) compressionLevel = -1; - ulong len = nbytes + nbytes / 100 + 13; - QByteArray bazip; - int res; - do { - bazip.resize(len + 4); - res = ::compress2((uchar*)bazip.data()+4, &len, data, nbytes, compressionLevel); - - switch (res) { - case Z_OK: - bazip.resize(len + 4); - bazip[0] = (nbytes & 0xff000000) >> 24; - bazip[1] = (nbytes & 0x00ff0000) >> 16; - bazip[2] = (nbytes & 0x0000ff00) >> 8; - bazip[3] = (nbytes & 0x000000ff); - break; - case Z_MEM_ERROR: - qWarning("qCompress: Z_MEM_ERROR: Not enough memory"); - bazip.resize(0); - break; - case Z_BUF_ERROR: - len *= 2; - break; + QArrayDataPointer out = [&] { + constexpr qsizetype SingleAllocLimit = 256 * 1024; // the maximum size for which we use + // zlib's compressBound() to guarantee + // the output buffer size is sufficient + // to hold result + qsizetype capacity = HeaderSize; + if (nbytes < SingleAllocLimit) { + // use maximum size + capacity += compressBound(uLong(nbytes)); // cannot overflow (both times)! + return QArrayDataPointer{QTypedArrayData<char>::allocate(capacity)}; } - } while (res == Z_BUF_ERROR); - return bazip; + // for larger buffers, assume it compresses optimally, and + // grow geometrically from there: + constexpr qsizetype MaxCompressionFactor = 1024; // max theoretical factor is 1032 + // cf. http://www.zlib.org/zlib_tech.html, + // but use a nearby power-of-two (faster) + capacity += std::max(qsizetype(compressBound(uLong(SingleAllocLimit))), + nbytes / MaxCompressionFactor); + return QArrayDataPointer{QTypedArrayData<char>::allocate(capacity, QArrayData::Grow)}; + }(); + + if (out.data() == nullptr) // allocation failed + return tooMuchData(ZLibOp::Compression); + + qToBigEndian(qt_saturate<CompressSizeHint_t>(nbytes), out.data()); + out.size = HeaderSize; + + return xxflate(ZLibOp::Compression, std::move(out), {data, nbytes}, + [=] (z_stream *zs) { return deflateInit(zs, compressionLevel); }, + [] (z_stream *zs, size_t inputLeft) { + return deflate(zs, inputLeft ? Z_NO_FLUSH : Z_FINISH); + }, + [] (z_stream *zs) { deflateEnd(zs); }); } #endif @@ -576,18 +738,21 @@ QByteArray qCompress(const uchar* data, qsizetype nbytes, int compressionLevel) data that was compressed using zlib, you first need to prepend a four byte header to the byte array containing the data. The header must contain the expected length (in bytes) of the uncompressed data, - expressed as an unsigned, big-endian, 32-bit integer. + expressed as an unsigned, big-endian, 32-bit integer. This number is + just a hint for the initial size of the output buffer size, + though. If the indicated size is too small to hold the result, the + output buffer size will still be increased until either the output + fits or the system runs out of memory. So, despite the 32-bit + header, this function, on 64-bit platforms, can produce more than + 4GiB of output. + + \note In Qt versions prior to Qt 6.5, more than 2GiB of data + worked unreliably; in Qt versions prior to Qt 6.0, not at all. \sa qCompress() */ #ifndef QT_NO_COMPRESS -static QByteArray invalidCompressedData() -{ - qWarning("qUncompress: Input data is corrupted"); - return QByteArray(); -} - /*! \relates QByteArray \overload @@ -597,64 +762,39 @@ static QByteArray invalidCompressedData() */ QByteArray qUncompress(const uchar* data, qsizetype nbytes) { - if (!data) { - qWarning("qUncompress: Data is null"); - return QByteArray(); - } - if (nbytes <= 4) { - if (nbytes < 4 || (data[0]!=0 || data[1]!=0 || data[2]!=0 || data[3]!=0)) - qWarning("qUncompress: Input data is corrupted"); - return QByteArray(); - } - size_t expectedSize = size_t((data[0] << 24) | (data[1] << 16) | - (data[2] << 8) | (data[3] )); - size_t len = qMax(expectedSize, 1ul); - const size_t maxPossibleSize = MaxAllocSize - sizeof(QByteArray::Data); - if (Q_UNLIKELY(len >= maxPossibleSize)) { - // QByteArray does not support that huge size anyway. - return invalidCompressedData(); - } + if (!data) + return dataIsNull(ZLibOp::Decompression); - QByteArray::DataPointer d(QByteArray::Data::allocate(len)); - if (Q_UNLIKELY(d.data() == nullptr)) + if (nbytes < 0) + return lengthIsNegative(ZLibOp::Decompression); + + constexpr qsizetype HeaderSize = sizeof(CompressSizeHint_t); + if (nbytes < HeaderSize) return invalidCompressedData(); - forever { - const auto alloc = len; - int res = ::uncompress((uchar*)d.data(), reinterpret_cast<uLongf*>(&len), - data+4, nbytes-4); - - switch (res) { - case Z_OK: { - Q_ASSERT(len <= alloc); - Q_UNUSED(alloc); - d.data()[len] = '\0'; - d.size = len; - return QByteArray(d); - } + const auto expectedSize = qFromBigEndian<CompressSizeHint_t>(data); + if (nbytes == HeaderSize) { + if (expectedSize != 0) + return invalidCompressedData(); + return QByteArray(); + } - case Z_MEM_ERROR: - qWarning("qUncompress: Z_MEM_ERROR: Not enough memory"); - return QByteArray(); + constexpr auto MaxDecompressedSize = size_t(MaxByteArraySize); + if constexpr (MaxDecompressedSize < std::numeric_limits<CompressSizeHint_t>::max()) { + if (expectedSize > MaxDecompressedSize) + return tooMuchData(ZLibOp::Decompression); + } - case Z_BUF_ERROR: - len *= 2; - if (Q_UNLIKELY(len >= maxPossibleSize)) { - // QByteArray does not support that huge size anyway. - return invalidCompressedData(); - } else { - // grow the block - d->reallocate(d->allocatedCapacity()*2, QArrayData::Grow); - if (Q_UNLIKELY(d.data() == nullptr)) - return invalidCompressedData(); - } - continue; + // expectedSize may be truncated, so always use at least nbytes + // (larger by at most 1%, according to zlib docs) + qsizetype capacity = std::max(qsizetype(expectedSize), // cannot overflow! + nbytes); - case Z_DATA_ERROR: - qWarning("qUncompress: Z_DATA_ERROR: Input data is corrupted"); - return QByteArray(); - } - } + QArrayDataPointer d(QTypedArrayData<char>::allocate(capacity, QArrayData::KeepSize)); + return xxflate(ZLibOp::Decompression, std::move(d), {data + HeaderSize, nbytes - HeaderSize}, + [] (z_stream *zs) { return inflateInit(zs); }, + [] (z_stream *zs, size_t) { return inflate(zs, Z_NO_FLUSH); }, + [] (z_stream *zs) { inflateEnd(zs); }); } #endif |