From c6651f91b8f31d94ef37aa41cc2fd76d97e990e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Abecasis?= Date: Tue, 20 Oct 2009 16:28:22 +0200 Subject: Adding a test case for large file support The test case creates a (tentatively sparse) very large file with scattered data and uses it to test various aspects of large file support in QFile. Reviewed-by: Thiago Macieira --- tests/auto/qfile/largefile/largefile.pro | 4 + tests/auto/qfile/largefile/tst_largefile.cpp | 492 +++++++++++++++++++++++++++ tests/auto/qfile/qfile.pro | 2 +- 3 files changed, 497 insertions(+), 1 deletion(-) create mode 100644 tests/auto/qfile/largefile/largefile.pro create mode 100644 tests/auto/qfile/largefile/tst_largefile.cpp diff --git a/tests/auto/qfile/largefile/largefile.pro b/tests/auto/qfile/largefile/largefile.pro new file mode 100644 index 0000000000..0f968659a1 --- /dev/null +++ b/tests/auto/qfile/largefile/largefile.pro @@ -0,0 +1,4 @@ +load(qttest_p4) + +QT = core +SOURCES += tst_largefile.cpp diff --git a/tests/auto/qfile/largefile/tst_largefile.cpp b/tests/auto/qfile/largefile/tst_largefile.cpp new file mode 100644 index 0000000000..398ca3f34d --- /dev/null +++ b/tests/auto/qfile/largefile/tst_largefile.cpp @@ -0,0 +1,492 @@ +/**************************************************************************** +** +** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +** All rights reserved. +** Contact: Nokia Corporation (qt-info@nokia.com) +** +** This file is part of the test suite of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** No Commercial Usage +** This file contains pre-release code and may not be distributed. +** You may use this file in accordance with the terms and conditions +** contained in the Technology Preview License Agreement accompanying +** this package. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 2.1 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 2.1 requirements +** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Nokia gives you certain additional +** rights. These rights are described in the Nokia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** If you have questions regarding the use of this file, please contact +** Nokia at qt-info@nokia.com. +** +** +** +** +** +** +** +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include + +#include +#include +#include +#include + +#include + +#include +#include + +#ifdef Q_OS_WIN + +#include +#include + +#ifndef FSCTL_SET_SPARSE +// MinGW doesn't define this. +#define FSCTL_SET_SPARSE (0x900C4) +#endif + +#endif // Q_OS_WIN + +class tst_LargeFile + : public QObject +{ + Q_OBJECT + +public: + tst_LargeFile() + : blockSize(1 << 12) + , maxSizeBits() + , largeFile("qt_largefile.tmp") + , fd_(-1) + , stream_(0) + { + #if defined(QT_LARGEFILE_SUPPORT) && !defined(Q_OS_MAC) + maxSizeBits = 36; // 64 GiB + #elif defined(Q_OS_MAC) + // HFS+ does not support sparse files, so we limit file size for the test + // on Mac OS. + maxSizeBits = 32; // 4 GiB + #else + maxSizeBits = 24; // 16 MiB + #endif + } + +private: + void sparseFileData(); + QByteArray const &getDataBlock(int index, qint64 position); + +private slots: + // The LargeFile test case was designed to be run in order as a single unit + + void initTestCase(); + void cleanupTestCase(); + + void init(); + void cleanup(); + + // Create and fill large file + void createSparseFile(); + void fillFileSparsely(); + void closeSparseFile(); + + // Verify file was created + void fileCreated(); + + // Positioning in large files + void filePositioning(); + void fdPositioning(); + void streamPositioning(); + + // Read data from file + void openFileForReading(); + void readFile(); + + // Map/unmap large file + void mapFile(); + void mapOffsetOverflow(); + + void closeFile() { largeFile.close(); } + + // Test data + void fillFileSparsely_data() { sparseFileData(); } + void filePositioning_data() { sparseFileData(); } + void fdPositioning_data() { sparseFileData(); } + void streamPositioning_data() { sparseFileData(); } + void readFile_data() { sparseFileData(); } + void mapFile_data() { sparseFileData(); } + +private: + const int blockSize; + int maxSizeBits; + + QFile largeFile; + + QVector generatedBlocks; + + int fd_; + FILE *stream_; +}; + +/* + Convenience function to hide reinterpret_cast when copying a POD directly + into a QByteArray. + */ +template +static inline void appendRaw(QByteArray &array, T data) +{ + array.append(reinterpret_cast(&data), sizeof(T)); +} + +/* + Pad array with filler up to size. On return, array.size() returns size. + */ +static inline void topUpWith(QByteArray &array, QByteArray filler, int size) +{ + Q_ASSERT(filler.size() > 0); + + for (int i = (size - array.size()) / filler.size(); i > 0; --i) + array.append(filler); + + if (array.size() < size) { + Q_ASSERT(size - array.size() < filler.size()); + array.append(filler.left(size - array.size())); + } +} + +/* + Generate a unique data block containing identifiable data. Unaligned, + overlapping and partial blocks should not compare equal. + */ +static inline QByteArray generateDataBlock(int blockSize, QString text, qint64 userBits = -1) +{ + QByteArray block; + block.reserve(blockSize); + + // Use of counter and randomBits means content of block will be dependent + // on the generation order. For (file-)systems that do not support sparse + // files, these can be removed so the test file can be reused and doesn't + // have to be generated for every run. + + static qint64 counter = 0; + + qint64 randomBits = ((qint64)qrand() << 32) + | ((qint64)qrand() & 0x00000000ffffffff); + + appendRaw(block, randomBits); + appendRaw(block, userBits); + appendRaw(block, counter); + appendRaw(block, (qint32)0xdeadbeef); + appendRaw(block, blockSize); + + QByteArray userContent = text.toUtf8(); + appendRaw(block, userContent.size()); + block.append(userContent); + appendRaw(block, (qint64)0); + + // size, so far + appendRaw(block, block.size()); + + QByteArray filler("0123456789"); + block.append(filler.right(10 - block.size() % 10)); + topUpWith(block, filler, blockSize - 2 * sizeof(qint64)); + + appendRaw(block, counter); + appendRaw(block, userBits); + appendRaw(block, randomBits); + + Q_ASSERT( block.size() >= blockSize ); + block.resize(blockSize); + + ++counter; + return block; +} + +/* + Generates data blocks the first time they are requested. Keeps copies for reuse. + */ +QByteArray const &tst_LargeFile::getDataBlock(int index, qint64 position) +{ + if (index >= generatedBlocks.size()) + generatedBlocks.resize(index + 1); + + if (generatedBlocks[index].isNull()) { + QString text = QString("Current %1-byte block (index = %2) " + "starts %3 bytes into the file '%4'.") + .arg(blockSize) + .arg(index) + .arg(position) + .arg(largeFile.fileName()); + + generatedBlocks[index] = generateDataBlock(blockSize, text, (qint64)1 << index); + } + + return generatedBlocks[index]; +} + +void tst_LargeFile::initTestCase() +{ + QVERIFY( !largeFile.exists() || largeFile.remove() ); +} + +void tst_LargeFile::cleanupTestCase() +{ + largeFile.close(); + QVERIFY( !largeFile.exists() || largeFile.remove() ); +} + +void tst_LargeFile::init() +{ + fd_ = -1; + stream_ = 0; +} + +void tst_LargeFile::cleanup() +{ + if (-1 != fd_) + QT_CLOSE(fd_); + if (stream_) + ::fclose(stream_); +} + +void tst_LargeFile::sparseFileData() +{ + QTest::addColumn("index"); + QTest::addColumn("position"); + QTest::addColumn("block"); + + QTest::newRow(QString("block[%1] @%2)") + .arg(0).arg(0) + .toLocal8Bit().constData()) + << 0 << (qint64)0 << getDataBlock(0, 0); + + // While on Linux sparse files scale well, on Windows, testing at every + // power of 2 leads to very large files. i += 4 gives us a good coverage + // without taxing too much on resources. + for (int index = 12; index <= maxSizeBits; index += 4) { + qint64 position = (qint64)1 << index; + QByteArray block = getDataBlock(index, position); + + QTest::newRow( + QString("block[%1] @%2)") + .arg(index).arg(position) + .toLocal8Bit().constData()) + << index << position << block; + } +} + +void tst_LargeFile::createSparseFile() +{ +#if defined(Q_OS_WIN) + // On Windows platforms, we must explicitly set the file to be sparse, + // so disk space is not allocated for the full file when writing to it. + HANDLE handle = ::CreateFileA(largeFile.fileName().toLocal8Bit().constData(), + GENERIC_WRITE, 0, 0, CREATE_ALWAYS, 0, 0); + QVERIFY( INVALID_HANDLE_VALUE != handle ); + + DWORD bytes; + if (!::DeviceIoControl(handle, FSCTL_SET_SPARSE, NULL, 0, NULL, 0, + &bytes, NULL)) { + QWARN("Unable to set test file as sparse. " + "Limiting test file to 16MiB."); + maxSizeBits = 24; + } + + int fd = ::_open_osfhandle((intptr_t)handle, 0); + QVERIFY( -1 != fd ); + QVERIFY( largeFile.open(fd, QIODevice::WriteOnly) ); +#else // !Q_OS_WIN + QVERIFY( largeFile.open(QIODevice::WriteOnly) ); +#endif +} + +void tst_LargeFile::closeSparseFile() +{ +#if defined(Q_OS_WIN) + int fd = largeFile.handle(); +#endif + + largeFile.close(); + +#if defined(Q_OS_WIN) + if (-1 != fd) + ::_close(fd); +#endif +} + +void tst_LargeFile::fillFileSparsely() +{ + QFETCH( qint64, position ); + QFETCH( QByteArray, block ); + QCOMPARE( block.size(), blockSize ); + + QVERIFY( largeFile.seek(position) ); + QCOMPARE( largeFile.pos(), position ); + + QCOMPARE( largeFile.write(block), (qint64)blockSize ); + QCOMPARE( largeFile.pos(), position + blockSize ); + QVERIFY( largeFile.flush() ); +} + +void tst_LargeFile::fileCreated() +{ + QFileInfo info(largeFile); + + QVERIFY( info.exists() ); + QVERIFY( info.isFile() ); + QVERIFY( info.size() >= ((qint64)1 << maxSizeBits) + blockSize ); +} + +void tst_LargeFile::filePositioning() +{ + QFETCH( qint64, position ); + + QFile file(largeFile.fileName()); + QVERIFY( file.open(QIODevice::ReadOnly) ); + + QVERIFY( file.seek(position) ); + QCOMPARE( file.pos(), position ); +} + +void tst_LargeFile::fdPositioning() +{ + QFETCH( qint64, position ); + + fd_ = QT_OPEN(largeFile.fileName().toLocal8Bit().constData(), + QT_OPEN_RDONLY | QT_OPEN_LARGEFILE); + QVERIFY( -1 != fd_ ); + + QFile file; + QVERIFY( file.open(fd_, QIODevice::ReadOnly) ); + QCOMPARE( file.pos(), (qint64)0 ); + QVERIFY( file.seek(position) ); + QCOMPARE( file.pos(), position ); + + file.close(); + + QCOMPARE( QT_LSEEK(fd_, QT_OFF_T(0), SEEK_SET), QT_OFF_T(0) ); + QCOMPARE( QT_LSEEK(fd_, QT_OFF_T(position), SEEK_SET), QT_OFF_T(position) ); + + QVERIFY( file.open(fd_, QIODevice::ReadOnly) ); + QCOMPARE( QT_LSEEK(fd_, QT_OFF_T(0), SEEK_CUR), QT_OFF_T(position) ); + QCOMPARE( file.pos(), position ); + QVERIFY( file.seek(0) ); + QCOMPARE( file.pos(), (qint64)0 ); + + file.close(); + + QVERIFY( !QT_CLOSE(fd_) ); + fd_ = -1; +} + +void tst_LargeFile::streamPositioning() +{ + QFETCH( qint64, position ); + + stream_ = QT_FOPEN(largeFile.fileName().toLocal8Bit().constData(), "rb"); + QVERIFY( 0 != stream_ ); + + QFile file; + QVERIFY( file.open(stream_, QIODevice::ReadOnly) ); + QCOMPARE( file.pos(), (qint64)0 ); + QVERIFY( file.seek(position) ); + QCOMPARE( file.pos(), position ); + + file.close(); + + QVERIFY( !QT_FSEEK(stream_, QT_OFF_T(0), SEEK_SET) ); + QCOMPARE( QT_FTELL(stream_), QT_OFF_T(0) ); + QVERIFY( !QT_FSEEK(stream_, QT_OFF_T(position), SEEK_SET) ); + QCOMPARE( QT_FTELL(stream_), QT_OFF_T(position) ); + + QVERIFY( file.open(stream_, QIODevice::ReadOnly) ); + QCOMPARE( QT_FTELL(stream_), QT_OFF_T(position) ); + QCOMPARE( file.pos(), position ); + QVERIFY( file.seek(0) ); + QCOMPARE( file.pos(), (qint64)0 ); + + file.close(); + + QVERIFY( !::fclose(stream_) ); + stream_ = 0; +} + +void tst_LargeFile::openFileForReading() +{ + QVERIFY( largeFile.open(QIODevice::ReadOnly) ); +} + +void tst_LargeFile::readFile() +{ + QFETCH( qint64, position ); + QFETCH( QByteArray, block ); + QCOMPARE( block.size(), blockSize ); + + QVERIFY( largeFile.size() >= position + blockSize ); + + QVERIFY( largeFile.seek(position) ); + QCOMPARE( largeFile.pos(), position ); + + QCOMPARE( largeFile.read(blockSize), block ); + QCOMPARE( largeFile.pos(), position + blockSize ); +} + +void tst_LargeFile::mapFile() +{ + QFETCH( qint64, position ); + QFETCH( QByteArray, block ); + QCOMPARE( block.size(), blockSize ); + + // Keep full block mapped to facilitate OS and/or internal reuse by Qt. + uchar *baseAddress = largeFile.map(position, blockSize); + QVERIFY( baseAddress ); + QVERIFY( qEqual(block.begin(), block.end(), reinterpret_cast(baseAddress)) ); + + for (int offset = 1; offset < blockSize; ++offset) { + uchar *address = largeFile.map(position + offset, blockSize - offset); + + QVERIFY( address ); + if ( !qEqual(block.begin() + offset, block.end(), reinterpret_cast(address)) ) { + qDebug() << "Expected:" << block.toHex(); + qDebug() << "Actual :" << QByteArray(reinterpret_cast(address), blockSize).toHex(); + QVERIFY(false); + } + + QVERIFY( largeFile.unmap( address ) ); + } + + QVERIFY( largeFile.unmap( baseAddress ) ); +} + +void tst_LargeFile::mapOffsetOverflow() +{ + // Out-of-range mappings should fail, and not silently clip the offset + for (int i = 50; i < 63; ++i) { + uchar *address = 0; + + address = largeFile.map(((qint64)1 << i), blockSize); + QVERIFY( !address ); + + address = largeFile.map(((qint64)1 << i) + blockSize, blockSize); + QVERIFY( !address ); + } +} + +QTEST_APPLESS_MAIN(tst_LargeFile) +#include "tst_largefile.moc" + diff --git a/tests/auto/qfile/qfile.pro b/tests/auto/qfile/qfile.pro index eebfcdafe9..f70f75025b 100644 --- a/tests/auto/qfile/qfile.pro +++ b/tests/auto/qfile/qfile.pro @@ -5,5 +5,5 @@ wince*:{ SUBDIRS = test stdinprocess } - +SUBDIRS += largefile -- cgit v1.2.3