From 189e9c93d7ed42202ad51507c8944d64e9a7888d Mon Sep 17 00:00:00 2001 From: Thiago Macieira Date: Wed, 28 Jun 2017 23:36:19 -0700 Subject: QTemporaryFile: Add support for Linux's O_TMPFILE That means a file is never created, unless you ask for the name. There's no chance of left-over temporary files being left behind. QSaveFile also benefits from this, since the save file is not present on disk until commit(). Unfortunately, QSaveFile must go through a temporary name because linkat(2) cannot overwrite -- we need rename(2) for that (for now). [ChangeLog][Important Behavior Changes][QTemporaryFile] On Linux, QTemporaryFile will attempt to create unnamed temporary files. If that succeeds, open() will return true but exists() will be false. If you call fileName() or any function that calls it, QTemporaryFile will give the file a name, so most applications will not see a difference. Change-Id: I1eba2b016de74620bfc8fffd14cc843e5b0919d0 Reviewed-by: Simon Hausmann --- src/corelib/global/minimum-linux.S | 1 + src/corelib/io/qfile.cpp | 8 +- src/corelib/io/qtemporaryfile.cpp | 187 +++++++++++++++++++-- src/corelib/io/qtemporaryfile_p.h | 18 ++ .../io/qtemporaryfile/tst_qtemporaryfile.cpp | 81 ++++++++- .../qnetworkdiskcache/tst_qnetworkdiskcache.cpp | 1 + 6 files changed, 276 insertions(+), 20 deletions(-) diff --git a/src/corelib/global/minimum-linux.S b/src/corelib/global/minimum-linux.S index aa67be44ca..6b7fb4f63b 100644 --- a/src/corelib/global/minimum-linux.S +++ b/src/corelib/global/minimum-linux.S @@ -81,6 +81,7 @@ * Feature Added in version Macro * - inotify_init1 before 2.6.12-rc12 * - futex(2) before 2.6.12-rc12 + * - linkat(2) 2.6.17 O_TMPFILE * - FUTEX_PRIVATE_FLAG 2.6.22 * - O_CLOEXEC 2.6.23 * - eventfd 2.6.23 diff --git a/src/corelib/io/qfile.cpp b/src/corelib/io/qfile.cpp index d32c2ed9fd..bac995ff25 100644 --- a/src/corelib/io/qfile.cpp +++ b/src/corelib/io/qfile.cpp @@ -556,7 +556,9 @@ bool QFile::rename(const QString &newName) { Q_D(QFile); - if (d->fileName.isEmpty()) { + + // if this is a QTemporaryFile, the virtual fileName() call here may do something + if (fileName().isEmpty()) { qWarning("QFile::rename: Empty or null file name"); return false; } @@ -717,7 +719,7 @@ bool QFile::link(const QString &linkName) { Q_D(QFile); - if (d->fileName.isEmpty()) { + if (fileName().isEmpty()) { qWarning("QFile::link: Empty or null file name"); return false; } @@ -763,7 +765,7 @@ bool QFile::copy(const QString &newName) { Q_D(QFile); - if (d->fileName.isEmpty()) { + if (fileName().isEmpty()) { qWarning("QFile::copy: Empty or null file name"); return false; } diff --git a/src/corelib/io/qtemporaryfile.cpp b/src/corelib/io/qtemporaryfile.cpp index 4e8e40123c..3ecc24a5db 100644 --- a/src/corelib/io/qtemporaryfile.cpp +++ b/src/corelib/io/qtemporaryfile.cpp @@ -1,6 +1,7 @@ /**************************************************************************** ** ** Copyright (C) 2016 The Qt Company Ltd. +** Copyright (C) 2017 Intel Corporation. ** Contact: https://www.qt.io/licensing/ ** ** This file is part of the QtCore module of the Qt Toolkit. @@ -264,6 +265,53 @@ static bool createFileFromTemplate(NativeFileHandle &file, QTemporaryFileName &t return false; } +enum class CreateUnnamedFileStatus { + Success = 0, + NotSupported, + OtherError +}; + +static CreateUnnamedFileStatus +createUnnamedFile(NativeFileHandle &file, QTemporaryFileName &tfn, quint32 mode, QSystemError *error) +{ +#ifdef LINUX_UNNAMED_TMPFILE + // first, check if we have /proc, otherwise can't make the file exist later + // (no error message set, as caller will try regular temporary file) + if (!qt_haveLinuxProcfs()) + return CreateUnnamedFileStatus::NotSupported; + + const char *p = "."; + int lastSlash = tfn.path.lastIndexOf('/'); + if (lastSlash != -1) { + tfn.path[lastSlash] = '\0'; + p = tfn.path.data(); + } + + file = QT_OPEN(p, O_TMPFILE | QT_OPEN_RDWR | QT_OPEN_LARGEFILE, + static_cast(mode)); + if (file != -1) + return CreateUnnamedFileStatus::Success; + + if (errno == EOPNOTSUPP || errno == EISDIR) { + // fs or kernel doesn't support O_TMPFILE, so + // put the slash back so we may try a regular file + if (lastSlash != -1) + tfn.path[lastSlash] = '/'; + return CreateUnnamedFileStatus::NotSupported; + } + + // real error + *error = QSystemError(errno, QSystemError::NativeError); + return CreateUnnamedFileStatus::OtherError; +#else + Q_UNUSED(file); + Q_UNUSED(tfn); + Q_UNUSED(mode); + Q_UNUSED(error); + return CreateUnnamedFileStatus::NotSupported; +#endif +} + //************* QTemporaryFileEngine QTemporaryFileEngine::~QTemporaryFileEngine() { @@ -313,19 +361,24 @@ bool QTemporaryFileEngine::open(QIODevice::OpenMode openMode) NativeFileHandle &file = d->fd; #endif - if (!createFileFromTemplate(file, tfn, fileMode, error)) { + CreateUnnamedFileStatus st = createUnnamedFile(file, tfn, fileMode, &error); + if (st == CreateUnnamedFileStatus::Success) { + unnamedFile = true; + d->fileEntry.clear(); + } else if (st == CreateUnnamedFileStatus::NotSupported && + createFileFromTemplate(file, tfn, fileMode, error)) { + filePathIsTemplate = false; + unnamedFile = false; + d->fileEntry = QFileSystemEntry(tfn.path, QFileSystemEntry::FromNativePath()); + } else { setError(QFile::OpenError, error.toString()); return false; } - d->fileEntry = QFileSystemEntry(tfn.path, QFileSystemEntry::FromNativePath()); - #if !defined(Q_OS_WIN) || defined(Q_OS_WINRT) d->closeFileHandle = true; #endif - filePathIsTemplate = false; - d->openMode = openMode; d->lastFlushFailed = false; d->tried_stat = 0; @@ -340,7 +393,7 @@ bool QTemporaryFileEngine::remove() // we must explicitly call QFSFileEngine::close() before we remove it. d->unmapAll(); QFSFileEngine::close(); - if (QFSFileEngine::remove()) { + if (isUnnamedFile() || QFSFileEngine::remove()) { d->fileEntry.clear(); // If a QTemporaryFile is constructed using a template file path, the path // is generated in QTemporaryFileEngine::open() and then filePathIsTemplate @@ -355,12 +408,22 @@ bool QTemporaryFileEngine::remove() bool QTemporaryFileEngine::rename(const QString &newName) { + if (isUnnamedFile()) { + bool ok = materializeUnnamedFile(newName, DontOverwrite); + QFSFileEngine::close(); + return ok; + } QFSFileEngine::close(); return QFSFileEngine::rename(newName); } bool QTemporaryFileEngine::renameOverwrite(const QString &newName) { + if (isUnnamedFile()) { + bool ok = materializeUnnamedFile(newName, Overwrite); + QFSFileEngine::close(); + return ok; + } QFSFileEngine::close(); return QFSFileEngine::renameOverwrite(newName); } @@ -373,6 +436,88 @@ bool QTemporaryFileEngine::close() return true; } +QString QTemporaryFileEngine::fileName(QAbstractFileEngine::FileName file) const +{ + if (isUnnamedFile()) { + if (file == LinkName) { + // we know our file isn't (won't be) a symlink + return QString(); + } + + // for all other cases, materialize the file + const_cast(this)->materializeUnnamedFile(templateName, NameIsTemplate); + } + return QFSFileEngine::fileName(file); +} + +bool QTemporaryFileEngine::materializeUnnamedFile(const QString &newName, QTemporaryFileEngine::MaterializationMode mode) +{ + Q_ASSERT(isUnnamedFile()); + +#ifdef LINUX_UNNAMED_TMPFILE + Q_D(QFSFileEngine); + const QByteArray src = "/proc/self/fd/" + QByteArray::number(d->fd); + auto materializeAt = [=](const QFileSystemEntry &dst) { + return ::linkat(AT_FDCWD, src, AT_FDCWD, dst.nativeFilePath(), AT_SYMLINK_FOLLOW) == 0; + }; +#else + auto materializeAt = [](const QFileSystemEntry &) { return false; }; +#endif + + auto success = [this](const QFileSystemEntry &entry) { + filePathIsTemplate = false; + unnamedFile = false; + d_func()->fileEntry = entry; + return true; + }; + + auto materializeAsTemplate = [=](const QString &newName) { + QTemporaryFileName tfn(newName); + static const int maxAttempts = 16; + for (int attempt = 0; attempt < maxAttempts; ++attempt) { + tfn.generateNext(); + QFileSystemEntry entry(tfn.path, QFileSystemEntry::FromNativePath()); + if (materializeAt(entry)) + return success(entry); + } + return false; + }; + + if (mode == NameIsTemplate) { + if (materializeAsTemplate(newName)) + return true; + } else { + // Use linkat to materialize the file + QFileSystemEntry dst(newName); + if (materializeAt(dst)) + return success(dst); + + if (errno == EEXIST && mode == Overwrite) { + // retry by first creating a temporary file in the right dir + if (!materializeAsTemplate(templateName)) + return false; + + // then rename the materialized file to target (same as renameOverwrite) + QFSFileEngine::close(); + return QFSFileEngine::renameOverwrite(newName); + } + } + + // failed + setError(QFile::RenameError, QSystemError(errno, QSystemError::NativeError).toString()); + return false; +} + +bool QTemporaryFileEngine::isUnnamedFile() const +{ +#ifdef LINUX_UNNAMED_TMPFILE + Q_ASSERT(unnamedFile == d_func()->fileEntry.isEmpty()); + return unnamedFile; +#else + return false; +#endif +} + //************* QTemporaryFilePrivate QTemporaryFilePrivate::QTemporaryFilePrivate() @@ -409,6 +554,17 @@ void QTemporaryFilePrivate::resetFileEngine() const tef->initialize(fileName, 0600, false); } +void QTemporaryFilePrivate::materializeUnnamedFile() +{ +#ifdef LINUX_UNNAMED_TMPFILE + if (!fileName.isEmpty() || !fileEngine) + return; + + auto *tef = static_cast(fileEngine); + fileName = tef->fileName(QAbstractFileEngine::DefaultName); +#endif +} + QString QTemporaryFilePrivate::defaultTemplateName() { QString baseName; @@ -617,6 +773,10 @@ void QTemporaryFile::setAutoRemove(bool b) QString QTemporaryFile::fileName() const { Q_D(const QTemporaryFile); + auto tef = static_cast(d->fileEngine); + if (tef && tef->isReallyOpen()) + const_cast(d)->materializeUnnamedFile(); + if(d->fileName.isEmpty()) return QString(); return d->engine()->fileName(QAbstractFileEngine::DefaultName); @@ -770,11 +930,10 @@ QTemporaryFile *QTemporaryFile::createNativeFile(QFile &file) bool QTemporaryFile::open(OpenMode flags) { Q_D(QTemporaryFile); - if (!d->fileName.isEmpty()) { - if (static_cast(d->engine())->isReallyOpen()) { - setOpenMode(flags); - return true; - } + auto tef = static_cast(d->fileEngine); + if (tef && tef->isReallyOpen()) { + setOpenMode(flags); + return true; } // reset the engine state so it creates a new, unique file name from the template; @@ -785,7 +944,11 @@ bool QTemporaryFile::open(OpenMode flags) d->resetFileEngine(); if (QFile::open(flags)) { - d->fileName = d->fileEngine->fileName(QAbstractFileEngine::DefaultName); + tef = static_cast(d->fileEngine); + if (tef->isUnnamedFile()) + d->fileName.clear(); + else + d->fileName = tef->fileName(QAbstractFileEngine::DefaultName); return true; } return false; diff --git a/src/corelib/io/qtemporaryfile_p.h b/src/corelib/io/qtemporaryfile_p.h index 97ffc48507..46a0d7aba3 100644 --- a/src/corelib/io/qtemporaryfile_p.h +++ b/src/corelib/io/qtemporaryfile_p.h @@ -58,6 +58,17 @@ #include "private/qfile_p.h" #include "qtemporaryfile.h" +#ifdef Q_OS_LINUX +# include +# ifdef O_TMPFILE +// some early libc support had the wrong values for O_TMPFILE +// (see https://bugzilla.gnome.org/show_bug.cgi?id=769453#c18) +# if (O_TMPFILE & O_DIRECTORY) == O_DIRECTORY +# define LINUX_UNNAMED_TMPFILE +# endif +# endif +#endif + QT_BEGIN_NAMESPACE struct QTemporaryFileName @@ -83,6 +94,7 @@ public: QAbstractFileEngine *engine() const override; void resetFileEngine() const; + void materializeUnnamedFile(); bool autoRemove = true; QString templateName = defaultTemplateName(); @@ -124,11 +136,17 @@ public: bool rename(const QString &newName) override; bool renameOverwrite(const QString &newName) override; bool close() override; + QString fileName(FileName file) const override; + + enum MaterializationMode { Overwrite, DontOverwrite, NameIsTemplate }; + bool materializeUnnamedFile(const QString &newName, MaterializationMode mode); + bool isUnnamedFile() const; const QString &templateName; quint32 fileMode; bool filePathIsTemplate; bool filePathWasTemplate; + bool unnamedFile = false; }; #endif // QT_NO_TEMPORARYFILE diff --git a/tests/auto/corelib/io/qtemporaryfile/tst_qtemporaryfile.cpp b/tests/auto/corelib/io/qtemporaryfile/tst_qtemporaryfile.cpp index 4481def214..11c24ca86f 100644 --- a/tests/auto/corelib/io/qtemporaryfile/tst_qtemporaryfile.cpp +++ b/tests/auto/corelib/io/qtemporaryfile/tst_qtemporaryfile.cpp @@ -1,6 +1,7 @@ /**************************************************************************** ** ** Copyright (C) 2016 The Qt Company Ltd. +** Copyright (C) 2017 Intel Corporation. ** Contact: https://www.qt.io/licensing/ ** ** This file is part of the test suite of the Qt Toolkit. @@ -32,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -63,7 +65,7 @@ private slots: void fileNameIsEmpty(); void autoRemove(); void nonWritableCurrentDir(); - void write(); + void io(); void openCloseOpenClose(); void removeAndReOpen(); void size(); @@ -286,6 +288,18 @@ void tst_QTemporaryFile::autoRemove() fileName = file.fileName(); file.close(); } + QVERIFY(!fileName.isEmpty()); + QVERIFY(!QFile::exists(fileName)); + + // same, but gets the file name after closing + { + QTemporaryFile file("tempXXXXXX"); + file.setAutoRemove(true); + QVERIFY(file.open()); + file.close(); + fileName = file.fileName(); + } + QVERIFY(!fileName.isEmpty()); QVERIFY(!QFile::exists(fileName)); // Test if disabling auto remove works. @@ -296,6 +310,19 @@ void tst_QTemporaryFile::autoRemove() fileName = file.fileName(); file.close(); } + QVERIFY(!fileName.isEmpty()); + QVERIFY(QFile::exists(fileName)); + QVERIFY(QFile::remove(fileName)); + + // same, but gets the file name after closing + { + QTemporaryFile file("tempXXXXXX"); + file.setAutoRemove(false); + QVERIFY(file.open()); + file.close(); + fileName = file.fileName(); + } + QVERIFY(!fileName.isEmpty()); QVERIFY(QFile::exists(fileName)); QVERIFY(QFile::remove(fileName)); @@ -346,17 +373,51 @@ void tst_QTemporaryFile::nonWritableCurrentDir() #endif } -void tst_QTemporaryFile::write() +void tst_QTemporaryFile::io() { QByteArray data("OLE\nOLE\nOLE"); QTemporaryFile file; + QDateTime before = QDateTime::currentDateTimeUtc().addMSecs(-250); + + // discard msec component (round down) - not all FSs and OSs support them + before.setSecsSinceEpoch(before.toSecsSinceEpoch()); + QVERIFY(file.open()); + QVERIFY(file.readLink().isEmpty()); // it's not a link! + QFile::Permissions perm = file.permissions(); + QVERIFY(perm & QFile::ReadOwner); + QVERIFY(file.setPermissions(perm)); + + QCOMPARE(int(file.size()), 0); + QVERIFY(file.resize(data.size())); + QCOMPARE(int(file.size()), data.size()); QCOMPARE((int)file.write(data), data.size()); + QCOMPARE(int(file.size()), data.size()); + + QDateTime mtime = file.fileTime(QFile::FileModificationTime).toUTC(); + QDateTime btime = file.fileTime(QFile::FileBirthTime).toUTC(); + QDateTime ctime = file.fileTime(QFile::FileMetadataChangeTime).toUTC(); + QDateTime atime = file.fileTime(QFile::FileAccessTime).toUTC(); + + QDateTime after = QDateTime::currentDateTimeUtc().toUTC().addMSecs(250); + // round msecs up + after.setSecsSinceEpoch(after.toSecsSinceEpoch() + 1); + + // mtime must be valid, the rest could fail + QVERIFY(mtime <= after && mtime >= before); + QVERIFY(!btime.isValid() || (btime <= after && btime >= before)); + QVERIFY(!ctime.isValid() || (ctime <= after && ctime >= before)); + QVERIFY(!btime.isValid() || (btime <= after && btime >= before)); + + QVERIFY(file.setFileTime(before.addSecs(-10), QFile::FileModificationTime)); + mtime = file.fileTime(QFile::FileModificationTime).toUTC(); + QCOMPARE(mtime, before.addSecs(-10)); + file.reset(); QFile compare(file.fileName()); compare.open(QIODevice::ReadOnly); QCOMPARE(compare.readAll() , data); - file.close(); + QCOMPARE(compare.fileTime(QFile::FileModificationTime), mtime); } void tst_QTemporaryFile::openCloseOpenClose() @@ -407,17 +468,19 @@ void tst_QTemporaryFile::size() { QTemporaryFile file; QVERIFY(file.open()); - QVERIFY(file.exists()); QVERIFY(!file.isSequential()); QByteArray str("foobar"); file.write(str); - QVERIFY(QFile::exists(file.fileName())); + // On CE it takes more time for the filesystem to update // the information. Usually you have to close it or seek // to get latest information. flush() does not help either. QCOMPARE(file.size(), qint64(6)); file.seek(0); QCOMPARE(file.size(), qint64(6)); + + QVERIFY(QFile::exists(file.fileName())); + QVERIFY(file.exists()); } void tst_QTemporaryFile::resize() @@ -814,6 +877,14 @@ void tst_QTemporaryFile::QTBUG_4796() QCOMPARE(file5.open(), openResult); QCOMPARE(file6.open(), openResult); + // force the files to exist, if they are supposed to + QCOMPARE(!file1.fileName().isEmpty(), openResult); + QCOMPARE(!file2.fileName().isEmpty(), openResult); + QCOMPARE(!file3.fileName().isEmpty(), openResult); + QCOMPARE(!file4.fileName().isEmpty(), openResult); + QCOMPARE(!file5.fileName().isEmpty(), openResult); + QCOMPARE(!file6.fileName().isEmpty(), openResult); + QCOMPARE(file1.exists(), openResult); QCOMPARE(file2.exists(), openResult); QCOMPARE(file3.exists(), openResult); diff --git a/tests/auto/network/access/qnetworkdiskcache/tst_qnetworkdiskcache.cpp b/tests/auto/network/access/qnetworkdiskcache/tst_qnetworkdiskcache.cpp index af6f2d0e10..e996347a9a 100644 --- a/tests/auto/network/access/qnetworkdiskcache/tst_qnetworkdiskcache.cpp +++ b/tests/auto/network/access/qnetworkdiskcache/tst_qnetworkdiskcache.cpp @@ -281,6 +281,7 @@ void tst_QNetworkDiskCache::clear() // don't delete files that it didn't create QTemporaryFile file(cacheDirectory + "/XXXXXX"); if (file.open()) { + file.fileName(); // make sure it exists with a name QCOMPARE(countFiles(cacheDirectory).count(), NUM_SUBDIRECTORIES + 3); cache.clear(); QCOMPARE(countFiles(cacheDirectory).count(), NUM_SUBDIRECTORIES + 3); -- cgit v1.2.3