From 72cf2339edbb302b8b1dbe14c5475e8d2c3f62b1 Mon Sep 17 00:00:00 2001 From: Timur Pocheptsov Date: Wed, 12 Jul 2017 12:52:06 +0200 Subject: Introduce QHstsStore - the permanent store for HSTS policies The store is using QSettings under the hood. A user can enable/disable storing HSTS policies (via QNAM's setter method) and we take care of the rest - filling QHstsCache from the store, writing updated/observed targets, removing expired policies. Change-Id: I26e4a98761ddfe5005fedd18be56a6303fe7b35a Reviewed-by: Timur Pocheptsov Reviewed-by: Edward Welbourne --- src/network/access/access.pri | 6 +- src/network/access/qhsts.cpp | 61 +++++++- src/network/access/qhsts_p.h | 7 + src/network/access/qhstsstore.cpp | 202 +++++++++++++++++++++++++++ src/network/access/qhstsstore_p.h | 93 ++++++++++++ src/network/access/qnetworkaccessmanager.cpp | 44 +++++- src/network/access/qnetworkaccessmanager.h | 3 + src/network/access/qnetworkaccessmanager_p.h | 2 + tests/auto/network/access/hsts/tst_qhsts.cpp | 72 ++++++++++ 9 files changed, 483 insertions(+), 7 deletions(-) create mode 100644 src/network/access/qhstsstore.cpp create mode 100644 src/network/access/qhstsstore_p.h diff --git a/src/network/access/access.pri b/src/network/access/access.pri index 13d52ea44a..4281a870a7 100644 --- a/src/network/access/access.pri +++ b/src/network/access/access.pri @@ -41,7 +41,8 @@ HEADERS += \ access/qnetworkfile_p.h \ access/qhttp2protocolhandler_p.h \ access/qhsts_p.h \ - access/qhstspolicy.h + access/qhstspolicy.h \ + access/qhstsstore_p.h SOURCES += \ access/qftp.cpp \ @@ -76,7 +77,8 @@ SOURCES += \ access/qnetworkfile.cpp \ access/qhttp2protocolhandler.cpp \ access/qhsts.cpp \ - access/qhstspolicy.cpp + access/qhstspolicy.cpp \ + access/qhstsstore.cpp mac: LIBS_PRIVATE += -framework Security diff --git a/src/network/access/qhsts.cpp b/src/network/access/qhsts.cpp index ca9f3b977b..6a731afc2f 100644 --- a/src/network/access/qhsts.cpp +++ b/src/network/access/qhsts.cpp @@ -37,6 +37,7 @@ ** ****************************************************************************/ +#include "qhstsstore_p.h" #include "qhsts_p.h" #include "QtCore/private/qipaddress_p.h" @@ -80,14 +81,24 @@ void QHstsCache::updateFromHeaders(const QList> &h return; QHstsHeaderParser parser; - if (parser.parse(headers)) + if (parser.parse(headers)) { updateKnownHost(url.host(), parser.expirationDate(), parser.includeSubDomains()); + if (hstsStore) + hstsStore->synchronize(); + } } void QHstsCache::updateFromPolicies(const QVector &policies) { for (const auto &policy : policies) updateKnownHost(policy.host(), policy.expiry(), policy.includesSubDomains()); + + if (hstsStore && policies.size()) { + // These policies are coming either from store or from QNAM's setter + // function. As a result we can notice expired or new policies, time + // to sync ... + hstsStore->synchronize(); + } } void QHstsCache::updateKnownHost(const QUrl &url, const QDateTime &expires, @@ -97,6 +108,8 @@ void QHstsCache::updateKnownHost(const QUrl &url, const QDateTime &expires, return; updateKnownHost(url.host(), expires, includeSubDomains); + if (hstsStore) + hstsStore->synchronize(); } void QHstsCache::updateKnownHost(const QString &host, const QDateTime &expires, @@ -124,13 +137,20 @@ void QHstsCache::updateKnownHost(const QString &host, const QDateTime &expires, } knownHosts.insert(pos, hostName, newPolicy); + if (hstsStore) + hstsStore->addToObserved(newPolicy); return; } if (newPolicy.isExpired()) knownHosts.erase(pos); - else + else if (*pos != newPolicy) *pos = std::move(newPolicy); + else + return; + + if (hstsStore) + hstsStore->addToObserved(newPolicy); } bool QHstsCache::isKnownHost(const QUrl &url) const @@ -165,10 +185,15 @@ bool QHstsCache::isKnownHost(const QUrl &url) const while (nameToTest.fragment.size()) { auto const pos = knownHosts.find(nameToTest); if (pos != knownHosts.end()) { - if (pos.value().isExpired()) + if (pos.value().isExpired()) { knownHosts.erase(pos); - else if (!superDomainMatch || pos.value().includesSubDomains()) + if (hstsStore) { + // Inform our store that this policy has expired. + hstsStore->addToObserved(pos.value()); + } + } else if (!superDomainMatch || pos.value().includesSubDomains()) { return true; + } } const int dot = nameToTest.fragment.indexOf(QLatin1Char('.')); @@ -196,6 +221,34 @@ QVector QHstsCache::policies() const return values; } +void QHstsCache::setStore(QHstsStore *store) +{ + // Caller retains ownership of store, which must outlive this cache. + if (store != hstsStore) { + hstsStore = store; + + if (!hstsStore) + return; + + // First we augment our store with the policies we already know about + // (and thus the cached policy takes priority over whatever policy we + // had in the store for the same host, if any). + if (knownHosts.size()) { + const QVector observed(policies()); + for (const auto &policy : observed) + hstsStore->addToObserved(policy); + hstsStore->synchronize(); + } + + // Now we update the cache with anything we have not observed yet, but + // the store knows about (well, it can happen we synchronize again as a + // result if some policies managed to expire or if we add a new one + // from the store to cache): + const QVector restored(store->readPolicies()); + updateFromPolicies(restored); + } +} + // The parser is quite simple: 'nextToken' knowns exactly what kind of tokens // are valid and it will return false if something else was found; then // we immediately stop parsing. 'parseDirective' knows how these tokens can diff --git a/src/network/access/qhsts_p.h b/src/network/access/qhsts_p.h index ab3ca536fb..2feb73b446 100644 --- a/src/network/access/qhsts_p.h +++ b/src/network/access/qhsts_p.h @@ -51,6 +51,8 @@ // We mean it. // +#include + #include #include @@ -66,6 +68,8 @@ QT_BEGIN_NAMESPACE template class QList; template class QVector; +class QHstsStore; + class Q_AUTOTEST_EXPORT QHstsCache { public: @@ -80,6 +84,8 @@ public: QVector policies() const; + void setStore(QHstsStore *store); + private: void updateKnownHost(const QString &hostName, const QDateTime &expires, @@ -112,6 +118,7 @@ private: }; mutable QMap knownHosts; + QHstsStore *hstsStore = nullptr; }; class Q_AUTOTEST_EXPORT QHstsHeaderParser diff --git a/src/network/access/qhstsstore.cpp b/src/network/access/qhstsstore.cpp new file mode 100644 index 0000000000..239a52b7a4 --- /dev/null +++ b/src/network/access/qhstsstore.cpp @@ -0,0 +1,202 @@ +/**************************************************************************** +** +** Copyright (C) 2017 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the QtNetwork module of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include "qhstsstore_p.h" +#include "qhstspolicy.h" + +#include "qstandardpaths.h" +#include "qdatastream.h" +#include "qbytearray.h" +#include "qdatetime.h" +#include "qvariant.h" +#include "qstring.h" +#include "qdir.h" + +#include + +QT_BEGIN_NAMESPACE + +static QString host_name_to_settings_key(const QString &hostName) +{ + const QByteArray hostNameAsHex(hostName.toUtf8().toHex()); + return QString::fromLatin1(hostNameAsHex); +} + +static QString settings_key_to_host_name(const QString &key) +{ + const QByteArray hostNameAsUtf8(QByteArray::fromHex(key.toLatin1())); + return QString::fromUtf8(hostNameAsUtf8); +} + +QHstsStore::QHstsStore(const QString &dirName) + : store(absoluteFilePath(dirName), QSettings::IniFormat) +{ + // Disable fallbacks, we do not want to use anything but our own ini file. + store.setFallbacksEnabled(false); +} + +QHstsStore::~QHstsStore() +{ + synchronize(); +} + +QVector QHstsStore::readPolicies() +{ + // This function only attempts to read policies, making no decision about + // expired policies. It's up to a user (QHstsCache) to mark these policies + // for deletion and sync the store later. But we immediately remove keys/values + // (if the store isWritable) for the policies that we fail to read. + QVector policies; + + beginHstsGroups(); + + const QStringList keys = store.childKeys(); + for (const auto &key : keys) { + QHstsPolicy restoredPolicy; + if (deserializePolicy(key, restoredPolicy)) { + restoredPolicy.setHost(settings_key_to_host_name(key)); + policies.push_back(std::move(restoredPolicy)); + } else if (isWritable()) { + evictPolicy(key); + } + } + + endHstsGroups(); + + return policies; +} + +void QHstsStore::addToObserved(const QHstsPolicy &policy) +{ + observedPolicies.push_back(policy); +} + +void QHstsStore::synchronize() +{ + if (!isWritable()) + return; + + if (observedPolicies.size()) { + beginHstsGroups(); + for (const QHstsPolicy &policy : observedPolicies) { + const QString key(host_name_to_settings_key(policy.host())); + // If we fail to write a new, updated policy, we also remove the old one. + if (policy.isExpired() || !serializePolicy(key, policy)) + evictPolicy(key); + } + observedPolicies.clear(); + endHstsGroups(); + } + + store.sync(); +} + +bool QHstsStore::isWritable() const +{ + return store.isWritable(); +} + +QString QHstsStore::absoluteFilePath(const QString &dirName) +{ + const QDir dir(dirName.isEmpty() ? QStandardPaths::writableLocation(QStandardPaths::CacheLocation) + : dirName); + return dir.absoluteFilePath(QLatin1String("hstsstore")); +} + +void QHstsStore::beginHstsGroups() +{ + store.beginGroup(QLatin1String("StrictTransportSecurity")); + store.beginGroup(QLatin1String("Policies")); +} + +void QHstsStore::endHstsGroups() +{ + store.endGroup(); + store.endGroup(); +} + +bool QHstsStore::deserializePolicy(const QString &key, QHstsPolicy &policy) +{ + Q_ASSERT(store.contains(key)); + + const QVariant data(store.value(key)); + if (data.isNull() || !data.canConvert()) + return false; + + const QByteArray serializedData(data.toByteArray()); + QDataStream streamer(serializedData); + qint64 expiryInMS = 0; + streamer >> expiryInMS; + if (streamer.status() != QDataStream::Ok) + return false; + bool includesSubDomains = false; + streamer >> includesSubDomains; + if (streamer.status() != QDataStream::Ok) + return false; + + policy.setExpiry(QDateTime::fromMSecsSinceEpoch(expiryInMS)); + policy.setIncludesSubDomains(includesSubDomains); + + return true; +} + +bool QHstsStore::serializePolicy(const QString &key, const QHstsPolicy &policy) +{ + Q_ASSERT(store.isWritable()); + + QByteArray serializedData; + QDataStream streamer(&serializedData, QIODevice::WriteOnly); + streamer << policy.expiry().toMSecsSinceEpoch(); + streamer << policy.includesSubDomains(); + + if (streamer.status() != QDataStream::Ok) + return false; + + store.setValue(key, serializedData); + return true; +} + +void QHstsStore::evictPolicy(const QString &key) +{ + Q_ASSERT(store.isWritable()); + if (store.contains(key)) + store.remove(key); +} + +QT_END_NAMESPACE diff --git a/src/network/access/qhstsstore_p.h b/src/network/access/qhstsstore_p.h new file mode 100644 index 0000000000..13042839c4 --- /dev/null +++ b/src/network/access/qhstsstore_p.h @@ -0,0 +1,93 @@ +/**************************************************************************** +** +** Copyright (C) 2017 The Qt Company Ltd. +** Contact: https://www.qt.io/licensing/ +** +** This file is part of the QtNetwork module of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** Commercial License Usage +** Licensees holding valid commercial Qt licenses may use this file in +** accordance with the commercial license agreement provided with the +** Software or, alternatively, in accordance with the terms contained in +** a written agreement between you and The Qt Company. For licensing terms +** and conditions see https://www.qt.io/terms-conditions. For further +** information use the contact form at https://www.qt.io/contact-us. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 3 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL3 included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 3 requirements +** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 2.0 or (at your option) the GNU General +** Public license version 3 or any later version approved by the KDE Free +** Qt Foundation. The licenses are as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 +** included in the packaging of this file. Please review the following +** information to ensure the GNU General Public License requirements will +** be met: https://www.gnu.org/licenses/gpl-2.0.html and +** https://www.gnu.org/licenses/gpl-3.0.html. +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#ifndef QHSTSSTORE_P_H +#define QHSTSSTORE_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists for the convenience +// of the Network Access API. This header file may change from +// version to version without notice, or even be removed. +// +// We mean it. +// + +#include + +#include +#include + +QT_BEGIN_NAMESPACE + +class QHstsPolicy; +class QByteArray; +class QString; + +class Q_AUTOTEST_EXPORT QHstsStore +{ +public: + explicit QHstsStore(const QString &dirName); + ~QHstsStore(); + + QVector readPolicies(); + void addToObserved(const QHstsPolicy &policy); + void synchronize(); + + bool isWritable() const; + + static QString absoluteFilePath(const QString &dirName); +private: + void beginHstsGroups(); + bool serializePolicy(const QString &key, const QHstsPolicy &policy); + bool deserializePolicy(const QString &key, QHstsPolicy &policy); + void evictPolicy(const QString &key); + void endHstsGroups(); + + QVector observedPolicies; + QSettings store; + + Q_DISABLE_COPY(QHstsStore) +}; + +QT_END_NAMESPACE + +#endif // QHSTSSTORE_P_H diff --git a/src/network/access/qnetworkaccessmanager.cpp b/src/network/access/qnetworkaccessmanager.cpp index 79f0aa8038..eeee82a87c 100644 --- a/src/network/access/qnetworkaccessmanager.cpp +++ b/src/network/access/qnetworkaccessmanager.cpp @@ -729,6 +729,48 @@ bool QNetworkAccessManager::isStrictTransportSecurityEnabled() const return d->stsEnabled; } +/*! + \since 5.10 + + If \a enabled is \c true, the internal HSTS cache will use a persistent store + to read and write HSTS policies. \a storeDir defines where this store will be + located. The default location is defined by QStandardPaths::CacheLocation. + If there is no writable QStandartPaths::CacheLocation and \a storeDir is an + empty string, the store will be located in the program's working directory. + + \note If HSTS cache already contains HSTS policies by the time persistent + store is enabled, these policies will be preserved in the store. In case both + cache and store contain the same known hosts, policies from cache are considered + to be more up-to-date (and thus will overwrite the previous values in the store). + If this behavior is undesired, enable HSTS store before enabling Strict Tranport + Security. By default, the persistent store of HSTS policies is disabled. + + \sa isStrictTransportSecurityStoreEnabled(), setStrictTransportSecurityEnabled(), + QStandardPaths::standardLocations() +*/ + +void QNetworkAccessManager::enableStrictTransportSecurityStore(bool enabled, const QString &storeDir) +{ + Q_D(QNetworkAccessManager); + d->stsStore.reset(enabled ? new QHstsStore(storeDir) : nullptr); + d->stsCache.setStore(d->stsStore.data()); +} + +/*! + \since 5.10 + + Returns true if HSTS cache uses a permanent store to load and store HSTS + policies. + + \sa enableStrictTransportSecurityStore() +*/ + +bool QNetworkAccessManager::isStrictTransportSecurityStoreEnabled() const +{ + Q_D(const QNetworkAccessManager); + return bool(d->stsStore.data()); +} + /*! \since 5.9 @@ -744,7 +786,7 @@ bool QNetworkAccessManager::isStrictTransportSecurityEnabled() const policies, but this information can be overridden by "Strict-Transport-Security" response headers. - \sa addStrictTransportSecurityHosts(), QHstsPolicy + \sa addStrictTransportSecurityHosts(), enableStrictTransportSecurityStore(), QHstsPolicy */ void QNetworkAccessManager::addStrictTransportSecurityHosts(const QVector &knownHosts) diff --git a/src/network/access/qnetworkaccessmanager.h b/src/network/access/qnetworkaccessmanager.h index f035ac5b00..4806ec0475 100644 --- a/src/network/access/qnetworkaccessmanager.h +++ b/src/network/access/qnetworkaccessmanager.h @@ -42,6 +42,7 @@ #include #include +#include #include #include #ifndef QT_NO_SSL @@ -124,6 +125,8 @@ public: void setStrictTransportSecurityEnabled(bool enabled); bool isStrictTransportSecurityEnabled() const; + void enableStrictTransportSecurityStore(bool enabled, const QString &storeDir = QString()); + bool isStrictTransportSecurityStoreEnabled() const; void addStrictTransportSecurityHosts(const QVector &knownHosts); QVector strictTransportSecurityHosts() const; diff --git a/src/network/access/qnetworkaccessmanager_p.h b/src/network/access/qnetworkaccessmanager_p.h index 13a26a54f1..e5257251a4 100644 --- a/src/network/access/qnetworkaccessmanager_p.h +++ b/src/network/access/qnetworkaccessmanager_p.h @@ -56,6 +56,7 @@ #include "qnetworkaccesscache_p.h" #include "qnetworkaccessbackend_p.h" #include "qnetworkrequest.h" +#include "qhstsstore_p.h" #include "qhsts_p.h" #include "private/qobject_p.h" #include "QtNetwork/qnetworkproxy.h" @@ -211,6 +212,7 @@ public: Q_AUTOTEST_EXPORT static void clearConnectionCache(QNetworkAccessManager *manager); QHstsCache stsCache; + QScopedPointer stsStore; bool stsEnabled = false; #ifndef QT_NO_BEARERMANAGEMENT diff --git a/tests/auto/network/access/hsts/tst_qhsts.cpp b/tests/auto/network/access/hsts/tst_qhsts.cpp index 656516f46b..d72991a2eb 100644 --- a/tests/auto/network/access/hsts/tst_qhsts.cpp +++ b/tests/auto/network/access/hsts/tst_qhsts.cpp @@ -32,7 +32,9 @@ #include #include #include +#include +#include #include QT_USE_NAMESPACE @@ -46,6 +48,7 @@ private Q_SLOTS: void testMultilpeKnownHosts(); void testPolicyExpiration(); void testSTSHeaderParser(); + void testStore(); }; void tst_QHsts::testSingleKnownHost_data() @@ -313,6 +316,75 @@ void tst_QHsts::testSTSHeaderParser() QVERIFY(!parser.expirationDate().isValid()); } +const QLatin1String storeDir("."); + +struct TestStoreDeleter +{ + ~TestStoreDeleter() + { + QDir cwd; + if (!cwd.remove(QHstsStore::absoluteFilePath(storeDir))) + qWarning() << "tst_QHsts::testStore: failed to remove the hsts store file"; + } +}; + +void tst_QHsts::testStore() +{ + // Delete the store's file after we finish the test. + TestStoreDeleter cleaner; + + const QUrl exampleCom(QStringLiteral("http://example.com")); + const QUrl subDomain(QStringLiteral("http://subdomain.example.com")); + const QDateTime validDate(QDateTime::currentDateTimeUtc().addDays(1)); + + { + // We start from an empty cache and empty store: + QHstsCache cache; + QHstsStore store(storeDir); + cache.setStore(&store); + QVERIFY(!cache.isKnownHost(exampleCom)); + QVERIFY(!cache.isKnownHost(subDomain)); + // (1) This will also store the policy: + cache.updateKnownHost(exampleCom, validDate, true); + QVERIFY(cache.isKnownHost(exampleCom)); + QVERIFY(cache.isKnownHost(subDomain)); + } + { + // Test the policy stored at (1): + QHstsCache cache; + QHstsStore store(storeDir); + cache.setStore(&store); + QVERIFY(cache.isKnownHost(exampleCom)); + QVERIFY(cache.isKnownHost(subDomain)); + // (2) Remove subdomains: + cache.updateKnownHost(exampleCom, validDate, false); + QVERIFY(!cache.isKnownHost(subDomain)); + } + { + // Test the previous update (2): + QHstsCache cache; + QHstsStore store(storeDir); + cache.setStore(&store); + QVERIFY(cache.isKnownHost(exampleCom)); + QVERIFY(!cache.isKnownHost(subDomain)); + } + { + QHstsCache cache; + cache.updateKnownHost(subDomain, validDate, false); + QVERIFY(cache.isKnownHost(subDomain)); + QHstsStore store(storeDir); + // (3) This should store policy from cache, over old policy from store: + cache.setStore(&store); + } + { + // Test that (3) was stored: + QHstsCache cache; + QHstsStore store(storeDir); + cache.setStore(&store); + QVERIFY(cache.isKnownHost(subDomain)); + } +} + QTEST_MAIN(tst_QHsts) #include "tst_qhsts.moc" -- cgit v1.2.3