make VFS aware of exact vs. cumulative evaluation

the cumulative evaluation has a good chance to make a mess of the
virtual file contents created by the exact parsing, so better contain it
to its own namespace.

the ProFile cache also needs to keep the files separate. this
specifically addresses the side issue discussed in QTCREATORBUG-10779.
it also fixes attempts to deploy the wrong build when the variant is
selected through a cache file, as in QTCREATORBUG-15815.

in the project explorer, we don't track from which evaluation pass
particular files came from, so we try the cumulative first to get the
most contents, and fall back to the exact one if the former file is
empty (or does not exist at all).

Task-number: QTCREATORBUG-15815
Change-Id: I2c1eb16c97526fa275a1c6a2eae9266d385859ac
Reviewed-by: Tobias Hunger <tobias.hunger@qt.io>

8 files changed, 134 insertions, 47 deletions

diff --git a/src/shared/proparser/profileevaluator.cpp b/src/shared/proparser/profileevaluator.cpp
index 1df3dc4b51..995658cdd2 100644
--- a/src/shared/proparser/profileevaluator.cpp
+++ b/src/shared/proparser/profileevaluator.cpp
@@ -123,11 +123,12 @@ QStringList ProFileEvaluator::absoluteFileValues(
         const QString &variable, const QString &baseDirectory, const QStringList &searchDirs,
         const ProFile *pro) const
 {
+    QMakeVfs::VfsFlags flags = (d->m_cumulative ? QMakeVfs::VfsCumulative : QMakeVfs::VfsExact);
     QStringList result;
     foreach (const QString &el, pro ? values(variable, pro) : values(variable)) {
         QString absEl;
         if (IoUtils::isAbsolutePath(el)) {
-            if (m_vfs->exists(el)) {
+            if (m_vfs->exists(el, flags)) {
                 result << el;
                 goto next;
             }
@@ -135,7 +136,7 @@ QStringList ProFileEvaluator::absoluteFileValues(
         } else {
             foreach (const QString &dir, searchDirs) {
                 QString fn = QDir::cleanPath(dir + QLatin1Char('/') + el);
-                if (m_vfs->exists(fn)) {
+                if (m_vfs->exists(fn, flags)) {
                     result << fn;
                     goto next;
                 }
diff --git a/src/shared/proparser/qmakebuiltins.cpp b/src/shared/proparser/qmakebuiltins.cpp
index 3a8e3b110e..852c40d55d 100644
--- a/src/shared/proparser/qmakebuiltins.cpp
+++ b/src/shared/proparser/qmakebuiltins.cpp
@@ -414,10 +414,10 @@ QMakeEvaluator::VisitReturn QMakeEvaluator::parseJsonInto(const QByteArray &json
 
 QMakeEvaluator::VisitReturn
 QMakeEvaluator::writeFile(const QString &ctx, const QString &fn, QIODevice::OpenMode mode,
-                          bool exe, const QString &contents)
+                          QMakeVfs::VfsFlags flags, const QString &contents)
 {
     QString errStr;
-    if (!m_vfs->writeFile(fn, mode, exe, contents, &errStr)) {
+    if (!m_vfs->writeFile(fn, mode, flags, contents, &errStr)) {
         evalError(fL1S("Cannot write %1file %2: %3")
                   .arg(ctx, QDir::toNativeSeparators(fn), errStr));
         return ReturnFalse;
@@ -1714,7 +1714,7 @@ QMakeEvaluator::VisitReturn QMakeEvaluator::evaluateBuiltinConditional(
             return ReturnFalse;
         }
         QIODevice::OpenMode mode = QIODevice::Truncate;
-        bool exe = false;
+        QMakeVfs::VfsFlags flags = (m_cumulative ? QMakeVfs::VfsCumulative : QMakeVfs::VfsExact);
         QString contents;
         if (args.count() >= 2) {
             const ProStringList &vals = values(args.at(1).toKey());
@@ -1727,7 +1727,7 @@ QMakeEvaluator::VisitReturn QMakeEvaluator::evaluateBuiltinConditional(
                     if (m_tmp3 == QLatin1String("append")) {
                         mode = QIODevice::Append;
                     } else if (m_tmp3 == QLatin1String("exe")) {
-                        exe = true;
+                        flags |= QMakeVfs::VfsExecutable;
                     } else {
                         evalError(fL1S("write_file(): invalid flag %1.").arg(m_tmp3));
                         return ReturnFalse;
@@ -1737,7 +1737,7 @@ QMakeEvaluator::VisitReturn QMakeEvaluator::evaluateBuiltinConditional(
         }
         QString path = resolvePath(args.at(0).toQString(m_tmp1));
         path.detach(); // make sure to not leak m_tmp1 into the map of written files.
-        return writeFile(QString(), path, mode, exe, contents);
+        return writeFile(QString(), path, mode, flags, contents);
     }
     case T_TOUCH: {
         if (args.count() != 2) {
@@ -1921,6 +1921,7 @@ QMakeEvaluator::VisitReturn QMakeEvaluator::evaluateBuiltinConditional(
             varstr += QLatin1Char('\n');
         }
         QString fn;
+        QMakeVfs::VfsFlags flags = (m_cumulative ? QMakeVfs::VfsCumulative : QMakeVfs::VfsExact);
         if (target == TargetSuper) {
             if (m_superfile.isEmpty()) {
                 m_superfile = QDir::cleanPath(m_outputDir + QLatin1String("/.qmake.super"));
@@ -1944,12 +1945,12 @@ QMakeEvaluator::VisitReturn QMakeEvaluator::evaluateBuiltinConditional(
             fn = m_stashfile;
             if (fn.isEmpty())
                 fn = QDir::cleanPath(m_outputDir + QLatin1String("/.qmake.stash"));
-            if (!m_vfs->exists(fn)) {
+            if (!m_vfs->exists(fn, flags)) {
                 printf("Info: creating stash file %s\n", qPrintable(QDir::toNativeSeparators(fn)));
                 valuesRef(ProKey("_QMAKE_STASH_")) << ProString(fn);
             }
         }
-        return writeFile(fL1S("cache "), fn, QIODevice::Append, false, varstr);
+        return writeFile(fL1S("cache "), fn, QIODevice::Append, flags, varstr);
     }
     default:
         evalError(fL1S("Function '%1' is not implemented.").arg(function.toQString(m_tmp1)));
diff --git a/src/shared/proparser/qmakeevaluator.cpp b/src/shared/proparser/qmakeevaluator.cpp
index 3f78878e63..19e9f8b232 100644
--- a/src/shared/proparser/qmakeevaluator.cpp
+++ b/src/shared/proparser/qmakeevaluator.cpp
@@ -1082,6 +1082,7 @@ void QMakeEvaluator::loadDefaults()
 
 bool QMakeEvaluator::prepareProject(const QString &inDir)
 {
+    QMakeVfs::VfsFlags flags = (m_cumulative ? QMakeVfs::VfsCumulative : QMakeVfs::VfsExact);
     QString superdir;
     if (m_option->do_cache) {
         QString conffile;
@@ -1092,7 +1093,7 @@ bool QMakeEvaluator::prepareProject(const QString &inDir)
             superdir = m_outputDir;
             forever {
                 QString superfile = superdir + QLatin1String("/.qmake.super");
-                if (m_vfs->exists(superfile)) {
+                if (m_vfs->exists(superfile, flags)) {
                     m_superfile = QDir::cleanPath(superfile);
                     break;
                 }
@@ -1107,10 +1108,10 @@ bool QMakeEvaluator::prepareProject(const QString &inDir)
             QString dir = m_outputDir;
             forever {
                 conffile = sdir + QLatin1String("/.qmake.conf");
-                if (!m_vfs->exists(conffile))
+                if (!m_vfs->exists(conffile, flags))
                     conffile.clear();
                 cachefile = dir + QLatin1String("/.qmake.cache");
-                if (!m_vfs->exists(cachefile))
+                if (!m_vfs->exists(cachefile, flags))
                     cachefile.clear();
                 if (!conffile.isEmpty() || !cachefile.isEmpty()) {
                     if (dir != sdir)
@@ -1138,7 +1139,7 @@ bool QMakeEvaluator::prepareProject(const QString &inDir)
     QString dir = m_outputDir;
     forever {
         QString stashfile = dir + QLatin1String("/.qmake.stash");
-        if (dir == (!superdir.isEmpty() ? superdir : m_buildRoot) || m_vfs->exists(stashfile)) {
+        if (dir == (!superdir.isEmpty() ? superdir : m_buildRoot) || m_vfs->exists(stashfile, flags)) {
             m_stashfile = QDir::cleanPath(stashfile);
             break;
         }
@@ -1265,7 +1266,8 @@ bool QMakeEvaluator::loadSpec()
                 m_cachefile, QMakeHandler::EvalConfigFile, LoadProOnly) != ReturnTrue)
             return false;
     }
-    if (!m_stashfile.isEmpty() && m_vfs->exists(m_stashfile)) {
+    QMakeVfs::VfsFlags flags = (m_cumulative ? QMakeVfs::VfsCumulative : QMakeVfs::VfsExact);
+    if (!m_stashfile.isEmpty() && m_vfs->exists(m_stashfile, flags)) {
         valuesRef(ProKey("_QMAKE_STASH_")) << ProString(m_stashfile);
         if (evaluateFile(
                 m_stashfile, QMakeHandler::EvalConfigFile, LoadProOnly) != ReturnTrue)
diff --git a/src/shared/proparser/qmakeevaluator.h b/src/shared/proparser/qmakeevaluator.h
index 8f7a2072b5..d1e734346b 100644
--- a/src/shared/proparser/qmakeevaluator.h
+++ b/src/shared/proparser/qmakeevaluator.h
@@ -30,6 +30,7 @@
 #endif
 
 #include "qmakeparser.h"
+#include "qmakevfs.h"
 #include "ioutils.h"
 
 #include <qlist.h>
@@ -232,7 +233,7 @@ public:
     VisitReturn parseJsonInto(const QByteArray &json, const QString &into, ProValueMap *value);
 
     VisitReturn writeFile(const QString &ctx, const QString &fn, QIODevice::OpenMode mode,
-                          bool exe, const QString &contents);
+                          QMakeVfs::VfsFlags flags, const QString &contents);
 #ifndef QT_BOOTSTRAPPED
     void runProcess(QProcess *proc, const QString &command) const;
 #endif
diff --git a/src/shared/proparser/qmakeparser.cpp b/src/shared/proparser/qmakeparser.cpp
index 00f67e675b..5d1bd71043 100644
--- a/src/shared/proparser/qmakeparser.cpp
+++ b/src/shared/proparser/qmakeparser.cpp
@@ -167,7 +167,13 @@ ProFile *QMakeParser::parsedProFile(const QString &fileName, ParseFlags flags)
 #ifdef PROPARSER_THREAD_SAFE
         QMutexLocker locker(&m_cache->mutex);
 #endif
-        QHash<QString, ProFileCache::Entry>::Iterator it = m_cache->parsed_files.find(fileName);
+        QHash<QString, ProFileCache::Entry>::Iterator it;
+#ifdef PROEVALUATOR_DUAL_VFS
+        QString virtFileName = ((flags & ParseCumulative) ? '-' : '+') + fileName;
+        it = m_cache->parsed_files.find(virtFileName);
+        if (it == m_cache->parsed_files.end())
+#endif
+            it = m_cache->parsed_files.find(fileName);
         if (it != m_cache->parsed_files.end()) {
             ent = &*it;
 #ifdef PROPARSER_THREAD_SAFE
@@ -185,18 +191,27 @@ ProFile *QMakeParser::parsedProFile(const QString &fileName, ParseFlags flags)
             if ((pro = ent->pro))
                 pro->ref();
         } else if (!(flags & ParseOnlyCached)) {
-            ent = &m_cache->parsed_files[fileName];
+            QString contents;
+            QMakeVfs::VfsFlags vfsFlags =
+                    ((flags & ParseCumulative) ? QMakeVfs::VfsCumulative : QMakeVfs::VfsExact);
+            bool virt = false;
+#ifdef PROEVALUATOR_DUAL_VFS
+            virt = m_vfs->readVirtualFile(fileName, vfsFlags, &contents);
+            if (virt)
+                ent = &m_cache->parsed_files[virtFileName];
+            else
+#endif
+                ent = &m_cache->parsed_files[fileName];
 #ifdef PROPARSER_THREAD_SAFE
             ent->locker = new ProFileCache::Entry::Locker;
             locker.unlock();
 #endif
-            pro = new ProFile(fileName);
-            if (!read(pro, flags)) {
-                delete pro;
-                pro = 0;
-            } else {
+            if (virt || readFile(fileName, vfsFlags | QMakeVfs::VfsNoVirtual, flags, &contents)) {
+                pro = parsedProBlock(QStringRef(&contents), fileName, 1, FullGrammar);
                 pro->itemsRef()->squeeze();
                 pro->ref();
+            } else {
+                pro = 0;
             }
             ent->pro = pro;
 #ifdef PROPARSER_THREAD_SAFE
@@ -213,11 +228,13 @@ ProFile *QMakeParser::parsedProFile(const QString &fileName, ParseFlags flags)
             pro = 0;
         }
     } else if (!(flags & ParseOnlyCached)) {
-        pro = new ProFile(fileName);
-        if (!read(pro, flags)) {
-            delete pro;
+        QString contents;
+        QMakeVfs::VfsFlags vfsFlags =
+                ((flags & ParseCumulative) ? QMakeVfs::VfsCumulative : QMakeVfs::VfsExact);
+        if (readFile(fileName, vfsFlags, flags, &contents))
+            pro = parsedProBlock(QStringRef(&contents), fileName, 1, FullGrammar);
+        else
             pro = 0;
-        }
     } else {
         pro = 0;
     }
@@ -238,18 +255,17 @@ void QMakeParser::discardFileFromCache(const QString &fileName)
         m_cache->discardFile(fileName);
 }
 
-bool QMakeParser::read(ProFile *pro, ParseFlags flags)
+bool QMakeParser::readFile(
+        const QString &fn, QMakeVfs::VfsFlags vfsFlags, ParseFlags flags, QString *contents)
 {
-    QString content;
     QString errStr;
-    QMakeVfs::ReadResult result = m_vfs->readFile(pro->fileName(), &content, &errStr);
+    QMakeVfs::ReadResult result = m_vfs->readFile(fn, vfsFlags, contents, &errStr);
     if (result != QMakeVfs::ReadOk) {
         if (m_handler && ((flags & ParseReportMissing) || result != QMakeVfs::ReadNotFound))
             m_handler->message(QMakeParserHandler::ParserIoError,
-                               fL1S("Cannot read %1: %2").arg(pro->fileName(), errStr));
+                               fL1S("Cannot read %1: %2").arg(fn, errStr));
         return false;
     }
-    read(pro, QStringRef(&content), 1, FullGrammar);
     return true;
 }
 
diff --git a/src/shared/proparser/qmakeparser.h b/src/shared/proparser/qmakeparser.h
index 07a9f07753..43fa354b9e 100644
--- a/src/shared/proparser/qmakeparser.h
+++ b/src/shared/proparser/qmakeparser.h
@@ -26,6 +26,7 @@
 #pragma once
 
 #include "qmake_global.h"
+#include "qmakevfs.h"
 #include "proitems.h"
 
 #include <qhash.h>
@@ -75,7 +76,12 @@ public:
         ParseDefault = 0,
         ParseUseCache = 1,
         ParseOnlyCached = 2,
-        ParseReportMissing = 4
+        ParseReportMissing = 4,
+#ifdef PROEVALUATOR_DUAL_VFS
+        ParseCumulative = 8
+#else
+        ParseCumulative = 0
+#endif
     };
     Q_DECLARE_FLAGS(ParseFlags, ParseFlag)
 
@@ -126,7 +132,7 @@ private:
         ushort terminator; // '}' if replace function call is braced, ':' if test function
     };
 
-    bool read(ProFile *pro, ParseFlags flags);
+    bool readFile(const QString &fn, QMakeVfs::VfsFlags vfsFlags, QMakeParser::ParseFlags flags, QString *contents);
     void read(ProFile *pro, const QStringRef &content, int line, SubGrammar grammar);
 
     ALWAYS_INLINE void putTok(ushort *&tokPtr, ushort tok);
diff --git a/src/shared/proparser/qmakevfs.cpp b/src/shared/proparser/qmakevfs.cpp
index a46b211aef..6115a02457 100644
--- a/src/shared/proparser/qmakevfs.cpp
+++ b/src/shared/proparser/qmakevfs.cpp
@@ -44,20 +44,24 @@ QMakeVfs::QMakeVfs()
 {
 }
 
-bool QMakeVfs::writeFile(const QString &fn, QIODevice::OpenMode mode, bool exe,
+bool QMakeVfs::writeFile(const QString &fn, QIODevice::OpenMode mode, VfsFlags flags,
                          const QString &contents, QString *errStr)
 {
 #ifndef PROEVALUATOR_FULL
 # ifdef PROEVALUATOR_THREAD_SAFE
     QMutexLocker locker(&m_mutex);
 # endif
+#ifdef PROEVALUATOR_DUAL_VFS
+    QString *cont = &m_files[((flags & VfsCumulative) ? '-' : '+') + fn];
+#else
     QString *cont = &m_files[fn];
+    Q_UNUSED(flags)
+#endif
     if (mode & QIODevice::Append)
         *cont += contents;
     else
         *cont = contents;
     Q_UNUSED(errStr)
-    Q_UNUSED(exe)
     return true;
 #else
     QFileInfo qfi(fn);
@@ -69,7 +73,7 @@ bool QMakeVfs::writeFile(const QString &fn, QIODevice::OpenMode mode, bool exe,
     QFile cfile(fn);
     if (!(mode & QIODevice::Append) && cfile.open(QIODevice::ReadOnly | QIODevice::Text)) {
         if (cfile.readAll() == bytes) {
-            if (exe) {
+            if (flags & VfsExecutable) {
                 cfile.setPermissions(cfile.permissions()
                                      | QFile::ExeUser | QFile::ExeGroup | QFile::ExeOther);
             } else {
@@ -90,7 +94,7 @@ bool QMakeVfs::writeFile(const QString &fn, QIODevice::OpenMode mode, bool exe,
         *errStr = cfile.errorString();
         return false;
     }
-    if (exe)
+    if (flags & VfsExecutable)
         cfile.setPermissions(cfile.permissions()
                              | QFile::ExeUser | QFile::ExeGroup | QFile::ExeOther);
     return true;
@@ -98,29 +102,52 @@ bool QMakeVfs::writeFile(const QString &fn, QIODevice::OpenMode mode, bool exe,
 }
 
 #ifndef PROEVALUATOR_FULL
-bool QMakeVfs::readVirtualFile(const QString &fn, QString *contents)
+bool QMakeVfs::readVirtualFile(const QString &fn, VfsFlags flags, QString *contents)
 {
 # ifdef PROEVALUATOR_THREAD_SAFE
     QMutexLocker locker(&m_mutex);
 # endif
-    QHash<QString, QString>::ConstIterator it = m_files.constFind(fn);
+    QHash<QString, QString>::ConstIterator it;
+# ifdef PROEVALUATOR_DUAL_VFS
+    it = m_files.constFind(((flags & VfsCumulative) ? '-' : '+') + fn);
+    if (it != m_files.constEnd()) {
+        *contents = *it;
+        return true;
+    }
+# else
+    it = m_files.constFind(fn);
     if (it != m_files.constEnd()
         && it->constData() != m_magicMissing.constData()
         && it->constData() != m_magicExisting.constData()) {
         *contents = *it;
         return true;
     }
+    Q_UNUSED(flags)
+# endif
     return false;
 }
 #endif
 
-QMakeVfs::ReadResult QMakeVfs::readFile(const QString &fn, QString *contents, QString *errStr)
+QMakeVfs::ReadResult QMakeVfs::readFile(
+        const QString &fn, VfsFlags flags, QString *contents, QString *errStr)
 {
 #ifndef PROEVALUATOR_FULL
 # ifdef PROEVALUATOR_THREAD_SAFE
     QMutexLocker locker(&m_mutex);
 # endif
-    QHash<QString, QString>::ConstIterator it = m_files.constFind(fn);
+    QHash<QString, QString>::ConstIterator it;
+# ifdef PROEVALUATOR_DUAL_VFS
+    if (!(flags & VfsNoVirtual)) {
+        it = m_files.constFind(((flags & VfsCumulative) ? '-' : '+') + fn);
+        if (it != m_files.constEnd()) {
+            *contents = *it;
+            return ReadOk;
+        }
+    }
+# else
+    Q_UNUSED(flags)
+# endif
+    it = m_files.constFind(fn);
     if (it != m_files.constEnd()) {
         if (it->constData() == m_magicMissing.constData()) {
             *errStr = fL1S("No such file or directory");
@@ -131,6 +158,8 @@ QMakeVfs::ReadResult QMakeVfs::readFile(const QString &fn, QString *contents, QS
             return ReadOk;
         }
     }
+#else
+    Q_UNUSED(flags)
 #endif
 
     QFile file(fn);
@@ -159,15 +188,25 @@ QMakeVfs::ReadResult QMakeVfs::readFile(const QString &fn, QString *contents, QS
     return ReadOk;
 }
 
-bool QMakeVfs::exists(const QString &fn)
+bool QMakeVfs::exists(const QString &fn, VfsFlags flags)
 {
 #ifndef PROEVALUATOR_FULL
 # ifdef PROEVALUATOR_THREAD_SAFE
     QMutexLocker locker(&m_mutex);
 # endif
-    QHash<QString, QString>::ConstIterator it = m_files.constFind(fn);
+    QHash<QString, QString>::ConstIterator it;
+# ifdef PROEVALUATOR_DUAL_VFS
+    it = m_files.constFind(((flags & VfsCumulative) ? '-' : '+') + fn);
+    if (it != m_files.constEnd())
+        return true;
+# else
+    Q_UNUSED(flags)
+# endif
+    it = m_files.constFind(fn);
     if (it != m_files.constEnd())
         return it->constData() != m_magicMissing.constData();
+#else
+    Q_UNUSED(flags)
 #endif
     bool ex = IoUtils::fileType(fn) == IoUtils::FileIsRegular;
 #ifndef PROEVALUATOR_FULL
diff --git a/src/shared/proparser/qmakevfs.h b/src/shared/proparser/qmakevfs.h
index 801658c62f..02c0a6406b 100644
--- a/src/shared/proparser/qmakevfs.h
+++ b/src/shared/proparser/qmakevfs.h
@@ -36,6 +36,12 @@
 # endif
 #endif
 
+#ifdef PROEVALUATOR_DUAL_VFS
+# ifndef PROEVALUATOR_CUMULATIVE
+#  error PROEVALUATOR_DUAL_VFS requires PROEVALUATOR_CUMULATIVE
+# endif
+#endif
+
 QT_BEGIN_NAMESPACE
 
 class QMAKE_EXPORT QMakeVfs
@@ -47,14 +53,27 @@ public:
         ReadOtherError
     };
 
+    enum VfsFlag {
+        VfsExecutable = 1,
+        VfsExact = 0,
+#ifdef PROEVALUATOR_DUAL_VFS
+        VfsCumulative = 2,
+        VfsNoVirtual = 4
+#else
+        VfsCumulative = 0,
+        VfsNoVirtual = 0
+#endif
+    };
+    Q_DECLARE_FLAGS(VfsFlags, VfsFlag)
+
     QMakeVfs();
 
-    bool writeFile(const QString &fn, QIODevice::OpenMode mode, bool exe, const QString &contents, QString *errStr);
-    ReadResult readFile(const QString &fn, QString *contents, QString *errStr);
-    bool exists(const QString &fn);
+    bool writeFile(const QString &fn, QIODevice::OpenMode mode, VfsFlags flags, const QString &contents, QString *errStr);
+    ReadResult readFile(const QString &fn, VfsFlags flags, QString *contents, QString *errStr);
+    bool exists(const QString &fn, VfsFlags flags);
 
 #ifndef PROEVALUATOR_FULL
-    bool readVirtualFile(const QString &fn, QString *contents);
+    bool readVirtualFile(const QString &fn, VfsFlags flags, QString *contents);
 
     void invalidateCache();
     void invalidateContents();
@@ -71,4 +90,6 @@ private:
 #endif
 };
 
+Q_DECLARE_OPERATORS_FOR_FLAGS(QMakeVfs::VfsFlags)
+
 QT_END_NAMESPACE