qmake: Fix overlong command lines for static Qt builds on Windows

Linker response files for the MinGW and Unix makefile generators are
controlled by the variable QMAKE_LINK_OBJECT_MAX.  This variable holds a
number.  If the number of object files passed to the linker exceeds this
number, a linker response file containing object file paths is created.

This heuristic is extremely imprecise.  It doesn't take into account the
length of object file names nor the length of $$OBJECTS_DIR.

Also, when using a static Qt, a big part of the linker command line are
libraries.  A relatively small example can fail to link with "The
command line is too long" on Windows, even with the object files being
in a response file.

The MinGW makefile generator already reads the variable
QMAKE_RESPONSEFILE_THRESHOLD for compiler response files.  Re-use this
variable for the linker response file of the Unix and MinGW makefile
generators.

If QMAKE_RESPONSEFILE_THRESHOLD is set, use it to determine whether to
create a response file.  QMAKE_LINK_OBJECT_MAX is then ignored.  The
response file contains objects and libraries.

If QMAKE_RESPONSEFILE_THRESHOLD is not set, use QMAKE_LINK_OBJECT_MAX to
determine whether to create a response file.  The response file contains
only object files.

QMAKE_LINK_OBJECT_SCRIPT is used in both cases to specify a common base
name of all linker response files.

Pick-to: 6.2 6.3
Task-number: QTBUG-100559
Change-Id: I3c78354fa5ebb1a86438ec804679e0ee776c3f49
Reviewed-by: Alexandru Croitor <alexandru.croitor@qt.io>
Reviewed-by: Qt CI Bot <qt_ci_bot@qt-project.org>
Reviewed-by: Kai Koehne <kai.koehne@qt.io>

6 files changed, 81 insertions, 28 deletions

diff --git a/qmake/generators/makefile.cpp b/qmake/generators/makefile.cpp
index 59dc24f022..82f9c00aec 100644
--- a/qmake/generators/makefile.cpp
+++ b/qmake/generators/makefile.cpp
@@ -3498,7 +3498,7 @@ ProKey MakefileGenerator::fullTargetVariable() const
 QString MakefileGenerator::createResponseFile(
         const QString &baseName,
         const ProStringList &objList,
-        const QString &prefix)
+        const QString &prefix) const
 {
     QString fileName = baseName + '.' + var("QMAKE_ORIG_TARGET");
     if (!var("BUILD_NAME").isEmpty())
@@ -3531,4 +3531,46 @@ QString MakefileGenerator::createResponseFile(
     return fileName;
 }
 
+/*
+ * If the threshold for response files are overstepped, create a response file for the linker
+ * command line.
+ */
+MakefileGenerator::LinkerResponseFileInfo MakefileGenerator::maybeCreateLinkerResponseFile() const
+{
+    bool useLinkObjectMax = false;
+    bool ok;
+    int threshold = project->first("QMAKE_RESPONSEFILE_THRESHOLD").toInt(&ok);
+    if (!ok) {
+        threshold = project->first("QMAKE_LINK_OBJECT_MAX").toInt(&ok);
+        if (ok)
+            useLinkObjectMax = true;
+    }
+    if (!ok)
+        return {};
+
+    ProStringList linkerInputs = project->values("OBJECTS");
+    if (useLinkObjectMax) {
+        // When using QMAKE_LINK_OBJECT_MAX, the number of object files (regardless of their path
+        // length) decides whether to use a response file.  This is far from being a useful
+        // heuristic but let's keep this behavior for backwards compatibility.
+        if (linkerInputs.count() < threshold)
+            return {};
+    } else {
+        // When using QMAKE_REPONSEFILE_THRESHOLD, try to determine the command line length of the
+        // inputs.
+        linkerInputs += project->values("LIBS");
+        int totalLength = std::accumulate(linkerInputs.cbegin(), linkerInputs.cend(), 0,
+                                          [](int total, const ProString &input) {
+                                              return total + input.size() + 1;
+                                          });
+        if (totalLength < threshold)
+            return {};
+    }
+
+    return {
+        createResponseFile(fileVar("OBJECTS_DIR") + var("QMAKE_LINK_OBJECT_SCRIPT"), linkerInputs),
+        useLinkObjectMax
+    };
+}
+
 QT_END_NAMESPACE
diff --git a/qmake/generators/makefile.h b/qmake/generators/makefile.h
index 60431d622a..2927e2de56 100644
--- a/qmake/generators/makefile.h
+++ b/qmake/generators/makefile.h
@@ -266,7 +266,17 @@ protected:
                             const QString &fixedFile);
     QString createResponseFile(const QString &baseName,
                                const ProStringList &objList,
-                               const QString &prefix = QString());
+                               const QString &prefix = QString()) const;
+
+    struct LinkerResponseFileInfo
+    {
+        QString filePath;
+        bool onlyObjects;
+
+        bool isValid() const { return !filePath.isEmpty(); }
+    };
+
+    LinkerResponseFileInfo maybeCreateLinkerResponseFile() const;
 
 public:
     QMakeProject *projectFile() const;
diff --git a/qmake/generators/unix/unixmake.h b/qmake/generators/unix/unixmake.h
index 1f32e4341f..a149005e74 100644
--- a/qmake/generators/unix/unixmake.h
+++ b/qmake/generators/unix/unixmake.h
@@ -58,7 +58,7 @@ protected:
     void writeSubTargets(QTextStream &t, QList<SubTarget*> subtargets, int flags) override;
     void writeMakeParts(QTextStream &);
     bool writeMakefile(QTextStream &) override;
-    std::pair<bool, QString> writeObjectsPart(QTextStream &, bool do_incremental);
+    bool writeObjectsPart(QTextStream &, bool do_incremental);
 private:
     void init2();
     ProStringList libdirToFlags(const ProKey &key);
diff --git a/qmake/generators/unix/unixmake2.cpp b/qmake/generators/unix/unixmake2.cpp
index d5a057ef7b..60c5e8a220 100644
--- a/qmake/generators/unix/unixmake2.cpp
+++ b/qmake/generators/unix/unixmake2.cpp
@@ -244,8 +244,8 @@ UnixMakefileGenerator::writeMakeParts(QTextStream &t)
     t << "####### Files\n\n";
     // This is used by the dist target.
     t << "SOURCES       = " << fileVarList("SOURCES") << ' ' << fileVarList("GENERATED_SOURCES") << Qt::endl;
-    auto objectParts = writeObjectsPart(t, do_incremental);
-    src_incremental = objectParts.first;
+
+    src_incremental = writeObjectsPart(t, do_incremental);
     if(do_incremental && !src_incremental)
         do_incremental = false;
     t << "DIST          = " << valList(fileFixify(project->values("DISTFILES").toQStringList())) << " "
@@ -395,6 +395,7 @@ UnixMakefileGenerator::writeMakeParts(QTextStream &t)
             }
         }
     }
+    LinkerResponseFileInfo linkerResponseFile = maybeCreateLinkerResponseFile();
     QString deps = escapeDependencyPath(fileFixify(Option::output.fileName()));
     QString allDeps;
     if (!project->values("QMAKE_APP_FLAG").isEmpty() || project->first("TEMPLATE") == "aux") {
@@ -481,8 +482,13 @@ UnixMakefileGenerator::writeMakeParts(QTextStream &t)
                     t << mkdir_p_asstring(destdir) << "\n\t";
                 if (!project->isEmpty("QMAKE_PRE_LINK"))
                     t << var("QMAKE_PRE_LINK") << "\n\t";
-                t << "$(LINK) $(LFLAGS) " << var("QMAKE_LINK_O_FLAG") << "$(TARGET) "
-                  << objectParts.second << " $(OBJCOMP) $(LIBS)";
+                t << "$(LINK) $(LFLAGS) " << var("QMAKE_LINK_O_FLAG") << "$(TARGET) ";
+                if (!linkerResponseFile.isValid())
+                    t << " $(OBJECTS) $(OBJCOMP) $(LIBS)";
+                else if (linkerResponseFile.onlyObjects)
+                    t << " @" << linkerResponseFile.filePath << " $(OBJCOMP) $(LIBS)";
+                else
+                    t << " $(OBJCOMP) @" << linkerResponseFile.filePath;
                 if (!project->isEmpty("QMAKE_POST_LINK"))
                     t << "\n\t" << var("QMAKE_POST_LINK");
             }
@@ -557,7 +563,10 @@ UnixMakefileGenerator::writeMakeParts(QTextStream &t)
               << incr_deps << " $(SUBLIBS) " << target_deps << ' ' << depVar("POST_TARGETDEPS");
         } else {
             ProStringList &cmd = project->values("QMAKE_LINK_SHLIB_CMD");
-            cmd[0] = cmd.at(0).toQString().replace(QLatin1String("$(OBJECTS)"), objectParts.second);
+            if (linkerResponseFile.isValid()) {
+                cmd[0] = cmd.at(0).toQString().replace(QLatin1String("$(OBJECTS)"),
+                                                       "@" + linkerResponseFile.filePath);
+            }
             t << destdir_d << depVar("TARGET") << ": " << depVar("PRE_TARGETDEPS")
               << " $(OBJECTS) $(SUBLIBS) $(OBJCOMP) " << target_deps
               << ' ' << depVar("POST_TARGETDEPS");
@@ -1550,7 +1559,7 @@ UnixMakefileGenerator::writeLibtoolFile()
         "libdir='" << Option::fixPathToTargetOS(install_dir.toQString(), false) << "'\n";
 }
 
-std::pair<bool, QString> UnixMakefileGenerator::writeObjectsPart(QTextStream &t, bool do_incremental)
+bool UnixMakefileGenerator::writeObjectsPart(QTextStream &t, bool do_incremental)
 {
     bool src_incremental = false;
     QString objectsLinkLine;
@@ -1584,18 +1593,9 @@ std::pair<bool, QString> UnixMakefileGenerator::writeObjectsPart(QTextStream &t,
               << escapeFilePaths(incrs_out).join(QString(" \\\n\t\t")) << Qt::endl;
         }
     } else {
-        const ProString &objMax = project->first("QMAKE_LINK_OBJECT_MAX");
-        // Used all over the place in both deps and commands.
-        if (objMax.isEmpty() || project->values("OBJECTS").count() < objMax.toInt()) {
-            objectsLinkLine = "$(OBJECTS)";
-        } else {
-            const QString ld_response_file = createResponseFile(
-                        fileVar("OBJECTS_DIR") + var("QMAKE_LINK_OBJECT_SCRIPT"), objs);
-            objectsLinkLine = "@" + escapeFilePath(ld_response_file);
-        }
         t << "OBJECTS       = " << valList(escapeDependencyPaths(objs)) << Qt::endl;
     }
-    return std::make_pair(src_incremental, objectsLinkLine);
+    return src_incremental;
 }
 
 QT_END_NAMESPACE
diff --git a/qmake/generators/win32/mingw_make.cpp b/qmake/generators/win32/mingw_make.cpp
index 823b799eb2..59fcb34ad8 100644
--- a/qmake/generators/win32/mingw_make.cpp
+++ b/qmake/generators/win32/mingw_make.cpp
@@ -242,21 +242,18 @@ void MingwMakefileGenerator::writeLibsPart(QTextStream &t)
 
 void MingwMakefileGenerator::writeObjectsPart(QTextStream &t)
 {
-    const ProString &objmax = project->first("QMAKE_LINK_OBJECT_MAX");
-    if (objmax.isEmpty() || project->values("OBJECTS").count() < objmax.toInt()) {
+    linkerResponseFile = maybeCreateLinkerResponseFile();
+    if (!linkerResponseFile.isValid()) {
         objectsLinkLine = "$(OBJECTS)";
     } else if (project->isActiveConfig("staticlib") && project->first("TEMPLATE") == "lib") {
         // QMAKE_LIB is used for win32, including mingw, whereas QMAKE_AR is used on Unix.
         QString ar_cmd = var("QMAKE_LIB");
         if (ar_cmd.isEmpty())
             ar_cmd = "ar -rc";
-        const QString ar_response_file =
-                createResponseFile(var("QMAKE_LINK_OBJECT_SCRIPT"), project->values("OBJECTS"));
-        objectsLinkLine = ar_cmd + ' ' + var("DEST_TARGET") + " @" + escapeFilePath(ar_response_file);
+        objectsLinkLine = ar_cmd + ' ' + var("DEST_TARGET") + " @"
+            + escapeFilePath(linkerResponseFile.filePath);
     } else {
-        const QString ld_response_file =
-                createResponseFile(var("QMAKE_LINK_OBJECT_SCRIPT"), project->values("OBJECTS"));
-        objectsLinkLine = "@" + escapeFilePath(ld_response_file);
+        objectsLinkLine = "@" + escapeFilePath(linkerResponseFile.filePath);
     }
     Win32MakefileGenerator::writeObjectsPart(t);
 }
@@ -284,7 +281,10 @@ void MingwMakefileGenerator::writeBuildRulesPart(QTextStream &t)
             t << "\n\t" << objectsLinkLine << " " ;
         }
     } else {
-        t << "\n\t$(LINKER) $(LFLAGS) " << var("QMAKE_LINK_O_FLAG") << "$(DESTDIR_TARGET) " << objectsLinkLine << "  $(LIBS)";
+        t << "\n\t$(LINKER) $(LFLAGS) " << var("QMAKE_LINK_O_FLAG") << "$(DESTDIR_TARGET) "
+          << objectsLinkLine;
+        if (!linkerResponseFile.isValid() || linkerResponseFile.onlyObjects)
+            t << " $(LIBS)";
     }
     if(!project->isEmpty("QMAKE_POST_LINK"))
         t << "\n\t" <<var("QMAKE_POST_LINK");
diff --git a/qmake/generators/win32/mingw_make.h b/qmake/generators/win32/mingw_make.h
index 6c1f0086cc..645a73c374 100644
--- a/qmake/generators/win32/mingw_make.h
+++ b/qmake/generators/win32/mingw_make.h
@@ -59,6 +59,7 @@ private:
     LibFlagType parseLibFlag(const ProString &flag, ProString *arg) override;
 
     QString objectsLinkLine;
+    LinkerResponseFileInfo linkerResponseFile;
 };
 
 QT_END_NAMESPACE