Use XML transformation to convert to Qt/MSbuild

Convert projects based on custom build steps to Qt/MSBuild. The
converted projects do not include generated files for optimized
performance. The conversion uses only an XML transformation instead of
the Visual Studio SDK. This speeds up the import of .pro files,
especially for large projects as the performance of the VS SDK tends to
degrade with the number of calls to the COM interfaces.

Task-number: QTVSADDINBUG-442
Change-Id: If59a52f44eb2eb4dd99f386c5b695aa56025e900
Reviewed-by: Oliver Wolff <oliver.wolff@qt.io>

1 files changed, 339 insertions, 8 deletions

diff --git a/src/qtprojectlib/MsBuildProject.cs b/src/qtprojectlib/MsBuildProject.cs
index d9040187..36f051f0 100644
--- a/src/qtprojectlib/MsBuildProject.cs
+++ b/src/qtprojectlib/MsBuildProject.cs
@@ -35,6 +35,9 @@ using System.Xml;
 using System.Xml.Linq;
 using QtProjectLib.QtMsBuild;
 using System.Text.RegularExpressions;
+using Microsoft.Build.Construction;
+using Microsoft.Build.Execution;
+using Microsoft.Build.Evaluation;
 
 namespace QtProjectLib
 {
@@ -72,6 +75,17 @@ namespace QtProjectLib
             }
         }
 
+        public string ProjectXml
+        {
+            get
+            {
+                var xml = this[Files.Project].xml;
+                if (xml == null)
+                    return "";
+                return xml.ToString(SaveOptions.None);
+            }
+        }
+
         private static XNamespace ns = "http://schemas.microsoft.com/developer/msbuild/2003";
 
         public static MsBuildProject Load(string pathToProject)
@@ -261,29 +275,237 @@ namespace QtProjectLib
             });
         }
 
+        string AddGeneratedFilesPath(string includePathList)
+        {
+            HashSet<string> includes = new HashSet<string> {
+                    QtVSIPSettings.GetMocDirectory(),
+                    QtVSIPSettings.GetRccDirectory(),
+                    QtVSIPSettings.GetUicDirectory(),
+                };
+            foreach (var includePath in includePathList.Split(new char[] { ';' }))
+                includes.Add(includePath);
+            return string.Join<string>(";", includes);
+        }
+
+        string CustomBuildMocInput(XElement cbt)
+        {
+            var commandLine = (string)cbt.Element(ns + "Command");
+            Dictionary<QtMoc.Property, string> properties;
+            if (!QtMsBuildContainer.QtMocInstance.ParseCommandLine(commandLine, out properties))
+                return (string)cbt.Attribute("Include");
+            string ouputFile;
+            if (!properties.TryGetValue(QtMoc.Property.InputFile, out ouputFile))
+                return (string)cbt.Attribute("Include");
+            return ouputFile;
+        }
+
+        void RemoveGeneratedFiles(
+            List<CustomBuildEval> cbEvals,
+            string configName,
+            string itemName,
+            IEnumerable<XElement> projectItems,
+            IEnumerable<XElement> filterItems)
+        {
+            //remove items with generated files
+            var cbEval = cbEvals
+                .Where(x => x.ProjectConfig == configName && x.Identity == itemName)
+                .FirstOrDefault();
+            if (cbEval != null) {
+                var outputFiles = cbEval.Outputs.Split(new char[] { ';' });
+                foreach (var outputFile in outputFiles) {
+                    if (!string.IsNullOrEmpty(outputFile)) {
+                        var outputItems = new List<XElement>();
+                        outputItems.AddRange(projectItems
+                            .Where(x => HelperFunctions.PathEquals(
+                                outputFile, (string)x.Attribute("Include"))));
+                        outputItems.AddRange(filterItems
+                            .Where(x => HelperFunctions.PathEquals(
+                                outputFile, (string)x.Attribute("Include"))));
+                        foreach (var item in outputItems)
+                            item.Remove();
+                    }
+                }
+            }
+        }
+
         public bool ConvertCustomBuildToQtMsBuild()
         {
+            var cbEvals = EvaluateCustomBuild();
+
             var qtMsBuild = new QtMsBuildContainer(new MsBuildConverterProvider());
             qtMsBuild.BeginSetItemProperties();
 
-            var configurations = this[Files.Project].xml
+            var configNames = this[Files.Project].xml
                 .Elements(ns + "Project")
                 .Elements(ns + "ItemGroup")
                 .Elements(ns + "ProjectConfiguration")
-                .Select(x => x.Attribute("Include").Value);
+                .Select(x => (string)x.Attribute("Include"));
 
-            var mocCustomBuilds = GetCustomBuilds(QtMoc.ToolExecName);
-            var rccCustomBuilds = GetCustomBuilds(QtRcc.ToolExecName);
-            var uicCustomBuilds = GetCustomBuilds(QtUic.ToolExecName);
+            var projectItems = this[Files.Project].xml
+                .Elements(ns + "Project")
+                .Elements(ns + "ItemGroup")
+                .Elements()
+                .Where(x => ((string)x.Attribute("Include"))
+                .IndexOfAny(Path.GetInvalidPathChars()) == -1);
+
+
+            var filterItems = this[Files.Filters].xml
+                .Elements(ns + "Project")
+                .Elements(ns + "ItemGroup")
+                .Elements()
+                .Where(x => projectItems.Where(y =>
+                    ((string)x.Attribute("Include")).Equals((string)y.Attribute("Include"),
+                    StringComparison.InvariantCultureIgnoreCase)).Any());
+
+            var cppIncludePaths = this[Files.Project].xml
+                .Elements(ns + "Project")
+                .Elements(ns + "ItemDefinitionGroup")
+                .Elements(ns + "ClCompile")
+                .Elements(ns + "AdditionalIncludeDirectories");
+
+            //add generated files path to C++ additional include dirs
+            foreach (var cppIncludePath in cppIncludePaths)
+                cppIncludePath.Value = AddGeneratedFilesPath((string)cppIncludePath);
+
+            // replace each set of .moc.cbt custom build steps
+            // with a single .cpp custom build step
+            var mocCbtCustomBuilds = GetCustomBuilds(QtMoc.ToolExecName)
+                .Where(x =>
+                ((string)x.Attribute("Include")).EndsWith(".moc.cbt",
+                StringComparison.InvariantCultureIgnoreCase))
+                .GroupBy(cbt => CustomBuildMocInput(cbt));
+
+            List<XElement> cbtToRemove = new List<XElement>();
+            foreach (var cbtGroup in mocCbtCustomBuilds) {
+
+                //create new CustomBuild item for .cpp
+                var newCbt = new XElement(ns + "CustomBuild",
+                    new XAttribute("Include", cbtGroup.Key),
+                    new XElement(ns + "FileType", "Document"));
+
+                //add properties from .moc.cbt items
+                List<string> cbtPropertyNames = new List<string> {
+                    "AdditionalInputs",
+                    "Command",
+                    "Message",
+                    "Outputs",
+                };
+                foreach (var cbt in cbtGroup) {
+                    var enabledProperties = cbt.Elements().Where(x =>
+                        cbtPropertyNames.Contains(x.Name.LocalName)
+                        && !x.Parent.Elements(ns + "ExcludedFromBuild").Where(y =>
+                        (string)x.Attribute("Condition") == (string)y.Attribute("Condition"))
+                        .Any());
+                    foreach (var property in enabledProperties)
+                        newCbt.Add(new XElement(property));
+                    cbtToRemove.Add(cbt);
+                }
+                cbtGroup.First().AddBeforeSelf(newCbt);
+
+                //remove ClCompile item (cannot have duplicate items)
+                var cppMocItems = this[Files.Project].xml
+                    .Elements(ns + "Project")
+                    .Elements(ns + "ItemGroup")
+                    .Elements(ns + "ClCompile")
+                    .Where(x =>
+                        cbtGroup.Key.Equals((string)x.Attribute("Include"),
+                        StringComparison.InvariantCultureIgnoreCase));
+                foreach (var cppMocItem in cppMocItems)
+                    cppMocItem.Remove();
+
+                //change type of item in filter
+                cppMocItems = this[Files.Filters].xml
+                    .Elements(ns + "Project")
+                    .Elements(ns + "ItemGroup")
+                    .Elements(ns + "ClCompile")
+                    .Where(x =>
+                        cbtGroup.Key.Equals((string)x.Attribute("Include"),
+                        StringComparison.InvariantCultureIgnoreCase));
+                foreach (var cppMocItem in cppMocItems)
+                    cppMocItem.Name = ns + "CustomBuild";
+            }
+
+            //remove .moc.cbt CustomBuild items
+            cbtToRemove.ForEach(x => x.Remove());
 
-            if (!SetCommandLines(qtMsBuild, configurations, mocCustomBuilds, QtMoc.ItemTypeName))
+            //convert moc custom build steps
+            var mocCustomBuilds = GetCustomBuilds(QtMoc.ToolExecName);
+            if (!SetCommandLines(qtMsBuild, configNames, mocCustomBuilds, QtMoc.ItemTypeName))
                 return false;
+            foreach (var qtMoc in mocCustomBuilds.Elements(ns + QtMoc.ItemTypeName)) {
+                var itemName = (string)qtMoc.Attribute("Include");
+                var configName = (string)qtMoc.Attribute("ConfigName");
+
+                //remove items with generated files
+                RemoveGeneratedFiles(cbEvals, configName, itemName, projectItems, filterItems);
+
+                //set properties
+                qtMsBuild.SetItemProperty(qtMoc,
+                    QtMoc.Property.ExecutionDescription, "Moc'ing %(Identity)...");
+                qtMsBuild.SetItemProperty(qtMoc,
+                    QtMoc.Property.InputFile, "%(FullPath)");
+                if (!HelperFunctions.IsSourceFile(itemName)) {
+                    qtMsBuild.SetItemProperty(qtMoc,
+                        QtMoc.Property.OutputFile, string.Format(@"{0}\moc_%(Filename).cpp",
+                        QtVSIPSettings.GetMocDirectory()));
+                    qtMsBuild.SetItemProperty(qtMoc,
+                        QtMoc.Property.DynamicSource, "output");
+                } else {
+                    qtMsBuild.SetItemProperty(qtMoc,
+                        QtMoc.Property.OutputFile, string.Format(@"{0}\%(Filename).moc",
+                        QtVSIPSettings.GetMocDirectory()));
+                    qtMsBuild.SetItemProperty(qtMoc,
+                        QtMoc.Property.DynamicSource, "input");
+                }
+                var includePath = qtMsBuild.GetPropertyChangedValue(
+                    QtMoc.Property.IncludePath, itemName, configName);
+                if (!string.IsNullOrEmpty(includePath)) {
+                    qtMsBuild.SetItemProperty(qtMoc,
+                        QtMoc.Property.IncludePath, AddGeneratedFilesPath(includePath));
+                }
+            }
 
-            if (!SetCommandLines(qtMsBuild, configurations, rccCustomBuilds, QtRcc.ItemTypeName))
+            //convert rcc custom build steps
+            var rccCustomBuilds = GetCustomBuilds(QtRcc.ToolExecName);
+            if (!SetCommandLines(qtMsBuild, configNames, rccCustomBuilds, QtRcc.ItemTypeName))
                 return false;
+            foreach (var qtRcc in rccCustomBuilds.Elements(ns + QtRcc.ItemTypeName)) {
+                var itemName = (string)qtRcc.Attribute("Include");
+                var configName = (string)qtRcc.Attribute("ConfigName");
+
+                //remove items with generated files
+                RemoveGeneratedFiles(cbEvals, configName, itemName, projectItems, filterItems);
+
+                //set properties
+                qtMsBuild.SetItemProperty(qtRcc,
+                    QtRcc.Property.ExecutionDescription, "Rcc'ing %(Identity)...");
+                qtMsBuild.SetItemProperty(qtRcc,
+                    QtRcc.Property.InputFile, "%(FullPath)");
+                qtMsBuild.SetItemProperty(qtRcc,
+                    QtRcc.Property.OutputFile, string.Format(@"{0}\qrc_%(Filename).cpp",
+                    QtVSIPSettings.GetRccDirectory()));
+            }
 
-            if (!SetCommandLines(qtMsBuild, configurations, uicCustomBuilds, QtUic.ItemTypeName))
+            //convert uic custom build steps
+            var uicCustomBuilds = GetCustomBuilds(QtUic.ToolExecName);
+            if (!SetCommandLines(qtMsBuild, configNames, uicCustomBuilds, QtUic.ItemTypeName))
                 return false;
+            foreach (var qtUic in uicCustomBuilds.Elements(ns + QtUic.ItemTypeName)) {
+                var itemName = (string)qtUic.Attribute("Include");
+                var configName = (string)qtUic.Attribute("ConfigName");
+
+                //remove items with generated files
+                RemoveGeneratedFiles(cbEvals, configName, itemName, projectItems, filterItems);
+
+                //set properties
+                qtMsBuild.SetItemProperty(qtUic,
+                    QtUic.Property.ExecutionDescription, "Uic'ing %(Identity)...");
+                qtMsBuild.SetItemProperty(qtUic,
+                    QtUic.Property.InputFile, "%(FullPath)");
+                qtMsBuild.SetItemProperty(qtUic,
+                    QtUic.Property.OutputFile, string.Format(@"{0}\ui_%(Filename).h",
+                    QtVSIPSettings.GetRccDirectory()));
+            }
 
             qtMsBuild.EndSetItemProperties();
 
@@ -294,6 +516,115 @@ namespace QtProjectLib
             return true;
         }
 
+        bool TryReplaceTextInPlace(ref string text, Regex findWhat, string newText)
+        {
+            var match = findWhat.Match(text);
+            if (!match.Success)
+                return false;
+            do {
+                text = text.Remove(match.Index, match.Length).Insert(match.Index, newText);
+                match = findWhat.Match(text, match.Index);
+            } while (match.Success);
+
+            return true;
+        }
+
+        void ReplaceText(XElement xElem, Regex findWhat, string newText)
+        {
+            var elemValue = (string)xElem;
+            if (!string.IsNullOrEmpty(elemValue)
+                && TryReplaceTextInPlace(ref elemValue, findWhat, newText)) {
+                xElem.Value = elemValue;
+            }
+        }
+
+        void ReplaceText(XAttribute xAttr, Regex findWhat, string newText)
+        {
+            var attrValue = (string)xAttr;
+            if (!string.IsNullOrEmpty(attrValue)
+                && TryReplaceTextInPlace(ref attrValue, findWhat, newText)) {
+                xAttr.Value = attrValue;
+            }
+        }
+
+        public void ReplacePath(string oldPath, string newPath)
+        {
+            var findWhat = new Regex(oldPath
+                .Replace("\\", "[\\\\\\/]")
+                .Replace(".", "\\.")
+                .Replace("$", "\\$"),
+                RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
+
+            foreach (var xElem in this[Files.Project].xml.Descendants()) {
+                if (!xElem.HasElements)
+                    ReplaceText(xElem, findWhat, newPath);
+                foreach (var xAttr in xElem.Attributes())
+                    ReplaceText(xAttr, findWhat, newPath);
+            }
+        }
+
+        class CustomBuildEval
+        {
+            public string ProjectConfig { get; set; }
+            public string Identity { get; set; }
+            public string AdditionalInputs { get; set; }
+            public string Outputs { get; set; }
+            public string Message { get; set; }
+            public string Command { get; set; }
+        }
+
+        List<CustomBuildEval> EvaluateCustomBuild()
+        {
+            var eval = new List<CustomBuildEval>();
+            var evaluateTarget = new XElement(ns + "Target",
+                new XAttribute("Name", "EvaluateCustomBuild"),
+                new XElement(ns + "PropertyGroup",
+                    new XElement(ns + "CustomBuildEval", "@(CustomBuild->'" +
+                    "{%(Identity)}" +
+                    "{%(AdditionalInputs)}" +
+                    "{%(Outputs)}" +
+                    "{%(Message)}" +
+                    "{%(Command)}')")));
+            this[Files.Project].xml.Root.Add(evaluateTarget);
+
+            var projRoot = ProjectRootElement.Create(this[Files.Project].xml.CreateReader());
+
+            var pattern = new Regex(@"{([^}]+)}{([^}]+)}{([^}]+)}{([^}]+)}{([^}]+)}");
+
+            var projConfigs = this[Files.Project].xml
+                .Elements(ns + "Project")
+                .Elements(ns + "ItemGroup")
+                .Elements(ns + "ProjectConfiguration");
+            foreach (var projConfig in projConfigs) {
+                string configName = (string)projConfig.Attribute("Include");
+                var properties = new Dictionary<string, string>();
+                foreach (var configProp in projConfig.Elements())
+                    properties.Add(configProp.Name.LocalName, (string)configProp);
+                var projInst = new ProjectInstance(projRoot, properties,
+                    null, new ProjectCollection());
+                var buildRequest = new BuildRequestData(
+                    projInst, new string[] { "EvaluateCustomBuild" },
+                    null, BuildRequestDataFlags.ProvideProjectStateAfterBuild);
+                var buildResult = BuildManager.DefaultBuildManager.Build(
+                    new BuildParameters(), buildRequest);
+                string customBuildEval = buildResult.ProjectStateAfterBuild
+                    .GetPropertyValue("CustomBuildEval");
+                foreach (Match cbEval in pattern.Matches(customBuildEval)) {
+                    eval.Add(new CustomBuildEval {
+                        ProjectConfig = configName,
+                        Identity = cbEval.Groups[1].Value,
+                        AdditionalInputs = cbEval.Groups[2].Value,
+                        Outputs = cbEval.Groups[3].Value,
+                        Message = cbEval.Groups[4].Value,
+                        Command = cbEval.Groups[5].Value,
+                    });
+                }
+            }
+
+            evaluateTarget.Remove();
+            return eval;
+        }
+
         static Regex ConditionParser =
             new Regex(@"\'\$\(Configuration[^\)]*\)\|\$\(Platform[^\)]*\)\'\=\=\'([^\']+)\'");