fix generators_folder relative path to conanfile (#9668)

* fix generators_folder relative path to conanfile

* define generators_folder for package command

* change new CMake.install to use cmake --install feature

* quote path with spaces

* condition to layout() and apply same to imports_folder

* wip

* wip

* apply cmd_build layout only if not test

* wip

* wip

* marking test to use cmake 3.20

* keep legacy test_package layout

* wip

* wip

* wip

* lets try again cmake 3.19 for iOS build

* review

12 files changed, 177 insertions, 49 deletions

diff --git a/conan/tools/cmake/cmake.py b/conan/tools/cmake/cmake.py
index 6afe2af6..ddf94bd0 100644
--- a/conan/tools/cmake/cmake.py
+++ b/conan/tools/cmake/cmake.py
@@ -141,7 +141,20 @@ class CMake(object):
         if not self._conanfile.should_install:
             return
         mkdir(self._conanfile, self._conanfile.package_folder)
-        self._build(build_type=build_type, target="install")
+
+        bt = build_type or self._conanfile.settings.get_safe("build_type")
+        if not bt:
+            raise ConanException("build_type setting should be defined.")
+        is_multi = is_multi_configuration(self._generator)
+        build_config = "--config {}".format(bt) if bt and is_multi else ""
+
+        pkg_folder = args_to_string([self._conanfile.package_folder.replace("\\", "/")])
+        build_folder = args_to_string([self._conanfile.build_folder])
+        arg_list = ["--install", build_folder, build_config, "--prefix", pkg_folder]
+        arg_list = " ".join(filter(None, arg_list))
+        command = "%s %s" % (self._cmake_program, arg_list)
+        self._conanfile.output.info("CMake command: %s" % command)
+        self._conanfile.run(command)
 
     def test(self, build_type=None, target=None, output_on_failure=False):
         if not self._conanfile.should_test:
diff --git a/conans/client/cmd/build.py b/conans/client/cmd/build.py
index 309c85e1..b2f6ef74 100644
--- a/conans/client/cmd/build.py
+++ b/conans/client/cmd/build.py
@@ -41,12 +41,19 @@ def cmd_build(app, conanfile_path, base_path, source_folder, build_folder, packa
     try:
         # FIXME: Conan 2.0 all these build_folder, source_folder will disappear
         #  Only base_path and conanfile_path will remain
-        conan_file.folders.set_base_build(build_folder)
-        conan_file.folders.set_base_source(source_folder)
-        conan_file.folders.set_base_package(package_folder)
-        conan_file.folders.set_base_generators(base_path)
-
-        conan_file.folders.set_base_install(install_folder)
+        if hasattr(conan_file, "layout") and not test:
+            conanfile_folder = os.path.dirname(conanfile_path)
+            conan_file.folders.set_base_build(conanfile_folder)
+            conan_file.folders.set_base_source(conanfile_folder)
+            conan_file.folders.set_base_package(conanfile_folder)
+            conan_file.folders.set_base_generators(conanfile_folder)
+            conan_file.folders.set_base_install(conanfile_folder)
+        else:
+            conan_file.folders.set_base_build(build_folder)
+            conan_file.folders.set_base_source(source_folder)
+            conan_file.folders.set_base_package(package_folder)
+            conan_file.folders.set_base_generators(base_path)
+            conan_file.folders.set_base_install(install_folder)
 
         mkdir(conan_file.build_folder)
         os.chdir(conan_file.build_folder)
diff --git a/conans/client/cmd/create.py b/conans/client/cmd/create.py
index d9280725..7f7076c2 100644
--- a/conans/client/cmd/create.py
+++ b/conans/client/cmd/create.py
@@ -52,7 +52,8 @@ def create(app, ref, graph_info, remotes, update, build_modes,
                          build_modes=build_modes,
                          update=update,
                          keep_build=keep_build,
-                         recorder=recorder)
+                         recorder=recorder,
+                         conanfile_path=os.path.dirname(test_conanfile_path))
             out.info("Executing test_package %s" % repr(ref))
             try:
                 graph_info.graph_lock.relax()
diff --git a/conans/client/cmd/export_pkg.py b/conans/client/cmd/export_pkg.py
index 3c00d5df..ac862f32 100644
--- a/conans/client/cmd/export_pkg.py
+++ b/conans/client/cmd/export_pkg.py
@@ -10,7 +10,7 @@ from conans.model.ref import PackageReference
 
 
 def export_pkg(app, recorder, full_ref, source_folder, build_folder, package_folder, install_folder,
-               graph_info, force, remotes):
+               graph_info, force, remotes, source_conanfile_path):
     ref = full_ref.copy_clear_rev()
     cache, output, hook_manager = app.cache, app.out, app.hook_manager
     graph_manager = app.graph_manager
@@ -54,13 +54,22 @@ def export_pkg(app, recorder, full_ref, source_folder, build_folder, package_fol
     recipe_hash = layout.recipe_manifest().summary_hash
     conanfile.info.recipe_hash = recipe_hash
     conanfile.develop = True
-    conanfile.folders.set_base_build(build_folder)
-    conanfile.folders.set_base_source(source_folder)
-    conanfile.folders.set_base_package(dest_package_folder)
-    conanfile.folders.set_base_install(install_folder)
+    if hasattr(conanfile, "layout"):
+        conanfile_folder = os.path.dirname(source_conanfile_path)
+        conanfile.folders.set_base_build(conanfile_folder)
+        conanfile.folders.set_base_source(conanfile_folder)
+        conanfile.folders.set_base_package(dest_package_folder)
+        conanfile.folders.set_base_install(conanfile_folder)
+        conanfile.folders.set_base_generators(conanfile_folder)
+    else:
+        conanfile.folders.set_base_build(build_folder)
+        conanfile.folders.set_base_source(source_folder)
+        conanfile.folders.set_base_package(dest_package_folder)
+        conanfile.folders.set_base_install(install_folder)
 
     with layout.set_dirty_context_manager(pref):
         if package_folder:
+            # FIXME: To be removed in 2.0
             prev = packager.export_pkg(conanfile, package_id, package_folder, hook_manager,
                                        conan_file_path, ref)
         else:
diff --git a/conans/client/cmd/test.py b/conans/client/cmd/test.py
index bcd09657..e6f43f5e 100644
--- a/conans/client/cmd/test.py
+++ b/conans/client/cmd/test.py
@@ -37,7 +37,10 @@ def install_build_and_test(app, conanfile_abs_path, reference, graph_info,
                      manifest_interactive=manifest_interactive,
                      keep_build=keep_build,
                      recorder=recorder,
-                     require_overrides=require_overrides)
+                     require_overrides=require_overrides,
+                     conanfile_path=os.path.dirname(conanfile_abs_path),
+                     test=True  # To keep legacy test_package_layout
+                     )
         cmd_build(app, conanfile_abs_path, test_build_folder,
                   source_folder=base_folder, build_folder=test_build_folder,
                   package_folder=os.path.join(test_build_folder, "package"),
diff --git a/conans/client/conan_api.py b/conans/client/conan_api.py
index 53e04dfe..0508151d 100644
--- a/conans/client/conan_api.py
+++ b/conans/client/conan_api.py
@@ -453,7 +453,7 @@ class ConanAPIV1(object):
             export_pkg(self.app, recorder, new_ref, source_folder=source_folder,
                        build_folder=build_folder, package_folder=package_folder,
                        install_folder=install_folder, graph_info=graph_info, force=force,
-                       remotes=remotes)
+                       remotes=remotes, source_conanfile_path=conanfile_path)
             if lockfile_out:
                 lockfile_out = _make_abs_path(lockfile_out, cwd)
                 graph_lock_file = GraphLockFile(graph_info.profile_host, graph_info.profile_build,
@@ -617,7 +617,8 @@ class ConanAPIV1(object):
                          generators=generators,
                          no_imports=no_imports,
                          recorder=recorder,
-                         require_overrides=require_overrides)
+                         require_overrides=require_overrides,
+                         conanfile_path=os.path.dirname(conanfile_path))
 
             if lockfile_out:
                 lockfile_out = _make_abs_path(lockfile_out, cwd)
@@ -796,10 +797,22 @@ class ConanAPIV1(object):
         else:
             package_folder = _make_abs_path(package_folder, cwd, default=build_folder)
 
-        conanfile.folders.set_base_build(build_folder)
-        conanfile.folders.set_base_source(source_folder)
-        conanfile.folders.set_base_package(package_folder)
-        conanfile.folders.set_base_install(install_folder)
+        if hasattr(conanfile, "layout"):
+            dir_path = os.path.dirname(conanfile_path)
+            conanfile.folders.set_base_generators(dir_path)
+            conanfile.folders.set_base_build(dir_path)
+            conanfile.folders.set_base_source(dir_path)
+            # FIXME: this is bad, but conanfile.folders.package=xxx is broken atm, cant be used
+            # The "conan package" will create "include", "lib" subfolders directly at the root folder
+            # level, which pollutes the source repo itself. It is not possible to change this via
+            # self.folders.package="package" in layout(), because that breaks packaging and create
+            conanfile.folders.set_base_package(os.path.join(dir_path, "package"))
+            conanfile.folders.set_base_install(install_folder)
+        else:
+            conanfile.folders.set_base_build(build_folder)
+            conanfile.folders.set_base_source(source_folder)
+            conanfile.folders.set_base_package(package_folder)
+            conanfile.folders.set_base_install(install_folder)
 
         # Use the complete package layout for the local method
         if conanfile.folders.package:
diff --git a/conans/client/manager.py b/conans/client/manager.py
index b8f324e7..3a235d62 100644
--- a/conans/client/manager.py
+++ b/conans/client/manager.py
@@ -22,7 +22,8 @@ def deps_install(app, ref_or_path, install_folder, base_folder, graph_info, remo
                  build_modes=None, update=False, manifest_folder=None, manifest_verify=False,
                  manifest_interactive=False, generators=None, no_imports=False,
                  create_reference=None, keep_build=False, recorder=None, lockfile_node_id=None,
-                 is_build_require=False, add_txt_generator=True, require_overrides=None):
+                 is_build_require=False, add_txt_generator=True, require_overrides=None,
+                 conanfile_path=None, test=None):
 
     """ Fetch and build all dependencies for the given reference
     @param app: The ConanApp instance with all collaborators
@@ -94,9 +95,14 @@ def deps_install(app, ref_or_path, install_folder, base_folder, graph_info, remo
                                      interactive=manifest_interactive)
         manifest_manager.print_log()
 
-    conanfile.folders.set_base_install(install_folder)
-    conanfile.folders.set_base_imports(install_folder)
-    conanfile.folders.set_base_generators(base_folder)
+    if hasattr(conanfile, "layout") and not test:
+        conanfile.folders.set_base_install(conanfile_path)
+        conanfile.folders.set_base_imports(conanfile_path)
+        conanfile.folders.set_base_generators(conanfile_path)
+    else:
+        conanfile.folders.set_base_install(install_folder)
+        conanfile.folders.set_base_imports(install_folder)
+        conanfile.folders.set_base_generators(base_folder)
 
     output = conanfile.output if root_node.recipe != RECIPE_VIRTUAL else out
 
diff --git a/conans/test/functional/layout/test_editable_cmake.py b/conans/test/functional/layout/test_editable_cmake.py
index 731046c5..a1444a19 100644
--- a/conans/test/functional/layout/test_editable_cmake.py
+++ b/conans/test/functional/layout/test_editable_cmake.py
@@ -22,10 +22,10 @@ def editable_cmake(generator):
            path=os.path.join(c.current_folder, "pkg"))
 
     def build_dep():
-        c.run("install .")
-        c.run("build .")
-        c.run("install . -s build_type=Debug")
-        c.run("build .")
+        c.run("install . -if=install_release")
+        c.run("build . -if=install_release")
+        c.run("install . -s build_type=Debug -if=install_debug")
+        c.run("build . -if=install_debug")
 
     with c.chdir("dep"):
         c.run("editable add . dep/0.1@")
@@ -94,10 +94,10 @@ def editable_cmake_exe(generator):
     c.save(pkg_cmake("dep", "0.1", exe=True), path=os.path.join(c.current_folder, "dep"))
 
     def build_dep():
-        c.run("install . -o dep:shared=True")
-        c.run("build .")
-        c.run("install . -s build_type=Debug -o dep:shared=True")
-        c.run("build .")
+        c.run("install . -o dep:shared=True -if=install_release")
+        c.run("build . -if=install_release")
+        c.run("install . -s build_type=Debug -o dep:shared=True -if=install_debug")
+        c.run("build . -if=install_debug")
 
     with c.chdir("dep"):
         c.run("editable add . dep/0.1@")
diff --git a/conans/test/functional/layout/test_local_commands.py b/conans/test/functional/layout/test_local_commands.py
index e27a20a4..acbfd603 100644
--- a/conans/test/functional/layout/test_local_commands.py
+++ b/conans/test/functional/layout/test_local_commands.py
@@ -1,8 +1,11 @@
 import os
+import platform
 import re
+import textwrap
 
 from conans.model.ref import ConanFileReference, PackageReference
 from conans.test.assets.genconanfile import GenConanfile
+from conans.test.assets.pkg_cmake import pkg_cmake
 from conans.test.utils.tools import TestClient
 
 
@@ -156,7 +159,8 @@ def test_local_build_change_base():
     client.save({"conanfile.py": conan_file})
     client.run("install . -if=common")
     client.run("build . -if=common -bf=common")
-    dll = os.path.join(client.current_folder, "common", "my_build", "build_file.dll")
+    # -bf is ignored here, the layout defines it
+    dll = os.path.join(client.current_folder, "my_build", "build_file.dll")
     assert os.path.exists(dll)
 
 
@@ -359,3 +363,59 @@ def test_imports():
     bfolder = client.cache.package_layout(ref).build(pref)
     imports_folder = os.path.join(bfolder, "my_imports")
     assert "WARN: Imports folder: {}".format(imports_folder) in client.out
+
+
+def test_start_dir_failure():
+    c = TestClient()
+    c.save(pkg_cmake("dep", "0.1"))
+    c.run("install .")
+    build = "build" if platform.system() == "Windows" else "cmake-build-release"
+    expected_path = os.path.join(c.current_folder, build, "conan", "conan_toolchain.cmake")
+    assert os.path.exists(expected_path)
+    os.unlink(expected_path)
+    with c.chdir("build"):
+        c.run("install ..")
+    assert os.path.exists(expected_path)
+
+
+def test_importdir_failure():
+    c = TestClient()
+    conanfile = textwrap.dedent("""
+        from conans import ConanFile
+        from conan.tools.files import save
+        class Cosumer(ConanFile):
+            requires = "dep/0.1"
+            settings = "build_type"
+            def layout(self):
+                self.folders.imports = "Import" + str(self.settings.build_type)
+                self.folders.build = "build"
+            def build(self):
+                self.output.info("saving file")
+                save(self, "mybuild.txt", "mybuild")
+            def imports(self):
+                self.copy("myfile.txt")
+        """)
+    c.save({"dep/conanfile.py": GenConanfile().with_package_file("myfile.txt", "mycontent"),
+            "consumer/conanfile.py": conanfile})
+    c.run("create dep dep/0.1@")
+    with c.chdir("consumer"):
+        c.run("install . -s build_type=Release")
+        expected_path_release = os.path.join(c.current_folder, "ImportRelease", "myfile.txt")
+        assert os.path.exists(expected_path_release)
+        c.run("build .")
+        expected_build_file = os.path.join(c.current_folder, "build", "mybuild.txt")
+        assert os.path.exists(expected_build_file)
+        c.run("install . -s build_type=Debug")
+        expected_path_debug = os.path.join(c.current_folder, "ImportDebug", "myfile.txt")
+        assert os.path.exists(expected_path_debug)
+
+        os.unlink(expected_path_release)
+        os.unlink(expected_path_debug)
+        os.unlink(expected_build_file)
+        with c.chdir("build"):
+            c.run("install .. -s build_type=Release")
+            assert os.path.exists(expected_path_release)
+            c.run("build ..")
+            assert os.path.exists(expected_build_file)
+            c.run("install .. -s build_type=Debug")
+            assert os.path.exists(expected_path_debug)
diff --git a/conans/test/functional/toolchains/cmake/test_v2_cmake_template.py b/conans/test/functional/toolchains/cmake/test_v2_cmake_template.py
index 85552f60..d84a04b9 100644
--- a/conans/test/functional/toolchains/cmake/test_v2_cmake_template.py
+++ b/conans/test/functional/toolchains/cmake/test_v2_cmake_template.py
@@ -1,3 +1,7 @@
+import os
+import re
+
+from conans.model.ref import ConanFileReference, PackageReference
 from conans.test.utils.tools import TestClient
 
 
@@ -7,6 +11,14 @@ def test_cmake_lib_template():
     # Local flow works
     client.run("install . -if=install")
     client.run("build . -if=install")
+    client.run("package . -if=install")
+    assert os.path.exists(os.path.join(client.current_folder, "package", "include", "hello.h"))
+
+    client.run("export-pkg . hello/0.1@ -if=install")
+    package_id = re.search(r"Packaging to (\S+)", str(client.out)).group(1)
+    pref = PackageReference(ConanFileReference.loads("hello/0.1"), package_id)
+    package_folder = client.cache.package_layout(pref.ref).package(pref)
+    assert os.path.exists(os.path.join(package_folder, "include", "hello.h"))
 
     # Create works
     client.run("create .")
diff --git a/conans/test/functional/toolchains/ios/test_using_cmake.py b/conans/test/functional/toolchains/ios/test_using_cmake.py
index 90f65062..7fb5e0d9 100644
--- a/conans/test/functional/toolchains/ios/test_using_cmake.py
+++ b/conans/test/functional/toolchains/ios/test_using_cmake.py
@@ -55,6 +55,7 @@ class ToolchainiOSTestCase(unittest.TestCase):
             """)
         })
 
+    @pytest.mark.tool_cmake(version="3.19")
     def test_xcode_generator(self):
         """ Simplest approach:
             https://cmake.org/cmake/help/latest/manual/cmake-toolchains.7.html#cross-compiling-for-ios-tvos-or-watchos
diff --git a/conans/test/functional/toolchains/microsoft/test_msbuild.py b/conans/test/functional/toolchains/microsoft/test_msbuild.py
index 4c9ff513..633d51a0 100644
--- a/conans/test/functional/toolchains/microsoft/test_msbuild.py
+++ b/conans/test/functional/toolchains/microsoft/test_msbuild.py
@@ -1,15 +1,11 @@
 import os
 import platform
-import shutil
 import textwrap
 import unittest
 
 import pytest
 from parameterized import parameterized, parameterized_class
 
-
-from conans.client.tools import chdir
-from conans.util.files import mkdir
 from conan.tools.microsoft.visual import vcvars_command
 from conans.client.tools import vs_installation_path
 from conans.test.assets.sources import gen_function_cpp
@@ -155,7 +151,7 @@ myapp_vcxproj = r"""<?xml version="1.0" encoding="utf-8"?>
     Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')"
     Label="LocalAppDataPlatform" />
   </ImportGroup>
-   <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='ReleaseShared|Win32'">
+   <ImportGroup Label="PropertySheets" Condition="'$(Configuration)|$(Platform)'=='ReleaseShared|x64'">
     <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props"
     Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')"
     Label="LocalAppDataPlatform" />
@@ -306,16 +302,19 @@ myapp_vcxproj = r"""<?xml version="1.0" encoding="utf-8"?>
 </Project>
 """
 
+
 @pytest.mark.tool_visual_studio(version='15')
 @pytest.mark.skipif(platform.system() != "Windows", reason="Only for windows")
 def test_msvc_runtime_flag_vs2017():
     check_msvc_runtime_flag("15", "19.1")
 
+
 @pytest.mark.tool_visual_studio(version='17')
 @pytest.mark.skipif(platform.system() != "Windows", reason="Only for windows")
 def test_msvc_runtime_flag_vs2022():
     check_msvc_runtime_flag("17", "19.3")
 
+
 def check_msvc_runtime_flag(vs_version, msvc_version):
     client = TestClient()
     conanfile = textwrap.dedent("""
@@ -341,11 +340,13 @@ def check_msvc_runtime_flag(vs_version, msvc_version):
                '-s compiler.cppstd=14'.format(msvc_version=msvc_version))
     assert "MSVC FLAG=MD!!" in client.out
 
+
 vs_versions = [{"vs_version": "15", "msvc_version": "19.1", "ide_year": "2017", "toolset": "v141"}]
 
 if "17" in tools_locations['visual_studio'] and not tools_locations['visual_studio']['17'].get('disabled', False):
     vs_versions.append({"vs_version": "17", "msvc_version": "19.3", "ide_year": "2022", "toolset": "v143"})
 
+
 @parameterized_class(vs_versions)
 @pytest.mark.skipif(platform.system() != "Windows", reason="Only for windows")
 @pytest.mark.tool_visual_studio
@@ -362,6 +363,7 @@ class WinTest(unittest.TestCase):
 
             def layout(self):
                 self.folders.generators = "conan"
+                self.folders.build = "."
 
             def generate(self):
                 tc = MSBuildToolchain(self)
@@ -387,11 +389,14 @@ class WinTest(unittest.TestCase):
             def imports(self):
                 if self.options["hello"].shared and self.settings.build_type == "Release":
                     configuration = "ReleaseShared"
+                    if self.settings.arch == "x86_64":
+                        dst = "x64/%s" % configuration
+                    else:
+                        dst = configuration
                 else:
                     configuration = self.settings.build_type
-                self.copy("*.dll", src="bin",
-                          dst="%s/%s" % (self.settings.arch, configuration),
-                          keep_path=False)
+                    dst = "%s/%s" % (self.settings.arch, configuration)
+                self.copy("*.dll", src="bin", dst=dst, keep_path=False)
 
             def build(self):
                 msbuild = MSBuild(self)
@@ -413,12 +418,7 @@ class WinTest(unittest.TestCase):
             command_str = "%s\\MyApp.exe" % configuration
         else:
             command_str = "x64\\%s\\MyApp.exe" % configuration
-        # To run the app without VS IDE, we need to copy the .exe to the DLLs folder
-        new_cmd = "conan\\%s\\%s\\MyApp.exe" % (arch, configuration)
-        with chdir(client.current_folder):
-            mkdir(os.path.dirname(new_cmd))
-            shutil.copy(command_str, new_cmd)
-        client.run_command(new_cmd)
+        client.run_command(command_str)
 
     @parameterized.expand([("Visual Studio", "15", "MT", "17"),
                            ("msvc", "19.1", "static", "17"),
@@ -588,9 +588,12 @@ class WinTest(unittest.TestCase):
                           {"DEFINITIONS_BOTH": "True",
                            "DEFINITIONS_CONFIG": build_type})
 
-            new_cmd = "conan\\%s\\%s\\MyApp.exe" % (arch, configuration)
+            if arch == "x86":
+                command_str = "%s\\MyApp.exe" % configuration
+            else:
+                command_str = "x64\\%s\\MyApp.exe" % configuration
             vcvars = vcvars_command(version=self.vs_version, architecture="amd64")
-            cmd = ('%s && dumpbin /dependents "%s"' % (vcvars, new_cmd))
+            cmd = ('%s && dumpbin /dependents "%s"' % (vcvars, command_str))
             client.run_command(cmd)
             if shared:
                 self.assertIn("hello.dll", client.out)