Define EmbeddedPython_EXECUTABLE CMake variable

Previously, there was no way to access the Python executable that was
aware of the full environment with `pip` packages. See the comment in
`embedde_python.cmake` for more details about the core/full split.

Author: dean0x7d

changelog.md

@@ -1,10 +1,11 @@
 # Changelog
 
-## v1.8.0 | In development
+## v1.8.0 | 2023-07-12
 
 - Added support for building with `openssl` v3 for Python 3.10 and newer.
 - `openssl` v3.0.8 is now the default for Python >= 3.10 and `openssl` v1.1.1u is the default for Python 3.9.
 - The `openssl_variant` is now obsolete. The flag is still there for backwards compatibility but it no longer does anything.
+- The `EmbeddedPython_EXECUTABLE` CMake variable is now defined and points to the executable that's aware of the full embedded Python environment with `pip` packages. `Python_EXECUTABLE` points to the `core` executable that's only aware of the standard library. This split ensures that we compile and link against `core` in order to avoid rebuilding the world when only the Python environment packages change but the version/headers/libs stay the same.  
 
 ## v1.7.0 | 2023-06-22
 

conanfile.py

@@ -35,6 +35,7 @@ class EmbeddedPython(ConanFile):
         "openssl_variant": "lowercase",
     }
     short_paths = True  # some of the pip packages go over the 260 char path limit on Windows
+    exports_sources = "embedded_python.cmake"
 
     def requirements(self):
         self.requires(f"embedded_python-core/1.2.0@{self.user}/{self.channel}")
@@ -172,6 +173,7 @@ def build(self):
         self._build_bootstrap()
 
     def package(self):
+        files.copy(self, "embedded_python.cmake", src=self.build_folder, dst=self.package_folder)
         files.copy(self, "embedded_python*", src=self.core_pkg, dst=self.package_folder)
         prefix = pathlib.Path(self.package_folder, "embedded_python")
         if self.settings.os == "Windows":

core/conanfile.py

@@ -26,7 +26,7 @@ class EmbeddedPythonCore(ConanFile):
     default_options = {
         "zip_stdlib": "stored",
     }
-    exports_sources = "embedded_python_tools.py", "embedded_python.cmake"
+    exports_sources = "embedded_python_tools.py", "embedded_python-core.cmake"
 
     def validate(self):
         minimum_python = "3.9.8"
@@ -87,7 +87,7 @@ def int_pyversion(self):
 
     def generate(self):
         files.replace_in_file(
-            self, "embedded_python.cmake", "${self.pyversion}", str(self.pyversion)
+            self, "embedded_python-core.cmake", "${self.pyversion}", str(self.pyversion)
         )
         if self.settings.os == "Windows":
             return
@@ -236,7 +236,7 @@ def is_landmark(filepath):
     def package(self):
         src = self.build_folder
         dst = pathlib.Path(self.package_folder, "embedded_python")
-        files.copy(self, "embedded_python.cmake", src, dst=self.package_folder)
+        files.copy(self, "embedded_python-core.cmake", src, dst=self.package_folder)
         files.copy(self, "embedded_python_tools.py", src, dst=self.package_folder)
         license_folder = pathlib.Path(self.package_folder, "licenses")
 
@@ -278,8 +278,8 @@ def package(self):
 
     def package_info(self):
         self.env_info.PYTHONPATH.append(self.package_folder)
-        self.cpp_info.set_property("cmake_build_modules", ["embedded_python.cmake"])
-        self.cpp_info.build_modules = ["embedded_python.cmake"]
+        self.cpp_info.set_property("cmake_build_modules", ["embedded_python-core.cmake"])
+        self.cpp_info.build_modules = ["embedded_python-core.cmake"]
         prefix = pathlib.Path(self.package_folder) / "embedded_python"
         self.cpp_info.includedirs = [str(prefix / "include")]
         if self.settings.os == "Windows":

core/embedded_python.cmake core/embedded_python-core.cmake

@@ -3,6 +3,8 @@ set(Python_ROOT_DIR "${CMAKE_CURRENT_LIST_DIR}/embedded_python")
 
 if(WIN32) # Extra hint to speed up the find (not needed for correctness)
     set(Python_EXECUTABLE "${Python_ROOT_DIR}/python.exe")
+else()
+    set(Python_EXECUTABLE "${Python_ROOT_DIR}/bin/python3")
 endif()
 
 find_package(Python ${self.pyversion} EXACT REQUIRED COMPONENTS Interpreter Development)

core/test_package/CMakeLists.txt

@@ -3,8 +3,9 @@ project(test_package)
 
 find_package(embedded_python-core)
 
-if(NOT Python_EXECUTABLE MATCHES "embedded_python")
-    message(FATAL_ERROR "`cmake` failed to find the correct Python")
+message(STATUS "Python_EXECUTABLE: ${Python_EXECUTABLE}")
+if(NOT Python_EXECUTABLE MATCHES "embedded_python-core")
+    message(FATAL_ERROR "CMake failed to find the correct Python")
 endif()
 
 add_executable(test_package src/main.cpp)

embedded_python.cmake

@@ -0,0 +1,17 @@
+# There is one important thing we want to achieve with the `embedded_python`/`embedded_python-core`
+# split: we want to avoid recompiling the world when only the Python environment packages change
+# but the version/headers/libs stay the same. To do that we must ensure that everything is built
+# against `embedded_python-core` while the full `embedded_python` is provided for runtime.
+#
+# `embedded_python-core.cmake` calls `find_package(Python Interpreter Development)` which provides
+# `Python_EXECUTABLE` plus the includes, libs, and other variables needed to build against Python.
+# This means that `Python_EXECUTABLE` points to an executable that's only aware of the standard
+# library. On top of that, `embedded_python.cmake` adds `EmbeddedPython_EXECUTABLE` which is aware
+# of the full environment with `pip` packages. Note that we do no provide any include or lib dirs
+# since those are already provided by `core`.
+
+if(WIN32)
+    set(EmbeddedPython_EXECUTABLE "${CMAKE_CURRENT_LIST_DIR}/embedded_python/python.exe")
+else()
+    set(EmbeddedPython_EXECUTABLE "${CMAKE_CURRENT_LIST_DIR}/embedded_python/bin/python3")
+endif()

test_package/CMakeLists.txt

@@ -3,8 +3,16 @@ project(test_package)
 
 find_package(embedded_python)
 
-if(NOT Python_EXECUTABLE MATCHES "embedded_python")
-    message(FATAL_ERROR "`cmake` failed to find the correct Python")
+message(STATUS "Python_EXECUTABLE: ${Python_EXECUTABLE}")
+if(NOT Python_EXECUTABLE MATCHES "embedded_python-core")
+    message(FATAL_ERROR "CMake failed to find the correct Python")
+endif()
+message(STATUS "EmbeddedPython_EXECUTABLE: ${EmbeddedPython_EXECUTABLE}")
+if(EmbeddedPython_EXECUTABLE MATCHES "embedded_python-core")
+    message(FATAL_ERROR "CMake found the `-core` package instead of the full environment")
+endif()
+if(NOT EmbeddedPython_EXECUTABLE MATCHES "embedded_python")
+    message(FATAL_ERROR "CMake failed to find the correct Python")
 endif()
 
 add_executable(test_package ../core/test_package/src/main.cpp)