From b60e2923fc402ffbdea13267571ff60f0c0ab5fb Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Thu, 27 Feb 2025 15:33:46 -0600 Subject: [PATCH 1/3] Refactor static / dynamic linking into build options --- ci-targets.yaml | 24 ++++++++--------- cpython-unix/build-cpython.sh | 2 +- cpython-unix/build-main.py | 16 +++++++++-- cpython-unix/build.py | 51 ++++++++++++++++++++++++++--------- pythonbuild/cpython.py | 12 ++++----- 5 files changed, 72 insertions(+), 33 deletions(-) diff --git a/ci-targets.yaml b/ci-targets.yaml index ec08bd9a..4979ff68 100644 --- a/ci-targets.yaml +++ b/ci-targets.yaml @@ -257,9 +257,9 @@ linux: - "3.12" - "3.13" build_options: - - debug - - noopt - - lto + - debug+static + - noopt+static + - lto+static run: true x86_64_v2-unknown-linux-musl: @@ -273,9 +273,9 @@ linux: - "3.12" - "3.13" build_options: - - debug - - noopt - - lto + - debug+static + - noopt+static + - lto+static run: true x86_64_v3-unknown-linux-musl: @@ -289,9 +289,9 @@ linux: - "3.12" - "3.13" build_options: - - debug - - noopt - - lto + - debug+static + - noopt+static + - lto+static run: true x86_64_v4-unknown-linux-musl: @@ -305,9 +305,9 @@ linux: - "3.12" - "3.13" build_options: - - debug - - noopt - - lto + - debug+static + - noopt+static + - lto+static run: true windows: diff --git a/cpython-unix/build-cpython.sh b/cpython-unix/build-cpython.sh index ea1b1c0a..d58cbdc5 100755 --- a/cpython-unix/build-cpython.sh +++ b/cpython-unix/build-cpython.sh @@ -381,7 +381,7 @@ CONFIGURE_FLAGS=" --without-ensurepip ${EXTRA_CONFIGURE_FLAGS}" -if [ "${CC}" = "musl-clang" ]; then +if [ -n "${CPYTHON_STATIC}" ]; then CFLAGS="${CFLAGS} -static" CPPFLAGS="${CPPFLAGS} -static" LDFLAGS="${LDFLAGS} -static" diff --git a/cpython-unix/build-main.py b/cpython-unix/build-main.py index dd4d2d7c..743a5bb3 100755 --- a/cpython-unix/build-main.py +++ b/cpython-unix/build-main.py @@ -45,6 +45,7 @@ def main(): print("Unsupported build platform: %s" % sys.platform) return 1 + # Note these arguments must be synced with `build.py` parser = argparse.ArgumentParser() parser.add_argument( @@ -54,10 +55,17 @@ def main(): help="Target host triple to build for", ) - optimizations = {"debug", "noopt", "pgo", "lto", "pgo+lto"} + # Construct possible options, we use a set here for canonical ordering + options = set() + options.update({"debug", "noopt", "pgo", "lto", "pgo+lto"}) + options.update({f"freethreaded+{option}" for option in options}) + link_modes = {"static", "shared"} + options.update( + {f"{option}+{link_mode}" for link_mode in link_modes for option in options} + ) parser.add_argument( "--options", - choices=optimizations.union({f"freethreaded+{o}" for o in optimizations}), + choices=options, default="noopt", help="Build options to apply when compiling Python", ) @@ -138,6 +146,10 @@ def main(): env = dict(os.environ) + # Default to dynamic linking if no link mode is specified + if not any(link_mode in args.options for link_mode in link_modes): + args.options += "+shared" + env["PYBUILD_HOST_PLATFORM"] = host_platform env["PYBUILD_TARGET_TRIPLE"] = target_triple env["PYBUILD_BUILD_OPTIONS"] = args.options diff --git a/cpython-unix/build.py b/cpython-unix/build.py index da66ba4e..bba204fd 100755 --- a/cpython-unix/build.py +++ b/cpython-unix/build.py @@ -488,6 +488,7 @@ def python_build_info( target_triple, musl, lto, + static, extensions, extra_metadata, ): @@ -506,7 +507,7 @@ def python_build_info( ) ) - if not musl: + if not static: bi["core"]["shared_lib"] = "install/lib/libpython%s%s.so.1.0" % ( version, binary_suffix, @@ -735,6 +736,7 @@ def build_cpython( python_archive, python_version=python_version, target_triple=target_triple, + build_options=parsed_build_options, extension_modules=ems, ) @@ -825,6 +827,8 @@ def build_cpython( env["CPYTHON_OPTIMIZED"] = "1" if "lto" in parsed_build_options: env["CPYTHON_LTO"] = "1" + if "static" in parsed_build_options: + env["CPYTHON_STATIC"] = "1" add_target_env(env, host_platform, target_triple, build_env) @@ -834,19 +838,26 @@ def build_cpython( crt_features = [] if host_platform == "linux64": - if "musl" in target_triple: + if "static" in parsed_build_options: crt_features.append("static") else: extension_module_loading.append("shared-library") - crt_features.append("glibc-dynamic") - glibc_max_version = build_env.get_file("glibc_version.txt").strip() - if not glibc_max_version: - raise Exception("failed to retrieve glibc max symbol version") + if "musl" in target_triple: + crt_features.append("musl-dynamic") + # TODO: Determine the dynamic musl libc version - crt_features.append( - "glibc-max-symbol-version:%s" % glibc_max_version.decode("ascii") - ) + else: + crt_features.append("glibc-dynamic") + + glibc_max_version = build_env.get_file("glibc_version.txt").strip() + if not glibc_max_version: + raise Exception("failed to retrieve glibc max symbol version") + + crt_features.append( + "glibc-max-symbol-version:%s" + % glibc_max_version.decode("ascii") + ) python_symbol_visibility = "global-default" @@ -874,7 +885,9 @@ def build_cpython( "python_stdlib_test_packages": sorted(STDLIB_TEST_PACKAGES), "python_symbol_visibility": python_symbol_visibility, "python_extension_module_loading": extension_module_loading, - "libpython_link_mode": "static" if "musl" in target_triple else "shared", + "libpython_link_mode": ( + "static" if "static" in parsed_build_options else "shared" + ), "crt_features": crt_features, "run_tests": "build/run_tests.py", "build_info": python_build_info( @@ -884,6 +897,7 @@ def build_cpython( target_triple, "musl" in target_triple, "lto" in parsed_build_options, + "static" in parsed_build_options, enabled_extensions, extra_metadata, ), @@ -946,6 +960,7 @@ def main(): print("unable to connect to Docker: %s" % e, file=sys.stderr) return 1 + # Note these arguments must be synced with `build-main.py` parser = argparse.ArgumentParser() parser.add_argument( "--host-platform", required=True, help="Platform we are building from" @@ -955,13 +970,25 @@ def main(): required=True, help="Host triple that we are building Python for", ) - optimizations = {"debug", "noopt", "pgo", "lto", "pgo+lto"} + + # Construct possible options + options = set() + options.update({"debug", "noopt", "pgo", "lto", "pgo+lto"}) + options.update({f"freethreaded+{option}" for option in options}) + options.update( + { + f"{option}+{link_mode}" + for link_mode in {"static", "shared"} + for option in options + } + ) parser.add_argument( "--options", - choices=optimizations.union({f"freethreaded+{o}" for o in optimizations}), + choices=options, default="noopt", help="Build options to apply when compiling Python", ) + parser.add_argument( "--toolchain", action="store_true", diff --git a/pythonbuild/cpython.py b/pythonbuild/cpython.py index 50ad6f70..e59eecd3 100644 --- a/pythonbuild/cpython.py +++ b/pythonbuild/cpython.py @@ -227,6 +227,7 @@ def derive_setup_local( cpython_source_archive, python_version, target_triple, + build_options, extension_modules, ): """Derive the content of the Modules/Setup.local file.""" @@ -466,12 +467,11 @@ def derive_setup_local( enabled_extensions[name]["setup_line"] = name.encode("ascii") continue - # musl is static only. Ignore build-mode override. - if "musl" in target_triple: - section = "static" - else: - section = info.get("build-mode", "static") - + # Force static linking if we're doing a fully static build, otherwise, + # respect the `build-mode` falling back to `static` if not defined. + section = ( + "static" if "static" in build_options else info.get("build-mode", "static") + ) enabled_extensions[name]["build-mode"] = section # Presumably this means the extension comes from the distribution's From 61bc835b0377081cd0f165ce077e6e1614a01821 Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Wed, 5 Mar 2025 15:13:25 -0600 Subject: [PATCH 2/3] Remove the `+shared` build option; enable implicitly --- cpython-unix/build-main.py | 9 +-------- cpython-unix/build.py | 8 +------- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/cpython-unix/build-main.py b/cpython-unix/build-main.py index 743a5bb3..6bf91b49 100755 --- a/cpython-unix/build-main.py +++ b/cpython-unix/build-main.py @@ -59,10 +59,7 @@ def main(): options = set() options.update({"debug", "noopt", "pgo", "lto", "pgo+lto"}) options.update({f"freethreaded+{option}" for option in options}) - link_modes = {"static", "shared"} - options.update( - {f"{option}+{link_mode}" for link_mode in link_modes for option in options} - ) + options.update({f"{option}+static" for option in options}) parser.add_argument( "--options", choices=options, @@ -146,10 +143,6 @@ def main(): env = dict(os.environ) - # Default to dynamic linking if no link mode is specified - if not any(link_mode in args.options for link_mode in link_modes): - args.options += "+shared" - env["PYBUILD_HOST_PLATFORM"] = host_platform env["PYBUILD_TARGET_TRIPLE"] = target_triple env["PYBUILD_BUILD_OPTIONS"] = args.options diff --git a/cpython-unix/build.py b/cpython-unix/build.py index bba204fd..12aa47f4 100755 --- a/cpython-unix/build.py +++ b/cpython-unix/build.py @@ -975,13 +975,7 @@ def main(): options = set() options.update({"debug", "noopt", "pgo", "lto", "pgo+lto"}) options.update({f"freethreaded+{option}" for option in options}) - options.update( - { - f"{option}+{link_mode}" - for link_mode in {"static", "shared"} - for option in options - } - ) + options.update({f"{option}+static" for option in options}) parser.add_argument( "--options", choices=options, From 1e5c7d70eab74503ec53a5e16396adb39eedcc8d Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Wed, 5 Mar 2025 15:17:14 -0600 Subject: [PATCH 3/3] Update release triples --- src/release.rs | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/release.rs b/src/release.rs index f0c70f62..ed9fe2fb 100644 --- a/src/release.rs +++ b/src/release.rs @@ -166,6 +166,7 @@ pub static RELEASE_TRIPLES: Lazy> = Lazy:: "freethreaded+lto", "freethreaded+noopt", ]; + let linux_suffixes_nopgo_static = vec!["debug+static", "lto+static", "noopt+static"]; h.insert( "aarch64-unknown-linux-gnu", @@ -297,8 +298,8 @@ pub static RELEASE_TRIPLES: Lazy> = Lazy:: h.insert( "x86_64-unknown-linux-musl", TripleRelease { - suffixes: linux_suffixes_nopgo.clone(), - install_only_suffix: "lto", + suffixes: linux_suffixes_nopgo_static.clone(), + install_only_suffix: "lto+static", python_version_requirement: Some(VersionSpecifier::from_str("<3.14").unwrap()), conditional_suffixes: vec![], }, @@ -306,8 +307,8 @@ pub static RELEASE_TRIPLES: Lazy> = Lazy:: h.insert( "x86_64_v2-unknown-linux-musl", TripleRelease { - suffixes: linux_suffixes_nopgo.clone(), - install_only_suffix: "lto", + suffixes: linux_suffixes_nopgo_static.clone(), + install_only_suffix: "lto+static", python_version_requirement: Some(VersionSpecifier::from_str("<3.14").unwrap()), conditional_suffixes: vec![], }, @@ -315,8 +316,8 @@ pub static RELEASE_TRIPLES: Lazy> = Lazy:: h.insert( "x86_64_v3-unknown-linux-musl", TripleRelease { - suffixes: linux_suffixes_nopgo.clone(), - install_only_suffix: "lto", + suffixes: linux_suffixes_nopgo_static.clone(), + install_only_suffix: "lto+static", python_version_requirement: Some(VersionSpecifier::from_str("<3.14").unwrap()), conditional_suffixes: vec![], }, @@ -324,8 +325,8 @@ pub static RELEASE_TRIPLES: Lazy> = Lazy:: h.insert( "x86_64_v4-unknown-linux-musl", TripleRelease { - suffixes: linux_suffixes_nopgo.clone(), - install_only_suffix: "lto", + suffixes: linux_suffixes_nopgo_static.clone(), + install_only_suffix: "lto+static", python_version_requirement: Some(VersionSpecifier::from_str("<3.14").unwrap()), conditional_suffixes: vec![], },