Skip to content

Commit ba35cc4

Browse files
tirantiran
committed
bpo-45950: Bootstrap Python again
The build system now uses a `_bootstrap_python` interpreter for freezing and deepfreezing again. Cross building depends on a build Python interpreter. `_bootstrap_python` and `_freeze_module` are no longer compiled with LTO. Link time optimization is very slow.
1 parent ccb73a0 commit ba35cc4

9 files changed

+185
-71
lines changed

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ Lib/distutils/command/*.pdb
6060
Lib/lib2to3/*.pickle
6161
Lib/test/data/*
6262
!Lib/test/data/README
63+
/_bootstrap_python
6364
/Makefile
6465
/Makefile.pre
6566
Mac/Makefile

‎Makefile.pre.in

+42-21
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,9 @@ CONFIGURE_CFLAGS_NODIST=@CFLAGS_NODIST@
9090
# Use it when a linker flag should _not_ be part of the distutils LDFLAGS
9191
# once Python is installed (bpo-35257)
9292
CONFIGURE_LDFLAGS_NODIST=@LDFLAGS_NODIST@
93+
# LDFLAGS_NOLTO is an extra flag to disable lto. It is used to speed up building
94+
# of _bootstrap_python and _freeze_module tools, which don't need LTO.
95+
CONFIGURE_LDFLAGS_NOLTO=@LDFLAGS_NOLTO@
9396
CONFIGURE_CPPFLAGS= @CPPFLAGS@
9497
CONFIGURE_LDFLAGS= @LDFLAGS@
9598
# Avoid assigning CFLAGS, LDFLAGS, etc. so users can use them on the
@@ -103,6 +106,7 @@ PY_CFLAGS_NODIST=$(CONFIGURE_CFLAGS_NODIST) $(CFLAGS_NODIST) -I$(srcdir)/Include
103106
PY_CPPFLAGS= $(BASECPPFLAGS) -I. -I$(srcdir)/Include $(CONFIGURE_CPPFLAGS) $(CPPFLAGS)
104107
PY_LDFLAGS= $(CONFIGURE_LDFLAGS) $(LDFLAGS)
105108
PY_LDFLAGS_NODIST=$(CONFIGURE_LDFLAGS_NODIST) $(LDFLAGS_NODIST)
109+
PY_LDFLAGS_NOLTO=$(PY_LDFLAGS) $(CONFIGURE_LDFLAGS_NOLTO) $(LDFLAGS_NODIST)
106110
NO_AS_NEEDED= @NO_AS_NEEDED@
107111
CCSHARED= @CCSHARED@
108112
# LINKFORSHARED are the flags passed to the $(CC) command that links
@@ -279,6 +283,9 @@ BUILDPYTHON= python$(BUILDEXE)
279283
PYTHON_FOR_REGEN?=@PYTHON_FOR_REGEN@
280284
UPDATE_FILE=$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/update_file.py
281285
PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@
286+
# Standard builds use _bootstrap_python for freezing, cross compiling
287+
# uses build Python, which must have the same version and bytecode,
288+
PYTHON_FOR_FREEZE?=@PYTHON_FOR_FREEZE@
282289
_PYTHON_HOST_PLATFORM=@_PYTHON_HOST_PLATFORM@
283290
BUILD_GNU_TYPE= @build@
284291
HOST_GNU_TYPE= @host@
@@ -938,75 +945,88 @@ regen-test-frozenmain: $(BUILDPYTHON)
938945
Programs/_testembed: Programs/_testembed.o $(LIBRARY_DEPS)
939946
$(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS)
940947

948+
############################################################################
949+
# "Bootstrap Python" used to run deepfreeze.py
950+
951+
BOOTSTRAP_HEADERS = \
952+
Python/frozen_modules/importlib._bootstrap.h \
953+
Python/frozen_modules/importlib._bootstrap_external.h
954+
955+
Programs/_bootstrap_python.o: Programs/_bootstrap_python.c $(BOOTSTRAP_HEADERS) $(PYTHON_HEADERS)
956+
957+
_bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modules/getpath.o Modules/Setup.local
958+
$(LINKCC) $(PY_LDFLAGS_NOLTO) -o $@ $(LIBRARY_OBJS_OMIT_FROZEN) \
959+
Programs/_bootstrap_python.o Modules/getpath.o $(LIBS) $(MODLIBS) $(SYSLIBS)
960+
941961
############################################################################
942962
# Deepfreeze targets
943963

944964
.PHONY: regen-deepfreeze
945965
regen-deepfreeze: $(DEEPFREEZE_OBJS)
946966

947-
DEEPFREEZE_DEPS=$(srcdir)/Tools/scripts/deepfreeze.py
967+
DEEPFREEZE_DEPS=$(srcdir)/Tools/scripts/deepfreeze.py _bootstrap_python
948968

949969
# BEGIN: deepfreeze modules
950970

951971
Python/deepfreeze/importlib._bootstrap.c: Python/frozen_modules/importlib._bootstrap.h $(DEEPFREEZE_DEPS)
952-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/importlib._bootstrap.h -m importlib._bootstrap -o Python/deepfreeze/importlib._bootstrap.c
972+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/importlib._bootstrap.h -m importlib._bootstrap -o Python/deepfreeze/importlib._bootstrap.c
953973

954974
Python/deepfreeze/importlib._bootstrap_external.c: Python/frozen_modules/importlib._bootstrap_external.h $(DEEPFREEZE_DEPS)
955-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/importlib._bootstrap_external.h -m importlib._bootstrap_external -o Python/deepfreeze/importlib._bootstrap_external.c
975+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/importlib._bootstrap_external.h -m importlib._bootstrap_external -o Python/deepfreeze/importlib._bootstrap_external.c
956976

957977
Python/deepfreeze/zipimport.c: Python/frozen_modules/zipimport.h $(DEEPFREEZE_DEPS)
958-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/zipimport.h -m zipimport -o Python/deepfreeze/zipimport.c
978+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/zipimport.h -m zipimport -o Python/deepfreeze/zipimport.c
959979

960980
Python/deepfreeze/abc.c: Python/frozen_modules/abc.h $(DEEPFREEZE_DEPS)
961-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/abc.h -m abc -o Python/deepfreeze/abc.c
981+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/abc.h -m abc -o Python/deepfreeze/abc.c
962982

963983
Python/deepfreeze/codecs.c: Python/frozen_modules/codecs.h $(DEEPFREEZE_DEPS)
964-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/codecs.h -m codecs -o Python/deepfreeze/codecs.c
984+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/codecs.h -m codecs -o Python/deepfreeze/codecs.c
965985

966986
Python/deepfreeze/io.c: Python/frozen_modules/io.h $(DEEPFREEZE_DEPS)
967-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/io.h -m io -o Python/deepfreeze/io.c
987+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/io.h -m io -o Python/deepfreeze/io.c
968988

969989
Python/deepfreeze/_collections_abc.c: Python/frozen_modules/_collections_abc.h $(DEEPFREEZE_DEPS)
970-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/_collections_abc.h -m _collections_abc -o Python/deepfreeze/_collections_abc.c
990+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/_collections_abc.h -m _collections_abc -o Python/deepfreeze/_collections_abc.c
971991

972992
Python/deepfreeze/_sitebuiltins.c: Python/frozen_modules/_sitebuiltins.h $(DEEPFREEZE_DEPS)
973-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/_sitebuiltins.h -m _sitebuiltins -o Python/deepfreeze/_sitebuiltins.c
993+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/_sitebuiltins.h -m _sitebuiltins -o Python/deepfreeze/_sitebuiltins.c
974994

975995
Python/deepfreeze/genericpath.c: Python/frozen_modules/genericpath.h $(DEEPFREEZE_DEPS)
976-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/genericpath.h -m genericpath -o Python/deepfreeze/genericpath.c
996+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/genericpath.h -m genericpath -o Python/deepfreeze/genericpath.c
977997

978998
Python/deepfreeze/ntpath.c: Python/frozen_modules/ntpath.h $(DEEPFREEZE_DEPS)
979-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/ntpath.h -m ntpath -o Python/deepfreeze/ntpath.c
999+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/ntpath.h -m ntpath -o Python/deepfreeze/ntpath.c
9801000

9811001
Python/deepfreeze/posixpath.c: Python/frozen_modules/posixpath.h $(DEEPFREEZE_DEPS)
982-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/posixpath.h -m posixpath -o Python/deepfreeze/posixpath.c
1002+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/posixpath.h -m posixpath -o Python/deepfreeze/posixpath.c
9831003

9841004
Python/deepfreeze/os.c: Python/frozen_modules/os.h $(DEEPFREEZE_DEPS)
985-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/os.h -m os -o Python/deepfreeze/os.c
1005+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/os.h -m os -o Python/deepfreeze/os.c
9861006

9871007
Python/deepfreeze/site.c: Python/frozen_modules/site.h $(DEEPFREEZE_DEPS)
988-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/site.h -m site -o Python/deepfreeze/site.c
1008+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/site.h -m site -o Python/deepfreeze/site.c
9891009

9901010
Python/deepfreeze/stat.c: Python/frozen_modules/stat.h $(DEEPFREEZE_DEPS)
991-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/stat.h -m stat -o Python/deepfreeze/stat.c
1011+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/stat.h -m stat -o Python/deepfreeze/stat.c
9921012

9931013
Python/deepfreeze/__hello__.c: Python/frozen_modules/__hello__.h $(DEEPFREEZE_DEPS)
994-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__hello__.h -m __hello__ -o Python/deepfreeze/__hello__.c
1014+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__hello__.h -m __hello__ -o Python/deepfreeze/__hello__.c
9951015

9961016
Python/deepfreeze/__phello__.c: Python/frozen_modules/__phello__.h $(DEEPFREEZE_DEPS)
997-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.h -m __phello__ -o Python/deepfreeze/__phello__.c
1017+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.h -m __phello__ -o Python/deepfreeze/__phello__.c
9981018

9991019
Python/deepfreeze/__phello__.ham.c: Python/frozen_modules/__phello__.ham.h $(DEEPFREEZE_DEPS)
1000-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.ham.h -m __phello__.ham -o Python/deepfreeze/__phello__.ham.c
1020+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.ham.h -m __phello__.ham -o Python/deepfreeze/__phello__.ham.c
10011021

10021022
Python/deepfreeze/__phello__.ham.eggs.c: Python/frozen_modules/__phello__.ham.eggs.h $(DEEPFREEZE_DEPS)
1003-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.ham.eggs.h -m __phello__.ham.eggs -o Python/deepfreeze/__phello__.ham.eggs.c
1023+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.ham.eggs.h -m __phello__.ham.eggs -o Python/deepfreeze/__phello__.ham.eggs.c
10041024

10051025
Python/deepfreeze/__phello__.spam.c: Python/frozen_modules/__phello__.spam.h $(DEEPFREEZE_DEPS)
1006-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.spam.h -m __phello__.spam -o Python/deepfreeze/__phello__.spam.c
1026+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/__phello__.spam.h -m __phello__.spam -o Python/deepfreeze/__phello__.spam.c
10071027

10081028
Python/deepfreeze/frozen_only.c: Python/frozen_modules/frozen_only.h $(DEEPFREEZE_DEPS)
1009-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/frozen_only.h -m frozen_only -o Python/deepfreeze/frozen_only.c
1029+
$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/frozen_only.h -m frozen_only -o Python/deepfreeze/frozen_only.c
10101030

10111031
# END: deepfreeze modules
10121032

@@ -2295,6 +2315,7 @@ clean-retain-profile: pycremoval
22952315
find build -name '*.py[co]' -exec rm -f {} ';' || true
22962316
-rm -f pybuilddir.txt
22972317
-rm -f Lib/lib2to3/*Grammar*.pickle
2318+
-rm -f _bootstrap_python
22982319
-rm -f Programs/_testembed Programs/_freeze_module
22992320
-rm -f Python/deepfreeze/*.[co]
23002321
-rm -f Python/frozen_modules/*.h

‎Misc/NEWS.d/next/Build/2021-12-01-17-28-39.bpo-45950.eEVLoz.rst

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
The build system now uses a :program:`_bootstrap_python` interpreter for
2+
freezing and deepfreezing again. To speed up build process the build tools
3+
:program:`_bootstrap_python` and :program:`_freeze_module` are no longer
4+
build with LTO.

‎Programs/_bootstrap_python.c

+105
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
2+
/* Frozen modules bootstrap
3+
*
4+
* Limited and restricted Python interpreter to run
5+
* "Tools/scripts/deepfreeze.py" on systems with no or older Python
6+
* interpreter.
7+
*/
8+
9+
#include "Python.h"
10+
#include "pycore_import.h"
11+
12+
/* Includes for frozen modules: */
13+
#include "Python/frozen_modules/importlib._bootstrap.h"
14+
#include "Python/frozen_modules/importlib._bootstrap_external.h"
15+
/* End includes */
16+
17+
/* Note that a negative size indicates a package. */
18+
19+
static const struct _frozen bootstrap_modules[] = {
20+
{"_frozen_importlib", _Py_M__importlib__bootstrap, (int)sizeof(_Py_M__importlib__bootstrap)},
21+
{"_frozen_importlib_external", _Py_M__importlib__bootstrap_external, (int)sizeof(_Py_M__importlib__bootstrap_external)},
22+
{0, 0, 0} /* bootstrap sentinel */
23+
};
24+
static const struct _frozen stdlib_modules[] = {
25+
{0, 0, 0} /* stdlib sentinel */
26+
};
27+
static const struct _frozen test_modules[] = {
28+
{0, 0, 0} /* test sentinel */
29+
};
30+
const struct _frozen *_PyImport_FrozenBootstrap = bootstrap_modules;
31+
const struct _frozen *_PyImport_FrozenStdlib = stdlib_modules;
32+
const struct _frozen *_PyImport_FrozenTest = test_modules;
33+
34+
static const struct _module_alias aliases[] = {
35+
{"_frozen_importlib", "importlib._bootstrap"},
36+
{"_frozen_importlib_external", "importlib._bootstrap_external"},
37+
{0, 0} /* aliases sentinel */
38+
};
39+
const struct _module_alias *_PyImport_FrozenAliases = aliases;
40+
41+
/* Embedding apps may change this pointer to point to their favorite
42+
collection of frozen modules: */
43+
44+
const struct _frozen *PyImport_FrozenModules = NULL;
45+
46+
int
47+
#ifdef MS_WINDOWS
48+
wmain(int argc, wchar_t **argv)
49+
#else
50+
main(int argc, char **argv)
51+
#endif
52+
{
53+
PyStatus status;
54+
55+
PyConfig config;
56+
PyConfig_InitIsolatedConfig(&config);
57+
// don't warn, pybuilddir.txt does not exist yet
58+
config.pathconfig_warnings = 0;
59+
// parse arguments
60+
config.parse_argv = 1;
61+
// add current script dir to sys.path
62+
config.isolated = 0;
63+
64+
#ifdef MS_WINDOWS
65+
status = PyConfig_SetArgv(&config, argc, argv);
66+
#else
67+
status = PyConfig_SetBytesArgv(&config, argc, argv);
68+
#endif
69+
if (PyStatus_Exception(status)) {
70+
goto error;
71+
}
72+
73+
status = PyConfig_Read(&config);
74+
if (config.run_filename == NULL) {
75+
status = PyStatus_Error("Run filename expected");
76+
goto error;
77+
}
78+
79+
#define CLEAR(ATTR) \
80+
do { \
81+
PyMem_RawFree(ATTR); \
82+
ATTR = NULL; \
83+
} while (0)
84+
85+
// isolate from system Python
86+
CLEAR(config.base_prefix);
87+
CLEAR(config.prefix);
88+
CLEAR(config.base_exec_prefix);
89+
CLEAR(config.exec_prefix);
90+
91+
status = Py_InitializeFromConfig(&config);
92+
if (PyStatus_Exception(status)) {
93+
goto error;
94+
}
95+
PyConfig_Clear(&config);
96+
97+
return Py_RunMain();
98+
99+
error:
100+
PyConfig_Clear(&config);
101+
if (PyStatus_IsExit(status)) {
102+
return status.exitcode;
103+
}
104+
Py_ExitStatusException(status);
105+
}

‎Python/bootstrap_frozen.c

-45
This file was deleted.

‎Tools/scripts/deepfreeze.py

+5-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
1+
"""Deep freeze
2+
3+
The script is executed by _bootstrap_python interpreter. Shared library
4+
extension modules are not available.
5+
"""
16
import argparse
27
import ast
38
import builtins
@@ -8,7 +13,6 @@
813
import sys
914
import time
1015
import types
11-
import unicodedata
1216
from typing import Dict, FrozenSet, Tuple, TextIO
1317

1418
import umarshal

‎Tools/scripts/freeze_modules.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -589,7 +589,7 @@ def regen_makefile(modules):
589589
])
590590
deepfreezerules.append(f'{cfile}: {frozen_header} $(DEEPFREEZE_DEPS)')
591591
deepfreezerules.append(
592-
f"\t$(PYTHON_FOR_REGEN) "
592+
f"\t$(PYTHON_FOR_FREEZE) "
593593
f"$(srcdir)/Tools/scripts/deepfreeze.py "
594594
f"{frozen_header} -m {src.frozenid} -o {cfile}")
595595
deepfreezerules.append('')

0 commit comments

Comments
 (0)