diff --git a/CHANGELOG.md b/CHANGELOG.md index be69f5e8d0..7cd292d227 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,7 @@ A brief description of the categories of changes: * Make `//python/pip_install:pip_repository_bzl` `bzl_library` target internal as all of the publicly available symbols (etc. `package_annotation`) are re-exported via `//python:pip_bzl` `bzl_library`. +* Gazelle Python extension no longer has runtime dependencies. Using `GAZELLE_PYTHON_RUNTIME_DEPS` from `@rules_python_gazelle_plugin//:def.bzl` is no longer necessary. ### Fixed diff --git a/examples/build_file_generation/BUILD.bazel b/examples/build_file_generation/BUILD.bazel index 79f62519df..a03af54a1a 100644 --- a/examples/build_file_generation/BUILD.bazel +++ b/examples/build_file_generation/BUILD.bazel @@ -6,7 +6,6 @@ load("@bazel_gazelle//:def.bzl", "gazelle") load("@pip//:requirements.bzl", "all_whl_requirements") load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") load("@rules_python//python:pip.bzl", "compile_pip_requirements") -load("@rules_python_gazelle_plugin//:def.bzl", "GAZELLE_PYTHON_RUNTIME_DEPS") load("@rules_python_gazelle_plugin//manifest:defs.bzl", "gazelle_python_manifest") load("@rules_python_gazelle_plugin//modules_mapping:def.bzl", "modules_mapping") @@ -56,7 +55,6 @@ gazelle_python_manifest( # See https://github.com/bazelbuild/bazel-gazelle/blob/master/extend.rst#example gazelle( name = "gazelle", - data = GAZELLE_PYTHON_RUNTIME_DEPS, gazelle = "@rules_python_gazelle_plugin//python:gazelle_binary", ) diff --git a/examples/bzlmod_build_file_generation/BUILD.bazel b/examples/bzlmod_build_file_generation/BUILD.bazel index 9b2e5bdce4..67288d6f43 100644 --- a/examples/bzlmod_build_file_generation/BUILD.bazel +++ b/examples/bzlmod_build_file_generation/BUILD.bazel @@ -9,7 +9,6 @@ load("@bazel_gazelle//:def.bzl", "gazelle") load("@pip//:requirements.bzl", "all_whl_requirements") load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test") load("@rules_python//python:pip.bzl", "compile_pip_requirements") -load("@rules_python_gazelle_plugin//:def.bzl", "GAZELLE_PYTHON_RUNTIME_DEPS") load("@rules_python_gazelle_plugin//manifest:defs.bzl", "gazelle_python_manifest") load("@rules_python_gazelle_plugin//modules_mapping:def.bzl", "modules_mapping") @@ -70,7 +69,6 @@ gazelle_python_manifest( # See: https://github.com/bazelbuild/bazel-gazelle#fix-and-update gazelle( name = "gazelle", - data = GAZELLE_PYTHON_RUNTIME_DEPS, gazelle = "@rules_python_gazelle_plugin//python:gazelle_binary", ) diff --git a/gazelle/README.md b/gazelle/README.md index b8be32ff44..c32f0d8258 100644 --- a/gazelle/README.md +++ b/gazelle/README.md @@ -7,7 +7,9 @@ Gazelle may be run by Bazel using the gazelle rule, or it may be installed and r This directory contains a plugin for [Gazelle](https://github.com/bazelbuild/bazel-gazelle) -that generates BUILD files content for Python code. +that generates BUILD files content for Python code. When Gazelle is run as a command line tool with this plugin, it embeds a Python interpreter resolved during the plugin build. +The behavior of the plugin is slightly different with different version of the interpreter as the Python `stdlib` changes with every minor version release. +Distributors of Gazelle binaries should, therefore, build a Gazelle binary for each OS+CPU architecture+Minor Python version combination they are targeting. The following instructions are for when you use [bzlmod](https://docs.bazel.build/versions/5.0.0/bzlmod.html). Please refer to older documentation that includes instructions on how to use Gazelle @@ -125,7 +127,6 @@ with the rules_python extension included. This typically goes in your root ```starlark load("@bazel_gazelle//:def.bzl", "gazelle") -load("@rules_python_gazelle_plugin//:def.bzl", "GAZELLE_PYTHON_RUNTIME_DEPS") # Our gazelle target points to the python gazelle binary. # This is the simple case where we only need one language supported. @@ -134,7 +135,6 @@ load("@rules_python_gazelle_plugin//:def.bzl", "GAZELLE_PYTHON_RUNTIME_DEPS") # See https://github.com/bazelbuild/bazel-gazelle/blob/master/extend.rst#example gazelle( name = "gazelle", - data = GAZELLE_PYTHON_RUNTIME_DEPS, gazelle = "@rules_python_gazelle_plugin//python:gazelle_binary", ) ``` diff --git a/gazelle/def.bzl b/gazelle/def.bzl index 80b11576e6..084b5a4a05 100644 --- a/gazelle/def.bzl +++ b/gazelle/def.bzl @@ -16,6 +16,4 @@ """ GAZELLE_PYTHON_RUNTIME_DEPS = [ - "@rules_python_gazelle_plugin//python:parse", - "@rules_python_gazelle_plugin//python:std_modules", ] diff --git a/gazelle/python/BUILD.bazel b/gazelle/python/BUILD.bazel index 4cb755de25..507d69e9d7 100644 --- a/gazelle/python/BUILD.bazel +++ b/gazelle/python/BUILD.bazel @@ -16,10 +16,7 @@ go_library( "std_modules.go", "target.go", ], - data = [ - ":parse", - ":std_modules", - ], + embedsrcs = [":helper.zip"], importpath = "github.com/bazelbuild/rules_python/gazelle/python", visibility = ["//visibility:public"], deps = [ @@ -36,20 +33,24 @@ go_library( "@com_github_emirpasic_gods//lists/singlylinkedlist", "@com_github_emirpasic_gods//sets/treeset", "@com_github_emirpasic_gods//utils", - "@io_bazel_rules_go//go/runfiles", ], ) py_binary( - name = "parse", - srcs = ["parse.py"], + name = "helper", + srcs = [ + "__main__.py", + "parse.py", + "std_modules.py", + ], + main = "__main__.py", visibility = ["//visibility:public"], ) -py_binary( - name = "std_modules", - srcs = ["std_modules.py"], - visibility = ["//visibility:public"], +filegroup( + name = "helper.zip", + srcs = [":helper"], + output_group = "python_zip_file", ) go_test( @@ -57,12 +58,12 @@ go_test( srcs = ["python_test.go"], data = [ ":gazelle_binary", - ":parse", - ":std_modules", + ":helper", ] + glob(["testdata/**"]), deps = [ "@bazel_gazelle//testtools:go_default_library", "@com_github_ghodss_yaml//:yaml", + "@io_bazel_rules_go//go/runfiles:go_default_library", "@io_bazel_rules_go//go/tools/bazel:go_default_library", ], ) diff --git a/gazelle/python/__main__.py b/gazelle/python/__main__.py new file mode 100644 index 0000000000..2f5a4a16ca --- /dev/null +++ b/gazelle/python/__main__.py @@ -0,0 +1,31 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# parse.py is a long-living program that communicates over STDIN and STDOUT. +# STDIN receives parse requests, one per line. It outputs the parsed modules and +# comments from all the files from each request. + +import parse +import std_modules +import sys + +if __name__ == "__main__": + if len(sys.argv) < 2: + sys.exit("Please provide subcommand, either print or std_modules") + if sys.argv[1] == "parse": + sys.exit(parse.main(sys.stdin, sys.stdout)) + elif sys.argv[1] == "std_modules": + sys.exit(std_modules.main(sys.stdin, sys.stdout)) + else: + sys.exit("Unknown subcommand: " + sys.argv[1]) diff --git a/gazelle/python/lifecycle.go b/gazelle/python/lifecycle.go index 592b322a3c..6d628e9137 100644 --- a/gazelle/python/lifecycle.go +++ b/gazelle/python/lifecycle.go @@ -16,14 +16,37 @@ package python import ( "context" + _ "embed" "github.com/bazelbuild/bazel-gazelle/language" + "log" + "os" +) + +var ( + //go:embed helper.zip + helperZip []byte + helperPath string ) type LifeCycleManager struct { language.BaseLifecycleManager + pyzFilePath string } func (l *LifeCycleManager) Before(ctx context.Context) { + helperPath = os.Getenv("GAZELLE_PYTHON_HELPER") + if helperPath == "" { + pyzFile, err := os.CreateTemp("", "python_zip_") + if err != nil { + log.Fatalf("failed to write parser zip: %v", err) + } + defer pyzFile.Close() + helperPath = pyzFile.Name() + l.pyzFilePath = helperPath + if _, err := pyzFile.Write(helperZip); err != nil { + log.Fatalf("cannot write %q: %v", helperPath, err) + } + } startParserProcess(ctx) startStdModuleProcess(ctx) } @@ -34,4 +57,7 @@ func (l *LifeCycleManager) DoneGeneratingRules() { func (l *LifeCycleManager) AfterResolvingDeps(ctx context.Context) { shutdownStdModuleProcess() + if l.pyzFilePath != "" { + os.Remove(l.pyzFilePath) + } } diff --git a/gazelle/python/parser.go b/gazelle/python/parser.go index 60a3c24269..ad55e03a01 100644 --- a/gazelle/python/parser.go +++ b/gazelle/python/parser.go @@ -17,6 +17,7 @@ package python import ( "bufio" "context" + _ "embed" "encoding/json" "fmt" "io" @@ -26,7 +27,6 @@ import ( "strings" "sync" - "github.com/bazelbuild/rules_go/go/runfiles" "github.com/emirpasic/gods/sets/treeset" godsutils "github.com/emirpasic/gods/utils" ) @@ -38,21 +38,9 @@ var ( ) func startParserProcess(ctx context.Context) { - rfiles, err := runfiles.New() - if err != nil { - log.Printf("failed to create a runfiles object: %v\n", err) - os.Exit(1) - } - - parseScriptRunfile, err := rfiles.Rlocation("rules_python_gazelle_plugin/python/parse") - if err != nil { - log.Printf("failed to initialize parser: %v\n", err) - os.Exit(1) - } - - cmd := exec.CommandContext(ctx, parseScriptRunfile) - cmd.Env = append(os.Environ(), rfiles.Env()...) - + // due to #691, we need a system interpreter to boostrap, part of which is + // to locate the hermetic interpreter. + cmd := exec.CommandContext(ctx, "python3", helperPath, "parse") cmd.Stderr = os.Stderr stdin, err := cmd.StdinPipe() diff --git a/gazelle/python/python_test.go b/gazelle/python/python_test.go index 79450ad584..74bd85bce6 100644 --- a/gazelle/python/python_test.go +++ b/gazelle/python/python_test.go @@ -31,6 +31,7 @@ import ( "time" "github.com/bazelbuild/bazel-gazelle/testtools" + "github.com/bazelbuild/rules_go/go/runfiles" "github.com/bazelbuild/rules_go/go/tools/bazel" "github.com/ghodss/yaml" ) @@ -159,6 +160,11 @@ func testPath(t *testing.T, name string, files []bazel.RunfileEntry) { cmd.Stdout = &stdout cmd.Stderr = &stderr cmd.Dir = workspaceRoot + helperScript, err := runfiles.Rlocation("rules_python_gazelle_plugin/python/helper") + if err != nil { + t.Fatalf("failed to initialize Python heler: %v", err) + } + cmd.Env = append(os.Environ(), "GAZELLE_PYTHON_HELPER="+helperScript) if err := cmd.Run(); err != nil { var e *exec.ExitError if !errors.As(err, &e) { diff --git a/gazelle/python/std_modules.go b/gazelle/python/std_modules.go index a87deec366..dd59cd8832 100644 --- a/gazelle/python/std_modules.go +++ b/gazelle/python/std_modules.go @@ -17,6 +17,7 @@ package python import ( "bufio" "context" + _ "embed" "fmt" "io" "log" @@ -25,8 +26,6 @@ import ( "strconv" "strings" "sync" - - "github.com/bazelbuild/rules_go/go/runfiles" ) var ( @@ -39,23 +38,12 @@ var ( func startStdModuleProcess(ctx context.Context) { stdModulesSeen = make(map[string]struct{}) - rfiles, err := runfiles.New() - if err != nil { - log.Printf("failed to create a runfiles object: %v\n", err) - os.Exit(1) - } - - stdModulesScriptRunfile, err := rfiles.Rlocation("rules_python_gazelle_plugin/python/std_modules") - if err != nil { - log.Printf("failed to initialize std_modules: %v\n", err) - os.Exit(1) - } - - cmd := exec.CommandContext(ctx, stdModulesScriptRunfile) - + // due to #691, we need a system interpreter to boostrap, part of which is + // to locate the hermetic interpreter. + cmd := exec.CommandContext(ctx, "python3", helperPath, "std_modules") cmd.Stderr = os.Stderr // All userland site-packages should be ignored. - cmd.Env = append([]string{"PYTHONNOUSERSITE=1"}, rfiles.Env()...) + cmd.Env = []string{"PYTHONNOUSERSITE=1"} stdin, err := cmd.StdinPipe() if err != nil {