From b4797ae1ade3eaabd58e1f6e56c96e902238a7e2 Mon Sep 17 00:00:00 2001
From: Zhongpeng Lin <zplin@uber.com>
Date: Wed, 11 Oct 2023 15:47:55 -0700
Subject: [PATCH 1/8] Embed Python scripts
revert changes in pythonconfig
remove runtime deps
---
gazelle/def.bzl | 2 --
gazelle/python/BUILD.bazel | 22 +++-------------------
gazelle/python/parse.py | 0
gazelle/python/parser.go | 24 +++++++++++-------------
gazelle/python/std_modules.go | 21 +++++----------------
5 files changed, 19 insertions(+), 50 deletions(-)
mode change 100644 => 100755 gazelle/python/parse.py
diff --git a/gazelle/def.bzl b/gazelle/def.bzl
index 80b11576e6..084b5a4a05 100644
--- a/gazelle/def.bzl
+++ b/gazelle/def.bzl
@@ -16,6 +16,4 @@
"""
GAZELLE_PYTHON_RUNTIME_DEPS = [
- "@rules_python_gazelle_plugin//python:parse",
- "@rules_python_gazelle_plugin//python:std_modules",
]
diff --git a/gazelle/python/BUILD.bazel b/gazelle/python/BUILD.bazel
index 4cb755de25..15e6bc1aea 100644
--- a/gazelle/python/BUILD.bazel
+++ b/gazelle/python/BUILD.bazel
@@ -1,6 +1,5 @@
load("@bazel_gazelle//:def.bzl", "gazelle_binary")
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
-load("@rules_python//python:defs.bzl", "py_binary")
go_library(
name = "python",
@@ -16,9 +15,9 @@ go_library(
"std_modules.go",
"target.go",
],
- data = [
- ":parse",
- ":std_modules",
+ embedsrcs = [
+ "parse.py",
+ "std_modules.py",
],
importpath = "github.com/bazelbuild/rules_python/gazelle/python",
visibility = ["//visibility:public"],
@@ -36,29 +35,14 @@ go_library(
"@com_github_emirpasic_gods//lists/singlylinkedlist",
"@com_github_emirpasic_gods//sets/treeset",
"@com_github_emirpasic_gods//utils",
- "@io_bazel_rules_go//go/runfiles",
],
)
-py_binary(
- name = "parse",
- srcs = ["parse.py"],
- visibility = ["//visibility:public"],
-)
-
-py_binary(
- name = "std_modules",
- srcs = ["std_modules.py"],
- visibility = ["//visibility:public"],
-)
-
go_test(
name = "python_test",
srcs = ["python_test.go"],
data = [
":gazelle_binary",
- ":parse",
- ":std_modules",
] + glob(["testdata/**"]),
deps = [
"@bazel_gazelle//testtools:go_default_library",
diff --git a/gazelle/python/parse.py b/gazelle/python/parse.py
old mode 100644
new mode 100755
diff --git a/gazelle/python/parser.go b/gazelle/python/parser.go
index 60a3c24269..86275ebea7 100644
--- a/gazelle/python/parser.go
+++ b/gazelle/python/parser.go
@@ -17,16 +17,17 @@ package python
import (
"bufio"
"context"
+ _ "embed"
"encoding/json"
"fmt"
"io"
"log"
"os"
"os/exec"
+ "path"
"strings"
"sync"
- "github.com/bazelbuild/rules_go/go/runfiles"
"github.com/emirpasic/gods/sets/treeset"
godsutils "github.com/emirpasic/gods/utils"
)
@@ -35,23 +36,19 @@ var (
parserStdin io.WriteCloser
parserStdout io.Reader
parserMutex sync.Mutex
+ //go:embed parse.py
+ parser []byte
+ parserPath = path.Join(os.TempDir(), "parse.py")
)
func startParserProcess(ctx context.Context) {
- rfiles, err := runfiles.New()
- if err != nil {
- log.Printf("failed to create a runfiles object: %v\n", err)
+ // "python -c" doesn't like parse.py for some reason, possibly due to the
+ // thread pool. So we need to write the code to a tmp file and execute it.
+ if err := os.WriteFile(parserPath, parser, 0644); err != nil {
+ log.Printf("cannot write %q: %s", parserPath, err.Error())
os.Exit(1)
}
-
- parseScriptRunfile, err := rfiles.Rlocation("rules_python_gazelle_plugin/python/parse")
- if err != nil {
- log.Printf("failed to initialize parser: %v\n", err)
- os.Exit(1)
- }
-
- cmd := exec.CommandContext(ctx, parseScriptRunfile)
- cmd.Env = append(os.Environ(), rfiles.Env()...)
+ cmd := exec.CommandContext(ctx, "python3", parserPath)
cmd.Stderr = os.Stderr
@@ -86,6 +83,7 @@ func shutdownParserProcess() {
if err := parserStdin.Close(); err != nil {
fmt.Fprintf(os.Stderr, "error closing parser: %v", err)
}
+ //os.Remove(parserPath)
}
// python3Parser implements a parser for Python files that extracts the modules
diff --git a/gazelle/python/std_modules.go b/gazelle/python/std_modules.go
index a87deec366..d4f755717f 100644
--- a/gazelle/python/std_modules.go
+++ b/gazelle/python/std_modules.go
@@ -17,6 +17,7 @@ package python
import (
"bufio"
"context"
+ _ "embed"
"fmt"
"io"
"log"
@@ -25,8 +26,6 @@ import (
"strconv"
"strings"
"sync"
-
- "github.com/bazelbuild/rules_go/go/runfiles"
)
var (
@@ -34,28 +33,18 @@ var (
stdModulesStdout io.Reader
stdModulesMutex sync.Mutex
stdModulesSeen map[string]struct{}
+ //go:embed std_modules.py
+ stdModue []byte
)
func startStdModuleProcess(ctx context.Context) {
stdModulesSeen = make(map[string]struct{})
- rfiles, err := runfiles.New()
- if err != nil {
- log.Printf("failed to create a runfiles object: %v\n", err)
- os.Exit(1)
- }
-
- stdModulesScriptRunfile, err := rfiles.Rlocation("rules_python_gazelle_plugin/python/std_modules")
- if err != nil {
- log.Printf("failed to initialize std_modules: %v\n", err)
- os.Exit(1)
- }
-
- cmd := exec.CommandContext(ctx, stdModulesScriptRunfile)
+ cmd := exec.CommandContext(ctx, "python3", "-c", string(stdModue))
cmd.Stderr = os.Stderr
// All userland site-packages should be ignored.
- cmd.Env = append([]string{"PYTHONNOUSERSITE=1"}, rfiles.Env()...)
+ cmd.Env = []string{"PYTHONNOUSERSITE=1"}
stdin, err := cmd.StdinPipe()
if err != nil {
From e95db20567c5daf27460d2ad9d641a7807956c3d Mon Sep 17 00:00:00 2001
From: Zhongpeng Lin <zplin@uber.com>
Date: Thu, 12 Oct 2023 14:34:52 -0700
Subject: [PATCH 2/8] embedding python zip files
---
gazelle/def.bzl | 2 ++
gazelle/python/BUILD.bazel | 28 ++++++++++++++++++++++++----
gazelle/python/parser.go | 6 ++----
gazelle/python/std_modules.go | 12 +++++++++---
4 files changed, 37 insertions(+), 11 deletions(-)
diff --git a/gazelle/def.bzl b/gazelle/def.bzl
index 084b5a4a05..80b11576e6 100644
--- a/gazelle/def.bzl
+++ b/gazelle/def.bzl
@@ -16,4 +16,6 @@
"""
GAZELLE_PYTHON_RUNTIME_DEPS = [
+ "@rules_python_gazelle_plugin//python:parse",
+ "@rules_python_gazelle_plugin//python:std_modules",
]
diff --git a/gazelle/python/BUILD.bazel b/gazelle/python/BUILD.bazel
index 15e6bc1aea..eba7cb0380 100644
--- a/gazelle/python/BUILD.bazel
+++ b/gazelle/python/BUILD.bazel
@@ -1,5 +1,6 @@
load("@bazel_gazelle//:def.bzl", "gazelle_binary")
load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+load("@rules_python//python:defs.bzl", "py_binary")
go_library(
name = "python",
@@ -15,10 +16,7 @@ go_library(
"std_modules.go",
"target.go",
],
- embedsrcs = [
- "parse.py",
- "std_modules.py",
- ],
+ embedsrcs = [":zip_files"],
importpath = "github.com/bazelbuild/rules_python/gazelle/python",
visibility = ["//visibility:public"],
deps = [
@@ -35,7 +33,29 @@ go_library(
"@com_github_emirpasic_gods//lists/singlylinkedlist",
"@com_github_emirpasic_gods//sets/treeset",
"@com_github_emirpasic_gods//utils",
+ "@io_bazel_rules_go//go/runfiles",
+ ],
+)
+
+py_binary(
+ name = "parse",
+ srcs = ["parse.py"],
+ visibility = ["//visibility:public"],
+)
+
+py_binary(
+ name = "std_modules",
+ srcs = ["std_modules.py"],
+ visibility = ["//visibility:public"],
+)
+
+filegroup(
+ name = "zip_files",
+ srcs = [
+ ":parse",
+ ":std_modules"
],
+ output_group = "python_zip_file",
)
go_test(
diff --git a/gazelle/python/parser.go b/gazelle/python/parser.go
index 86275ebea7..c4fe9b40fe 100644
--- a/gazelle/python/parser.go
+++ b/gazelle/python/parser.go
@@ -36,14 +36,12 @@ var (
parserStdin io.WriteCloser
parserStdout io.Reader
parserMutex sync.Mutex
- //go:embed parse.py
+ //go:embed parse.zip
parser []byte
- parserPath = path.Join(os.TempDir(), "parse.py")
+ parserPath = path.Join(os.TempDir(), "parse.zip")
)
func startParserProcess(ctx context.Context) {
- // "python -c" doesn't like parse.py for some reason, possibly due to the
- // thread pool. So we need to write the code to a tmp file and execute it.
if err := os.WriteFile(parserPath, parser, 0644); err != nil {
log.Printf("cannot write %q: %s", parserPath, err.Error())
os.Exit(1)
diff --git a/gazelle/python/std_modules.go b/gazelle/python/std_modules.go
index d4f755717f..fc48b14f35 100644
--- a/gazelle/python/std_modules.go
+++ b/gazelle/python/std_modules.go
@@ -23,6 +23,7 @@ import (
"log"
"os"
"os/exec"
+ "path"
"strconv"
"strings"
"sync"
@@ -33,14 +34,19 @@ var (
stdModulesStdout io.Reader
stdModulesMutex sync.Mutex
stdModulesSeen map[string]struct{}
- //go:embed std_modules.py
- stdModue []byte
+ //go:embed std_modules.zip
+ stdModule []byte
+ stdModulePath = path.Join(os.TempDir(), "std_modules.zip")
)
func startStdModuleProcess(ctx context.Context) {
stdModulesSeen = make(map[string]struct{})
- cmd := exec.CommandContext(ctx, "python3", "-c", string(stdModue))
+ if err := os.WriteFile(stdModulePath, stdModule, 0644); err != nil {
+ log.Printf("cannot write %q: %s", stdModulePath, err.Error())
+ os.Exit(1)
+ }
+ cmd := exec.CommandContext(ctx, "python3", stdModulePath)
cmd.Stderr = os.Stderr
// All userland site-packages should be ignored.
From 982d730647aa9730653456f4c41f42792d375405 Mon Sep 17 00:00:00 2001
From: Zhongpeng Lin <zplin@uber.com>
Date: Fri, 13 Oct 2023 21:00:32 -0700
Subject: [PATCH 3/8] embed pyz file
---
gazelle/python/BUILD.bazel | 25 ++++++++++---------------
gazelle/python/__main__.py | 31 +++++++++++++++++++++++++++++++
gazelle/python/lifecycle.go | 18 ++++++++++++++++++
gazelle/python/parser.go | 11 +----------
gazelle/python/python_test.go | 13 ++++---------
gazelle/python/std_modules.go | 10 +---------
6 files changed, 65 insertions(+), 43 deletions(-)
create mode 100644 gazelle/python/__main__.py
diff --git a/gazelle/python/BUILD.bazel b/gazelle/python/BUILD.bazel
index eba7cb0380..ca3944e658 100644
--- a/gazelle/python/BUILD.bazel
+++ b/gazelle/python/BUILD.bazel
@@ -16,7 +16,7 @@ go_library(
"std_modules.go",
"target.go",
],
- embedsrcs = [":zip_files"],
+ embedsrcs = [":helper.zip"],
importpath = "github.com/bazelbuild/rules_python/gazelle/python",
visibility = ["//visibility:public"],
deps = [
@@ -33,28 +33,23 @@ go_library(
"@com_github_emirpasic_gods//lists/singlylinkedlist",
"@com_github_emirpasic_gods//sets/treeset",
"@com_github_emirpasic_gods//utils",
- "@io_bazel_rules_go//go/runfiles",
],
)
py_binary(
- name = "parse",
- srcs = ["parse.py"],
- visibility = ["//visibility:public"],
-)
-
-py_binary(
- name = "std_modules",
- srcs = ["std_modules.py"],
+ name = "helper",
+ srcs = [
+ "__main__.py",
+ "parse.py",
+ "std_modules.py",
+ ],
+ main = "__main__.py",
visibility = ["//visibility:public"],
)
filegroup(
- name = "zip_files",
- srcs = [
- ":parse",
- ":std_modules"
- ],
+ name = "helper.zip",
+ srcs = [":helper"],
output_group = "python_zip_file",
)
diff --git a/gazelle/python/__main__.py b/gazelle/python/__main__.py
new file mode 100644
index 0000000000..2f5a4a16ca
--- /dev/null
+++ b/gazelle/python/__main__.py
@@ -0,0 +1,31 @@
+# Copyright 2023 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# parse.py is a long-living program that communicates over STDIN and STDOUT.
+# STDIN receives parse requests, one per line. It outputs the parsed modules and
+# comments from all the files from each request.
+
+import parse
+import std_modules
+import sys
+
+if __name__ == "__main__":
+ if len(sys.argv) < 2:
+ sys.exit("Please provide subcommand, either print or std_modules")
+ if sys.argv[1] == "parse":
+ sys.exit(parse.main(sys.stdin, sys.stdout))
+ elif sys.argv[1] == "std_modules":
+ sys.exit(std_modules.main(sys.stdin, sys.stdout))
+ else:
+ sys.exit("Unknown subcommand: " + sys.argv[1])
diff --git a/gazelle/python/lifecycle.go b/gazelle/python/lifecycle.go
index 592b322a3c..910763c8dc 100644
--- a/gazelle/python/lifecycle.go
+++ b/gazelle/python/lifecycle.go
@@ -16,7 +16,16 @@ package python
import (
"context"
+ _ "embed"
"github.com/bazelbuild/bazel-gazelle/language"
+ "log"
+ "os"
+)
+
+var (
+ //go:embed helper.zip
+ pythonZip []byte
+ pyzPath string
)
type LifeCycleManager struct {
@@ -24,6 +33,14 @@ type LifeCycleManager struct {
}
func (l *LifeCycleManager) Before(ctx context.Context) {
+ pyzFile, err := os.CreateTemp("", "python_zip_")
+ if err != nil {
+ log.Fatalf("failed to write parser zip: %v", err)
+ }
+ pyzPath = pyzFile.Name()
+ if _, err := pyzFile.Write(pythonZip); err != nil {
+ log.Fatalf("cannot write %q: %v", pyzPath, err)
+ }
startParserProcess(ctx)
startStdModuleProcess(ctx)
}
@@ -34,4 +51,5 @@ func (l *LifeCycleManager) DoneGeneratingRules() {
func (l *LifeCycleManager) AfterResolvingDeps(ctx context.Context) {
shutdownStdModuleProcess()
+ os.Remove(pyzPath)
}
diff --git a/gazelle/python/parser.go b/gazelle/python/parser.go
index c4fe9b40fe..91c5ea928a 100644
--- a/gazelle/python/parser.go
+++ b/gazelle/python/parser.go
@@ -24,7 +24,6 @@ import (
"log"
"os"
"os/exec"
- "path"
"strings"
"sync"
@@ -36,17 +35,10 @@ var (
parserStdin io.WriteCloser
parserStdout io.Reader
parserMutex sync.Mutex
- //go:embed parse.zip
- parser []byte
- parserPath = path.Join(os.TempDir(), "parse.zip")
)
func startParserProcess(ctx context.Context) {
- if err := os.WriteFile(parserPath, parser, 0644); err != nil {
- log.Printf("cannot write %q: %s", parserPath, err.Error())
- os.Exit(1)
- }
- cmd := exec.CommandContext(ctx, "python3", parserPath)
+ cmd := exec.CommandContext(ctx, "python3", pyzPath, "parse")
cmd.Stderr = os.Stderr
@@ -81,7 +73,6 @@ func shutdownParserProcess() {
if err := parserStdin.Close(); err != nil {
fmt.Fprintf(os.Stderr, "error closing parser: %v", err)
}
- //os.Remove(parserPath)
}
// python3Parser implements a parser for Python files that extracts the modules
diff --git a/gazelle/python/python_test.go b/gazelle/python/python_test.go
index 79450ad584..23225c15a7 100644
--- a/gazelle/python/python_test.go
+++ b/gazelle/python/python_test.go
@@ -21,18 +21,15 @@ package python_test
import (
"bytes"
- "context"
"errors"
+ "github.com/bazelbuild/bazel-gazelle/testtools"
+ "github.com/bazelbuild/rules_go/go/tools/bazel"
+ "github.com/ghodss/yaml"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
- "time"
-
- "github.com/bazelbuild/bazel-gazelle/testtools"
- "github.com/bazelbuild/rules_go/go/tools/bazel"
- "github.com/ghodss/yaml"
)
const (
@@ -152,9 +149,7 @@ func testPath(t *testing.T, name string, files []bazel.RunfileEntry) {
args := []string{"-build_file_name=BUILD,BUILD.bazel"}
- ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
- t.Cleanup(cancel)
- cmd := exec.CommandContext(ctx, gazellePath, args...)
+ cmd := exec.Command(gazellePath, args...)
var stdout, stderr bytes.Buffer
cmd.Stdout = &stdout
cmd.Stderr = &stderr
diff --git a/gazelle/python/std_modules.go b/gazelle/python/std_modules.go
index fc48b14f35..5aa2f62222 100644
--- a/gazelle/python/std_modules.go
+++ b/gazelle/python/std_modules.go
@@ -23,7 +23,6 @@ import (
"log"
"os"
"os/exec"
- "path"
"strconv"
"strings"
"sync"
@@ -34,19 +33,12 @@ var (
stdModulesStdout io.Reader
stdModulesMutex sync.Mutex
stdModulesSeen map[string]struct{}
- //go:embed std_modules.zip
- stdModule []byte
- stdModulePath = path.Join(os.TempDir(), "std_modules.zip")
)
func startStdModuleProcess(ctx context.Context) {
stdModulesSeen = make(map[string]struct{})
- if err := os.WriteFile(stdModulePath, stdModule, 0644); err != nil {
- log.Printf("cannot write %q: %s", stdModulePath, err.Error())
- os.Exit(1)
- }
- cmd := exec.CommandContext(ctx, "python3", stdModulePath)
+ cmd := exec.CommandContext(ctx, "python3", pyzPath, "std_modules")
cmd.Stderr = os.Stderr
// All userland site-packages should be ignored.
From 007c1237c3d8898d0424d16b2e302afc26296564 Mon Sep 17 00:00:00 2001
From: Zhongpeng Lin <zplin@uber.com>
Date: Fri, 13 Oct 2023 21:14:41 -0700
Subject: [PATCH 4/8] no runtime deps anymore
---
gazelle/def.bzl | 2 --
1 file changed, 2 deletions(-)
diff --git a/gazelle/def.bzl b/gazelle/def.bzl
index 80b11576e6..084b5a4a05 100644
--- a/gazelle/def.bzl
+++ b/gazelle/def.bzl
@@ -16,6 +16,4 @@
"""
GAZELLE_PYTHON_RUNTIME_DEPS = [
- "@rules_python_gazelle_plugin//python:parse",
- "@rules_python_gazelle_plugin//python:std_modules",
]
From db35bdbf3dccbf701f76a8187e3e66b456f0d8d2 Mon Sep 17 00:00:00 2001
From: Zhongpeng Lin <zplin@uber.com>
Date: Sun, 15 Oct 2023 10:34:09 -0700
Subject: [PATCH 5/8] comments
change log
change log
---
CHANGELOG.md | 1 +
examples/build_file_generation/BUILD.bazel | 2 --
.../bzlmod_build_file_generation/BUILD.bazel | 2 --
gazelle/README.md | 2 --
gazelle/python/BUILD.bazel | 2 ++
gazelle/python/lifecycle.go | 28 ++++++++++++-------
gazelle/python/parse.py | 0
gazelle/python/parser.go | 5 ++--
gazelle/python/python_test.go | 19 ++++++++++---
gazelle/python/std_modules.go | 5 ++--
10 files changed, 42 insertions(+), 24 deletions(-)
mode change 100755 => 100644 gazelle/python/parse.py
diff --git a/CHANGELOG.md b/CHANGELOG.md
index be69f5e8d0..1629e1e708 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -24,6 +24,7 @@ A brief description of the categories of changes:
* Make `//python/pip_install:pip_repository_bzl` `bzl_library` target internal
as all of the publicly available symbols (etc. `package_annotation`) are
re-exported via `//python:pip_bzl` `bzl_library`.
+* Gazelle Python extension no longer have runtime dependencies.
### Fixed
diff --git a/examples/build_file_generation/BUILD.bazel b/examples/build_file_generation/BUILD.bazel
index 79f62519df..a03af54a1a 100644
--- a/examples/build_file_generation/BUILD.bazel
+++ b/examples/build_file_generation/BUILD.bazel
@@ -6,7 +6,6 @@ load("@bazel_gazelle//:def.bzl", "gazelle")
load("@pip//:requirements.bzl", "all_whl_requirements")
load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")
load("@rules_python//python:pip.bzl", "compile_pip_requirements")
-load("@rules_python_gazelle_plugin//:def.bzl", "GAZELLE_PYTHON_RUNTIME_DEPS")
load("@rules_python_gazelle_plugin//manifest:defs.bzl", "gazelle_python_manifest")
load("@rules_python_gazelle_plugin//modules_mapping:def.bzl", "modules_mapping")
@@ -56,7 +55,6 @@ gazelle_python_manifest(
# See https://github.com/bazelbuild/bazel-gazelle/blob/master/extend.rst#example
gazelle(
name = "gazelle",
- data = GAZELLE_PYTHON_RUNTIME_DEPS,
gazelle = "@rules_python_gazelle_plugin//python:gazelle_binary",
)
diff --git a/examples/bzlmod_build_file_generation/BUILD.bazel b/examples/bzlmod_build_file_generation/BUILD.bazel
index 9b2e5bdce4..67288d6f43 100644
--- a/examples/bzlmod_build_file_generation/BUILD.bazel
+++ b/examples/bzlmod_build_file_generation/BUILD.bazel
@@ -9,7 +9,6 @@ load("@bazel_gazelle//:def.bzl", "gazelle")
load("@pip//:requirements.bzl", "all_whl_requirements")
load("@rules_python//python:defs.bzl", "py_binary", "py_library", "py_test")
load("@rules_python//python:pip.bzl", "compile_pip_requirements")
-load("@rules_python_gazelle_plugin//:def.bzl", "GAZELLE_PYTHON_RUNTIME_DEPS")
load("@rules_python_gazelle_plugin//manifest:defs.bzl", "gazelle_python_manifest")
load("@rules_python_gazelle_plugin//modules_mapping:def.bzl", "modules_mapping")
@@ -70,7 +69,6 @@ gazelle_python_manifest(
# See: https://github.com/bazelbuild/bazel-gazelle#fix-and-update
gazelle(
name = "gazelle",
- data = GAZELLE_PYTHON_RUNTIME_DEPS,
gazelle = "@rules_python_gazelle_plugin//python:gazelle_binary",
)
diff --git a/gazelle/README.md b/gazelle/README.md
index b8be32ff44..fe275e97c9 100644
--- a/gazelle/README.md
+++ b/gazelle/README.md
@@ -125,7 +125,6 @@ with the rules_python extension included. This typically goes in your root
```starlark
load("@bazel_gazelle//:def.bzl", "gazelle")
-load("@rules_python_gazelle_plugin//:def.bzl", "GAZELLE_PYTHON_RUNTIME_DEPS")
# Our gazelle target points to the python gazelle binary.
# This is the simple case where we only need one language supported.
@@ -134,7 +133,6 @@ load("@rules_python_gazelle_plugin//:def.bzl", "GAZELLE_PYTHON_RUNTIME_DEPS")
# See https://github.com/bazelbuild/bazel-gazelle/blob/master/extend.rst#example
gazelle(
name = "gazelle",
- data = GAZELLE_PYTHON_RUNTIME_DEPS,
gazelle = "@rules_python_gazelle_plugin//python:gazelle_binary",
)
```
diff --git a/gazelle/python/BUILD.bazel b/gazelle/python/BUILD.bazel
index ca3944e658..507d69e9d7 100644
--- a/gazelle/python/BUILD.bazel
+++ b/gazelle/python/BUILD.bazel
@@ -58,10 +58,12 @@ go_test(
srcs = ["python_test.go"],
data = [
":gazelle_binary",
+ ":helper",
] + glob(["testdata/**"]),
deps = [
"@bazel_gazelle//testtools:go_default_library",
"@com_github_ghodss_yaml//:yaml",
+ "@io_bazel_rules_go//go/runfiles:go_default_library",
"@io_bazel_rules_go//go/tools/bazel:go_default_library",
],
)
diff --git a/gazelle/python/lifecycle.go b/gazelle/python/lifecycle.go
index 910763c8dc..6d628e9137 100644
--- a/gazelle/python/lifecycle.go
+++ b/gazelle/python/lifecycle.go
@@ -24,22 +24,28 @@ import (
var (
//go:embed helper.zip
- pythonZip []byte
- pyzPath string
+ helperZip []byte
+ helperPath string
)
type LifeCycleManager struct {
language.BaseLifecycleManager
+ pyzFilePath string
}
func (l *LifeCycleManager) Before(ctx context.Context) {
- pyzFile, err := os.CreateTemp("", "python_zip_")
- if err != nil {
- log.Fatalf("failed to write parser zip: %v", err)
- }
- pyzPath = pyzFile.Name()
- if _, err := pyzFile.Write(pythonZip); err != nil {
- log.Fatalf("cannot write %q: %v", pyzPath, err)
+ helperPath = os.Getenv("GAZELLE_PYTHON_HELPER")
+ if helperPath == "" {
+ pyzFile, err := os.CreateTemp("", "python_zip_")
+ if err != nil {
+ log.Fatalf("failed to write parser zip: %v", err)
+ }
+ defer pyzFile.Close()
+ helperPath = pyzFile.Name()
+ l.pyzFilePath = helperPath
+ if _, err := pyzFile.Write(helperZip); err != nil {
+ log.Fatalf("cannot write %q: %v", helperPath, err)
+ }
}
startParserProcess(ctx)
startStdModuleProcess(ctx)
@@ -51,5 +57,7 @@ func (l *LifeCycleManager) DoneGeneratingRules() {
func (l *LifeCycleManager) AfterResolvingDeps(ctx context.Context) {
shutdownStdModuleProcess()
- os.Remove(pyzPath)
+ if l.pyzFilePath != "" {
+ os.Remove(l.pyzFilePath)
+ }
}
diff --git a/gazelle/python/parse.py b/gazelle/python/parse.py
old mode 100755
new mode 100644
diff --git a/gazelle/python/parser.go b/gazelle/python/parser.go
index 91c5ea928a..ad55e03a01 100644
--- a/gazelle/python/parser.go
+++ b/gazelle/python/parser.go
@@ -38,8 +38,9 @@ var (
)
func startParserProcess(ctx context.Context) {
- cmd := exec.CommandContext(ctx, "python3", pyzPath, "parse")
-
+ // due to #691, we need a system interpreter to boostrap, part of which is
+ // to locate the hermetic interpreter.
+ cmd := exec.CommandContext(ctx, "python3", helperPath, "parse")
cmd.Stderr = os.Stderr
stdin, err := cmd.StdinPipe()
diff --git a/gazelle/python/python_test.go b/gazelle/python/python_test.go
index 23225c15a7..74bd85bce6 100644
--- a/gazelle/python/python_test.go
+++ b/gazelle/python/python_test.go
@@ -21,15 +21,19 @@ package python_test
import (
"bytes"
+ "context"
"errors"
- "github.com/bazelbuild/bazel-gazelle/testtools"
- "github.com/bazelbuild/rules_go/go/tools/bazel"
- "github.com/ghodss/yaml"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
+ "time"
+
+ "github.com/bazelbuild/bazel-gazelle/testtools"
+ "github.com/bazelbuild/rules_go/go/runfiles"
+ "github.com/bazelbuild/rules_go/go/tools/bazel"
+ "github.com/ghodss/yaml"
)
const (
@@ -149,11 +153,18 @@ func testPath(t *testing.T, name string, files []bazel.RunfileEntry) {
args := []string{"-build_file_name=BUILD,BUILD.bazel"}
- cmd := exec.Command(gazellePath, args...)
+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+ t.Cleanup(cancel)
+ cmd := exec.CommandContext(ctx, gazellePath, args...)
var stdout, stderr bytes.Buffer
cmd.Stdout = &stdout
cmd.Stderr = &stderr
cmd.Dir = workspaceRoot
+ helperScript, err := runfiles.Rlocation("rules_python_gazelle_plugin/python/helper")
+ if err != nil {
+ t.Fatalf("failed to initialize Python heler: %v", err)
+ }
+ cmd.Env = append(os.Environ(), "GAZELLE_PYTHON_HELPER="+helperScript)
if err := cmd.Run(); err != nil {
var e *exec.ExitError
if !errors.As(err, &e) {
diff --git a/gazelle/python/std_modules.go b/gazelle/python/std_modules.go
index 5aa2f62222..dd59cd8832 100644
--- a/gazelle/python/std_modules.go
+++ b/gazelle/python/std_modules.go
@@ -38,8 +38,9 @@ var (
func startStdModuleProcess(ctx context.Context) {
stdModulesSeen = make(map[string]struct{})
- cmd := exec.CommandContext(ctx, "python3", pyzPath, "std_modules")
-
+ // due to #691, we need a system interpreter to boostrap, part of which is
+ // to locate the hermetic interpreter.
+ cmd := exec.CommandContext(ctx, "python3", helperPath, "std_modules")
cmd.Stderr = os.Stderr
// All userland site-packages should be ignored.
cmd.Env = []string{"PYTHONNOUSERSITE=1"}
From a81ddaf892789e9b8fd3603bdc81a2794470a329 Mon Sep 17 00:00:00 2001
From: Zhongpeng Lin <zplin@uber.com>
Date: Mon, 16 Oct 2023 08:42:28 -0700
Subject: [PATCH 6/8] Update CHANGELOG.md
Co-authored-by: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
---
CHANGELOG.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 1629e1e708..7cd292d227 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -24,7 +24,7 @@ A brief description of the categories of changes:
* Make `//python/pip_install:pip_repository_bzl` `bzl_library` target internal
as all of the publicly available symbols (etc. `package_annotation`) are
re-exported via `//python:pip_bzl` `bzl_library`.
-* Gazelle Python extension no longer have runtime dependencies.
+* Gazelle Python extension no longer has runtime dependencies. Using `GAZELLE_PYTHON_RUNTIME_DEPS` from `@rules_python_gazelle_plugin//:def.bzl` is no longer necessary.
### Fixed
From e444f682e398a240546ce2a768318bffe8a14d5b Mon Sep 17 00:00:00 2001
From: Zhongpeng Lin <zplin@uber.com>
Date: Mon, 16 Oct 2023 09:19:27 -0700
Subject: [PATCH 7/8] document Python version
---
gazelle/README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/gazelle/README.md b/gazelle/README.md
index fe275e97c9..a57fa7393e 100644
--- a/gazelle/README.md
+++ b/gazelle/README.md
@@ -7,7 +7,7 @@ Gazelle may be run by Bazel using the gazelle rule, or it may be installed and r
This directory contains a plugin for
[Gazelle](https://github.com/bazelbuild/bazel-gazelle)
-that generates BUILD files content for Python code.
+that generates BUILD files content for Python code. When Gazelle is run as a command line tool with this plugin, it embeds with a Python interpreter, which is the one Bazel resolves to when it is building the plugin. The behavior of the plugin is different with different version of the interpreter. Distributors of Gazelle binaries need to build a Gazelle binary for each OS+CPU architecture+Python version combination they are targeting.
The following instructions are for when you use [bzlmod](https://docs.bazel.build/versions/5.0.0/bzlmod.html).
Please refer to older documentation that includes instructions on how to use Gazelle
From 03e89f33b1bf2786e2722f5996c25ebec49c77bf Mon Sep 17 00:00:00 2001
From: Ignas Anikevicius <240938+aignas@users.noreply.github.com>
Date: Tue, 17 Oct 2023 08:48:32 +0900
Subject: [PATCH 8/8] Update gazelle/README.md
---
gazelle/README.md | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/gazelle/README.md b/gazelle/README.md
index a57fa7393e..c32f0d8258 100644
--- a/gazelle/README.md
+++ b/gazelle/README.md
@@ -7,7 +7,9 @@ Gazelle may be run by Bazel using the gazelle rule, or it may be installed and r
This directory contains a plugin for
[Gazelle](https://github.com/bazelbuild/bazel-gazelle)
-that generates BUILD files content for Python code. When Gazelle is run as a command line tool with this plugin, it embeds with a Python interpreter, which is the one Bazel resolves to when it is building the plugin. The behavior of the plugin is different with different version of the interpreter. Distributors of Gazelle binaries need to build a Gazelle binary for each OS+CPU architecture+Python version combination they are targeting.
+that generates BUILD files content for Python code. When Gazelle is run as a command line tool with this plugin, it embeds a Python interpreter resolved during the plugin build.
+The behavior of the plugin is slightly different with different version of the interpreter as the Python `stdlib` changes with every minor version release.
+Distributors of Gazelle binaries should, therefore, build a Gazelle binary for each OS+CPU architecture+Minor Python version combination they are targeting.
The following instructions are for when you use [bzlmod](https://docs.bazel.build/versions/5.0.0/bzlmod.html).
Please refer to older documentation that includes instructions on how to use Gazelle