From 431cc2bcd85ed2414d69515994dfb1c0e4dae24c Mon Sep 17 00:00:00 2001
From: Jacob Quinn <quinn.jacobd@gmail.com>
Date: Tue, 22 Nov 2022 14:18:53 -0700
Subject: [PATCH 1/5] Support a `[sources]` section in Project.toml for
specifying relative path and repo locations for dependencies
---
docs/src/toml-files.md | 15 ++-
src/API.jl | 35 +++++-
src/Operations.jl | 100 ++++++++++++++----
src/Types.jl | 21 +++-
src/precompile.jl | 2 +-
src/project.jl | 62 +++++++++--
test/runtests.jl | 5 +-
test/sources.jl | 27 +++++
.../WithSources/LocalPkg/Project.toml | 4 +
.../WithSources/LocalPkg/src/LocalPkg.jl | 5 +
test/test_packages/WithSources/Project.toml | 7 ++
.../WithSources/TestWithUnreg/Project.toml | 15 +++
.../TestWithUnreg/src/TestWithUnreg.jl | 5 +
.../TestWithUnreg/test/runtests.jl | 2 +
14 files changed, 270 insertions(+), 35 deletions(-)
create mode 100644 test/sources.jl
create mode 100644 test/test_packages/WithSources/LocalPkg/Project.toml
create mode 100644 test/test_packages/WithSources/LocalPkg/src/LocalPkg.jl
create mode 100644 test/test_packages/WithSources/Project.toml
create mode 100644 test/test_packages/WithSources/TestWithUnreg/Project.toml
create mode 100644 test/test_packages/WithSources/TestWithUnreg/src/TestWithUnreg.jl
create mode 100644 test/test_packages/WithSources/TestWithUnreg/test/runtests.jl
diff --git a/docs/src/toml-files.md b/docs/src/toml-files.md
index 6b6645729e..37645080f2 100644
--- a/docs/src/toml-files.md
+++ b/docs/src/toml-files.md
@@ -91,6 +91,19 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
Typically it is not needed to manually add entries to the `[deps]` section; this is instead
handled by Pkg operations such as `add`.
+### The `[sources]` section
+
+Specifiying a path or repo (+ branch) for a dependency is done in the `[sources]` section.
+These are especially useful for controlling unregistered dependencies without having to bundle a
+corresponding manifest file.
+
+```toml
+[sources]
+Example = {url = "https://github.com/JuliaLang/Example.jl", rev = "custom_branch"}
+SomeDependency = {path = "deps/SomeDependency.jl"}
+```
+
+Note that this information is only used when this environment is active, i.e. it is not used if this project is a package that is being used as a dependency.
### The `[compat]` section
@@ -135,7 +148,7 @@ Julia will then preferentially use the version-specific manifest file if availab
For example, if both `Manifest-v1.11.toml` and `Manifest.toml` exist, Julia 1.11 will prioritize using `Manifest-v1.11.toml`.
However, Julia versions 1.10, 1.12, and all others will default to using `Manifest.toml`.
This feature allows for easier management of different instantiated versions of dependencies for various Julia versions.
-Note that there can only be one `Project.toml` file. While `Manifest-v{major}.{minor}.toml` files are not automatically
+Note that there can only be one `Project.toml` file. While `Manifest-v{major}.{minor}.toml` files are not automatically
created by Pkg, users can manually rename a `Manifest.toml` file to match
the versioned format, and Pkg will subsequently maintain it through its operations.
diff --git a/src/API.jl b/src/API.jl
index 950d3b002c..9b22a22b02 100644
--- a/src/API.jl
+++ b/src/API.jl
@@ -181,6 +181,31 @@ for f in (:develop, :add, :rm, :up, :pin, :free, :test, :build, :status, :why, :
end
end
+function update_source_if_set(project, pkg)
+ source = get(project.sources, pkg.name, nothing)
+ source === nothing && return
+ # This should probably not modify the dicts directly...
+ if pkg.repo.source !== nothing
+ source["url"] = pkg.repo.source
+ end
+ if pkg.repo.rev !== nothing
+ source["rev"] = pkg.repo.rev
+ end
+ if pkg.path !== nothing
+ source["path"] = pkg.path
+ end
+ path, repo = get_path_repo(project, pkg.name)
+ if path !== nothing
+ pkg.path = path
+ end
+ if repo.source !== nothing
+ pkg.repo.source = repo.source
+ end
+ if repo.rev !== nothing
+ pkg.repo.rev = repo.rev
+ end
+end
+
function develop(ctx::Context, pkgs::Vector{PackageSpec}; shared::Bool=true,
preserve::PreserveLevel=Operations.default_preserve(), platform::AbstractPlatform=HostPlatform(), kwargs...)
require_not_empty(pkgs, :develop)
@@ -212,6 +237,7 @@ function develop(ctx::Context, pkgs::Vector{PackageSpec}; shared::Bool=true,
new_git = handle_repos_develop!(ctx, pkgs, shared)
+
for pkg in pkgs
if Types.collides_with_project(ctx.env, pkg)
pkgerror("package $(err_rep(pkg)) has the same name or UUID as the active project")
@@ -219,6 +245,7 @@ function develop(ctx::Context, pkgs::Vector{PackageSpec}; shared::Bool=true,
if length(findall(x -> x.uuid == pkg.uuid, pkgs)) > 1
pkgerror("it is invalid to specify multiple packages with the same UUID: $(err_rep(pkg))")
end
+ update_source_if_set(ctx.env.project, pkg)
end
Operations.develop(ctx, pkgs, new_git; preserve=preserve, platform=platform)
@@ -272,6 +299,7 @@ function add(ctx::Context, pkgs::Vector{PackageSpec}; preserve::PreserveLevel=Op
if length(findall(x -> x.uuid == pkg.uuid, pkgs)) > 1
pkgerror("it is invalid to specify multiple packages with the same UUID: $(err_rep(pkg))")
end
+ update_source_if_set(ctx.env.project, pkg)
end
Operations.add(ctx, pkgs, new_git; preserve, platform, target)
@@ -311,12 +339,14 @@ end
function append_all_pkgs!(pkgs, ctx, mode)
if mode == PKGMODE_PROJECT || mode == PKGMODE_COMBINED
for (name::String, uuid::UUID) in ctx.env.project.deps
- push!(pkgs, PackageSpec(name=name, uuid=uuid))
+ path, repo = get_path_repo(ctx.env.project, name)
+ push!(pkgs, PackageSpec(name=name, uuid=uuid, path=path, repo=repo))
end
end
if mode == PKGMODE_MANIFEST || mode == PKGMODE_COMBINED
for (uuid, entry) in ctx.env.manifest
- push!(pkgs, PackageSpec(name=entry.name, uuid=uuid))
+ path, repo = get_path_repo(ctx.env.project, entry.name)
+ push!(pkgs, PackageSpec(name=entry.name, uuid=uuid, path=path, repo=repo))
end
end
return
@@ -347,6 +377,7 @@ function up(ctx::Context, pkgs::Vector{PackageSpec};
manifest_resolve!(ctx.env.manifest, pkgs)
ensure_resolved(ctx, ctx.env.manifest, pkgs)
end
+
Operations.up(ctx, pkgs, level; skip_writing_project, preserve)
return
end
diff --git a/src/Operations.jl b/src/Operations.jl
index 42db0ec638..f58825a4ff 100644
--- a/src/Operations.jl
+++ b/src/Operations.jl
@@ -71,14 +71,15 @@ function load_direct_deps(env::EnvCache, pkgs::Vector{PackageSpec}=PackageSpec[]
pkgs = copy(pkgs)
for (name::String, uuid::UUID) in env.project.deps
findfirst(pkg -> pkg.uuid == uuid, pkgs) === nothing || continue # do not duplicate packages
+ path, repo = get_path_repo(env.project, name)
entry = manifest_info(env.manifest, uuid)
push!(pkgs, entry === nothing ?
- PackageSpec(;uuid=uuid, name=name) :
+ PackageSpec(;uuid=uuid, name=name, path=path, repo=repo) :
PackageSpec(;
uuid = uuid,
name = name,
- path = entry.path,
- repo = entry.repo,
+ path = path === nothing ? entry.path : path,
+ repo = repo == GitRepo() ? entry.repo : repo,
pinned = entry.pinned,
tree_hash = entry.tree_hash, # TODO should tree_hash be changed too?
version = load_version(entry.version, isfixed(entry), preserve),
@@ -108,6 +109,19 @@ end
function load_all_deps(env::EnvCache, pkgs::Vector{PackageSpec}=PackageSpec[];
preserve::PreserveLevel=PRESERVE_ALL)
pkgs = load_manifest_deps(env.manifest, pkgs; preserve=preserve)
+ # Sources takes presedence over the manifest...
+ for pkg in pkgs
+ path, repo = get_path_repo(env.project, pkg.name)
+ if path !== nothing
+ pkg.path = path
+ end
+ if repo.source !== nothing
+ pkg.repo.source = repo.source
+ end
+ if repo.rev !== nothing
+ pkg.repo.rev = repo.rev
+ end
+ end
return load_direct_deps(env, pkgs; preserve=preserve)
end
@@ -244,8 +258,9 @@ function collect_project(pkg::PackageSpec, path::String)
pkgerror("julia version requirement from Project.toml's compat section not satisfied for package $(err_rep(pkg)) at `$path`")
end
for (name, uuid) in project.deps
+ path, repo = get_path_repo(project, name)
vspec = get_compat(project, name)
- push!(deps, PackageSpec(name, uuid, vspec))
+ push!(deps, PackageSpec(name=name, uuid=uuid, version=vspec, path=path, repo=repo))
end
for (name, uuid) in project.weakdeps
vspec = get_compat(project, name)
@@ -302,6 +317,11 @@ function collect_fixed!(env::EnvCache, pkgs::Vector{PackageSpec}, names::Dict{UU
names[pkg.uuid] = pkg.name
end
for pkg in pkgs
+ # add repo package if necessary
+ if (pkg.repo.rev !== nothing || pkg.repo.source !== nothing) && pkg.tree_hash === nothing
+ # ensure revved package is installed
+ Types.handle_repo_add!(Types.Context(env=env), pkg)
+ end
path = project_rel_path(env, source_path(env.manifest_file, pkg))
if !isdir(path)
pkgerror("expected package $(err_rep(pkg)) to exist at path `$path`")
@@ -1134,7 +1154,7 @@ function build_versions(ctx::Context, uuids::Set{UUID}; verbose=false)
fancyprint && show_progress(ctx.io, bar)
let log_file=log_file
- sandbox(ctx, pkg, source_path, builddir(source_path), build_project_override; preferences=build_project_preferences) do
+ sandbox(ctx, pkg, builddir(source_path), build_project_override; preferences=build_project_preferences) do
flush(ctx.io)
ok = open(log_file, "w") do log
std = verbose ? ctx.io : log
@@ -1225,6 +1245,9 @@ function rm(ctx::Context, pkgs::Vector{PackageSpec}; mode::PackageMode)
filter!(ctx.env.project.compat) do (name, _)
name == "julia" || name in keys(ctx.env.project.deps) || name in keys(ctx.env.project.extras) || name in keys(ctx.env.project.weakdeps)
end
+ filter!(ctx.env.project.sources) do (name, _)
+ name in keys(ctx.env.project.deps) || name in keys(ctx.env.project.extras)
+ end
deps_names = union(keys(ctx.env.project.deps), keys(ctx.env.project.extras))
filter!(ctx.env.project.targets) do (target, deps)
!isempty(filter!(in(deps_names), deps))
@@ -1237,8 +1260,8 @@ function rm(ctx::Context, pkgs::Vector{PackageSpec}; mode::PackageMode)
show_update(ctx.env, ctx.registries; io=ctx.io)
end
-update_package_add(ctx::Context, pkg::PackageSpec, ::Nothing, is_dep::Bool) = pkg
-function update_package_add(ctx::Context, pkg::PackageSpec, entry::PackageEntry, is_dep::Bool)
+update_package_add(ctx::Context, pkg::PackageSpec, ::Nothing, source_path, source_repo, is_dep::Bool) = pkg
+function update_package_add(ctx::Context, pkg::PackageSpec, entry::PackageEntry, source_path, source_repo, is_dep::Bool)
if entry.pinned
if pkg.version == VersionSpec()
println(ctx.io, "`$(pkg.name)` is pinned at `v$(entry.version)`: maintaining pinned version")
@@ -1381,7 +1404,8 @@ function add(ctx::Context, pkgs::Vector{PackageSpec}, new_git=Set{UUID}();
for (i, pkg) in pairs(pkgs)
entry = manifest_info(ctx.env.manifest, pkg.uuid)
is_dep = any(uuid -> uuid == pkg.uuid, [uuid for (name, uuid) in ctx.env.project.deps])
- pkgs[i] = update_package_add(ctx, pkg, entry, is_dep)
+ source_path, source_repo = get_path_repo(ctx.env.project, pkg.name)
+ pkgs[i] = update_package_add(ctx, pkg, entry, source_path, source_repo, is_dep)
end
names = (p.name for p in pkgs)
@@ -1455,14 +1479,19 @@ end
# load version constraint
# if version isa VersionNumber -> set tree_hash too
-up_load_versions!(ctx::Context, pkg::PackageSpec, ::Nothing, level::UpgradeLevel) = false
-function up_load_versions!(ctx::Context, pkg::PackageSpec, entry::PackageEntry, level::UpgradeLevel)
+up_load_versions!(ctx::Context, pkg::PackageSpec, ::Nothing, source_path, source_repo, level::UpgradeLevel) = false
+function up_load_versions!(ctx::Context, pkg::PackageSpec, entry::PackageEntry, source_path, source_repo, level::UpgradeLevel)
+ # With [sources], `pkg` can have a path or repo here
entry.version !== nothing || return false # no version to set
if entry.pinned || level == UPLEVEL_FIXED
pkg.version = entry.version
pkg.tree_hash = entry.tree_hash
- elseif entry.repo.source !== nothing # repo packages have a version but are treated special
- pkg.repo = entry.repo
+ elseif entry.repo.source !== nothing || source_repo.source !== nothing # repo packages have a version but are treated specially
+ if source_repo.source !== nothing
+ pkg.repo = source_repo
+ else
+ pkg.repo = entry.repo
+ end
if level == UPLEVEL_MAJOR
# Updating a repo package is equivalent to adding it
new = Types.handle_repo_add!(ctx, pkg)
@@ -1470,6 +1499,7 @@ function up_load_versions!(ctx::Context, pkg::PackageSpec, entry::PackageEntry,
if pkg.tree_hash != entry.tree_hash
# TODO parse find_installed and set new version
end
+
return new
else
pkg.version = entry.version
@@ -1489,8 +1519,12 @@ end
up_load_manifest_info!(pkg::PackageSpec, ::Nothing) = nothing
function up_load_manifest_info!(pkg::PackageSpec, entry::PackageEntry)
pkg.name = entry.name # TODO check name is same
- pkg.repo = entry.repo # TODO check that repo is same
- pkg.path = entry.path
+ if pkg.repo == GitRepo()
+ pkg.repo = entry.repo # TODO check that repo is same
+ end
+ if pkg.path === nothing
+ pkg.path = entry.path
+ end
pkg.pinned = entry.pinned
# `pkg.version` and `pkg.tree_hash` is set by `up_load_versions!`
end
@@ -1558,12 +1592,15 @@ function up(ctx::Context, pkgs::Vector{PackageSpec}, level::UpgradeLevel;
# TODO check all pkg.version == VersionSpec()
# set version constraints according to `level`
for pkg in pkgs
- new = up_load_versions!(ctx, pkg, manifest_info(ctx.env.manifest, pkg.uuid), level)
+ source_path, source_repo = get_path_repo(ctx.env.project, pkg.name)
+ entry = manifest_info(ctx.env.manifest, pkg.uuid)
+ new = up_load_versions!(ctx, pkg, entry, source_path, source_repo, level)
new && push!(new_git, pkg.uuid) #TODO put download + push! in utility function
end
# load rest of manifest data (except for version info)
for pkg in pkgs
- up_load_manifest_info!(pkg, manifest_info(ctx.env.manifest, pkg.uuid))
+ entry = manifest_info(ctx.env.manifest, pkg.uuid)
+ up_load_manifest_info!(pkg, entry)
end
if preserve !== nothing
pkgs, deps_map = targeted_resolve_up(ctx.env, ctx.registries, pkgs, preserve, ctx.julia_version)
@@ -1653,7 +1690,11 @@ end
# TODO: this is two technically different operations with the same name
# split into two subfunctions ...
function free(ctx::Context, pkgs::Vector{PackageSpec}; err_if_free=true)
- foreach(pkg -> update_package_free!(ctx.registries, pkg, manifest_info(ctx.env.manifest, pkg.uuid), err_if_free), pkgs)
+ for pkg in pkgs
+ entry = manifest_info(ctx.env.manifest, pkg.uuid)
+ delete!(ctx.env.project.sources, pkg.name)
+ update_package_free!(ctx.registries, pkg, entry, err_if_free)
+ end
if any(pkg -> pkg.version == VersionSpec(), pkgs)
pkgs = load_direct_deps(ctx.env, pkgs)
@@ -1744,8 +1785,9 @@ function sandbox_preserve(env::EnvCache, target::PackageSpec, test_project::Stri
env.manifest.manifest_format = v"2.0"
end
# preserve important nodes
+ project = read_project(test_project)
keep = [target.uuid]
- append!(keep, collect(values(read_project(test_project).deps)))
+ append!(keep, collect(values(project.deps)))
record_project_hash(env)
# prune and return
return prune_manifest(env.manifest, keep)
@@ -1760,8 +1802,17 @@ function abspath!(env::EnvCache, manifest::Manifest)
return manifest
end
+function abspath!(env::EnvCache, project::Project)
+ for (key, entry) in project.sources
+ if haskey(entry, "path")
+ entry["path"] = project_rel_path(env, entry["path"])
+ end
+ end
+ return project
+end
+
# ctx + pkg used to compute parent dep graph
-function sandbox(fn::Function, ctx::Context, target::PackageSpec, target_path::String,
+function sandbox(fn::Function, ctx::Context, target::PackageSpec,
sandbox_path::String, sandbox_project_override;
preferences::Union{Nothing,Dict{String,Any}} = nothing,
force_latest_compatible_version::Bool=false,
@@ -1782,16 +1833,20 @@ function sandbox(fn::Function, ctx::Context, target::PackageSpec, target_path::S
sandbox_project_override = Project()
end
end
+ abspath!(ctx.env, sandbox_project_override)
Types.write_project(sandbox_project_override, tmp_project)
# create merged manifest
# - copy over active subgraph
# - abspath! to maintain location of all deved nodes
- working_manifest = abspath!(ctx.env, sandbox_preserve(ctx.env, target, tmp_project))
+ working_manifest = sandbox_preserve(ctx.env, target, tmp_project)
+ abspath!(ctx.env, working_manifest)
+
# - copy over fixed subgraphs from test subgraph
# really only need to copy over "special" nodes
sandbox_env = Types.EnvCache(projectfile_path(sandbox_path))
abspath!(sandbox_env, sandbox_env.manifest)
+ abspath!(sandbox_env, sandbox_env.project)
for (uuid, entry) in sandbox_env.manifest.deps
entry_working = get(working_manifest, uuid, nothing)
if entry_working === nothing
@@ -1896,6 +1951,7 @@ function gen_target_project(ctx::Context, pkg::PackageSpec, source_path::String,
source_env = EnvCache(projectfile_path(source_path))
# collect regular dependencies
test_project.deps = source_env.project.deps
+ test_project.sources = source_env.project.sources
# collect test dependencies
for name in get(source_env.project.targets, target, String[])
uuid = nothing
@@ -1975,11 +2031,11 @@ function test(ctx::Context, pkgs::Vector{PackageSpec};
end
# now we sandbox
printpkgstyle(ctx.io, :Testing, pkg.name)
- sandbox(ctx, pkg, source_path, testdir(source_path), test_project_override; preferences=test_project_preferences, force_latest_compatible_version, allow_earlier_backwards_compatible_versions, allow_reresolve) do
+ sandbox(ctx, pkg, testdir(source_path), test_project_override; preferences=test_project_preferences, force_latest_compatible_version, allow_earlier_backwards_compatible_versions, allow_reresolve) do
test_fn !== nothing && test_fn()
sandbox_ctx = Context(;io=ctx.io)
status(sandbox_ctx.env, sandbox_ctx.registries; mode=PKGMODE_COMBINED, io=sandbox_ctx.io, ignore_indent = false, show_usagetips = false)
- flags = gen_subprocess_flags(source_path; coverage, julia_args)
+ flags = gen_subprocess_flags(source_path; coverage,julia_args)
if should_autoprecompile()
cacheflags = Base.CacheFlags(parse(UInt8, read(`$(Base.julia_cmd()) $(flags) --eval 'show(ccall(:jl_cache_flags, UInt8, ()))'`, String)))
diff --git a/src/Types.jl b/src/Types.jl
index b73c486dfa..706f91dc40 100644
--- a/src/Types.jl
+++ b/src/Types.jl
@@ -25,7 +25,7 @@ export UUID, SHA1, VersionRange, VersionSpec,
project_resolve!, project_deps_resolve!, manifest_resolve!, registry_resolve!, stdlib_resolve!, handle_repos_develop!, handle_repos_add!, ensure_resolved,
registered_name,
manifest_info,
- read_project, read_package, read_manifest,
+ read_project, read_package, read_manifest, get_path_repo,
PackageMode, PKGMODE_MANIFEST, PKGMODE_PROJECT, PKGMODE_COMBINED,
UpgradeLevel, UPLEVEL_FIXED, UPLEVEL_PATCH, UPLEVEL_MINOR, UPLEVEL_MAJOR,
PreserveLevel, PRESERVE_ALL_INSTALLED, PRESERVE_ALL, PRESERVE_DIRECT, PRESERVE_SEMVER, PRESERVE_TIERED,
@@ -257,6 +257,7 @@ Base.@kwdef mutable struct Project
extras::Dict{String,UUID} = Dict{String,UUID}()
targets::Dict{String,Vector{String}} = Dict{String,Vector{String}}()
compat::Dict{String,Compat} = Dict{String,Compat}()
+ sources::Dict{String,Dict{String, String}} = Dict{String,Dict{String, String}}()
end
Base.:(==)(t1::Project, t2::Project) = all(x -> (getfield(t1, x) == getfield(t2, x))::Bool, fieldnames(Project))
Base.hash(t::Project, h::UInt) = foldr(hash, [getfield(t, x) for x in fieldnames(Project)], init=h)
@@ -1102,6 +1103,24 @@ function manifest_info(manifest::Manifest, uuid::UUID)::Union{PackageEntry,Nothi
end
function write_env(env::EnvCache; update_undo=true,
skip_writing_project::Bool=false)
+ # Verify that the generated manifest is consistent with `sources`
+ for (pkg, uuid) in env.project.deps
+ path, repo = get_path_repo(env.project, pkg)
+ entry = manifest_info(env.manifest, uuid)
+ if path !== nothing
+ @assert entry.path == path
+ end
+ if repo != GitRepo()
+ @show repo, entry.repo
+ @assert entry.repo.source == repo.source
+ if repo.rev !== nothing
+ @assert entry.repo.rev == repo.rev
+ end
+ if entry.repo.subdir !== nothing
+ @assert entry.repo.subdir == repo.subdir
+ end
+ end
+ end
if (env.project != env.original_project) && (!skip_writing_project)
write_project(env)
end
diff --git a/src/precompile.jl b/src/precompile.jl
index 35dde8b3dd..b6a0339651 100644
--- a/src/precompile.jl
+++ b/src/precompile.jl
@@ -159,6 +159,6 @@ let
end
if Base.generating_output()
- pkg_precompile()
+ # pkg_precompile()
end
end
diff --git a/src/project.jl b/src/project.jl
index 4e407ba705..c466048169 100644
--- a/src/project.jl
+++ b/src/project.jl
@@ -1,8 +1,24 @@
#########
# UTILS #
#########
-listed_deps(project::Project) =
- append!(collect(keys(project.deps)), collect(keys(project.extras)), collect(keys(project.weakdeps)))
+listed_deps(project::Project; include_weak::Bool) =
+ vcat(collect(keys(project.deps)), collect(keys(project.extras)), include_weak ? collect(keys(project.weakdeps)) : String[])
+
+function get_path_repo(project::Project, name::String)
+ source = get(project.sources, name, nothing)
+ if source === nothing
+ return nothing, GitRepo()
+ end
+ path = get(source, "path", nothing)::Union{String, Nothing}
+ url = get(source, "url", nothing)::Union{String, Nothing}
+ rev = get(source, "rev", nothing)::Union{String, Nothing}
+ subdir = get(source, "subdir", nothing)::Union{String, Nothing}
+ if path !== nothing && url !== nothing
+ pkgerror("`path` and `url` are conflicting specifications")
+ end
+ repo = GitRepo(url, rev, subdir)
+ return path, repo
+end
###########
# READING #
@@ -74,6 +90,25 @@ end
read_project_compat(raw, project::Project) =
pkgerror("Expected `compat` section to be a key-value list")
+read_project_sources(::Nothing, project::Project) = Dict{String,Any}()
+function read_project_sources(raw::Dict{String,Any}, project::Project)
+ valid_keys = ("path", "url", "rev")
+ sources = Dict{String,Any}()
+ for (name, source) in raw
+ if !(source isa AbstractDict)
+ pkgerror("Expected `source` section to be a table")
+ end
+ for key in keys(source)
+ key in valid_keys || pkgerror("Invalid key `$key` in `source` section")
+ end
+ if haskey(source, "path") && (haskey(source, "url") || haskey(source, "rev"))
+ pkgerror("Both `path` and `url` or `rev` are specified in `source` section")
+ end
+ sources[name] = source
+ end
+ return sources
+end
+
function validate(project::Project; file=nothing)
# deps
location_string = file === nothing ? "" : " at $(repr(file))."
@@ -100,7 +135,7 @@ function validate(project::Project; file=nothing)
end
=#
# targets
- listed = listed_deps(project)
+ listed = listed_deps(project; include_weak=true)
for (target, deps) in project.targets, dep in deps
if length(deps) != length(unique(deps))
pkgerror("A dependency was named twice in target `$target`")
@@ -110,11 +145,17 @@ function validate(project::Project; file=nothing)
""" * location_string)
end
# compat
- for (name, version) in project.compat
+ for name in keys(project.compat)
name == "julia" && continue
name in listed ||
pkgerror("Compat `$name` not listed in `deps`, `weakdeps` or `extras` section" * location_string)
end
+ # sources
+ listed_nonweak = listed_deps(project; include_weak=false)
+ for name in keys(project.sources)
+ name in listed_nonweak ||
+ pkgerror("Sources for `$name` not listed in `deps` or `extras` section" * location_string)
+ end
end
function Project(raw::Dict; file=nothing)
@@ -128,6 +169,7 @@ function Project(raw::Dict; file=nothing)
project.deps = read_project_deps(get(raw, "deps", nothing), "deps")
project.weakdeps = read_project_deps(get(raw, "weakdeps", nothing), "weakdeps")
project.exts = get(Dict{String, String}, raw, "extensions")
+ project.sources = read_project_sources(get(raw, "sources", nothing), project)
project.extras = read_project_deps(get(raw, "extras", nothing), "extras")
project.compat = read_project_compat(get(raw, "compat", nothing), project)
project.targets = read_project_targets(get(raw, "targets", nothing), project)
@@ -183,13 +225,14 @@ function destructure(project::Project)::Dict
entry!("path", project.path)
entry!("deps", merge(project.deps, project._deps_weak))
entry!("weakdeps", project.weakdeps)
+ entry!("sources", project.sources)
entry!("extras", project.extras)
entry!("compat", Dict(name => x.str for (name, x) in project.compat))
entry!("targets", project.targets)
return raw
end
-const _project_key_order = ["name", "uuid", "keywords", "license", "desc", "deps", "weakdeps", "extensions", "compat"]
+const _project_key_order = ["name", "uuid", "keywords", "license", "desc", "deps", "weakdeps", "sources", "extensions", "compat"]
project_key_order(key::String) =
something(findfirst(x -> x == key, _project_key_order), length(_project_key_order) + 1)
@@ -200,7 +243,14 @@ end
write_project(project::Project, project_file::AbstractString) =
write_project(destructure(project), project_file)
function write_project(io::IO, project::Dict)
- TOML.print(io, project, sorted=true, by=key -> (project_key_order(key), key)) do x
+ inline_tables = Base.IdSet{Dict}()
+ if haskey(project, "sources")
+ for source in values(project["sources"])
+ source isa Dict || error("Expected `sources` to be a table")
+ push!(inline_tables, source)
+ end
+ end
+ TOML.print(io, project; inline_tables, sorted=true, by=key -> (project_key_order(key), key)) do x
x isa UUID || x isa VersionNumber || pkgerror("unhandled type `$(typeof(x))`")
return string(x)
end
diff --git a/test/runtests.jl b/test/runtests.jl
index db273666bd..00ab937081 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -60,7 +60,7 @@ Logging.with_logger((islogging || Pkg.DEFAULT_IO[] == devnull) ? Logging.Console
@testset "Pkg" begin
try
- @testset "$f" for f in [
+ @testset "$f" for f in [
"new.jl",
"pkg.jl",
"repl.jl",
@@ -76,7 +76,8 @@ Logging.with_logger((islogging || Pkg.DEFAULT_IO[] == devnull) ? Logging.Console
"misc.jl",
"force_latest_compatible_version.jl",
"manifests.jl",
- "project_manifest.jl"
+ "project_manifest.jl",
+ "sources.jl"
]
@info "==== Testing `test/$f`"
flush(Pkg.DEFAULT_IO[])
diff --git a/test/sources.jl b/test/sources.jl
new file mode 100644
index 0000000000..0a0d285a76
--- /dev/null
+++ b/test/sources.jl
@@ -0,0 +1,27 @@
+module SourcesTest
+
+import ..Pkg # ensure we are using the correct Pkg
+using Test, Pkg
+using ..Utils
+
+temp_pkg_dir() do project_path
+ @testset "test Project.toml [sources]" begin
+ mktempdir() do dir
+ path = abspath(joinpath(dirname(pathof(Pkg)), "../test", "test_packages", "WithSources"))
+ cp(path, joinpath(dir, "WithSources"))
+ cd(joinpath(dir, "WithSources")) do
+ with_current_env() do
+ Pkg.resolve()
+ end
+ end
+
+ cd(joinpath(dir, "WithSources", "TestWithUnreg")) do
+ with_current_env() do
+ Pkg.test()
+ end
+ end
+ end
+ end
+end
+
+end # module
diff --git a/test/test_packages/WithSources/LocalPkg/Project.toml b/test/test_packages/WithSources/LocalPkg/Project.toml
new file mode 100644
index 0000000000..21503e8c0e
--- /dev/null
+++ b/test/test_packages/WithSources/LocalPkg/Project.toml
@@ -0,0 +1,4 @@
+name = "LocalPkg"
+uuid = "fcf55292-0d03-4e8a-9e0b-701580031fc3"
+authors = ["KristofferC <kristoffer.carlsson@juliacomputing.com>"]
+version = "0.1.0"
diff --git a/test/test_packages/WithSources/LocalPkg/src/LocalPkg.jl b/test/test_packages/WithSources/LocalPkg/src/LocalPkg.jl
new file mode 100644
index 0000000000..42feb00fa8
--- /dev/null
+++ b/test/test_packages/WithSources/LocalPkg/src/LocalPkg.jl
@@ -0,0 +1,5 @@
+module LocalPkg
+
+greet() = print("Hello World!")
+
+end # module LocalPkg
diff --git a/test/test_packages/WithSources/Project.toml b/test/test_packages/WithSources/Project.toml
new file mode 100644
index 0000000000..a2d3ffb7bd
--- /dev/null
+++ b/test/test_packages/WithSources/Project.toml
@@ -0,0 +1,7 @@
+[deps]
+Example = "7876af07-990d-54b4-ab0e-23690620f79a"
+LocalPkg = "fcf55292-0d03-4e8a-9e0b-701580031fc3"
+
+[sources]
+Example = {url = "https://github.com/JuliaLang/Example.jl"}
+LocalPkg = {path = "LocalPkg"}
diff --git a/test/test_packages/WithSources/TestWithUnreg/Project.toml b/test/test_packages/WithSources/TestWithUnreg/Project.toml
new file mode 100644
index 0000000000..0dee30048a
--- /dev/null
+++ b/test/test_packages/WithSources/TestWithUnreg/Project.toml
@@ -0,0 +1,15 @@
+name = "TestWithUnreg"
+uuid = "b22516cc-aa7b-4e1f-935d-71632a5f0028"
+authors = ["KristofferC <kristoffer.carlsson@juliacomputing.com>"]
+version = "0.1.0"
+
+[extras]
+LocalPkg = "fcf55292-0d03-4e8a-9e0b-701580031fc3"
+Unregistered = "dcb67f36-efa0-11e8-0cef-2fc465ed98ae"
+
+[sources]
+LocalPkg = {path = "../LocalPkg"}
+Unregistered = {url = "https://github.com/00vareladavid/Unregistered.jl", rev = "1b7a462"}
+
+[targets]
+test = ["LocalPkg", "Unregistered"]
diff --git a/test/test_packages/WithSources/TestWithUnreg/src/TestWithUnreg.jl b/test/test_packages/WithSources/TestWithUnreg/src/TestWithUnreg.jl
new file mode 100644
index 0000000000..a6d1d49f94
--- /dev/null
+++ b/test/test_packages/WithSources/TestWithUnreg/src/TestWithUnreg.jl
@@ -0,0 +1,5 @@
+module TestWithUnreg
+
+greet() = print("Hello World!")
+
+end # module TestWithUnreg
diff --git a/test/test_packages/WithSources/TestWithUnreg/test/runtests.jl b/test/test_packages/WithSources/TestWithUnreg/test/runtests.jl
new file mode 100644
index 0000000000..20d92266bc
--- /dev/null
+++ b/test/test_packages/WithSources/TestWithUnreg/test/runtests.jl
@@ -0,0 +1,2 @@
+using LocalPkg
+using Unregistered
From 16a85d4055e9d4567531c302ff2766630c0b1ff4 Mon Sep 17 00:00:00 2001
From: Kristoffer <kcarlsson89@gmail.com>
Date: Fri, 1 Mar 2024 17:43:52 +0100
Subject: [PATCH 2/5] more tests + add sources when add/dev
---
src/API.jl | 2 ++
src/Types.jl | 8 +++++++-
test/sources.jl | 12 ++++++++++++
test/test_packages/WithSources/BadManifest.toml | 17 +++++++++++++++++
4 files changed, 38 insertions(+), 1 deletion(-)
create mode 100644 test/test_packages/WithSources/BadManifest.toml
diff --git a/src/API.jl b/src/API.jl
index 9b22a22b02..a06d7f907f 100644
--- a/src/API.jl
+++ b/src/API.jl
@@ -101,6 +101,7 @@ Base.@kwdef struct ProjectInfo
version::Union{Nothing,VersionNumber}
ispackage::Bool
dependencies::Dict{String,UUID}
+ sources::Dict{String,Dict{String,String}}
path::String
end
@@ -113,6 +114,7 @@ function project(env::EnvCache)::ProjectInfo
version = pkg === nothing ? nothing : pkg.version::VersionNumber,
ispackage = pkg !== nothing,
dependencies = env.project.deps,
+ sources = env.project.sources,
path = env.project_file
)
end
diff --git a/src/Types.jl b/src/Types.jl
index 706f91dc40..81686bf79a 100644
--- a/src/Types.jl
+++ b/src/Types.jl
@@ -1111,7 +1111,6 @@ function write_env(env::EnvCache; update_undo=true,
@assert entry.path == path
end
if repo != GitRepo()
- @show repo, entry.repo
@assert entry.repo.source == repo.source
if repo.rev !== nothing
@assert entry.repo.rev == repo.rev
@@ -1120,6 +1119,13 @@ function write_env(env::EnvCache; update_undo=true,
@assert entry.repo.subdir == repo.subdir
end
end
+ if entry.path !== nothing
+ env.project.sources[pkg] = Dict("path" => entry.path)
+ elseif entry.repo != GitRepo()
+ d = Dict("url" => entry.repo.source)
+ entry.repo.rev !== nothing && (d["rev"] = entry.repo.rev)
+ env.project.sources[pkg] = d
+ end
end
if (env.project != env.original_project) && (!skip_writing_project)
write_project(env)
diff --git a/test/sources.jl b/test/sources.jl
index 0a0d285a76..49d256e479 100644
--- a/test/sources.jl
+++ b/test/sources.jl
@@ -12,6 +12,18 @@ temp_pkg_dir() do project_path
cd(joinpath(dir, "WithSources")) do
with_current_env() do
Pkg.resolve()
+ @test !isempty(Pkg.project().sources["Example"])
+ project_backup = cp("Project.toml", "Project.toml.bak"; force=true)
+ Pkg.free("Example")
+ @test !haskey(Pkg.project().sources, "Example")
+ cp("Project.toml.bak", "Project.toml"; force=true)
+ Pkg.add(; url="https://github.com/JuliaLang/Example.jl/", rev="78406c204b8")
+ @test Pkg.project().sources["Example"] == Dict("url" => "https://github.com/JuliaLang/Example.jl/", "rev" => "78406c204b8")
+ cp("Project.toml.bak", "Project.toml"; force=true)
+ cp("BadManifest.toml", "Manifest.toml"; force=true)
+ Pkg.resolve()
+ @test Pkg.project().sources["Example"] == Dict("url" => "https://github.com/JuliaLang/Example.jl")
+ @test Pkg.project().sources["LocalPkg"] == Dict("path" => "LocalPkg")
end
end
diff --git a/test/test_packages/WithSources/BadManifest.toml b/test/test_packages/WithSources/BadManifest.toml
new file mode 100644
index 0000000000..9b9f2cddd0
--- /dev/null
+++ b/test/test_packages/WithSources/BadManifest.toml
@@ -0,0 +1,17 @@
+# This file is machine-generated - editing it directly is not advised
+
+julia_version = "1.12.0-DEV"
+manifest_format = "2.0"
+project_hash = "65567d20ea33b8d376b53b9e9e96938f75d0fab1"
+
+[[deps.Example]]
+git-tree-sha1 = "e64c907c88640c370060a89aaae1b641eedc3dfc"
+repo-rev = "master"
+repo-url = "https://what.is.this.url???"
+uuid = "7876af07-990d-54b4-ab0e-23690620f79a"
+version = "0.5.4"
+
+[[deps.LocalPkg]]
+path = "what/is/this/path???"
+uuid = "fcf55292-0d03-4e8a-9e0b-701580031fc3"
+version = "0.1.0"
From c09520e4b7ff1c54f1bfb322f4d402ccfb271496 Mon Sep 17 00:00:00 2001
From: Kristoffer <kcarlsson89@gmail.com>
Date: Fri, 1 Mar 2024 17:51:16 +0100
Subject: [PATCH 3/5] add a Changelog entry
---
CHANGELOG.md | 5 +++++
1 file changed, 5 insertions(+)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index cf81dbb33e..f2f388e56c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,8 @@
+Pkg v1.12 Release Notes
+
+- It is now possible to specify "sources" for packages in a `[sources]` section in Project.toml.
+ This can be used to add non-registered normal or test dependencies. Packages will also automatically be added to `[sources]` when they are added by url or devved.
+
Pkg v1.11 Release Notes
=======================
From 878acf57968a9dfcfcc586808336d37959f959d6 Mon Sep 17 00:00:00 2001
From: Kristoffer <kcarlsson89@gmail.com>
Date: Fri, 1 Mar 2024 18:26:16 +0100
Subject: [PATCH 4/5] remove other source if already set
---
src/API.jl | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/src/API.jl b/src/API.jl
index a06d7f907f..e2042f0643 100644
--- a/src/API.jl
+++ b/src/API.jl
@@ -189,12 +189,16 @@ function update_source_if_set(project, pkg)
# This should probably not modify the dicts directly...
if pkg.repo.source !== nothing
source["url"] = pkg.repo.source
+ delete!(source, "path")
end
if pkg.repo.rev !== nothing
source["rev"] = pkg.repo.rev
+ delete!(source, "path")
end
if pkg.path !== nothing
source["path"] = pkg.path
+ delete!(source, "url")
+ delete!(source, "rev")
end
path, repo = get_path_repo(project, pkg.name)
if path !== nothing
From 6e7ef20a6e6f313dec4e6e8c948020708b309ebe Mon Sep 17 00:00:00 2001
From: Kristoffer <kcarlsson89@gmail.com>
Date: Tue, 5 Mar 2024 11:40:48 +0100
Subject: [PATCH 5/5] back out of auto adding to sources
---
CHANGELOG.md | 3 ++-
src/API.jl | 4 ----
src/Types.jl | 7 -------
3 files changed, 2 insertions(+), 12 deletions(-)
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f2f388e56c..8a9e6390e9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,7 +1,8 @@
Pkg v1.12 Release Notes
+=======================
- It is now possible to specify "sources" for packages in a `[sources]` section in Project.toml.
- This can be used to add non-registered normal or test dependencies. Packages will also automatically be added to `[sources]` when they are added by url or devved.
+ This can be used to add non-registered normal or test dependencies.
Pkg v1.11 Release Notes
=======================
diff --git a/src/API.jl b/src/API.jl
index e2042f0643..a06d7f907f 100644
--- a/src/API.jl
+++ b/src/API.jl
@@ -189,16 +189,12 @@ function update_source_if_set(project, pkg)
# This should probably not modify the dicts directly...
if pkg.repo.source !== nothing
source["url"] = pkg.repo.source
- delete!(source, "path")
end
if pkg.repo.rev !== nothing
source["rev"] = pkg.repo.rev
- delete!(source, "path")
end
if pkg.path !== nothing
source["path"] = pkg.path
- delete!(source, "url")
- delete!(source, "rev")
end
path, repo = get_path_repo(project, pkg.name)
if path !== nothing
diff --git a/src/Types.jl b/src/Types.jl
index 81686bf79a..4f5da0f6ad 100644
--- a/src/Types.jl
+++ b/src/Types.jl
@@ -1119,13 +1119,6 @@ function write_env(env::EnvCache; update_undo=true,
@assert entry.repo.subdir == repo.subdir
end
end
- if entry.path !== nothing
- env.project.sources[pkg] = Dict("path" => entry.path)
- elseif entry.repo != GitRepo()
- d = Dict("url" => entry.repo.source)
- entry.repo.rev !== nothing && (d["rev"] = entry.repo.rev)
- env.project.sources[pkg] = d
- end
end
if (env.project != env.original_project) && (!skip_writing_project)
write_project(env)