Support a [sources] section in Project.toml for specifying paths and repo locations for dependencies (#3783)

* Support a `[sources]` section in Project.toml for specifying relative path and repo locations for dependencies

---------

Co-authored-by: Jacob Quinn <[email protected]>

Author: KristofferC

Diff for: CHANGELOG.md

@@ -1,3 +1,9 @@
+Pkg v1.12 Release Notes
+=======================
+
+- It is now possible to specify "sources" for packages in a `[sources]` section in Project.toml.
+  This can be used to add non-registered normal or test dependencies.
+
 Pkg v1.11 Release Notes
 =======================
 

Diff for: docs/src/toml-files.md

@@ -91,6 +91,19 @@ Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 Typically it is not needed to manually add entries to the `[deps]` section; this is instead
 handled by Pkg operations such as `add`.
 
+### The `[sources]` section
+
+Specifiying a path or repo (+ branch) for a dependency is done in the `[sources]` section.
+These are especially useful for controlling unregistered dependencies without having to bundle a
+corresponding manifest file.
+
+```toml
+[sources]
+Example = {url = "https://github.com/JuliaLang/Example.jl", rev = "custom_branch"}
+SomeDependency = {path = "deps/SomeDependency.jl"}
+```
+
+Note that this information is only used when this environment is active, i.e. it is not used if this project is a package that is being used as a dependency.
 
 ### The `[compat]` section
 
@@ -135,7 +148,7 @@ Julia will then preferentially use the version-specific manifest file if availab
 For example, if both `Manifest-v1.11.toml` and `Manifest.toml` exist, Julia 1.11 will prioritize using `Manifest-v1.11.toml`.
 However, Julia versions 1.10, 1.12, and all others will default to using `Manifest.toml`.
 This feature allows for easier management of different instantiated versions of dependencies for various Julia versions.
-Note that there can only be one `Project.toml` file. While `Manifest-v{major}.{minor}.toml` files are not automatically 
+Note that there can only be one `Project.toml` file. While `Manifest-v{major}.{minor}.toml` files are not automatically
 created by Pkg, users can manually rename a `Manifest.toml` file to match
 the versioned format, and Pkg will subsequently maintain it through its operations.
 

Diff for: src/API.jl

@@ -101,6 +101,7 @@ Base.@kwdef struct ProjectInfo
     version::Union{Nothing,VersionNumber}
     ispackage::Bool
     dependencies::Dict{String,UUID}
+    sources::Dict{String,Dict{String,String}}
     path::String
 end
 
@@ -113,6 +114,7 @@ function project(env::EnvCache)::ProjectInfo
         version      = pkg === nothing ? nothing : pkg.version::VersionNumber,
         ispackage    = pkg !== nothing,
         dependencies = env.project.deps,
+        sources      = env.project.sources,
         path         = env.project_file
     )
 end
@@ -181,6 +183,31 @@ for f in (:develop, :add, :rm, :up, :pin, :free, :test, :build, :status, :why, :
     end
 end
 
+function update_source_if_set(project, pkg)
+    source = get(project.sources, pkg.name, nothing)
+    source === nothing && return
+    # This should probably not modify the dicts directly...
+    if pkg.repo.source !== nothing
+        source["url"] = pkg.repo.source
+    end
+    if pkg.repo.rev !== nothing
+        source["rev"] = pkg.repo.rev
+    end
+    if pkg.path !== nothing
+        source["path"] = pkg.path
+    end
+    path, repo = get_path_repo(project, pkg.name)
+    if path !== nothing
+        pkg.path = path
+    end
+    if repo.source !== nothing
+        pkg.repo.source = repo.source
+    end
+    if repo.rev !== nothing
+        pkg.repo.rev = repo.rev
+    end
+end
+
 function develop(ctx::Context, pkgs::Vector{PackageSpec}; shared::Bool=true,
                  preserve::PreserveLevel=Operations.default_preserve(), platform::AbstractPlatform=HostPlatform(), kwargs...)
     require_not_empty(pkgs, :develop)
@@ -212,13 +239,15 @@ function develop(ctx::Context, pkgs::Vector{PackageSpec}; shared::Bool=true,
 
     new_git = handle_repos_develop!(ctx, pkgs, shared)
 
+
     for pkg in pkgs
         if Types.collides_with_project(ctx.env, pkg)
             pkgerror("package $(err_rep(pkg)) has the same name or UUID as the active project")
         end
         if length(findall(x -> x.uuid == pkg.uuid, pkgs)) > 1
             pkgerror("it is invalid to specify multiple packages with the same UUID: $(err_rep(pkg))")
         end
+        update_source_if_set(ctx.env.project, pkg)
     end
 
     Operations.develop(ctx, pkgs, new_git; preserve=preserve, platform=platform)
@@ -272,6 +301,7 @@ function add(ctx::Context, pkgs::Vector{PackageSpec}; preserve::PreserveLevel=Op
         if length(findall(x -> x.uuid == pkg.uuid, pkgs)) > 1
             pkgerror("it is invalid to specify multiple packages with the same UUID: $(err_rep(pkg))")
         end
+        update_source_if_set(ctx.env.project, pkg)
     end
 
     Operations.add(ctx, pkgs, new_git; preserve, platform, target)
@@ -311,12 +341,14 @@ end
 function append_all_pkgs!(pkgs, ctx, mode)
     if mode == PKGMODE_PROJECT || mode == PKGMODE_COMBINED
         for (name::String, uuid::UUID) in ctx.env.project.deps
-            push!(pkgs, PackageSpec(name=name, uuid=uuid))
+            path, repo = get_path_repo(ctx.env.project, name)
+            push!(pkgs, PackageSpec(name=name, uuid=uuid, path=path, repo=repo))
         end
     end
     if mode == PKGMODE_MANIFEST || mode == PKGMODE_COMBINED
         for (uuid, entry) in ctx.env.manifest
-            push!(pkgs, PackageSpec(name=entry.name, uuid=uuid))
+            path, repo = get_path_repo(ctx.env.project, entry.name)
+            push!(pkgs, PackageSpec(name=entry.name, uuid=uuid, path=path, repo=repo))
         end
     end
     return
@@ -347,6 +379,7 @@ function up(ctx::Context, pkgs::Vector{PackageSpec};
         manifest_resolve!(ctx.env.manifest, pkgs)
         ensure_resolved(ctx, ctx.env.manifest, pkgs)
     end
+
     Operations.up(ctx, pkgs, level; skip_writing_project, preserve)
     return
 end

Diff for: src/Operations.jl

@@ -71,14 +71,15 @@ function load_direct_deps(env::EnvCache, pkgs::Vector{PackageSpec}=PackageSpec[]
     pkgs = copy(pkgs)
     for (name::String, uuid::UUID) in env.project.deps
         findfirst(pkg -> pkg.uuid == uuid, pkgs) === nothing || continue # do not duplicate packages
+        path, repo = get_path_repo(env.project, name)
         entry = manifest_info(env.manifest, uuid)
         push!(pkgs, entry === nothing ?
-              PackageSpec(;uuid=uuid, name=name) :
+              PackageSpec(;uuid=uuid, name=name, path=path, repo=repo) :
               PackageSpec(;
                 uuid      = uuid,
                 name      = name,
-                path      = entry.path,
-                repo      = entry.repo,
+                path      = path === nothing ? entry.path : path,
+                repo      = repo == GitRepo() ? entry.repo : repo,
                 pinned    = entry.pinned,
                 tree_hash = entry.tree_hash, # TODO should tree_hash be changed too?
                 version   = load_version(entry.version, isfixed(entry), preserve),
@@ -108,6 +109,19 @@ end
 function load_all_deps(env::EnvCache, pkgs::Vector{PackageSpec}=PackageSpec[];
                        preserve::PreserveLevel=PRESERVE_ALL)
     pkgs = load_manifest_deps(env.manifest, pkgs; preserve=preserve)
+    # Sources takes presedence over the manifest...
+    for pkg in pkgs
+        path, repo = get_path_repo(env.project, pkg.name)
+        if path !== nothing
+            pkg.path = path
+        end
+        if repo.source !== nothing
+            pkg.repo.source = repo.source
+        end
+        if repo.rev !== nothing
+            pkg.repo.rev = repo.rev
+        end
+    end
     return load_direct_deps(env, pkgs; preserve=preserve)
 end
 
@@ -244,8 +258,9 @@ function collect_project(pkg::PackageSpec, path::String)
         pkgerror("julia version requirement from Project.toml's compat section not satisfied for package $(err_rep(pkg)) at `$path`")
     end
     for (name, uuid) in project.deps
+        path, repo = get_path_repo(project, name)
         vspec = get_compat(project, name)
-        push!(deps, PackageSpec(name, uuid, vspec))
+        push!(deps, PackageSpec(name=name, uuid=uuid, version=vspec, path=path, repo=repo))
     end
     for (name, uuid) in project.weakdeps
         vspec = get_compat(project, name)
@@ -302,6 +317,11 @@ function collect_fixed!(env::EnvCache, pkgs::Vector{PackageSpec}, names::Dict{UU
         names[pkg.uuid] = pkg.name
     end
     for pkg in pkgs
+        # add repo package if necessary
+        if (pkg.repo.rev !== nothing || pkg.repo.source !== nothing) && pkg.tree_hash === nothing
+            # ensure revved package is installed
+            Types.handle_repo_add!(Types.Context(env=env), pkg)
+        end
         path = project_rel_path(env, source_path(env.manifest_file, pkg))
         if !isdir(path)
             pkgerror("expected package $(err_rep(pkg)) to exist at path `$path`")
@@ -1135,7 +1155,7 @@ function build_versions(ctx::Context, uuids::Set{UUID}; verbose=false)
         fancyprint && show_progress(ctx.io, bar)
 
         let log_file=log_file
-            sandbox(ctx, pkg, source_path, builddir(source_path), build_project_override; preferences=build_project_preferences) do
+            sandbox(ctx, pkg, builddir(source_path), build_project_override; preferences=build_project_preferences) do
                 flush(ctx.io)
                 ok = open(log_file, "w") do log
                     std = verbose ? ctx.io : log
@@ -1226,6 +1246,9 @@ function rm(ctx::Context, pkgs::Vector{PackageSpec}; mode::PackageMode)
     filter!(ctx.env.project.compat) do (name, _)
         name == "julia" || name in keys(ctx.env.project.deps) || name in keys(ctx.env.project.extras) || name in keys(ctx.env.project.weakdeps)
     end
+    filter!(ctx.env.project.sources) do (name, _)
+        name in keys(ctx.env.project.deps) || name in keys(ctx.env.project.extras)
+    end
     deps_names = union(keys(ctx.env.project.deps), keys(ctx.env.project.extras))
     filter!(ctx.env.project.targets) do (target, deps)
         !isempty(filter!(in(deps_names), deps))
@@ -1238,8 +1261,8 @@ function rm(ctx::Context, pkgs::Vector{PackageSpec}; mode::PackageMode)
     show_update(ctx.env, ctx.registries; io=ctx.io)
 end
 
-update_package_add(ctx::Context, pkg::PackageSpec, ::Nothing, is_dep::Bool) = pkg
-function update_package_add(ctx::Context, pkg::PackageSpec, entry::PackageEntry, is_dep::Bool)
+update_package_add(ctx::Context, pkg::PackageSpec, ::Nothing, source_path, source_repo, is_dep::Bool) = pkg
+function update_package_add(ctx::Context, pkg::PackageSpec, entry::PackageEntry, source_path, source_repo, is_dep::Bool)
     if entry.pinned
         if pkg.version == VersionSpec()
             println(ctx.io, "`$(pkg.name)` is pinned at `v$(entry.version)`: maintaining pinned version")
@@ -1382,7 +1405,8 @@ function add(ctx::Context, pkgs::Vector{PackageSpec}, new_git=Set{UUID}();
     for (i, pkg) in pairs(pkgs)
         entry = manifest_info(ctx.env.manifest, pkg.uuid)
         is_dep = any(uuid -> uuid == pkg.uuid, [uuid for (name, uuid) in ctx.env.project.deps])
-        pkgs[i] = update_package_add(ctx, pkg, entry, is_dep)
+        source_path, source_repo = get_path_repo(ctx.env.project, pkg.name)
+        pkgs[i] = update_package_add(ctx, pkg, entry, source_path, source_repo, is_dep)
     end
 
     names = (p.name for p in pkgs)
@@ -1456,21 +1480,27 @@ end
 
 # load version constraint
 # if version isa VersionNumber -> set tree_hash too
-up_load_versions!(ctx::Context, pkg::PackageSpec, ::Nothing, level::UpgradeLevel) = false
-function up_load_versions!(ctx::Context, pkg::PackageSpec, entry::PackageEntry, level::UpgradeLevel)
+up_load_versions!(ctx::Context, pkg::PackageSpec, ::Nothing, source_path, source_repo, level::UpgradeLevel) = false
+function up_load_versions!(ctx::Context, pkg::PackageSpec, entry::PackageEntry, source_path, source_repo, level::UpgradeLevel)
+    # With [sources], `pkg` can have a path or repo here
     entry.version !== nothing || return false # no version to set
     if entry.pinned || level == UPLEVEL_FIXED
         pkg.version = entry.version
         pkg.tree_hash = entry.tree_hash
-    elseif entry.repo.source !== nothing # repo packages have a version but are treated special
-        pkg.repo = entry.repo
+    elseif entry.repo.source !== nothing || source_repo.source !== nothing # repo packages have a version but are treated specially
+        if source_repo.source !== nothing
+            pkg.repo = source_repo
+        else
+            pkg.repo = entry.repo
+        end
         if level == UPLEVEL_MAJOR
             # Updating a repo package is equivalent to adding it
             new = Types.handle_repo_add!(ctx, pkg)
             pkg.version = entry.version
             if pkg.tree_hash != entry.tree_hash
                 # TODO parse find_installed and set new version
             end
+
             return new
         else
             pkg.version = entry.version
@@ -1490,8 +1520,12 @@ end
 up_load_manifest_info!(pkg::PackageSpec, ::Nothing) = nothing
 function up_load_manifest_info!(pkg::PackageSpec, entry::PackageEntry)
     pkg.name = entry.name # TODO check name is same
-    pkg.repo = entry.repo # TODO check that repo is same
-    pkg.path = entry.path
+    if pkg.repo == GitRepo()
+        pkg.repo = entry.repo # TODO check that repo is same
+    end
+    if pkg.path === nothing
+        pkg.path = entry.path
+    end
     pkg.pinned = entry.pinned
     # `pkg.version` and `pkg.tree_hash` is set by `up_load_versions!`
 end
@@ -1559,12 +1593,15 @@ function up(ctx::Context, pkgs::Vector{PackageSpec}, level::UpgradeLevel;
     # TODO check all pkg.version == VersionSpec()
     # set version constraints according to `level`
     for pkg in pkgs
-        new = up_load_versions!(ctx, pkg, manifest_info(ctx.env.manifest, pkg.uuid), level)
+        source_path, source_repo = get_path_repo(ctx.env.project, pkg.name)
+        entry = manifest_info(ctx.env.manifest, pkg.uuid)
+        new = up_load_versions!(ctx, pkg, entry, source_path, source_repo, level)
         new && push!(new_git, pkg.uuid) #TODO put download + push! in utility function
     end
     # load rest of manifest data (except for version info)
     for pkg in pkgs
-        up_load_manifest_info!(pkg, manifest_info(ctx.env.manifest, pkg.uuid))
+        entry = manifest_info(ctx.env.manifest, pkg.uuid)
+        up_load_manifest_info!(pkg, entry)
     end
     if preserve !== nothing
         pkgs, deps_map = targeted_resolve_up(ctx.env, ctx.registries, pkgs, preserve, ctx.julia_version)
@@ -1654,7 +1691,11 @@ end
 # TODO: this is two technically different operations with the same name
 # split into two subfunctions ...
 function free(ctx::Context, pkgs::Vector{PackageSpec}; err_if_free=true)
-    foreach(pkg -> update_package_free!(ctx.registries, pkg, manifest_info(ctx.env.manifest, pkg.uuid), err_if_free), pkgs)
+    for pkg in pkgs
+        entry = manifest_info(ctx.env.manifest, pkg.uuid)
+        delete!(ctx.env.project.sources, pkg.name)
+        update_package_free!(ctx.registries, pkg, entry, err_if_free)
+    end
 
     if any(pkg -> pkg.version == VersionSpec(), pkgs)
         pkgs = load_direct_deps(ctx.env, pkgs)
@@ -1745,8 +1786,9 @@ function sandbox_preserve(env::EnvCache, target::PackageSpec, test_project::Stri
         env.manifest.manifest_format = v"2.0"
     end
     # preserve important nodes
+    project = read_project(test_project)
     keep = [target.uuid]
-    append!(keep, collect(values(read_project(test_project).deps)))
+    append!(keep, collect(values(project.deps)))
     record_project_hash(env)
     # prune and return
     return prune_manifest(env.manifest, keep)
@@ -1761,8 +1803,17 @@ function abspath!(env::EnvCache, manifest::Manifest)
     return manifest
 end
 
+function abspath!(env::EnvCache, project::Project)
+    for (key, entry) in project.sources
+        if haskey(entry, "path")
+            entry["path"] = project_rel_path(env, entry["path"])
+        end
+    end
+    return project
+end
+
 # ctx + pkg used to compute parent dep graph
-function sandbox(fn::Function, ctx::Context, target::PackageSpec, target_path::String,
+function sandbox(fn::Function, ctx::Context, target::PackageSpec,
                  sandbox_path::String, sandbox_project_override;
                  preferences::Union{Nothing,Dict{String,Any}} = nothing,
                  force_latest_compatible_version::Bool=false,
@@ -1783,16 +1834,20 @@ function sandbox(fn::Function, ctx::Context, target::PackageSpec, target_path::S
                 sandbox_project_override = Project()
             end
         end
+        abspath!(ctx.env, sandbox_project_override)
         Types.write_project(sandbox_project_override, tmp_project)
 
         # create merged manifest
         # - copy over active subgraph
         # - abspath! to maintain location of all deved nodes
-        working_manifest = abspath!(ctx.env, sandbox_preserve(ctx.env, target, tmp_project))
+        working_manifest = sandbox_preserve(ctx.env, target, tmp_project)
+        abspath!(ctx.env, working_manifest)
+
         # - copy over fixed subgraphs from test subgraph
         # really only need to copy over "special" nodes
         sandbox_env = Types.EnvCache(projectfile_path(sandbox_path))
         abspath!(sandbox_env, sandbox_env.manifest)
+        abspath!(sandbox_env, sandbox_env.project)
         for (uuid, entry) in sandbox_env.manifest.deps
             entry_working = get(working_manifest, uuid, nothing)
             if entry_working === nothing
@@ -1897,6 +1952,7 @@ function gen_target_project(ctx::Context, pkg::PackageSpec, source_path::String,
     source_env = EnvCache(projectfile_path(source_path))
     # collect regular dependencies
     test_project.deps = source_env.project.deps
+    test_project.sources = source_env.project.sources
     # collect test dependencies
     for name in get(source_env.project.targets, target, String[])
         uuid = nothing
@@ -1976,11 +2032,11 @@ function test(ctx::Context, pkgs::Vector{PackageSpec};
         end
         # now we sandbox
         printpkgstyle(ctx.io, :Testing, pkg.name)
-        sandbox(ctx, pkg, source_path, testdir(source_path), test_project_override; preferences=test_project_preferences, force_latest_compatible_version, allow_earlier_backwards_compatible_versions, allow_reresolve) do
+        sandbox(ctx, pkg, testdir(source_path), test_project_override; preferences=test_project_preferences, force_latest_compatible_version, allow_earlier_backwards_compatible_versions, allow_reresolve) do
             test_fn !== nothing && test_fn()
             sandbox_ctx = Context(;io=ctx.io)
             status(sandbox_ctx.env, sandbox_ctx.registries; mode=PKGMODE_COMBINED, io=sandbox_ctx.io, ignore_indent = false, show_usagetips = false)
-            flags = gen_subprocess_flags(source_path; coverage, julia_args)
+            flags = gen_subprocess_flags(source_path; coverage,julia_args)
 
             if should_autoprecompile()
                 cacheflags = Base.CacheFlags(parse(UInt8, read(`$(Base.julia_cmd()) $(flags) --eval 'show(ccall(:jl_cache_flags, UInt8, ()))'`, String)))