pass correct paths through to docstrings

fix #22105

Author: vtjnash

Makefile

@@ -186,6 +186,7 @@ $(build_private_libdir)/%.$(SHLIB_EXT): $(build_private_libdir)/%.o
 
 CORE_SRCS := $(addprefix $(JULIAHOME)/, \
 		base/boot.jl base/coreimg.jl \
+		base/docs/core.jl \
 		base/abstractarray.jl \
 		base/array.jl \
 		base/bool.jl \

base/boot.jl

@@ -337,15 +337,15 @@ end
 
 # docsystem basics
 macro doc(x...)
-    atdoc(x...)
+    atdoc(__source__, x...)
 end
 macro __doc__(x)
     Expr(:escape, Expr(:block, Expr(:meta, :doc), x))
 end
 macro doc_str(s)
     Expr(:escape, s)
 end
-atdoc     = (str, expr) -> Expr(:escape, expr)
+atdoc     = (source, str, expr) -> Expr(:escape, expr)
 atdoc!(λ) = global atdoc = λ
 
 

base/docs/Docs.jl

@@ -169,10 +169,10 @@ _docstr(doc::DocStr, data) = (doc.data = merge(data, doc.data); doc)
 macro ref(x)
     binding = bindingexpr(namify(x))
     typesig = signature(x)
-    esc(docexpr(binding, typesig))
+    esc(docexpr(__source__, binding, typesig))
 end
 
-docexpr(args...) = Expr(:call, docstr, args...)
+docexpr(__source__, args...) = Expr(:call, docstr, args...)
 
 function formatdoc(d::DocStr)
     buffer = IOBuffer()
@@ -482,26 +482,27 @@ isfield(x) = isexpr(x, :.) &&
 # =========================
 
 """
-    Docs.metadata(expr)
+    Docs.metadata(source, expr)
 
 Build a `Dict` expression containing metadata captured from the expression `expr`.
 
 Fields that may be included in the returned `Dict`:
 
-- `:path`:       String representing the file where `expr` is defined.
+- `:path`:       Symbol representing the file where `expr` is defined.
 - `:linenumber`: Linenumber where `expr` is defined.
 - `:module`:     Module where the docstring is defined.
 - `:fields`:     `Dict` of all field docs found in `expr`. Only for concrete types.
 """
-function metadata(expr)
+function metadata(__source__, expr)
     args = []
     # Filename and linenumber of the docstring.
-    push!(args, :($(Pair)(:path, $(Base).@__FILE__)))
-    push!(args, :($(Pair)(:linenumber, $(unsafe_load(cglobal(:jl_lineno, Cint))))))
+    __file__ = isa(__source__.file, Symbol) ? String(__source__.file) : ""
+    push!(args, Pair(:path, __file__))
+    push!(args, Pair(:linenumber, __source__.line))
     # Module in which the docstring is defined.
     push!(args, :($(Pair)(:module, $(current_module)())))
-    # Field docs for concrete types.
     if isexpr(expr, :type)
+        # Field docs for concrete types.
         fields = []
         tmp = nothing
         for each in expr.args[3].args
@@ -518,37 +519,36 @@ function metadata(expr)
     :($(Dict)($(args...)))
 end
 
-function keyworddoc(str, def)
-    docstr = esc(docexpr(lazy_iterpolate(str), metadata(def)))
-    :($(keywords)[$(esc(quot(def.name)))] = $docstr)
+function keyworddoc(__source__, str, def)
+    docstr = esc(docexpr(__source__, lazy_iterpolate(str), metadata(__source__, def)))
+    return :($(keywords)[$(esc(quot(def.name)))] = $docstr)
 end
 
-function objectdoc(str, def, expr, sig = :(Union{}))
+function objectdoc(__source__, str, def, expr, sig = :(Union{}))
     binding = esc(bindingexpr(namify(expr)))
-    docstr  = esc(docexpr(lazy_iterpolate(str), metadata(expr)))
+    docstr  = esc(docexpr(__source__, lazy_iterpolate(str), metadata(__source__, expr)))
     quote
         $(esc(def))
         $(doc!)($binding, $docstr, $(esc(sig)))
     end
 end
 
-function calldoc(str, def)
+function calldoc(__source__, str, def)
     args = def.args[2:end]
     if isempty(args) || all(validcall, args)
-        objectdoc(str, nothing, def, signature(def))
+        objectdoc(__source__, str, nothing, def, signature(def))
     else
         docerror(def)
     end
 end
 validcall(x) = isa(x, Symbol) || isexpr(x, (:(::), :..., :kw, :parameters))
 
-function moduledoc(meta, def, def′)
+function moduledoc(__source__, meta, def, def′)
     name  = namify(def′)
     docex = Expr(:call, doc!, bindingexpr(name),
-        docexpr(lazy_iterpolate(meta), metadata(name))
-    )
+        docexpr(__source__, lazy_iterpolate(meta), metadata(__source__, name)))
     if def === nothing
-        esc(:(eval($name, $(quot(docex)))))
+        esc(:($eval($name, $(quot(docex)))))
     else
         def = unblock(def)
         block = def.args[3].args
@@ -562,18 +562,18 @@ function moduledoc(meta, def, def′)
 end
 
 # Shares a single doc, `meta`, between several expressions from the tuple expression `ex`.
-function multidoc(meta, ex, define)
+function multidoc(__source__, meta, ex, define)
     out = Expr(:toplevel)
-    str = docexpr(lazy_iterpolate(meta), metadata(ex))
+    str = docexpr(__source__, lazy_iterpolate(meta), metadata(__source__, ex))
     ref = Ref{DocStr}()
     for (n, arg) in enumerate(ex.args)
         # The first `arg` to be documented needs to also create the docstring for the group.
         # Subsequent `arg`s just need `ref` to be able to find the docstring without having
         # to create an entirely new one each.
         docstr = n === 1 ? :($(ref)[] = $str) : :($(ref)[])
-        push!(out.args, :(@doc($docstr, $arg, $define)))
+        push!(out.args, docm(__source__, docstr, arg, define))
     end
-    esc(out)
+    return out
 end
 
 """
@@ -649,7 +649,7 @@ isquotedmacrocall(x) =
 isbasicdoc(x) = isexpr(x, :.) || isa(x, Union{QuoteNode, Symbol})
 is_signature(x) = isexpr(x, :call) || (isexpr(x, :(::), 2) && isexpr(x.args[1], :call)) || isexpr(x, :where)
 
-function docm(meta, ex, define = true)
+function docm(source::LineNumberNode, meta, ex, define = true)
     # Some documented expressions may be decorated with macro calls which obscure the actual
     # expression. Expand the macro calls and remove extra blocks.
     x = unblock(macroexpand(ex))
@@ -665,7 +665,7 @@ function docm(meta, ex, define = true)
     #   "..."
     #   kw"if", kw"else"
     #
-    isa(x, Base.BaseDocs.Keyword) ? keyworddoc(meta, x) :
+    isa(x, Base.BaseDocs.Keyword) ? keyworddoc(source, meta, x) :
 
     # Method / macro definitions and "call" syntax.
     #
@@ -675,38 +675,38 @@ function docm(meta, ex, define = true)
     #   function f end
     #   f(...)
     #
-    isexpr(x, FUNC_HEADS) && is_signature(x.args[1])   ? objectdoc(meta, def, x, signature(x)) :
-    isexpr(x, :function)  && !isexpr(x.args[1], :call) ? objectdoc(meta, def, x) :
-    isexpr(x, :call)                                   ? calldoc(meta, x) :
+    isexpr(x, FUNC_HEADS) && is_signature(x.args[1])   ? objectdoc(source, meta, def, x, signature(x)) :
+    isexpr(x, :function)  && !isexpr(x.args[1], :call) ? objectdoc(source, meta, def, x) :
+    isexpr(x, :call)                                   ? calldoc(source, meta, x) :
 
     # Type definitions.
     #
     #   type T ... end
     #   abstract T
     #   bitstype N T
     #
-    isexpr(x, [:type, :abstract, :bitstype]) ? objectdoc(meta, def, x) :
+    isexpr(x, [:type, :abstract, :bitstype]) ? objectdoc(source, meta, def, x) :
 
     # "Bindings". Names that resolve to objects with different names, ie.
     #
     #   const T = S
     #   T = S
     #   global T = S
     #
-    isexpr(x, BINDING_HEADS) && !isexpr(x.args[1], :call) ? objectdoc(meta, def, x) :
+    isexpr(x, BINDING_HEADS) && !isexpr(x.args[1], :call) ? objectdoc(source, meta, def, x) :
 
     # Quoted macrocall syntax. `:@time` / `:(Base.@time)`.
-    isquotedmacrocall(x) ? objectdoc(meta, def, x) :
+    isquotedmacrocall(x) ? objectdoc(source, meta, def, x) :
     # Modules and baremodules.
-    isexpr(x, :module) ? moduledoc(meta, def, x) :
+    isexpr(x, :module) ? moduledoc(source, meta, def, x) :
     # Document several expressions with the same docstring. `a, b, c`.
-    isexpr(x, :tuple) ? multidoc(meta, x, define) :
+    isexpr(x, :tuple) ? multidoc(source, meta, x, define) :
     # Errors generated by calling `macroexpand` are passed back to the call site.
     isexpr(x, :error) ? esc(x) :
     # When documenting macro-generated code we look for embedded `@__doc__` calls.
     __doc__!(meta, x, define) ? esc(x) :
     # Any "basic" expression such as a bare function or module name or numeric literal.
-    isbasicdoc(x) ? objectdoc(meta, nothing, x) :
+    isbasicdoc(x) ? objectdoc(source, meta, nothing, x) :
 
     # All other expressions are undocumentable and should be handled on a case-by-case basis
     # with `@__doc__`. Unbound string literals are also undocumentable since they cannot be
@@ -725,9 +725,9 @@ function docerror(ex)
     :($(error)($txt, "\n"))
 end
 
-function docm(ex)
+function docm(source::LineNumberNode, ex)
     if isexpr(ex, :->)
-        docm(ex.args...)
+        docm(source, ex.args...)
     elseif haskey(keywords, ex)
         parsedoc(keywords[ex])
     elseif isa(ex, Union{Expr, Symbol})
@@ -764,7 +764,7 @@ function loaddocs(docs)
     for (mod, ex, str, file, line) in docs
         data = Dict(:path => string(file), :linenumber => line)
         doc = docstr(str, data)
-        docstring = eval(mod, Expr(:body, Expr(:return, Expr(:call, QuoteNode(docm), QuoteNode(doc), QuoteNode(ex), false)))) # expand the real @doc macro now (using a hack because macroexpand takes current-module as an implicit argument)
+        docstring = eval(mod, Expr(:body, Expr(:return, Expr(:call, QuoteNode(docm), QuoteNode(LineNumberNode(line, file)), QuoteNode(doc), QuoteNode(ex), false)))) # expand the real @doc macro now (using a hack because macroexpand takes current-module as an implicit argument)
         eval(mod, Expr(:macrocall, unescape, nothing, docstring))
     end
     empty!(docs)

base/docs/core.jl

@@ -4,12 +4,8 @@ module CoreDocs
 
 import ..esc, ..push!, ..getindex, ..current_module, ..unsafe_load, ..Csize_t
 
-function doc!(str, ex)
-    ptr  = unsafe_load(Core.Intrinsics.cglobal(:jl_filename, Ptr{UInt8}))
-    len  = ccall(:strlen, Csize_t, (Ptr{UInt8},), ptr)
-    file = ccall(:jl_symbol_n, Any, (Ptr{UInt8}, Csize_t), ptr, len)
-    line = unsafe_load(Core.Intrinsics.cglobal(:jl_lineno, Int32)) # Cint
-    push!(DOCS, (current_module(), ex, str, file, line))
+function doc!(source::LineNumberNode, str, ex)
+    push!(DOCS, (current_module(), ex, str, source.file, source.line))
 end
 const DOCS = Array{Any, 1}()
 
@@ -18,11 +14,12 @@ isexpr(x, h) = isa(x, Expr) && x.head === h
 lazy_iterpolate(s::AbstractString) = Expr(:call, Core.svec, s)
 lazy_iterpolate(x) = isexpr(x, :string) ? Expr(:call, Core.svec, x.args...) : x
 
-function docm(str, x)
-    out = esc(Expr(:call, doc!, lazy_iterpolate(str), Expr(:quote, x)))
+function docm(source::LineNumberNode, str, x)
+    out = esc(Expr(:call, doc!, QuoteNode(source), lazy_iterpolate(str), Expr(:quote, x)))
     isexpr(x, :module) ? Expr(:toplevel, out, esc(x)) :
     isexpr(x, :call) ? out : Expr(:block, esc(x), out)
 end
-docm(x) = isexpr(x, :->) ? docm(x.args[1], x.args[2].args[2]) : error("invalid '@doc'.")
+docm(source::LineNumberNode, x) =
+    isexpr(x, :->) ? docm(source, x.args[1], x.args[2].args[2]) : error("invalid '@doc'.")
 
 end

base/markdown/Markdown.jl

@@ -37,25 +37,23 @@ function mdexpr(s, flavor = :julia)
     esc(toexpr(md))
 end
 
-function docexpr(s, flavor = :julia)
-    quote
-        let md = $(mdexpr(s, flavor))
-            md.meta[:path] = @__FILE__
-            md.meta[:module] = current_module()
-            md
-        end
-    end
+function docexpr(source::LineNumberNode, s, flavor = :julia)
+    :($doc_str($(mdexpr(s, flavor)), $(QuoteNode(source))))
 end
 
 macro md_str(s, t...)
     mdexpr(s, t...)
 end
 
-doc_str(md, file, mod) = (md.meta[:path] = file; md.meta[:module] = mod; md)
-doc_str(md::AbstractString, file, mod) = doc_str(parse(md), file, mod)
+function doc_str(md, source::LineNumberNode)
+    md.meta[:path] = isa(source.file, Symbol) ? String(source.file) : ""
+    md.meta[:module] = current_module()
+    md
+end
+doc_str(md::AbstractString, source::LineNumberNode) = doc_str(parse(md), source)
 
 macro doc_str(s::AbstractString, t...)
-    :($(doc_str)($(mdexpr(s, t...)), $(Base).@__FILE__, $(current_module)()))
+    docexpr(__source__, s, t...)
 end
 
 function Base.display(d::Base.REPL.REPLDisplay, md::Vector{MD})

base/precompile.jl

@@ -743,7 +743,7 @@ precompile(Tuple{getfield(Base.Docs, Symbol("#@repl")), Base.TTY, Symbol})
 precompile(Tuple{typeof(Base.Docs.repl), Base.TTY, Symbol})
 precompile(Tuple{typeof(Base.Docs._repl), Symbol})
 precompile(Tuple{getfield(Core, Symbol("#@doc")), Symbol})
-precompile(Tuple{typeof(Base.Docs.docm), Symbol})
+precompile(Tuple{typeof(Base.Docs.docm), LineNumberNode, Symbol})
 precompile(Tuple{typeof(Base._setindex!), Base.Dict{Any, Any}, Base.Markdown.Config, Symbol, Int64})
 precompile(Tuple{Type{Base.Markdown.MD}})
 precompile(Tuple{typeof(Base.setindex!), Base.Dict{Any, Any}, Base.Markdown.Config, Symbol})
@@ -1720,8 +1720,8 @@ precompile(Tuple{typeof(Base.Distributed.send_msg), Base.Distributed.Worker, Bas
 precompile(Tuple{typeof(Base.Distributed.send_msg_), Base.Distributed.Worker, Base.Distributed.MsgHeader, Base.Distributed.RemoteDoMsg, Bool})
 precompile(Tuple{typeof(Base.Distributed.terminate_all_workers)})
 precompile(Tuple{typeof(Base.Distributed.test_existing_ref), Base.Distributed.Future})
-precompile(Tuple{typeof(Base.Docs.docm), String, Expr})
-precompile(Tuple{typeof(Base.Docs.docm), String, Expr, Bool})
+precompile(Tuple{typeof(Base.Docs.docm), LineNumberNode, String, Expr})
+precompile(Tuple{typeof(Base.Docs.docm), LineNumberNode, String, Expr, Bool})
 precompile(Tuple{typeof(Base.Docs.keyworddoc), String, Base.BaseDocs.Keyword})
 precompile(Tuple{typeof(Base.Docs.objectdoc), String, Expr, Expr, Expr})
 precompile(Tuple{typeof(Base.FastMath.make_fastmath), Expr})

test/docs.jl

@@ -481,24 +481,26 @@ end
 
 # Issue #16359. Error message for invalid doc syntax.
 
-for each in [ # valid syntax
-        :(f()),
-        :(f(x)),
-        :(f(x::Int)),
-        :(f(x...)),
-        :(f(x = 1)),
-        :(f(; x = 1))
-    ]
-    @test Meta.isexpr(Docs.docm("...", each), :block)
-end
-for each in [ # invalid syntax
-        :(f("...")),
-        :(f(1, 2)),
-        :(f(() -> ()))
-    ]
-    result = Docs.docm("...", each)
-    @test Meta.isexpr(result, :call)
-    @test result.args[1] === error
+let __source__ = LineNumberNode(0)
+    for each in [ # valid syntax
+            :(f()),
+            :(f(x)),
+            :(f(x::Int)),
+            :(f(x...)),
+            :(f(x = 1)),
+            :(f(; x = 1))
+        ]
+        @test Meta.isexpr(Docs.docm(__source__, "...", each), :block)
+    end
+    for each in [ # invalid syntax
+            :(f("...")),
+            :(f(1, 2)),
+            :(f(() -> ()))
+        ]
+        result = Docs.docm(__source__, "...", each)
+        @test Meta.isexpr(result, :call)
+        @test result.args[1] === error
+    end
 end
 
 # Issue #15424. Non-markdown docstrings.
@@ -1012,3 +1014,21 @@ end
     """
 )
 
+# issue #22105
+module I22105
+    lineno = @__LINE__
+    """foo docs"""
+    function foo end
+end
+
+let foo_docs = meta(I22105)[@var(I22105.foo)].docs
+    @test length(foo_docs) === 1
+    @test isa(first(foo_docs), Pair)
+    local docstr = first(foo_docs).second
+    @test isa(docstr, DocStr)
+    @test docstr.data[:path] == Base.source_path()
+    @test docstr.data[:linenumber] == I22105.lineno + 1
+    @test docstr.data[:module] === I22105
+    @test docstr.data[:typesig] === Union{}
+    @test docstr.data[:binding] == Binding(I22105, :foo)
+end