var"ident" syntax for non-standard identifiers (JuliaLang#32408)

Allow identifiers like Symbol("#example#") to be represented in julia
code with the syntax `var"#example#"`. This needed support in the parser
rather than being a string macro so that it could be used in any context
where a normal identifier is allowed.

Show nonstandard identifiers in Exprs with var"ident" syntax

Author: c42f

NEWS.md

@@ -12,6 +12,7 @@ New language features
 Language changes
 ----------------
 
+* The syntax `var"#example#"` is used to print and parse non-standard variable names ([#32408]).
 
 Multi-threading changes
 -----------------------

base/docs/basedocs.jl

@@ -934,6 +934,29 @@ using the syntax `T{p1, p2, ...}`.
 """
 kw"where"
 
+"""
+    var
+
+The syntax `var"#example#"` refers to a variable named `Symbol("#example#")`,
+even though `#example#` is not a valid Julia identifier name.
+
+This can be useful for interoperability with programming languages which have
+different rules for the construction of valid identifiers. For example, to
+refer to the `R` variable `draw.segments`, you can use `var"draw.segments"` in
+your Julia code.
+
+It is also used to `show` julia source code which has gone through macro
+hygiene or otherwise contains variable names which can't be parsed normally.
+
+Note that this syntax requires parser support so it is expanded directly by the
+parser rather than being implemented as a normal string macro `@var_str`.
+
+!!! compat "Julia 1.3"
+    This syntax requires at least Julia 1.3.
+
+"""
+kw"var\"name\"", kw"@var_str"
+
 """
     ans
 

base/show.jl

@@ -490,48 +490,28 @@ function show_type_name(io::IO, tn::Core.TypeName)
         end
     end
     sym = (globfunc ? globname : tn.name)::Symbol
-    if get(io, :compact, false)
-        if globfunc
-            return print(io, "typeof(", sym, ")")
-        else
-            return print(io, sym)
-        end
-    end
-    sym_str = string(sym)
-    hidden = !globfunc && '#' ∈ sym_str
+    globfunc && print(io, "typeof(")
     quo = false
-    if hidden
-        print(io, "getfield(")
-    elseif globfunc
-        print(io, "typeof(")
-    end
-    # Print module prefix unless type is visible from module passed to IOContext
-    # If :module is not set, default to Main. nothing can be used to force printing prefix
-    from = get(io, :module, Main)
-    if isdefined(tn, :module) && (hidden || from === nothing || !isvisible(sym, tn.module, from))
-        show(io, tn.module)
-        if !hidden
+    if !get(io, :compact, false)
+        # Print module prefix unless type is visible from module passed to
+        # IOContext If :module is not set, default to Main. nothing can be used
+        # to force printing prefix
+        from = get(io, :module, Main)
+        if isdefined(tn, :module) && (from === nothing || !isvisible(sym, tn.module, from))
+            show(io, tn.module)
             print(io, ".")
-            if globfunc && !is_id_start_char(first(sym_str))
-                print(io, ":")
-                if sym == :(==)
-                    print(io, "(")
+            if globfunc && !is_id_start_char(first(string(sym)))
+                print(io, ':')
+                if sym in quoted_syms
+                    print(io, '(')
                     quo = true
                 end
             end
         end
     end
-    if hidden
-        print(io, ", Symbol(\"", sym_str, "\"))")
-    else
-        print(io, sym_str)
-        if globfunc
-            print(io, ")")
-            if quo
-                print(io, ")")
-            end
-        end
-    end
+    show_sym(io, sym)
+    quo      && print(io, ")")
+    globfunc && print(io, ")")
 end
 
 function show_datatype(io::IO, x::DataType)
@@ -814,7 +794,7 @@ julia> Base.isoperator(:+), Base.isoperator(:f)
 (true, false)
 ```
 """
-isoperator(s::Symbol) = ccall(:jl_is_operator, Cint, (Cstring,), s) != 0
+isoperator(s::Union{Symbol,AbstractString}) = ccall(:jl_is_operator, Cint, (Cstring,), s) != 0
 
 """
     isunaryoperator(s::Symbol)
@@ -981,7 +961,7 @@ end
 function show_call(io::IO, head, func, func_args, indent)
     op, cl = expr_calls[head]
     if (isa(func, Symbol) && func !== :(:) && !(head === :. && isoperator(func))) ||
-            (isa(func, Expr) && (func.head == :. || func.head == :curly)) ||
+            (isa(func, Expr) && (func.head == :. || func.head == :curly || func.head == :macroname)) ||
             isa(func, GlobalRef)
         show_unquoted(io, func, indent)
     else
@@ -1003,10 +983,24 @@ function show_call(io::IO, head, func, func_args, indent)
     end
 end
 
+# Print `sym` as it would appear as an identifier name in code
+# * Print valid identifiers & operators literally; also macros names if allow_macroname=true
+# * Escape invalid identifiers with var"" syntax
+function show_sym(io::IO, sym; allow_macroname=false)
+    if isidentifier(sym) || isoperator(sym)
+        print(io, sym)
+    elseif allow_macroname && (sym_str = string(sym); startswith(sym_str, '@'))
+        print(io, '@')
+        show_sym(io, sym_str[2:end])
+    else
+        print(io, "var", repr(string(sym)))
+    end
+end
+
 ## AST printing ##
 
 show_unquoted(io::IO, val::SSAValue, ::Int, ::Int)      = print(io, "%", val.id)
-show_unquoted(io::IO, sym::Symbol, ::Int, ::Int)        = print(io, sym)
+show_unquoted(io::IO, sym::Symbol, ::Int, ::Int)        = show_sym(io, sym)
 show_unquoted(io::IO, ex::LineNumberNode, ::Int, ::Int) = show_linenumber(io, ex.line, ex.file)
 show_unquoted(io::IO, ex::GotoNode, ::Int, ::Int)       = print(io, "goto %", ex.label)
 function show_unquoted(io::IO, ex::GlobalRef, ::Int, ::Int)
@@ -1016,7 +1010,7 @@ function show_unquoted(io::IO, ex::GlobalRef, ::Int, ::Int)
     parens = quoted && (!isoperator(ex.name) || (ex.name in quoted_syms))
     quoted && print(io, ':')
     parens && print(io, '(')
-    print(io, ex.name)
+    show_sym(io, ex.name, allow_macroname=true)
     parens && print(io, ')')
     nothing
 end
@@ -1094,7 +1088,7 @@ end
 
 function show_import_path(io::IO, ex)
     if !isa(ex, Expr)
-        print(io, ex)
+        show_unquoted(io, ex)
     elseif ex.head === :(:)
         show_import_path(io, ex.args[1])
         print(io, ": ")
@@ -1105,18 +1099,21 @@ function show_import_path(io::IO, ex)
             show_import_path(io, ex.args[i])
         end
     elseif ex.head === :(.)
-        print(io, ex.args[1])
-        for i = 2:length(ex.args)
-            if ex.args[i-1] != :(.)
+        for i = 1:length(ex.args)
+            if i > 1 && ex.args[i-1] != :(.)
                 print(io, '.')
             end
-            print(io, ex.args[i])
+            show_sym(io, ex.args[i], allow_macroname=(i==length(ex.args)))
         end
     else
         show_unquoted(io, ex)
     end
 end
 
+# Wrap symbols for macro names to allow them to be printed literally
+allow_macroname(ex) = ex isa Symbol && first(string(ex)) == '@' ?
+                      Expr(:macroname, ex) : ex
+
 # TODO: implement interpolated strings
 function show_unquoted(io::IO, ex::Expr, indent::Int, prec::Int)
     head, args, nargs = ex.head, ex.args, length(ex.args)
@@ -1279,7 +1276,9 @@ function show_unquoted(io::IO, ex::Expr, indent::Int, prec::Int)
         print(io, "end")
 
     elseif (head === :function || head === :macro) && nargs == 1
-        print(io, head, ' ', args[1], " end")
+        print(io, head, ' ')
+        show_unquoted(io, args[1])
+        print(io, " end")
 
     elseif head === :do && nargs == 2
         show_unquoted(io, args[1], indent, -1)
@@ -1344,26 +1343,39 @@ function show_unquoted(io::IO, ex::Expr, indent::Int, prec::Int)
         print(io, head)
 
     elseif (nargs == 1 && head in (:return, :const)) ||
-                          head in (:local,  :global, :export)
+                          head in (:local,  :global)
         print(io, head, ' ')
         show_list(io, args, ", ", indent)
 
+    elseif head === :export
+        print(io, head, ' ')
+        show_list(io, allow_macroname.(args), ", ", indent)
+
     elseif head === :macrocall && nargs >= 2
         # first show the line number argument as a comment
         if isa(args[2], LineNumberNode) || is_expr(args[2], :line)
             print(io, args[2], ' ')
         end
         # Use the functional syntax unless specifically designated with prec=-1
         # and hide the line number argument from the argument list
+        mname = allow_macroname(args[1])
         if prec >= 0
-            show_call(io, :call, args[1], args[3:end], indent)
+            show_call(io, :call, mname, args[3:end], indent)
         else
             show_args = Vector{Any}(undef, nargs - 1)
-            show_args[1] = args[1]
+            show_args[1] = mname
             show_args[2:end] = args[3:end]
             show_list(io, show_args, ' ', indent)
         end
 
+    elseif head === :macroname && nargs == 1
+        arg1 = args[1]
+        if arg1 isa Symbol
+            show_sym(io, arg1, allow_macroname=true)
+        else
+            show_unquoted(io, arg1)
+        end
+
     elseif head === :line && 1 <= nargs <= 2
         show_linenumber(io, args...)
 
@@ -1406,11 +1418,7 @@ function show_unquoted(io::IO, ex::Expr, indent::Int, prec::Int)
         for x in args
             if !isa(x,AbstractString)
                 print(io, "\$(")
-                if isa(x,Symbol) && !(x in quoted_syms)
-                    print(io, x)
-                else
-                    show_unquoted(io, x)
-                end
+                show_unquoted(io, x)
                 print(io, ")")
             else
                 escape_string(io, x, "\"\$")

doc/src/base/base.md

@@ -251,6 +251,7 @@ Base.@nospecialize
 Base.@specialize
 Base.gensym
 Base.@gensym
+var"name"
 Base.@goto
 Base.@label
 Base.@simd

src/julia-parser.scm

@@ -1068,10 +1068,11 @@
 ;; -2^3 is parsed as -(2^3), so call parse-decl for the first argument,
 ;; and parse-unary from then on (to handle 2^-3)
 (define (parse-factor s)
-  (parse-factor-with-initial-ex s (parse-unary-prefix s)))
+  (let ((nxt (peek-token s)))
+    (parse-factor-with-initial-ex s (parse-unary-prefix s) nxt)))
 
-(define (parse-factor-with-initial-ex s ex0)
-  (let* ((ex (parse-decl-with-initial-ex s (parse-call-with-initial-ex s ex0)))
+(define (parse-factor-with-initial-ex s ex0 (tok #f))
+  (let* ((ex (parse-decl-with-initial-ex s (parse-call-with-initial-ex s ex0 tok)))
          (t  (peek-token s)))
     (if (is-prec-power? t)
         (begin (take-token s)
@@ -1100,10 +1101,11 @@
 ;; parse function call, indexing, dot, and transpose expressions
 ;; also handles looking for syntactic reserved words
 (define (parse-call s)
-  (parse-call-with-initial-ex s (parse-unary-prefix s)))
+  (let ((nxt (peek-token s)))
+    (parse-call-with-initial-ex s (parse-unary-prefix s) nxt)))
 
-(define (parse-call-with-initial-ex s ex)
-  (if (or (initial-reserved-word? ex) (eq? ex 'mutable) (eq? ex 'primitive) (eq? ex 'abstract))
+(define (parse-call-with-initial-ex s ex tok)
+  (if (or (initial-reserved-word? tok) (memq tok '(mutable primitive abstract)))
       (parse-resword s ex)
       (parse-call-chain s ex #f)))
 
@@ -2282,7 +2284,17 @@
                (begin (check-identifier t)
                       (if (closing-token? t)
                           (error (string "unexpected \"" (take-token s) "\"")))))
-           (take-token s))
+           (take-token s)
+           (if (and (eq? t 'var) (eqv? (peek-token s) #\") (not (ts:space? s)))
+             (begin
+               ;; var"funky identifier" syntax
+               (take-token s)
+               (let ((str (parse-raw-literal s #\"))
+                     (nxt (peek-token s)))
+                 (if (and (symbol? nxt) (not (operator? nxt)) (not (ts:space? s)))
+                   (error (string "suffix not allowed after `var\"" str "\"`")))
+                 (symbol str)))
+             t))
 
           ;; parens or tuple
           ((eqv? t #\( )

stdlib/Test/src/Test.jl

@@ -236,6 +236,7 @@ function eval_test(evaluated::Expr, quoted::Expr, source::LineNumberNode, negate
     evaled_args = evaluated.args
     quoted_args = quoted.args
     n = length(evaled_args)
+    kw_suffix = ""
     if evaluated.head == :comparison
         args = evaled_args
         while i < n
@@ -260,17 +261,9 @@ function eval_test(evaluated::Expr, quoted::Expr, source::LineNumberNode, negate
         func_sym = quoted_args[1]
         if isempty(kwargs)
             quoted = Expr(:call, func_sym, args...)
-        elseif func_sym === :≈
-            # in case of `≈(x, y, atol = z)`
-            # make the display like `Evaluated: x ≈ y (atol=z)`
-            kws = [Symbol(Expr(:kw, k, v), ",") for (k, v) in kwargs]
-            kws[end] = Symbol(Expr(:kw, kwargs[end]...))
-            kws[1] = Symbol("(", kws[1])
-            kws[end] = Symbol(kws[end], ")")
-            quoted = Expr(:comparison, args[1], func_sym, args[2], kws...)
-            if length(quoted.args) & 1 == 0  # hack to fit `show_unquoted`
-                push!(quoted.args, Symbol())
-            end
+        elseif func_sym === :≈ && !res
+            quoted = Expr(:call, func_sym, args...)
+            kw_suffix = " ($(join(["$k=$v" for (k, v) in kwargs], ", ")))"
         else
             kwargs_expr = Expr(:parameters, [Expr(:kw, k, v) for (k, v) in kwargs]...)
             quoted = Expr(:call, func_sym, kwargs_expr, args...)
@@ -286,7 +279,7 @@ function eval_test(evaluated::Expr, quoted::Expr, source::LineNumberNode, negate
 
     Returned(res,
              # stringify arguments in case of failure, for easy remote printing
-             res ? quoted : sprint(io->print(IOContext(io, :limit => true), quoted)),
+             res ? quoted : sprint(io->print(IOContext(io, :limit => true), quoted))*kw_suffix,
              source)
 end
 

test/show.jl

@@ -351,6 +351,22 @@ end
 @test sprint(show, :+) == ":+"
 @test sprint(show, :end) == ":end"
 
+# issue #32408: Printing of names which are invalid identifiers
+# Invalid identifiers which need `var` quoting:
+@test sprint(show, Expr(:call, :foo, Symbol("##")))   == ":(foo(var\"##\"))"
+@test sprint(show, Expr(:call, :foo, Symbol("a-b")))  == ":(foo(var\"a-b\"))"
+@test sprint(show, :(export var"#"))    == ":(export var\"#\")"
+@test sprint(show, :(import A: var"#")) == ":(import A: var\"#\")"
+@test sprint(show, :(macro var"#" end)) == ":(macro var\"#\" end)"
+@test sprint(show, :"x$(var"#")y") == ":(\"x\$(var\"#\")y\")"
+# Macro-like names outside macro calls
+@test sprint(show, Expr(:call, :foo, Symbol("@bar"))) == ":(foo(var\"@bar\"))"
+@test sprint(show, :(export @foo)) == ":(export @foo)"
+@test sprint(show, :(import A.B: c.@d)) == ":(import A.B: c.@d)"
+@test sprint(show, :(using A.@foo)) == ":(using A.@foo)"
+# Hidden macro names
+@test sprint(show, Expr(:macrocall, Symbol("@#"), nothing, :a)) == ":(@var\"#\" a)"
+
 # issue #12477
 @test sprint(show,  Union{Int64, Int32, Int16, Int8, Float64}) == "Union{Float64, Int16, Int32, Int64, Int8}"
 
@@ -580,9 +596,9 @@ function f13127()
     show(buf, f)
     String(take!(buf))
 end
-@test startswith(f13127(), "getfield($(@__MODULE__), Symbol(\"")
+@test startswith(f13127(), "$(@__MODULE__).var\"#f")
 
-@test startswith(sprint(show, typeof(x->x), context = :module=>@__MODULE__), "getfield($(@__MODULE__), Symbol(\"")
+@test startswith(sprint(show, typeof(x->x), context = :module=>@__MODULE__), "var\"")
 
 #test methodshow.jl functions
 @test Base.inbase(Base)
@@ -1146,7 +1162,7 @@ end
     @test repr(typeof(UnexportedOperators.:(==))) == "typeof($(curmod_prefix)UnexportedOperators.:(==))"
     anonfn = x->2x
     modname = string(@__MODULE__)
-    anonfn_type_repr = "getfield($modname, Symbol(\"$(typeof(anonfn).name.name)\"))"
+    anonfn_type_repr = "$modname.var\"$(typeof(anonfn).name.name)\""
     @test repr(typeof(anonfn)) == anonfn_type_repr
     @test repr(anonfn) == anonfn_type_repr * "()"
     @test repr("text/plain", anonfn) == "$(typeof(anonfn).name.mt.name) (generic function with 1 method)"