mapexpr argument for include: transform each parsed expression (#34595)

Author: stevengj

NEWS.md

@@ -51,6 +51,8 @@ New library functions
   `merge!` are still available for backward compatibility ([#34296]).
 * The new `isdisjoint` function indicates whether two collections are disjoint ([#34427]).
 * Add function `ismutable` and deprecate `isimmutable` to check whether something is mutable.([#34652])
+* `include` now accepts an optional `mapexpr` first argument to transform the parsed
+  expressions before they are evaluated ([#34595]).
 
 New library features
 --------------------

base/Base.jl

@@ -365,8 +365,8 @@ for m in methods(include)
 end
 # These functions are duplicated in client.jl/include(::String) for
 # nicer stacktraces. Modifications here have to be backported there
-include(mod::Module, path::AbstractString) = include(mod, convert(String, path))
-function include(mod::Module, _path::String)
+include(mod::Module, _path::AbstractString) = include(identity, mod, _path)
+function include(mapexpr::Function, mod::Module, _path::AbstractString)
     path, prev = _include_dependency(mod, _path)
     for callback in include_callbacks # to preserve order, must come before Core.include
         invokelatest(callback, mod, path)
@@ -376,7 +376,11 @@ function include(mod::Module, _path::String)
     local result
     try
         # result = Core.include(mod, path)
-        result = ccall(:jl_load_, Any, (Any, Any), mod, path)
+        if mapexpr === identity
+            result = ccall(:jl_load, Any, (Any, Cstring), mod, path)
+        else
+            result = ccall(:jl_load_rewrite, Any, (Any, Cstring, Any), mod, path, mapexpr)
+        end
     finally
         if prev === nothing
             delete!(tls, :SOURCE_PATH)

base/client.jl

@@ -423,8 +423,10 @@ end
 # MainInclude exists to hide Main.include and eval from `names(Main)`.
 baremodule MainInclude
 using ..Base
-# We inline the definition of include from loading.jl/include_relative to get one-frame stacktraces.
-# include(fname::AbstractString) = Main.Base.include(Main, fname)
+include(mapexpr::Function, fname::AbstractString) = Base.include(mapexpr, Main, fname)
+# We inline the definition of include from loading.jl/include_relative to get one-frame stacktraces
+# for the common case of include(fname).  Otherwise we would use:
+#    include(fname::AbstractString) = Base.include(Main, fname)
 function include(fname::AbstractString)
     mod = Main
     isa(fname, String) || (fname = Base.convert(String, fname)::String)
@@ -436,7 +438,7 @@ function include(fname::AbstractString)
     tls[:SOURCE_PATH] = path
     local result
     try
-        result = ccall(:jl_load_, Any, (Any, Any), mod, path)
+        result = ccall(:jl_load, Any, (Any, Cstring), mod, path)
     finally
         if prev === nothing
             Base.delete!(tls, :SOURCE_PATH)
@@ -459,16 +461,20 @@ definition of `eval`, which evaluates expressions in that module.
 MainInclude.eval
 
 """
-    include(path::AbstractString)
+    include([mapexpr::Function,] path::AbstractString)
 
 Evaluate the contents of the input source file in the global scope of the containing module.
-Every module (except those defined with `baremodule`) has its own 1-argument
+Every module (except those defined with `baremodule`) has its own
 definition of `include`, which evaluates the file in that module.
 Returns the result of the last evaluated expression of the input file. During including,
 a task-local include path is set to the directory containing the file. Nested calls to
 `include` will search relative to that path. This function is typically used to load source
 interactively, or to combine files in packages that are broken into multiple source files.
 
+The optional first argument `mapexpr` can be used to transform the included code before
+it is evaluated: for each parsed expression `expr` in `path`, the `include` function
+actually evaluates `mapexpr(expr)`.  If it is omitted, `mapexpr` defaults to [`identity`](@ref).
+
 Use [`Base.include`](@ref) to evaluate a file into another module.
 """
 MainInclude.include

base/loading.jl

@@ -1073,16 +1073,27 @@ end
 # relative-path load
 
 """
-    include_string(m::Module, code::AbstractString, filename::AbstractString="string")
+    include_string([mapexpr::Function,] m::Module, code::AbstractString, filename::AbstractString="string")
 
 Like [`include`](@ref), except reads code from the given string rather than from a file.
+
+The optional first argument `mapexpr` can be used to transform the included code before
+it is evaluated: for each parsed expression `expr` in `code`, the `include_string` function
+actually evaluates `mapexpr(expr)`.  If it is omitted, `mapexpr` defaults to [`identity`](@ref).
 """
-include_string(m::Module, txt::String, fname::String) =
-    ccall(:jl_load_file_string, Any, (Ptr{UInt8}, Csize_t, Cstring, Any),
-          txt, sizeof(txt), fname, m)
+function include_string(mapexpr::Function, m::Module, txt_::AbstractString, fname::AbstractString="string")
+    txt = String(txt_)
+    if mapexpr === identity
+        ccall(:jl_load_file_string, Any, (Ptr{UInt8}, Csize_t, Cstring, Any),
+            txt, sizeof(txt), String(fname), m)
+    else
+        ccall(:jl_load_rewrite_file_string, Any, (Ptr{UInt8}, Csize_t, Cstring, Any, Any),
+            txt, sizeof(txt), String(fname), m, mapexpr)
+    end
+end
 
 include_string(m::Module, txt::AbstractString, fname::AbstractString="string") =
-    include_string(m, String(txt), String(fname))
+    include_string(identity, m, txt, fname)
 
 function source_path(default::Union{AbstractString,Nothing}="")
     s = current_task().storage
@@ -1098,15 +1109,19 @@ function source_dir()
 end
 
 """
-    Base.include([m::Module,] path::AbstractString)
+    Base.include([mapexpr::Function,] [m::Module,] path::AbstractString)
 
 Evaluate the contents of the input source file in the global scope of module `m`.
-Every module (except those defined with [`baremodule`](@ref)) has its own 1-argument
-definition of `include`, which evaluates the file in that module.
+Every module (except those defined with [`baremodule`](@ref)) has its own
+definition of `include` omitting the `m` argument, which evaluates the file in that module.
 Returns the result of the last evaluated expression of the input file. During including,
 a task-local include path is set to the directory containing the file. Nested calls to
 `include` will search relative to that path. This function is typically used to load source
 interactively, or to combine files in packages that are broken into multiple source files.
+
+The optional first argument `mapexpr` can be used to transform the included code before
+it is evaluated: for each parsed expression `expr` in `path`, the `include` function
+actually evaluates `mapexpr(expr)`.  If it is omitted, `mapexpr` defaults to [`identity`](@ref).
 """
 Base.include # defined in sysimg.jl
 
@@ -1122,6 +1137,7 @@ function evalfile(path::AbstractString, args::Vector{String}=String[])
              :(const ARGS = $args),
              :(eval(x) = $(Expr(:core, :eval))(__anon__, x)),
              :(include(x) = $(Expr(:top, :include))(__anon__, x)),
+             :(include(mapexpr::Function, x) = $(Expr(:top, :include))(mapexpr, __anon__, x)),
              :(include($path))))
 end
 evalfile(path::AbstractString, args::Vector) = evalfile(path, String[args...])

src/ast.c

@@ -828,7 +828,8 @@ JL_DLLEXPORT jl_value_t *jl_parse_string(const char *str, size_t len,
 // parse and eval a whole file, possibly reading from a string (`content`)
 jl_value_t *jl_parse_eval_all(const char *fname,
                               const char *content, size_t contentlen,
-                              jl_module_t *inmodule)
+                              jl_module_t *inmodule,
+                              jl_value_t *mapexpr)
 {
     jl_ptls_t ptls = jl_get_ptls_states();
     if (ptls->in_pure_callback)
@@ -879,9 +880,16 @@ jl_value_t *jl_parse_eval_all(const char *fname,
                 JL_TIMING(LOWERING);
                 if (fl_ctx->T == fl_applyn(fl_ctx, 1, symbol_value(symbol(fl_ctx, "contains-macrocall")), expression)) {
                     form = scm_to_julia(fl_ctx, expression, inmodule);
+                    if (mapexpr)
+                        form = jl_call1(mapexpr, form);
                     form = jl_expand_macros(form, inmodule, NULL, 0);
                     expression = julia_to_scm(fl_ctx, form);
                 }
+                else if (mapexpr) {
+                    form = scm_to_julia(fl_ctx, expression, inmodule);
+                    form = jl_call1(mapexpr, form);
+                    expression = julia_to_scm(fl_ctx, form);
+                }
                 // expand non-final expressions in statement position (value unused)
                 expression =
                     fl_applyn(fl_ctx, 4,
@@ -925,10 +933,17 @@ jl_value_t *jl_parse_eval_all(const char *fname,
     return result;
 }
 
+JL_DLLEXPORT jl_value_t *jl_load_rewrite_file_string(const char *text, size_t len,
+                                                     char *filename, jl_module_t *inmodule,
+                                                     jl_value_t *mapexpr)
+{
+    return jl_parse_eval_all(filename, text, len, inmodule, mapexpr);
+}
+
 JL_DLLEXPORT jl_value_t *jl_load_file_string(const char *text, size_t len,
                                              char *filename, jl_module_t *inmodule)
 {
-    return jl_parse_eval_all(filename, text, len, inmodule);
+    return jl_parse_eval_all(filename, text, len, inmodule, NULL);
 }
 
 // returns either an expression or a thunk

src/jlfrontend.scm

@@ -180,7 +180,8 @@
           (loc  (if (and (pair? loc) (eq? (car loc) 'line))
                     (list loc)
                     '()))
-          (x    (if (eq? name 'x) 'y 'x)))
+          (x    (if (eq? name 'x) 'y 'x))
+          (mex  (if (eq? name 'mapexpr) 'map_expr 'mapexpr)))
      `(block
        (= (call eval ,x)
           (block
@@ -189,7 +190,11 @@
        (= (call include ,x)
           (block
            ,@loc
-           (call (top include) ,name ,x)))))
+           (call (top include) ,name ,x)))
+       (= (call include (:: ,mex (top Function)) ,x)
+          (block
+           ,@loc
+           (call (top include) ,mex ,name ,x)))))
    'none 0))
 
 ; run whole frontend on a string. useful for testing.

src/julia_internal.h

@@ -452,7 +452,8 @@ jl_value_t *jl_toplevel_eval_flex(jl_module_t *m, jl_value_t *e, int fast, int e
 jl_value_t *jl_eval_global_var(jl_module_t *m JL_PROPAGATES_ROOT, jl_sym_t *e);
 jl_value_t *jl_parse_eval_all(const char *fname,
                               const char *content, size_t contentlen,
-                              jl_module_t *inmodule);
+                              jl_module_t *inmodule,
+                              jl_value_t *mapexpr);
 jl_value_t *jl_interpret_toplevel_thunk(jl_module_t *m, jl_code_info_t *src);
 jl_value_t *jl_interpret_toplevel_expr_in(jl_module_t *m, jl_value_t *e,
                                           jl_code_info_t *src,

src/toplevel.c

@@ -863,13 +863,18 @@ JL_DLLEXPORT jl_value_t *jl_infer_thunk(jl_code_info_t *thk, jl_module_t *m)
     return (jl_value_t*)jl_any_type;
 }
 
-JL_DLLEXPORT jl_value_t *jl_load(jl_module_t *module, const char *fname)
+JL_DLLEXPORT jl_value_t *jl_load_rewrite(jl_module_t *module, const char *fname, jl_value_t *mapexpr)
 {
     uv_stat_t stbuf;
     if (jl_stat(fname, (char*)&stbuf) != 0 || (stbuf.st_mode & S_IFMT) != S_IFREG) {
         jl_errorf("could not open file %s", fname);
     }
-    return jl_parse_eval_all(fname, NULL, 0, module);
+    return jl_parse_eval_all(fname, NULL, 0, module, mapexpr);
+}
+
+JL_DLLEXPORT jl_value_t *jl_load(jl_module_t *module, const char *fname)
+{
+    return jl_load_rewrite(module, fname, NULL);
 }
 
 // load from filename given as a String object

test/loading.jl

@@ -660,3 +660,25 @@ Base.ACTIVE_PROJECT[] = saved_active_project
 module Foo; import Libdl; end
 import .Foo.Libdl; import Libdl
 @test Foo.Libdl === Libdl
+
+@testset "include with mapexpr" begin
+    let exprs = Any[]
+        @test 13 === include_string(@__MODULE__, "1+1\n3*4") do ex
+            ex isa LineNumberNode || push!(exprs, ex)
+            Meta.isexpr(ex, :call) ? :(1 + $ex) : ex
+        end
+        @test exprs == [:(1 + 1), :(3 * 4)]
+    end
+    # test using test_exec.jl, just because that is the shortest handy file
+    for incl in (include, (mapexpr,path) -> Base.include(mapexpr, @__MODULE__, path))
+        let exprs = Any[]
+            incl("test_exec.jl") do ex
+                ex isa LineNumberNode || push!(exprs, ex)
+                Meta.isexpr(ex, :macrocall) ? :nothing : ex
+            end
+            @test length(exprs) == 2 && exprs[1] == :(using Test)
+            @test Meta.isexpr(exprs[2], :macrocall) &&
+                  exprs[2].args[[1,3]] == [Symbol("@test"), :(1 == 2)]
+        end
+    end
+end