WIP: allow @generated begin ... end inside a function to provide an optional optimizer

use meta nodes instead of `stagedfunction` expression head

Author: JeffBezanson

base/docs/Docs.jl

@@ -642,7 +642,7 @@ finddoc(λ, def) = false
 
 # Predicates and helpers for `docm` expression selection:
 
-const FUNC_HEADS    = [:function, :stagedfunction, :macro, :(=)]
+const FUNC_HEADS    = [:function, :macro, :(=)]
 const BINDING_HEADS = [:typealias, :const, :global, :(=)]  # deprecation: remove `typealias` post-0.6
 # For the special `:@mac` / `:(Base.@mac)` syntax for documenting a macro after definition.
 isquotedmacrocall(x) =

base/expr.jl

@@ -333,9 +333,18 @@ function remove_linenums!(ex::Expr)
 end
 
 macro generated(f)
-     if isa(f, Expr) && (f.head === :function || is_short_function_def(f))
-        f.head = :stagedfunction
-        return Expr(:escape, f)
+    if isa(f, Expr) && (f.head === :function || is_short_function_def(f))
+        body = f.args[2]
+        lno = body.args[1]
+        return Expr(:escape,
+                    Expr(f.head, f.args[1],
+                         Expr(:block,
+                              lno,
+                              Expr(:meta, :generator, body),
+                              Expr(:meta, :generated_only),
+                              Expr(:return, nothing))))
+    elseif isa(f, Expr) && f.head === :block
+        return Expr(:escape, Expr(:meta, :generator, f))
     else
         error("invalid syntax; @generated must be used with a function definition")
     end

base/methodshow.jl

@@ -42,6 +42,16 @@ function argtype_decl(env, n, sig::DataType, i::Int, nargs, isva::Bool) # -> (ar
     return s, string_with_env(env, t)
 end
 
+function method_argnames(m::Method)
+    if !isdefined(m, :source)
+        gm = first(methods(m.generator))
+        return method_argnames(gm)[length(m.sparam_syms)+2 : end]
+    end
+    argnames = Vector{Any}(m.nargs)
+    ccall(:jl_fill_argnames, Void, (Any, Any), m.source, argnames)
+    return argnames
+end
+
 function arg_decl_parts(m::Method)
     tv = Any[]
     sig = m.sig
@@ -52,8 +62,7 @@ function arg_decl_parts(m::Method)
     file = m.file
     line = m.line
     if isdefined(m, :source) || isdefined(m, :generator)
-        argnames = Vector{Any}(m.nargs)
-        ccall(:jl_fill_argnames, Void, (Any, Any), isdefined(m, :source) ? m.source : m.generator.inferred, argnames)
+        argnames = method_argnames(m)
         show_env = ImmutableDict{Symbol, Any}()
         for t in tv
             show_env = ImmutableDict(show_env, :unionall_env => t)

base/reflection.jl

@@ -732,7 +732,8 @@ function length(mt::MethodTable)
 end
 isempty(mt::MethodTable) = (mt.defs === nothing)
 
-uncompressed_ast(m::Method) = uncompressed_ast(m, isdefined(m, :source) ? m.source : m.generator.inferred)
+uncompressed_ast(m::Method) = isdefined(m,:source) ? uncompressed_ast(m, m.source) :
+                                                     uncompressed_ast(first(methods(m.generator)))
 uncompressed_ast(m::Method, s::CodeInfo) = s
 uncompressed_ast(m::Method, s::Array{UInt8,1}) = ccall(:jl_uncompress_ast, Any, (Any, Any), m, s)::CodeInfo
 uncompressed_ast(m::Core.MethodInstance) = uncompressed_ast(m.def)

src/ast.c

@@ -55,7 +55,8 @@ jl_sym_t *meta_sym; jl_sym_t *compiler_temp_sym;
 jl_sym_t *inert_sym; jl_sym_t *vararg_sym;
 jl_sym_t *unused_sym; jl_sym_t *static_parameter_sym;
 jl_sym_t *polly_sym; jl_sym_t *inline_sym;
-jl_sym_t *propagate_inbounds_sym;
+jl_sym_t *propagate_inbounds_sym; jl_sym_t *generator_sym;
+jl_sym_t *generated_only_sym;
 jl_sym_t *isdefined_sym; jl_sym_t *nospecialize_sym;
 jl_sym_t *macrocall_sym;
 jl_sym_t *hygienicscope_sym;
@@ -343,6 +344,8 @@ void jl_init_frontend(void)
     hygienicscope_sym = jl_symbol("hygienic-scope");
     gc_preserve_begin_sym = jl_symbol("gc_preserve_begin");
     gc_preserve_end_sym = jl_symbol("gc_preserve_end");
+    generator_sym = jl_symbol("generator");
+    generated_only_sym = jl_symbol("generated_only");
 }
 
 JL_DLLEXPORT void jl_lisp_prompt(void)

src/ast.scm

@@ -358,6 +358,12 @@
   (and (if one (length= e 3) (length> e 2))
        (eq? (car e) 'meta) (eq? (cadr e) 'nospecialize)))
 
+(define (generator-meta? e)
+  (and (length= e 3) (eq? (car e) 'meta) (eq? (cadr e) 'generator)))
+
+(define (generated_only-meta? e)
+  (and (length= e 2) (eq? (car e) 'meta) (eq? (cadr e) 'generated_only)))
+
 ;; flatten nested expressions with the given head
 ;; (op (op a b) c) => (op a b c)
 (define (flatten-ex head e)

src/codegen.cpp

@@ -1199,8 +1199,6 @@ jl_llvm_functions_t jl_compile_linfo(jl_method_instance_t **pli, jl_code_info_t
                 li->inferred &&
                 // and there is something to delete (test this before calling jl_ast_flag_inlineable)
                 li->inferred != jl_nothing &&
-                // don't delete the code for the generator
-                li != li->def.method->generator &&
                 // don't delete inlineable code, unless it is constant
                 (li->jlcall_api == 2 || !jl_ast_flag_inlineable((jl_array_t*)li->inferred)) &&
                 // don't delete code when generating a precompile file
@@ -3829,11 +3827,10 @@ static jl_cgval_t emit_expr(jl_codectx_t &ctx, jl_value_t *expr)
         }
         Value *a1 = boxed(ctx, emit_expr(ctx, args[1]));
         Value *a2 = boxed(ctx, emit_expr(ctx, args[2]));
-        Value *mdargs[4] = {
+        Value *mdargs[3] = {
             /*argdata*/a1,
             /*code*/a2,
-            /*module*/literal_pointer_val(ctx, (jl_value_t*)ctx.module),
-            /*isstaged*/literal_pointer_val(ctx, args[3])
+            /*module*/literal_pointer_val(ctx, (jl_value_t*)ctx.module)
         };
         ctx.builder.CreateCall(prepare_call(jlmethod_func), makeArrayRef(mdargs));
         return ghostValue(jl_void_type);
@@ -6377,7 +6374,6 @@ static void init_julia_llvm_env(Module *m)
     mdargs.push_back(T_prjlvalue);
     mdargs.push_back(T_prjlvalue);
     mdargs.push_back(T_pjlvalue);
-    mdargs.push_back(T_pjlvalue);
     jlmethod_func =
         Function::Create(FunctionType::get(T_void, mdargs, false),
                          Function::ExternalLinkage,

src/dump.c

@@ -1439,7 +1439,7 @@ static jl_value_t *jl_deserialize_value_method(jl_serializer_state *s, jl_value_
     m->unspecialized = (jl_method_instance_t*)jl_deserialize_value(s, (jl_value_t**)&m->unspecialized);
     if (m->unspecialized)
         jl_gc_wb(m, m->unspecialized);
-    m->generator = (jl_method_instance_t*)jl_deserialize_value(s, (jl_value_t**)&m->generator);
+    m->generator = jl_deserialize_value(s, (jl_value_t**)&m->generator);
     if (m->generator)
         jl_gc_wb(m, m->generator);
     m->invokes.unknown = jl_deserialize_value(s, (jl_value_t**)&m->invokes);

src/interpreter.c

@@ -322,7 +322,7 @@ static jl_value_t *eval(jl_value_t *e, interpreter_state *s)
         JL_GC_PUSH2(&atypes, &meth);
         atypes = eval(args[1], s);
         meth = eval(args[2], s);
-        jl_method_def((jl_svec_t*)atypes, (jl_code_info_t*)meth, s->module, args[3]);
+        jl_method_def((jl_svec_t*)atypes, (jl_code_info_t*)meth, s->module);
         JL_GC_POP();
         return jl_nothing;
     }

src/jltypes.c

@@ -2061,7 +2061,7 @@ void jl_init_types(void)
                             jl_simplevector_type,
                             jl_any_type,
                             jl_any_type, // jl_method_instance_type
-                            jl_any_type, // jl_method_instance_type
+                            jl_any_type,
                             jl_array_any_type,
                             jl_any_type,
                             jl_int32_type,
@@ -2174,7 +2174,6 @@ void jl_init_types(void)
 #endif
     jl_svecset(jl_methtable_type->types, 8, jl_int32_type); // uint32_t
     jl_svecset(jl_method_type->types, 10, jl_method_instance_type);
-    jl_svecset(jl_method_type->types, 11, jl_method_instance_type);
     jl_svecset(jl_method_instance_type->types, 12, jl_voidpointer_type);
     jl_svecset(jl_method_instance_type->types, 13, jl_voidpointer_type);
     jl_svecset(jl_method_instance_type->types, 14, jl_voidpointer_type);