Merge branch 'master' into undotted_setindex

Author: adienes

JuliaLowering/src/eval.jl

@@ -765,17 +765,22 @@ end
 
 #-------------------------------------------------------------------------------
 # Our version of eval - should be upstreamed though?
-@fzone "JL: eval" function eval(mod::Module, ex::SyntaxTree;
+@fzone "JL: eval" function eval(mod::Module, @nospecialize(ex);
                                 soft_scope::Union{Nothing,Bool}=nothing,
                                 expr_compat_mode::Bool=false)
+    # Run the `eval` driver in the lowering world. Any internal operations
+    # are required to `invokelatest` before executing any code that dispatches
+    # on user code / types.
     ver = expr_compat_mode ? JL_OLD_SYNTAX_VERSION : JL_NEW_SYNTAX_VERSION
-    iter = lower_init(ex, ver)
-    _eval(mod, iter; soft_scope)
+    return invoke_in_lowering_world(_lower_and_eval, mod, ex, ver, soft_scope)
 end
 
-# Version of eval() taking `Expr` (or Expr tree leaves of any type)
-function eval(mod::Module, @nospecialize(ex); opts...)
-    eval(mod, expr_to_est(ex); opts...)
+# `ex` may be a `SyntaxTree` or an `Expr` (or `Expr` tree leaves of any type).
+function _lower_and_eval(mod::Module, @nospecialize(ex), ver::VersionNumber,
+                         soft_scope::Union{Nothing,Bool})
+    st = ex isa SyntaxTree ? ex : expr_to_est(ex)
+    iter = lower_init(st, ver)
+    return _eval(mod, iter; soft_scope)
 end
 
 function _eval(mod::Module, iter::LoweringIterator; soft_scope::Union{Nothing,Bool}=nothing)
@@ -797,7 +802,7 @@ function _eval(mod::Module, iter::LoweringIterator; soft_scope::Union{Nothing,Bo
             result = pop!(modules)
         else
             @assert type == :thunk
-            result = Core.eval(modules[end], thunk[2])
+            result = Base.invokelatest(Core.eval, modules[end], thunk[2])
         end
     end
     @assert length(modules) === 1

JuliaLowering/src/hooks.jl

@@ -13,6 +13,12 @@ function core_lowering_hook(@nospecialize(code), mod::Module, file::Union{String
         return Core.svec(code)
     end
 
+    if _has_v1_13_hooks && Core._lower === core_lowering_hook &&
+            unsafe_load(cglobal(:jl_lowering_world, Csize_t)) == 0
+        # Refuse to run as `Core._lower` without a pinned world
+        error("`Core._lower` was set without pinning the lowering world; use `JuliaLowering.activate!()`")
+    end
+
     # TODO: fix in base
     file = file isa Ptr{UInt8} ? unsafe_string(file) : file
     line = !(line isa Int) ? Int(line) : line
@@ -59,7 +65,10 @@ function activate!(enable=true)
 
     if enable
         Core._setlowerer!(core_lowering_hook)
+        ccall(:jl_set_lowering_world, Cvoid, (Csize_t,), Base.get_world_counter())
     else
         Core._setlowerer!(Base.fl_lower)
+        # Unlike JL, `jl_lower` dispatches the flisp wrapper at the latest world
+        ccall(:jl_set_lowering_world, Cvoid, (Csize_t,), 0)
     end
 end

JuliaLowering/src/macro_expansion.jl

@@ -155,7 +155,7 @@ function _eval_dot(world::UInt, mod, ex::SyntaxTree)
         ex = ex[1]
     end
     kind(ex) in KSet"Identifier Symbol" && mod isa Module ?
-        Base.invoke_in_world(world, getproperty, mod, Symbol(ex.name_val)) :
+        _invoke_in_world(world, getproperty, mod, Symbol(ex.name_val)) :
         nothing
 end
 
@@ -171,8 +171,8 @@ function eval_macro_name(ctx, mctx::MacroContext, st0::SyntaxTree)
         if kind(st) === K"Value"
             st.value
         elseif kind(st) === K"Identifier"
-            Base.invoke_in_world(ctx.world, getproperty,
-                                 syntax_module(st), Symbol(st.name_val))
+            _invoke_in_world(ctx.world, getproperty,
+                             syntax_module(st), Symbol(st.name_val))
         elseif kind(st) === K"." &&
                 # TODO: correct mod?
                 (ed = _eval_dot(ctx.world, mod, st); !isnothing(ed))
@@ -223,16 +223,15 @@ function expand_macro(ctx::MacroExpansionContext, st::SyntaxTree)
     macfunc = eval_macro_name(ctx, mctx, macname)
     raw_args = st[3:end]
 
-    # TODO: hasmethod always returns false for our `typemax(UInt)` meaning
-    # "latest world," which we shouldn't be using.
-    has_new_macro = ctx.world === typemax(UInt) ?
-        hasmethod(macfunc, Tuple{typeof(mctx), typeof.(raw_args)...}) :
-        hasmethod(macfunc, Tuple{typeof(mctx), typeof.(raw_args)...}; world=ctx.world)
+    # `ctx.world === typemax(UInt)` is our sentinel for "latest world"
+    macro_world = ctx.world === typemax(UInt) ? Base.get_world_counter() : ctx.world
+    has_new_macro = hasmethod(macfunc, Tuple{typeof(mctx), typeof.(raw_args)...}; world=macro_world)
 
     if has_new_macro
         macro_args = [mctx, raw_args...]
+        macro_mi = lookup_method_instance(macfunc, macro_args, macro_world)
         expanded = try
-            Base.invoke_in_world(ctx.world, macfunc, macro_args...)
+            _invoke_in_world(ctx.world, macfunc, macro_args...)
         catch exc
             newexc = exc isa MacroExpansionError ?
                 MacroExpansionError(mctx, exc.ex, exc.msg, exc.position, exc.err) :
@@ -253,8 +252,9 @@ function expand_macro(ctx::MacroExpansionContext, st::SyntaxTree)
             @jl_assert kind(arg) !== K"VERSION" arg # handled in EST conversion
             push!(macro_args, est_to_expr(arg))
         end
+        macro_mi = lookup_method_instance(macfunc, macro_args, macro_world)
         st_out = try
-            Base.invoke_in_world(ctx.world, macfunc, macro_args...)
+            _invoke_in_world(ctx.world, macfunc, macro_args...)
         catch exc
             if exc isa MethodError && exc.f === macfunc && !isempty(
                 methods_in_world(macfunc, Tuple{typeof(mctx), Vararg{Any}}, ctx.world, st))
@@ -270,7 +270,7 @@ function expand_macro(ctx::MacroExpansionContext, st::SyntaxTree)
     end
     # Module scope for the returned AST is the module where this particular
     # method was defined (may be different from `parentmodule(macfunc)`)
-    mod_for_ast = lookup_method_instance(macfunc, macro_args, ctx.world).def.module
+    mod_for_ast = macro_mi !== nothing ? macro_mi.def.module : parentmodule(macfunc)
     sc2 = SyntaxContext(
         ScopeLayer(mod_for_ast, sc_in.layer), st,
         (has_new_macro ? JL_NEW_SYNTAX_VERSION : JL_OLD_SYNTAX_VERSION), false)

JuliaLowering/src/precompile.jl

@@ -34,7 +34,7 @@ if Base.get_bool_env("JULIA_LOWERING_PRECOMPILE", true)
     end
 
     macro _precompile_plus1(ex)
-        :($ex + 1)
+        :($(esc(ex)) + 1)
     end
     _precompile_usemac(x) = @_precompile_plus1(x)
 
@@ -48,5 +48,5 @@ if Base.get_bool_env("JULIA_LOWERING_PRECOMPILE", true)
     _precompile_usemac(3)
     _precompile_genf(1.0)
     """
-    include_string(@__MODULE__, workload, @__FILE__)
+    include_string(@__MODULE__, workload, @__FILE__; expr_compat_mode=true)
 end

JuliaLowering/src/runtime.jl

@@ -7,15 +7,36 @@
 #-------------------------------------------------------------------------------
 # Functions/types used by code emitted from lowering, but not called by it directly
 
+@inline function _invoke_in_world(w::UInt, f::F, @nospecialize(args...)) where {F}
+    if ccall(:jl_is_in_pure_context, Int8, ()) != 0
+        # Similar to `Base.invoke_in_world` but also works inside generated-function
+        # expansion (see `jl_code_for_staged`)
+        return Core._call_in_world_total(w, f, args...)
+    end
+    return Base.invoke_in_world(w, f, args...)
+end
+
+# Re-dispatch `f(args...)` at the pinned lowering world (see `jl_lowering_world`)
+@inline function invoke_in_lowering_world(f::F, @nospecialize(args...)) where {F}
+    w = unsafe_load(cglobal(:jl_lowering_world, Csize_t))
+    if w == 0
+        # Fallback when the Base lowering hook is not set up
+        w = Base.tls_world_age()
+        # FIXME: as a side effect, enabling the Base lowering hook now affects
+        #        JuliaLowering execution not passing through the hook
+    end
+    return _invoke_in_world(w, f, args...)
+end
+
 # Return the current exception. In JuliaLowering we use this rather than the
 # special form `K"the_exception"` to reduce the number of special forms.
 Base.@assume_effects :removable function current_exception()
     @ccall jl_current_exception(current_task()::Any)::Any
 end
 
-function _interpolate_expr(@nospecialize(ex), depth, @nospecialize(vals::Tuple), val_i)
+function __interpolate_expr(@nospecialize(ex), depth, @nospecialize(vals::Tuple), val_i)
     if ex isa QuoteNode
-        out = _interpolate_expr(Expr(:inert, ex.value), depth, vals, val_i)
+        out = __interpolate_expr(Expr(:inert, ex.value), depth, vals, val_i)
         QuoteNode(only(out.args))
     elseif !(ex isa Expr)
         ex
@@ -30,18 +51,21 @@ function _interpolate_expr(@nospecialize(ex), depth, @nospecialize(vals::Tuple),
                     push!(cs_out, v)
                 end
             else
-                push!(cs_out, _interpolate_expr(e, inner_depth, vals, val_i))
+                push!(cs_out, __interpolate_expr(e, inner_depth, vals, val_i))
             end
         end
         Expr(ex.head, cs_out...)
     end
 end
-function interpolate_expr(@nospecialize(ex), @nospecialize(values...))
+function _interpolate_expr(@nospecialize(ex), @nospecialize(values::Tuple))
     @jl_assert !Meta.isexpr(ex, :$) (expr_to_est(ex), "expand_quote should handle this")
-    _interpolate_expr(ex, 0, values, Ref(0))
+    __interpolate_expr(ex, 0, values, Ref(0))
+end
+function interpolate_expr(@nospecialize(ex), @nospecialize(values...))
+    return invoke_in_lowering_world(_interpolate_expr, ex, values)
 end
 
-function _interpolate_syntax(st::SyntaxTree, depth, @nospecialize(vals), val_i)
+function __interpolate_syntax(st::SyntaxTree, depth, @nospecialize(vals), val_i)
     is_leaf(st) && return mkleaf(st)
     k = kind(st)
     inner_depth = k == K"syntaxquote" ? depth + 1 :
@@ -58,19 +82,22 @@ function _interpolate_syntax(st::SyntaxTree, depth, @nospecialize(vals), val_i)
                 push!(cs_out, v2)
             end
         else
-            push!(cs_out, _interpolate_syntax(c, inner_depth, vals, val_i))
+            push!(cs_out, __interpolate_syntax(c, inner_depth, vals, val_i))
         end
     end
     mknode(st, cs_out)
 end
-function interpolate_syntax(st::SyntaxTree, @nospecialize(vals...))
+function _interpolate_syntax(st::SyntaxTree, @nospecialize(vals::Tuple))
     st = copy_ast(ensure_macro_attributes!(SyntaxGraph()), st)
     val_i = Ref(0)
-    out = _interpolate_syntax((@ast st._graph st [K"None" st]), 0, vals, val_i)
+    out = __interpolate_syntax((@ast st._graph st [K"None" st]), 0, vals, val_i)
     @jl_assert val_i[] == length(vals) st
     @jl_assert numchildren(out) == 1 st
     out[1]
 end
+function interpolate_syntax(st::SyntaxTree, @nospecialize(vals...))
+    return invoke_in_lowering_world(_interpolate_syntax, st, vals)
+end
 
 #--------------------------------------------------
 # Functions called by closure conversion
@@ -246,6 +273,18 @@ function (g::GeneratedFunctionStub)(world::UInt, source::Method, @nospecialize a
 
     # Run code generator - this acts like a macro expander
     ex0 = g.gen(sc, args...)
+
+    # Note that we expand in `tls_world_age()` (see Core.GeneratedFunctionStub)
+    world = Base.tls_world_age()
+
+    # Lower the generated code in the lowering world
+    return invoke_in_lowering_world(_lower_generated_code, g, source, graph, sc,
+                                   __module__, world, ex0)
+end
+
+function _lower_generated_code(g::GeneratedFunctionStub, source::Method, graph,
+                               sc::SyntaxContext, __module__::Module,
+                               world::UInt, @nospecialize(ex0))
     if ex0 isa Expr
         ex0 = expr_to_est(
             graph, ex0, source_location(LineNumberNode, g.srcref))
@@ -261,8 +300,6 @@ function (g::GeneratedFunctionStub)(world::UInt, source::Method, @nospecialize a
     end
 
     @jl_assert base_layer(sc).mod == __module__ ex0
-    # Note that we expand in `tls_world_age()` (see Core.GeneratedFunctionStub)
-    world = Base.tls_world_age()
     ex0 = JuliaSyntax.fill_context!(ex0, sc)
     ctx1 = MacroExpansionContext(ex0, world, true)
     ex1 = expand_forms_1(ctx1, ex0)
@@ -317,7 +354,7 @@ end
 #
 # (This should do what fl_defined_julia_global does for flisp lowering)
 function is_defined_and_owned_global(mod, name, world::UInt=Base.get_world_counter())
-    return Base.invoke_in_world(world, Base.binding_kind, mod, name) === Base.PARTITION_KIND_GLOBAL
+    return _invoke_in_world(world, Base.binding_kind, mod, name) === Base.PARTITION_KIND_GLOBAL
 end
 
 # "Reserve" a binding: create the binding if it doesn't exist but do not assign

JuliaLowering/test/macros.jl

@@ -1510,6 +1510,21 @@ code = JuliaLowering.include_string(test_mod, """Mod1.@indirect_MODULE()""")
     end
 end
 
+@testset "(AI) old macro attribution survives a nested eval in its body (#32)" begin
+    Base.eval(test_mod, :(module MacDefMod
+        const secret = 99
+        macro getsecret()
+            __module__.eval(:(nested_eval_side_effect = 1 + 1))
+            return :(secret)   # bare name -> resolves in the defining module
+        end
+    end))
+    Core.@latestworld
+    # `secret` must resolve in MacDefMod (== mod_for_ast), matching flisp.
+    @test JuliaLowering.include_string(test_mod, "MacDefMod.@getsecret()") == 99
+    @test test_mod.nested_eval_side_effect == 2
+    @test fl_eval(test_mod, :(MacDefMod.@getsecret())) == 99
+end
+
 @testset "macros defining macros" begin
     @eval test_mod macro make_and_use_macro_toplevel()
         Expr(:toplevel,