Skip to content
This repository was archived by the owner on May 4, 2019. It is now read-only.

Commit 4d1a1f8

Browse files
nalimilannalimilan
committed
Rewrite broadcast() using Base implementation and lifting semantics
Remove the custom implementation of broadcast(), and just call the base method on the lifted method. For now, standard lifting semantics are always used, even for logical operators and isnull/get.
1 parent f530754 commit 4d1a1f8

File tree

4 files changed

+116
-198
lines changed

4 files changed

+116
-198
lines changed

‎REQUIRE

+1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
julia 0.4
2+
julia 0.5
23
Compat 0.9.4
34
Reexport

‎src/broadcast.jl

+99-171
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
1-
using Base: promote_eltype
2-
using Base.Cartesian
1+
using Base: _default_eltype
2+
using Compat
3+
34
if VERSION >= v"0.6.0-dev.693"
45
using Base.Broadcast: check_broadcast_indices, broadcast_indices
56
else
@@ -8,199 +9,126 @@ else
89
const broadcast_indices = broadcast_shape
910
end
1011

11-
if VERSION >= v"0.5.0-dev+5189"
12-
_to_shape(dims::Base.DimsOrInds) = map(_to_shape, dims)
13-
_to_shape(r::Base.OneTo) = Int(last(r))
12+
if !isdefined(Base.Broadcast, :ftype) # Julia < 0.6
13+
ftype(f, A) = typeof(f)
14+
ftype(f, A...) = typeof(a -> f(a...))
15+
ftype(T::DataType, A) = Type{T}
16+
ftype(T::DataType, A...) = Type{T}
1417
else
15-
_to_shape(x) = x
18+
using Base.Broadcast: ftype
1619
end
1720

18-
if VERSION < v"0.5.0-dev+5434"
19-
function gen_nullcheck(narrays::Int, nd::Int)
20-
e_nullcheck = macroexpand(:( @nref $nd isnull_1 d->j_d_1 ))
21-
for k = 2:narrays
22-
isnull = Symbol("isnull_$k")
23-
j_d_k = Symbol("j_d_$k")
24-
e_isnull_k = macroexpand(:( @nref $nd $(isnull) d->$(j_d_k) ))
25-
e_nullcheck = Expr(:||, e_nullcheck, e_isnull_k)
26-
end
27-
return e_nullcheck
21+
if !isdefined(Base.Broadcast, :ziptype) # Julia < 0.6
22+
if isdefined(Base, :Iterators)
23+
using Base.Iterators: Zip2
24+
else
25+
using Base: Zip2
2826
end
27+
ziptype(A) = Tuple{eltype(A)}
28+
ziptype(A, B) = Zip2{Tuple{eltype(A)}, Tuple{eltype(B)}}
29+
@inline ziptype(A, B, C, D...) = Zip{Tuple{eltype(A)}, ziptype(B, C, D...)}
30+
else
31+
using Base.Broadcast: ziptype
32+
end
2933

30-
function gen_broadcast_body(nd::Int, narrays::Int, f, lift::Bool)
31-
F = Expr(:quote, f)
32-
e_nullcheck = gen_nullcheck(narrays, nd)
33-
if lift
34-
return quote
35-
# set up aliases to facilitate subsequent Base.Cartesian magic
36-
B_isnull = B.isnull
37-
@nexprs $narrays k->(values_k = A_k.values)
38-
@nexprs $narrays k->(isnull_k = A_k.isnull)
39-
# check size
40-
@assert ndims(B) == $nd
41-
@ncall $narrays check_broadcast_shape size(B) k->A_k
42-
# main loops
43-
@nloops($nd, i, B,
44-
d->(@nexprs $narrays k->(j_d_k = size(A_k, d) == 1 ? 1 : i_d)), # pre
45-
begin # body
46-
if $e_nullcheck
47-
@inbounds (@nref $nd B_isnull i) = true
48-
else
49-
@nexprs $narrays k->(@inbounds v_k = @nref $nd values_k d->j_d_k)
50-
@inbounds (@nref $nd B i) = (@ncall $narrays $F v)
51-
end
52-
end
53-
)
54-
end
34+
@inline function broadcast_lift(f, x)
35+
if null_safe_op(f, eltype(x))
36+
return @compat Nullable(f(x.value), !isnull(x))
37+
else
38+
U = Core.Inference.return_type(f, Tuple{eltype(x)})
39+
if isnull(x)
40+
return Nullable{U}()
5541
else
56-
return Base.Broadcast.gen_broadcast_body_cartesian(nd, narrays, f)
42+
return Nullable(f(unsafe_get(x)))
5743
end
5844
end
45+
end
5946

60-
function gen_broadcast_function(nd::Int, narrays::Int, f, lift::Bool)
61-
As = [Symbol("A_"*string(i)) for i = 1:narrays]
62-
body = gen_broadcast_body(nd, narrays, f, lift)
63-
@eval let
64-
local _F_
65-
function _F_(B, $(As...))
66-
$body
67-
end
68-
_F_
47+
@inline function broadcast_lift(f, x1, x2)
48+
if null_safe_op(f, eltype(x1), eltype(x2))
49+
return @compat Nullable(f(x1.value, x2.value), !(isnull(x1) | isnull(x2)))
50+
else
51+
U = Core.Inference.return_type(f, Tuple{eltype(x1), eltype(x2)})
52+
if isnull(x1) | isnull(x2)
53+
return Nullable{U}()
54+
else
55+
return Nullable(f(unsafe_get(x1), unsafe_get(x2)))
6956
end
7057
end
58+
end
7159

72-
function Base.broadcast!(f, X::NullableArray; lift::Bool=false)
73-
broadcast!(f, X, X; lift=lift)
74-
end
60+
eltypes() = Tuple{}
61+
eltypes(x) = Tuple{eltype(x)}
62+
eltypes(x, xs...) = Tuple{eltype(x), eltypes(xs...).parameters...}
7563

76-
@eval let cache = Dict{Any, Dict{Bool, Dict{Int, Dict{Int, Any}}}}()
77-
"""
78-
broadcast!(f, B::NullableArray, As::NullableArray...; lift::Bool=false)
79-
80-
This method implements the same behavior as that of `broadcast!` when called on
81-
regular `Array` arguments. It also includes the `lift` keyword argument, which
82-
when set to true will lift `f` over the entries of the `As`.
83-
84-
Lifting is disabled by default. Note that this method's signature specifies
85-
the destination `B` array as well as the source `As` arrays as all
86-
`NullableArray`s. Thus, calling `broadcast!` on a arguments consisting
87-
of both `Array`s and `NullableArray`s will fall back to the implementation
88-
of `broadcast!` in `base/broadcast.jl`.
89-
"""
90-
function Base.broadcast!(f, B::NullableArray, As::NullableArray...; lift::Bool=false)
91-
nd = ndims(B)
92-
narrays = length(As)
93-
94-
cache_f = Base.@get! cache f Dict{Bool, Dict{Int, Dict{Int, Any}}}()
95-
cache_lift = Base.@get! cache_f lift Dict{Int, Dict{Int, Any}}()
96-
cache_f_na = Base.@get! cache_lift narrays Dict{Int, Any}()
97-
func = Base.@get! cache_f_na nd gen_broadcast_function(nd, narrays, f, lift)
98-
99-
func(B, As...)
100-
return B
101-
end
102-
end # let cache
103-
else
104-
using Base.Broadcast: newindexer, map_newindexer, newindex
105-
106-
function _nullcheck(nargs)
107-
nullcheck = :(isnull_1[I_1])
108-
for i in 2:nargs
109-
sym_isnull = Symbol("isnull_$i")
110-
sym_idx = Symbol("I_$i")
111-
nullcheck = Expr(:||, :($sym_isnull[$sym_idx]), nullcheck)
112-
end
113-
# if 0 argument arrays, treat nullcheck as though it returns false
114-
nargs >= 1 ? nullcheck : :(false)
115-
end
64+
"""
65+
broadcast_lift(f, xs...)
11666
117-
@generated function Base.Broadcast._broadcast!{K,ID,XT,nargs}(f,
118-
Z::NullableArray, keeps::K, Idefaults::ID, Xs::XT, ::Type{Val{nargs}}; lift=false)
119-
nullcheck = _nullcheck(nargs)
120-
quote
121-
T = eltype(Z)
122-
$(Expr(:meta, :noinline))
123-
# destructure keeps and Xs tuples (common to both lifted and non-lifted broadcast)
124-
@nexprs $nargs i->(keep_i = keeps[i])
125-
@nexprs $nargs i->(Idefault_i = Idefaults[i])
126-
if !lift
127-
# destructure the keeps and As tuples
128-
@nexprs $nargs i->(X_i = Xs[i])
129-
@simd for I in CartesianRange(indices(Z))
130-
# reverse-broadcast the indices
131-
@nexprs $nargs i->(I_i = newindex(I, keep_i, Idefault_i))
132-
# extract array values
133-
@nexprs $nargs i->(@inbounds val_i = X_i[I_i])
134-
# call the function and store the result
135-
@inbounds Z[I] = @ncall $nargs f val
136-
end
137-
else
138-
# destructure the indexmaps and Xs tuples
139-
@nexprs $nargs i->(values_i = Xs[i].values)
140-
@nexprs $nargs i->(isnull_i = Xs[i].isnull)
141-
@simd for I in CartesianRange(indices(Z))
142-
# reverse-broadcast the indices
143-
@nexprs $nargs i->(I_i = newindex(I, keep_i, Idefault_i))
144-
if $nullcheck
145-
# if any args are null, store null
146-
@inbounds Z.isnull[I] = true
147-
else
148-
# extract array values
149-
@nexprs $nargs i->(@inbounds val_i = values_i[I_i])
150-
# call the function and store the result
151-
@inbounds Z[I] = @ncall $nargs f val
152-
end
153-
end
154-
end
67+
Lift function `f`, passing it arguments `xs...`, using standard lifting semantics:
68+
for a function call `f(xs...)`, return null if any `x` in `xs` is null; otherwise,
69+
return `f` applied to values of `xs`.
70+
"""
71+
@inline function broadcast_lift(f, xs...)
72+
if null_safe_op(f, eltypes(xs).parameters...)
73+
# TODO: find a more efficient approach than mapreduce
74+
# (i.e. one which gets lowered to just isnull(x1) | isnull(x2) | ...)
75+
return @compat Nullable(f(unsafe_get.(xs)...), !mapreduce(isnull, |, xs))
76+
else
77+
U = Core.Inference.return_type(f, eltypes(xs...))
78+
# TODO: find a more efficient approach than mapreduce
79+
# (i.e. one which gets lowered to just isnull(x1) | isnull(x2) | ...)
80+
if mapreduce(isnull, |, xs)
81+
return Nullable{U}()
82+
else
83+
return Nullable(f(map(unsafe_get, xs)...))
15584
end
15685
end
86+
end
15787

158-
"""
159-
broadcast!(f, B::NullableArray, As::NullableArray...; lift::Bool=false)
160-
161-
This method implements the same behavior as that of `broadcast!` when called
162-
on regular `Array` arguments. It also includes the `lift` keyword argument,
163-
which when set to true will lift `f` over the entries of the `As`.
164-
165-
Lifting is disabled by default. Note that this method's signature specifies
166-
the destination `B` array as well as the source `As` arrays as all
167-
`NullableArray`s. Thus, calling `broadcast!` on a arguments consisting of
168-
both `Array`s and `NullableArray`s will fall back to the implementation of
169-
`broadcast!` in `base/broadcast.jl`.
170-
"""
171-
# Required to solve dispatch ambiguity between
172-
# broadcast!(f, X::AbstractArray, x::Number...)
173-
# broadcast!(f, Z::NullableArrays.NullableArray, Xs::NullableArrays.NullableArray...)
174-
@inline Base.broadcast!(f, Z::NullableArray; lift=false) =
175-
broadcast!(f, Z, Z; lift=lift)
176-
177-
@inline function Base.broadcast!(f, Z::NullableArray, Xs::NullableArray...;
178-
lift=false)
179-
nargs = length(Xs)
180-
shape = indices(Z)
181-
check_broadcast_indices(shape, Xs...)
182-
keeps, Idefaults = map_newindexer(shape, Xs)
183-
Base.Broadcast._broadcast!(f, Z, keeps, Idefaults, Xs, Val{nargs}; lift=lift)
184-
return Z
88+
"""
89+
broadcast(f, As::NullableArray...)
90+
91+
Call `broadcast` with nullable lifting semantics and return a `NullableArray`.
92+
Lifting means calling function `f` on the the values wrapped inside `Nullable` entries
93+
of the input arrays, and returning null if any entry is missing.
94+
95+
Note that this method's signature specifies the source `As` arrays as all
96+
`NullableArray`s. Thus, calling `broadcast` on arguments consisting
97+
of both `Array`s and `NullableArray`s will fall back to the standard implementation
98+
of `broadcast` (i.e. without lifting).
99+
"""
100+
function Base.broadcast{N}(f, As::Vararg{NullableArray, N})
101+
f2(x...) = broadcast_lift(f, x...)
102+
T = _default_eltype(Base.Generator{ziptype(As...), ftype(f2, As...)})
103+
if isleaftype(T) && !(T <: Nullable)
104+
dest = similar(Array{eltype(T)}, broadcast_indices(As...))
105+
else
106+
dest = similar(NullableArray{eltype(T)}, broadcast_indices(As...))
185107
end
108+
invoke(broadcast!, Tuple{Function, AbstractArray, Vararg{AbstractArray, N}}, f2, dest, As...)
186109
end
187110

188111
"""
189-
broadcast(f, As::NullableArray...;lift::Bool=false)
112+
broadcast!(f, dest::NullableArray, As::NullableArray...)
190113
191-
This method implements the same behavior as that of `broadcast` when called on
192-
regular `Array` arguments. It also includes the `lift` keyword argument, which
193-
when set to true will lift `f` over the entries of the `As`.
114+
Call `broadcast!` with nullable lifting semantics.
115+
Lifting means calling function `f` on the the values wrapped inside `Nullable` entries
116+
of the input arrays, and returning null if any entry is missing.
194117
195-
Lifting is disabled by default. Note that this method's signature specifies the
196-
source `As` arrays as all `NullableArray`s. Thus, calling `broadcast!` on
197-
arguments consisting of both `Array`s and `NullableArray`s will fall back to the
198-
implementation of `broadcast` in `base/broadcast.jl`.
118+
Note that this method's signature specifies the destination `dest` array as well as the
119+
source `As` arrays as all `NullableArray`s. Thus, calling `broadcast!` on a arguments
120+
consisting of both `Array`s and `NullableArray`s will fall back to the standard implementation
121+
of `broadcast!` (i.e. without lifting).
199122
"""
200-
@inline function Base.broadcast(f, Xs::NullableArray...;lift::Bool=false)
201-
return broadcast!(f, NullableArray(eltype(promote_eltype(Xs...)),
202-
_to_shape(broadcast_indices(Xs...))),
203-
Xs...; lift=lift)
123+
function Base.broadcast!{N}(f, dest::NullableArray, As::Vararg{NullableArray, N})
124+
f2(x...) = broadcast_lift(f, x...)
125+
invoke(broadcast!, Tuple{Function, AbstractArray, Vararg{AbstractArray, N}}, f2, dest, As...)
126+
end
127+
128+
# To fix ambiguity
129+
function Base.broadcast!(f, dest::NullableArray)
130+
f2(x...) = broadcast_lift(f, x...)
131+
invoke(broadcast!, Tuple{Function, AbstractArray, Vararg{AbstractArray, 0}}, f2, dest)
204132
end
205133

206134
# broadcasted ops

‎src/operators.jl

+2-4
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,7 @@ if VERSION < v"0.5.0-dev+5096"
2020
end
2121

2222
"""
23-
null_safe_op(f::Any, ::Type)::Bool
24-
null_safe_op(f::Any, ::Type, ::Type)::Bool
23+
null_safe_op(f::Any, ::Type...)::Bool
2524
2625
Returns whether an operation `f` can safely be applied to any value of the passed type(s).
2726
Returns `false` by default.
@@ -37,8 +36,7 @@ Types declared as safe can benefit from higher performance for operations on nul
3736
always computing the result even for null values, a branch is avoided, which helps
3837
vectorization.
3938
"""
40-
null_safe_op(f::Any, ::Type) = false
41-
null_safe_op(f::Any, ::Type, ::Type) = false
39+
null_safe_op(f::Any, ::Type...) = false
4240

4341
typealias SafeSignedInts Union{Int128,Int16,Int32,Int64,Int8}
4442
typealias SafeUnsignedInts Union{Bool,UInt128,UInt16,UInt32,UInt64,UInt8}

0 commit comments

Comments
 (0)