make map and collect more general and uniform by adding iterator traits

make HasEltype and HasLength the default

this is a possible fix for #15342

type-accumulating Dict constructor

some tests for type accumulation in `map` and `Dict`

Author: JeffBezanson

base/abstractarray.jl

@@ -1086,34 +1086,6 @@ foreach(f) = (f(); nothing)
 foreach(f, itr) = (for x in itr; f(x); end; nothing)
 foreach(f, itrs...) = (for z in zip(itrs...); f(z...); end; nothing)
 
-# generic map on any iterator
-function map(f, iters...)
-    result = []
-    len = length(iters)
-    states = [start(iters[idx]) for idx in 1:len]
-    nxtvals = cell(len)
-    cont = true
-    for idx in 1:len
-        if done(iters[idx], states[idx])
-            cont = false
-            break
-        end
-    end
-    while cont
-        for idx in 1:len
-            nxtvals[idx],states[idx] = next(iters[idx], states[idx])
-        end
-        push!(result, f(nxtvals...))
-        for idx in 1:len
-            if done(iters[idx], states[idx])
-                cont = false
-                break
-            end
-        end
-    end
-    result
-end
-
 ## map over arrays ##
 
 ## transform any set of dimensions
@@ -1174,39 +1146,6 @@ function mapslices(f, A::AbstractArray, dims::AbstractVector)
     return R
 end
 
-
-# using promote_type
-function promote_to!{T,F}(f::F, offs, dest::AbstractArray{T}, A::AbstractArray)
-    # map to dest array, checking the type of each result. if a result does not
-    # match, do a type promotion and re-dispatch.
-    for i = offs:length(A) #Fixme iter
-        @inbounds Ai = A[i]
-        el = f(Ai)
-        S = typeof(el)
-        if S === T || S <: T
-            @inbounds dest[i] = el::T
-        else
-            R = promote_type(T, S)
-            if R !== T
-                new = similar(dest, R)
-                copy!(new,1, dest,1, i-1)
-                new[i] = el
-                return promote_to!(f, i+1, new, A)
-            end
-            @inbounds dest[i] = el
-        end
-    end
-    return dest
-end
-
-function map_promote(f, A::AbstractArray)
-    if isempty(A); return similar(A, Bottom); end
-    first = f(A[1])
-    dest = similar(A, typeof(first))
-    dest[1] = first
-    return promote_to!(f, 2, dest, A)
-end
-
 # These are needed because map(eltype, As) is not inferrable
 promote_eltype_op(::Any) = (@_pure_meta; Bottom)
 promote_eltype_op{T}(op, ::AbstractArray{T}) = (@_pure_meta; promote_op(op, T))
@@ -1225,39 +1164,11 @@ function map!{F}(f::F, dest::AbstractArray, A::AbstractArray)
     return dest
 end
 
-function map_to!{T,F}(f::F, offs, st, dest::AbstractArray{T}, A)
-    # map to dest array, checking the type of each result. if a result does not
-    # match, widen the result type and re-dispatch.
-    i = offs
-    while !done(A, st)
-        @inbounds Ai, st = next(A, st)
-        el = f(Ai)
-        S = typeof(el)
-        if S === T || S <: T
-            @inbounds dest[i] = el::T
-            i += 1
-        else
-            R = typejoin(T, S)
-            new = similar(dest, R)
-            copy!(new,1, dest,1, i-1)
-            @inbounds new[i] = el
-            return map_to!(f, i+1, st, new, A)
-        end
-    end
-    return dest
-end
+# map on collections
+map(f, A::Union{AbstractArray,AbstractSet,Associative}) = collect_similar(A, Generator(f,A))
 
-function map(f, A::AbstractArray)
-    if isempty(A)
-        return isa(f,Type) ? similar(A,f) : similar(A)
-    end
-    st = start(A)
-    A1, st = next(A, st)
-    first = f(A1)
-    dest = similar(A, typeof(first))
-    dest[1] = first
-    return map_to!(f, 2, st, dest, A)
-end
+# default to returning an Array for `map` on general iterators
+map(f, A) = collect(Generator(f,A))
 
 ## 2 argument
 function map!{F}(f::F, dest::AbstractArray, A::AbstractArray, B::AbstractArray)
@@ -1267,34 +1178,6 @@ function map!{F}(f::F, dest::AbstractArray, A::AbstractArray, B::AbstractArray)
     return dest
 end
 
-function map_to!{T,F}(f::F, offs, dest::AbstractArray{T}, A::AbstractArray, B::AbstractArray)
-    for i = offs:length(A) #Fixme iter
-        @inbounds Ai, Bi = A[i], B[i]
-        el = f(Ai, Bi)
-        S = typeof(el)
-        if (S !== T) && !(S <: T)
-            R = typejoin(T, S)
-            new = similar(dest, R)
-            copy!(new,1, dest,1, i-1)
-            @inbounds new[i] = el
-            return map_to!(f, i+1, new, A, B)
-        end
-        @inbounds dest[i] = el::T
-    end
-    return dest
-end
-
-function map(f, A::AbstractArray, B::AbstractArray)
-    shp = promote_shape(size(A),size(B))
-    if prod(shp) == 0
-        return similar(A, promote_type(eltype(A),eltype(B)), shp)
-    end
-    first = f(A[1], B[1])
-    dest = similar(A, typeof(first), shp)
-    dest[1] = first
-    return map_to!(f, 2, dest, A, B)
-end
-
 ## N argument
 
 ith_all(i, ::Tuple{}) = ()
@@ -1310,32 +1193,7 @@ end
 
 map!{F}(f::F, dest::AbstractArray, As::AbstractArray...) = map_n!(f, dest, As)
 
-function map_to_n!{T,F}(f::F, offs, dest::AbstractArray{T}, As)
-    for i = offs:length(As[1])
-        el = f(ith_all(i, As)...)
-        S = typeof(el)
-        if (S !== T) && !(S <: T)
-            R = typejoin(T, S)
-            new = similar(dest, R)
-            copy!(new,1, dest,1, i-1)
-            @inbounds new[i] = el
-            return map_to_n!(f, i+1, new, As)
-        end
-        @inbounds dest[i] = el::T
-    end
-    return dest
-end
-
-function map(f, As::AbstractArray...)
-    shape = mapreduce(size, promote_shape, As)
-    if prod(shape) == 0
-        return similar(As[1], promote_eltype(As...), shape)
-    end
-    first = f(map(a->a[1], As)...)
-    dest = similar(As[1], typeof(first), shape)
-    dest[1] = first
-    return map_to_n!(f, 2, dest, As)
-end
+map(f, iters...) = collect(Generator(f, iters...))
 
 # multi-item push!, unshift! (built on top of type-specific 1-item version)
 # (note: must not cause a dispatch loop when 1-item case is not defined)

base/array.jl

@@ -199,35 +199,99 @@ convert{T,n,S}(::Type{Array{T,n}}, x::AbstractArray{S,n}) = copy!(Array(T, size(
 
 promote_rule{T,n,S}(::Type{Array{T,n}}, ::Type{Array{S,n}}) = Array{promote_type(T,S),n}
 
+## copying iterators to containers
+
+# make a collection similar to `c` and appropriate for collecting `itr`
+_similar_for(c::AbstractArray, T, itr, ::SizeUnknown) = similar(c, T, 0)
+_similar_for(c::AbstractArray, T, itr, ::HasLength) = similar(c, T, Int(length(itr)::Integer))
+_similar_for(c::AbstractArray, T, itr, ::HasShape) = similar(c, T, convert(Dims,size(itr)))
+_similar_for(c, T, itr, isz) = similar(c, T)
+
 """
     collect(element_type, collection)
 
 Return an array of type `Array{element_type,1}` of all items in a collection.
 """
-function collect{T}(::Type{T}, itr)
-    if applicable(length, itr)
-        # when length() isn't defined this branch might pollute the
-        # type of the other.
-        a = Array(T,length(itr)::Integer)
-        i = 0
-        for x in itr
-            a[i+=1] = x
-        end
-    else
-        a = Array(T,0)
-        for x in itr
-            push!(a,x)
-        end
-    end
-    return a
-end
+collect{T}(::Type{T}, itr) = collect(Generator(T, itr))
 
 """
     collect(collection)
 
 Return an array of all items in a collection. For associative collections, returns Pair{KeyType, ValType}.
 """
-collect(itr) = collect(eltype(itr), itr)
+collect(itr) = _collect(1:1 #= Array =#, itr, iteratoreltype(itr), iteratorsize(itr))
+
+collect_similar(cont, itr) = _collect(cont, itr, iteratoreltype(itr), iteratorsize(itr))
+
+_collect(cont, itr, ::HasEltype, isz::Union{HasLength,HasShape}) =
+    copy!(_similar_for(cont, eltype(itr), itr, isz), itr)
+
+function _collect(cont, itr, ::HasEltype, isz::SizeUnknown)
+    a = _similar_for(cont, eltype(itr), itr, isz)
+    for x in itr
+        push!(a,x)
+    end
+    return a
+end
+
+_collect(c, itr, ::EltypeUnknown, isz::SizeUnknown) = grow_to!(_similar_for(c, Union{}, itr, isz), itr)
+
+function _collect(c, itr, ::EltypeUnknown, isz::Union{HasLength,HasShape})
+    st = start(itr)
+    if done(itr,st)
+        return _similar_for(c, Union{}, itr, isz)
+    end
+    v1, st = next(itr, st)
+    collect_to_with_first!(_similar_for(c, typeof(v1), itr, isz), v1, itr, st)
+end
+
+function collect_to_with_first!(dest::AbstractArray, v1, itr, st)
+    dest[1] = v1
+    return collect_to!(dest, itr, 2, st)
+end
+
+function collect_to_with_first!(dest, v1, itr, st)
+    push!(dest, v1)
+    return grow_to!(dest, itr, st)
+end
+
+function collect_to!{T}(dest::AbstractArray{T}, itr, offs, st)
+    # collect to dest array, checking the type of each result. if a result does not
+    # match, widen the result type and re-dispatch.
+    i = offs
+    while !done(itr, st)
+        el, st = next(itr, st)
+        S = typeof(el)
+        if S === T || S <: T
+            @inbounds dest[i] = el::T
+            i += 1
+        else
+            R = typejoin(T, S)
+            new = similar(dest, R)
+            copy!(new,1, dest,1, i-1)
+            @inbounds new[i] = el
+            return collect_to!(new, itr, i+1, st)
+        end
+    end
+    return dest
+end
+
+function grow_to!(dest, itr, st = start(itr))
+    T = eltype(dest)
+    while !done(itr, st)
+        el, st = next(itr, st)
+        S = typeof(el)
+        if S === T || S <: T
+            push!(dest, el::T)
+        else
+            new = similar(dest, typejoin(T, S))
+            copy!(new, dest)
+            push!(new, el)
+            return grow_to!(new, itr, st)
+        end
+    end
+    return dest
+end
 
 ## Iteration ##
 start(A::Array) = 1

base/channels.jl

@@ -95,3 +95,5 @@ function done(c::Channel, state::Ref)
     end
 end
 next{T}(c::Channel{T}, state) = (v=get(state[]); state[]=nothing; (v, state))
+
+iteratorsize{C<:Channel}(::Type{C}) = SizeUnknown()

base/dict.jl

@@ -2,8 +2,6 @@
 
 # generic operations on associative collections
 
-abstract Associative{K,V}
-
 const secret_table_token = :__c782dbf1cf4d6a2e5e3865d7e95634f2e09b5902__
 
 haskey(d::Associative, k) = in(k,keys(d))
@@ -225,6 +223,16 @@ function merge!(d::Associative, others::Associative...)
     end
     return d
 end
+
+# very similar to `merge!`, but accepts any iterable and extends code
+# that would otherwise only use `copy!` with arrays.
+function copy!(dest::Union{Associative,AbstractSet}, src)
+    for x in src
+        push!(dest, x)
+    end
+    return dest
+end
+
 keytype{K,V}(::Type{Associative{K,V}}) = K
 keytype(a::Associative) = keytype(typeof(a))
 keytype{A<:Associative}(::Type{A}) = keytype(supertype(A))
@@ -449,19 +457,6 @@ copy(d::Dict) = Dict(d)
 
 const AnyDict = Dict{Any,Any}
 
-# TODO: this can probably be simplified using `eltype` as a THT (Tim Holy trait)
-Dict{K,V}(kv::Tuple{Vararg{Tuple{K,V}}})          = Dict{K,V}(kv)
-Dict{K  }(kv::Tuple{Vararg{Tuple{K,Any}}})        = Dict{K,Any}(kv)
-Dict{V  }(kv::Tuple{Vararg{Tuple{Any,V}}})        = Dict{Any,V}(kv)
-Dict{K,V}(kv::Tuple{Vararg{Pair{K,V}}})           = Dict{K,V}(kv)
-Dict{K  }(kv::Tuple{Vararg{Pair{K}}})             = Dict{K,Any}(kv)
-Dict{V  }(kv::Tuple{Vararg{Pair{TypeVar(:K),V}}}) = Dict{Any,V}(kv)
-Dict(     kv::Tuple{Vararg{Pair}})                = Dict{Any,Any}(kv)
-
-Dict{K,V}(kv::AbstractArray{Tuple{K,V}}) = Dict{K,V}(kv)
-Dict{K,V}(kv::AbstractArray{Pair{K,V}})  = Dict{K,V}(kv)
-Dict{K,V}(kv::Associative{K,V})          = Dict{K,V}(kv)
-
 Dict{K,V}(ps::Pair{K,V}...)            = Dict{K,V}(ps)
 Dict{K  }(ps::Pair{K}...,)             = Dict{K,Any}(ps)
 Dict{V  }(ps::Pair{TypeVar(:K),V}...,) = Dict{Any,V}(ps)
@@ -482,9 +477,27 @@ end
 
 dict_with_eltype{K,V}(kv, ::Type{Tuple{K,V}}) = Dict{K,V}(kv)
 dict_with_eltype{K,V}(kv, ::Type{Pair{K,V}}) = Dict{K,V}(kv)
-dict_with_eltype(kv, t) = Dict{Any,Any}(kv)
+dict_with_eltype(kv, t) = grow_to!(Dict{Union{},Union{}}(), kv)
+
+# this is a special case due to (1) allowing both Pairs and Tuples as elements,
+# and (2) Pair being invariant. a bit annoying.
+function grow_to!{K,V}(dest::Associative{K,V}, itr, st = start(itr))
+    while !done(itr, st)
+        (k,v), st = next(itr, st)
+        if isa(k,K) && isa(v,V)
+            dest[k] = v
+        else
+            new = similar(dest, Pair{typejoin(K,typeof(k)), typejoin(V,typeof(v))})
+            copy!(new, dest)
+            new[k] = v
+            return grow_to!(new, itr, st)
+        end
+    end
+    return dest
+end
 
 similar{K,V}(d::Dict{K,V}) = Dict{K,V}()
+similar{K,V}(d::Dict, ::Type{Pair{K,V}}) = Dict{K,V}()
 
 # conversion between Dict types
 function convert{K,V}(::Type{Dict{K,V}},d::Associative)

base/essentials.jl

@@ -6,6 +6,9 @@ typealias Callable Union{Function,DataType}
 
 const Bottom = Union{}
 
+abstract AbstractSet{T}
+abstract Associative{K,V}
+
 # The real @inline macro is not available until after array.jl, so this
 # internal macro splices the meta Expr directly into the function body.
 macro _inline_meta()