fix #10958: buggy handling of embedded NUL chars

(cherry picked from commit 1d90e97)
ref PR #10991

Conflicts:
	base/string.jl
	base/utf8.jl
	base/utf8proc.jl
	test/unicode.jl

Author: stevengj

base/string.jl

@@ -538,8 +538,6 @@ beginswith(a::Array{Uint8,1}, b::Array{Uint8,1}) =
 
 charwidth(c::Char) = max(0,int(ccall(:wcwidth, Int32, (Uint32,), c)))
 strwidth(s::String) = (w=0; for c in s; w += charwidth(c); end; w)
-strwidth(s::ByteString) = int(ccall(:u8_strwidth, Csize_t, (Ptr{Uint8},), s.data))
-# TODO: implement and use u8_strnwidth that takes a length argument
 
 ## libc character class predicates ##
 

base/utf8.jl

@@ -37,7 +37,8 @@ function endof(s::UTF8String)
     end
     i
 end
-length(s::UTF8String) = int(ccall(:u8_strlen, Csize_t, (Ptr{Uint8},), s.data))
+length(s::UTF8String) = int(ccall(:u8_charnum, Csize_t, (Ptr{Uint8}, Csize_t),
+                                  s.data, length(s.data)))
 
 function next(s::UTF8String, i::Int)
     # potentially faster version

base/utf8proc.jl

@@ -41,7 +41,6 @@ const UTF8PROC_CATEGORY_CS = 28
 const UTF8PROC_CATEGORY_CO = 29
 const UTF8PROC_CATEGORY_CN = 30
 
-const UTF8PROC_NULLTERM  = (1<<0)
 const UTF8PROC_STABLE    = (1<<1)
 const UTF8PROC_COMPAT    = (1<<2)
 const UTF8PROC_COMPOSE   = (1<<3)
@@ -60,10 +59,10 @@ const UTF8PROC_STRIPMARK = (1<<13)
 let
     const p = Array(Ptr{Uint8}, 1)
     global utf8proc_map
-    function utf8proc_map(s::String, flags::Integer)
+    function utf8proc_map(s::ByteString, flags::Integer)
         result = ccall(:utf8proc_map, Cssize_t,
                        (Ptr{Uint8}, Cssize_t, Ptr{Ptr{Uint8}}, Cint),
-                       s, 0, p, flags | UTF8PROC_NULLTERM)
+                       s, sizeof(s), p, flags)
         result < 0 && error(bytestring(ccall(:utf8proc_errmsg, Ptr{Uint8},
                                              (Cssize_t,), result)))
         a = ccall(:jl_ptr_to_array_1d, Vector{Uint8},
@@ -73,6 +72,8 @@ let
     end
 end
 
+utf8proc_map(s::String, flags::Integer) = utf8proc_map(bytestring(s), flags)
+
 function normalize_string(s::String; stable::Bool=false, compat::Bool=false, compose::Bool=true, decompose::Bool=false, stripignore::Bool=false, rejectna::Bool=false, newline2ls::Bool=false, newline2ps::Bool=false, newline2lf::Bool=false, stripcc::Bool=false, casefold::Bool=false, lump::Bool=false, stripmark::Bool=false)
     flags = 0
     stable && (flags = flags | UTF8PROC_STABLE)

test/unicode.jl

@@ -99,3 +99,8 @@ let c_ll = 'β', c_cn = '\u038B'
     # check codepoint with category code CN
     @test Base.UTF8proc.category_code(c_cn) == Base.UTF8proc.UTF8PROC_CATEGORY_CN
 end
+
+# handling of embedded NUL chars (#10958)
+@test length("\0w") == length("\0α") == 2
+@test strwidth("\0w") == strwidth("\0α") == 1
+@test normalize_string("\0W", casefold=true) == "\0w"