Make S"\xff" throw an error like bare "\xff" does.

We're getting very close to parity between the how
the parser interprets strings and how the julia
string implementation handles strings.

It would be nice if instead of this:

  julia> S"\xff"
  invalid UTF-8 sequence
  syntax error: error expanding macro S_str

when S_str throws an error that error message is
printed after the "syntax error:" prefix.

Author: StefanKarpinski

j/string.j

@@ -417,6 +417,13 @@ end
 
 unescape_string(s::String) = print_to_string(print_unescaped, s)
 
+## checking UTF-8 validity ##
+
+is_valid_utf8(s::ByteString) = is_valid_utf8(s.data)
+is_valid_utf8(a::Array{Uint8,1}) =
+    bool(ccall(:u8_isvalid, Int32, (Ptr{Uint8}, Int32), a, length(a)))
+check_utf8(s::ByteString) = is_valid_utf8(s) ? s : error("invalid UTF-8 sequence")
+
 ## string interpolation parsing ##
 
 function interp_parse(str::String, unescape::Function)
@@ -450,14 +457,14 @@ function interp_parse(str::String, unescape::Function)
     !isa(strs[1],String) ? expr(:call,:string,strs[1]) : strs[1]
 end
 
-interp_parse(str::String) = interp_parse(str, unescape_string)
+interp_parse(str::String) = interp_parse(str, s->check_utf8(unescape_string(s)))
 
 ## core string macros ##
 
 macro   str(s); interp_parse(s); end
 macro S_str(s); interp_parse(s); end
-macro I_str(s); interp_parse(s, unbackslash); end
-macro E_str(s); unescape_string(s); end
+macro I_str(s); interp_parse(s, s->unbackslash(s)); end
+macro E_str(s); check_utf8(unescape_string(s)); end
 
 ## shell-like command parsing ##
 

src/julia-parser.scm

@@ -916,7 +916,7 @@
 			      (loop (read-char (ts:port s))))))
 		 (let ((str (unescape-string (io.tostring! b))))
 		   (if (= (length str) 1)
-		       ;; one byte, e.g. '\xff'. maybe not valid utf-8, but we
+		       ;; one byte, e.g. '\xff'. maybe not valid UTF-8, but we
 		       ;; want to use the raw value as a codepoint in this case.
 		       (wchar (aref str 0))
 		       (if (or (not (= (string-length str) 1))
@@ -1020,7 +1020,7 @@
 		 `(macrocall str ,(car ps))
 		 (let ((str (unescape-string (car ps))))
 		   (if (not (string.isutf8 str))
-		       (error "invalid utf-8 sequence"))
+		       (error "invalid UTF-8 sequence"))
 		   str))))
 
 	  ;; macro call

src/julia.expmap

@@ -81,6 +81,7 @@
     ios_getc;
     ios_getutf8;
     jl_getutf8;
+    u8_isvalid;
     ios_close;
     ios_flush;
     ios_read;

src/support/utf8.h

@@ -119,7 +119,7 @@ size_t u8_vprintf(const char *fmt, va_list ap);
 size_t u8_printf(const char *fmt, ...);
 
 /* determine whether a sequence of bytes is valid UTF-8. length is in bytes */
-int u8_isvalid(const char *str, int length);
+DLLEXPORT int u8_isvalid(const char *str, int length);
 
 /* reverse a UTF-8 string. len is length in bytes. dest and src must both
    be allocated to at least len+1 bytes. returns 1 for error, 0 otherwise */