From c43ffb644362af712d98ba430e6e21524f0fff39 Mon Sep 17 00:00:00 2001
From: "Steven G. Johnson" <stevenj@alum.mit.edu>
Date: Tue, 30 Jul 2013 00:21:35 -0400
Subject: [PATCH] exported new readbytes and readbytes! functionality
---
base/exports.jl | 2 ++
base/fs.jl | 22 +++++++++++++++---
base/io.jl | 55 +++++++++++++++++++++++++++++----------------
base/iobuffer.jl | 44 +++++++++++++++++++++++++-----------
base/multi.jl | 4 ++++
doc/helpdb.jl | 20 +++++++++++++++++
doc/stdlib/base.rst | 10 +++++++++
7 files changed, 122 insertions(+), 35 deletions(-)
diff --git a/base/exports.jl b/base/exports.jl
index 3957c80b017ca..217c230157214 100644
--- a/base/exports.jl
+++ b/base/exports.jl
@@ -1074,6 +1074,8 @@ export
position,
read,
readall,
+ readbytes,
+ readbytes!,
readchomp,
readdir,
readline,
diff --git a/base/fs.jl b/base/fs.jl
index 41777ed4450f6..98e0a2eb11997 100644
--- a/base/fs.jl
+++ b/base/fs.jl
@@ -147,9 +147,12 @@ function read(f::File, ::Type{Uint8})
return uint8(ret)
end
-function read{T}(f::File, a::Array{T})
+function read{T}(f::File, a::Array{T}, nel=length(a))
+ if nel < 0 || nel > length(a)
+ throw(BoundsError())
+ end
if isbits(T)
- nb = length(a)*sizeof(T)
+ nb = nel*sizeof(T)
ret = ccall(:jl_fs_read, Int32, (Int32, Ptr{Void}, Csize_t),
f.handle, a, nb)
uv_error("write",ret == -1)
@@ -159,8 +162,21 @@ function read{T}(f::File, a::Array{T})
a
end
+nb_available(f::File) = filesize(f) - position(f)
+
+function readbytes!(f::File, b::Array{Uint8}, nb=length(b))
+ nr = min(nb, nb_available(f))
+ if length(b) < nr
+ resize!(b, nr)
+ end
+ read(f, b, nr)
+ return nr
+end
+readbytes(io::File) = read(io, Array(Uint8, nb_available(io)))
+readbytes(io::File, nb) = read(io, Array(Uint8, min(nb, nb_available(io))))
+
function readbytes(f::File)
- a = Array(Uint8, filesize(f) - position(f))
+ a = Array(Uint8, nb_available(f))
read(f,a)
a
end
diff --git a/base/io.jl b/base/io.jl
index 383aceb7eec50..493cf71a0cb20 100644
--- a/base/io.jl
+++ b/base/io.jl
@@ -165,17 +165,33 @@ function readuntil{T}(s::IO, delim::T)
end
readline(s::IO) = readuntil(s, '\n')
+readchomp(x) = chomp!(readall(x))
-function readall(s::IO)
- out = IOBuffer()
- while !eof(s)
+# read up to nb bytes into nb, returning # bytes read
+function readbytes!(s::IO, b::AbstractArray{Uint8}, nb=length(b))
+ olb = lb = length(b)
+ nr = 0
+ while !eof(s) && nr < nb
a = read(s, Uint8)
- write(out, a)
+ nr += 1
+ if nr > lb
+ lb = nr * 2
+ resize!(b, lb)
+ end
+ b[nr] = a
end
- takebuf_string(out)
+ if lb > olb
+ resize!(b, nr) # shrink to just contain input data if was resized
+ end
+ return nr
end
-readchomp(x) = chomp!(readall(x))
+# read up to nb bytes from s, returning a Vector{Uint8} of bytes read.
+function readbytes(s::IO, nb=typemax(Int))
+ b = Array(Uint8, min(nb, 65536))
+ nr = readbytes!(s, b, nb)
+ resize!(b, nr)
+end
function readall(s::IO)
b = readbytes(s)
@@ -414,21 +430,22 @@ function readuntil(s::IOStream, delim::Uint8)
ccall(:jl_readuntil, Array{Uint8,1}, (Ptr{Void}, Uint8), s.ios, delim)
end
-function readbytes(s::IOStream)
- n = 65536
- b = Array(Uint8, n)
- p = 1
- while true
- nr = int(ccall(:ios_readall, Uint,
- (Ptr{Void}, Ptr{Void}, Uint), s.ios, pointer(b,p), n))
- if eof(s)
- resize!(b, p+nr-1)
- break
+function readbytes!(s::IOStream, b::Array{Uint8}, nb=length(b))
+ olb = lb = length(b)
+ nr = 0
+ while !eof(s) && nr < nb
+ if lb < nr+1
+ lb = max(65536, (nr+1) * 2)
+ resize!(b, lb)
end
- p += nr
- resize!(b, p+n-1)
+ nr += int(ccall(:ios_readall, Uint,
+ (Ptr{Void}, Ptr{Void}, Uint),
+ s.ios, pointer(b, nr+1), lb - nr))
+ end
+ if lb > olb
+ resize!(b, nr) # shrink to just contain input data if was resized
end
- b
+ return nr
end
# based on code by Glen Hertz
diff --git a/base/iobuffer.jl b/base/iobuffer.jl
index 9656c2f134ad9..0a30c80f918b3 100644
--- a/base/iobuffer.jl
+++ b/base/iobuffer.jl
@@ -38,19 +38,28 @@ IOBuffer(readable::Bool,writable::Bool) = IOBuffer(Uint8[],readable,writable)
IOBuffer() = IOBuffer(Uint8[], true, true)
IOBuffer(maxsize::Int) = (x=IOBuffer(Array(Uint8,maxsize),true,true,maxsize); x.size=0; x)
-function read{T}(from::IOBuffer, a::Array{T})
- if !from.readable error("read failed") end
- if isbits(T)
- nb = length(a)*sizeof(T)
- if nb > nb_available(from)
- throw(EOFError())
- end
- ccall(:memcpy, Ptr{Void}, (Ptr{Void}, Ptr{Void}, Uint), a, pointer(from.data,from.ptr), nb)
- from.ptr += nb
- return a
- else
+read(from::IOBuffer, a::Array) = read_sub(from, a, 1, length(a))
+
+function read_sub{T}(from::IOBuffer, a::Array{T}, offs, nel)
+ if offs+nel-1 > length(a) || offs < 1 || nel < 0
+ throw(BoundsError())
+ end
+ if !isbits(T)
error("Read from IOBuffer only supports bits types or arrays of bits types; got "*string(T)*".")
end
+ read(from, pointer(a, offs), nel*sizeof(T))
+ return a
+end
+
+read(from::IOBuffer, p::Ptr, nb::Integer) = read(from, p, int(nb))
+function read(from::IOBuffer, p::Ptr, nb::Int)
+ if !from.readable error("read failed") end
+ if nb > nb_available(from)
+ throw(EOFError())
+ end
+ ccall(:memcpy, Ptr{Void}, (Ptr{Void}, Ptr{Void}, Uint), p, pointer(from.data,from.ptr), nb)
+ from.ptr += nb
+ p
end
function read(from::IOBuffer, ::Type{Uint8})
@@ -216,8 +225,17 @@ end
write(to::IOBuffer, p::Ptr) = write(to, convert(Uint, p))
-readbytes(io::IOBuffer,nb::Integer) = bytestring(read(io, Array(Uint8, nb)))
-readall(io::IOBuffer) = readbytes(io,nb_available(io))
+function readbytes!(io::IOBuffer, b::Array{Uint8}, nb=length(b))
+ nr = min(nb, nb_available(io))
+ if length(b) < nr
+ resize!(b, nr)
+ end
+ read_sub(io, b, 1, nr)
+ return nr
+end
+readbytes(io::IOBuffer) = read(io, Array(Uint8, nb_available(io)))
+readbytes(io::IOBuffer, nb) = read(io, Array(Uint8, min(nb, nb_available(io))))
+
function search(buf::IOBuffer, delim)
p = pointer(buf.data, buf.ptr)
q = ccall(:memchr,Ptr{Uint8},(Ptr{Uint8},Int32,Csize_t),p,delim,nb_available(buf))
diff --git a/base/multi.jl b/base/multi.jl
index 58b28d3ca9a08..90a28e910daf8 100644
--- a/base/multi.jl
+++ b/base/multi.jl
@@ -1006,6 +1006,10 @@ function create_worker(privhost, port, pubhost, stream, config)
if nread>0
try
line = readbytes(stream.buffer, nread)
+ if length(line) < nread
+ println(STDERR,"\tTruncated reply from worker $(wrker.id):\t",err)
+ return false
+ end
print("\tFrom worker $(wrker.id):\t",line)
catch err
println(STDERR,"\tError parsing reply from worker $(wrker.id):\t",err)
diff --git a/doc/helpdb.jl b/doc/helpdb.jl
index f9bb34b9ef77a..76fd651e85bbd 100644
--- a/doc/helpdb.jl
+++ b/doc/helpdb.jl
@@ -1667,6 +1667,26 @@
"),
+("I/O","Base","readbytes!","readbytes!(stream, b::Vector{Uint8}, nb=length(b))
+
+ Read up to nb bytes from the stream into b, returning the
+ number of bytes read (increasing the size of b as needed).
+
+"),
+
+("I/O","Base","readbytes","readbytes(stream, nb=typemax(Int))
+
+ Read at most nb bytes from the stream, returning a
+ Vector{Uint8} of the bytes read.
+
+"),
+
+("Text I/O","Base","readall","readall(stream)
+
+ Read the entire contents of an I/O stream as a string.
+
+"),
+
("Text I/O","Base","readline","readline(stream)
Read a single line of text, including a trailing newline character
diff --git a/doc/stdlib/base.rst b/doc/stdlib/base.rst
index def8ef1d0af30..c07d9908074a0 100644
--- a/doc/stdlib/base.rst
+++ b/doc/stdlib/base.rst
@@ -970,6 +970,16 @@ I/O
Read a series of values of the given type from a stream, in canonical binary representation. ``dims`` is either a tuple or a series of integer arguments specifying the size of ``Array`` to return.
+.. function:: readbytes!(stream, b::Vector{Uint8}, nb=length(b))
+
+ Read at most nb bytes from the stream into b, returning the
+ number of bytes read (increasing the size of b as needed).
+
+.. function:: readbytes(stream, nb=typemax(Int))
+
+ Read at most nb bytes from the stream, returning a
+ Vector{Uint8} of the bytes read.
+
.. function:: position(s)
Get the current position of a stream.