ICE rust-lang/rust#53443

Author: alecmocatta

src/lib.rs

@@ -124,8 +124,12 @@
 #![feature(rustc_private)]
 #![feature(specialization)]
 #![feature(try_from)]
+#![feature(existential_type)]
+#![feature(maybe_uninit)]
+#![feature(nll)]
 #![allow(dead_code)]
 #![allow(non_camel_case_types)]
+#![allow(private_in_public)]
 
 #[macro_use]
 extern crate quick_error;

src/record/mod.rs

@@ -20,6 +20,28 @@
 mod api;
 pub mod reader;
 mod triplet;
+pub mod schemas;
+pub mod types;
 
+use std::{fmt,fmt::Debug};
+use schema::types::ColumnDescPtr;
+use schema::types::ColumnPath;
+use std::collections::HashMap;
+use errors::ParquetError;
+use schema::types::Type;
+use record::reader::RRReader;
+use column::reader::ColumnReader;
 // pub use self::api::{List, ListAccessor, Map, MapAccessor, Row, RowAccessor};
-pub use self::triplet::TypedTripletIter;
+// pub use self::triplet::TypedTripletIter;
+
+pub trait DebugType {
+	fn fmt(f: &mut fmt::Formatter) -> Result<(), fmt::Error>;
+}
+
+pub trait Deserialize: Sized {
+	type Schema: Debug + DebugType;
+	type Reader: RRReader<Item = Self>;
+
+	fn parse(schema: &Type) -> Result<(String,Self::Schema),ParquetError>;
+	fn reader(schema: &Self::Schema, mut path: &mut Vec<String>, curr_def_level: i16, curr_rep_level: i16, paths: &mut HashMap<ColumnPath, (ColumnDescPtr,ColumnReader)>) -> Self::Reader;
+}

src/record/reader.rs

@@ -19,7 +19,6 @@
 //! [`Row`](`::record::api::Row`)s.
 
 use std::{collections::HashMap, fmt, rc::Rc};
-
 use basic::{LogicalType, Repetition};
 use errors::{ParquetError, Result};
 use file::reader::{FileReader, RowGroupReader};
@@ -31,9 +30,10 @@ use schema::types::{ColumnPath, SchemaDescPtr, SchemaDescriptor, Type, TypePtr};
 use record::triplet::TypedTripletIter;
 use data_type::{BoolType, Int32Type, Int64Type, Int96Type, FloatType, DoubleType, ByteArrayType, FixedLenByteArrayType, Int96};
 use basic::Type as PhysicalType;
-
-// /// Default batch size for a reader
-// const DEFAULT_BATCH_SIZE: usize = 1024;
+use super::{Deserialize,DebugType};
+use super::types::{Value,Group,Timestamp,List,Map,Root};
+use std::{convert::TryInto,marker::PhantomData};
+use std::error::Error;
 
 // /// Tree builder for `Reader` enum.
 // /// Serves as a container of options for building a reader tree and a builder, and
@@ -860,6 +860,243 @@ impl<K: RRReader, V: RRReader> RRReader for KeyValueReader<K, V> {
   }
 }
 
+pub struct GroupReader {
+  pub(super) def_level: i16,
+  pub(super) readers: Vec<ValueReader>,
+  pub(super) fields: Rc<HashMap<String,usize>>,
+}
+impl RRReader for GroupReader {
+  type Item = Group;
+
+  fn read_field(&mut self) -> Result<Self::Item> {
+    let mut fields = Vec::new();
+    for reader in self.readers.iter_mut() {
+      fields.push(reader.read_field()?);
+    }
+    Ok(Group(fields, self.fields.clone()))
+  }
+  fn advance_columns(&mut self) {
+    for reader in self.readers.iter_mut() {
+      reader.advance_columns();
+    }
+  }
+  fn has_next(&self) -> bool {
+    self.readers.first().unwrap().has_next()
+  }
+  fn current_def_level(&self) -> i16 {
+    match self.readers.first() {
+        Some(reader) => reader.current_def_level(),
+        None => panic!("Current definition level: empty group reader"),
+      }
+  }
+  fn current_rep_level(&self) -> i16 {
+    match self.readers.first() {
+      Some(reader) => reader.current_rep_level(),
+      None => panic!("Current repetition level: empty group reader"),
+    }
+  }
+}
+
+pub enum ValueReader {
+  Bool(<bool as Deserialize>::Reader),
+  U8(<u8 as Deserialize>::Reader),
+  I8(<i8 as Deserialize>::Reader),
+  U16(<u16 as Deserialize>::Reader),
+  I16(<i16 as Deserialize>::Reader),
+  U32(<u32 as Deserialize>::Reader),
+  I32(<i32 as Deserialize>::Reader),
+  U64(<u64 as Deserialize>::Reader),
+  I64(<i64 as Deserialize>::Reader),
+  F32(<f32 as Deserialize>::Reader),
+  F64(<f64 as Deserialize>::Reader),
+  Timestamp(<Timestamp as Deserialize>::Reader),
+  Array(<Vec<u8> as Deserialize>::Reader),
+  String(<String as Deserialize>::Reader),
+  List(Box<<List<Value> as Deserialize>::Reader>),
+  Map(Box<<Map<Value,Value> as Deserialize>::Reader>),
+  Group(<Group as Deserialize>::Reader),
+  Option(Box<<Option<Value> as Deserialize>::Reader>),
+}
+impl RRReader for ValueReader {
+  type Item = Value;
+
+  fn read_field(&mut self) -> Result<Self::Item> {
+    match self {
+      ValueReader::Bool(ref mut reader) => reader.read_field().map(Value::Bool),
+      ValueReader::U8(ref mut reader) => reader.read_field().map(Value::U8),
+      ValueReader::I8(ref mut reader) => reader.read_field().map(Value::I8),
+      ValueReader::U16(ref mut reader) => reader.read_field().map(Value::U16),
+      ValueReader::I16(ref mut reader) => reader.read_field().map(Value::I16),
+      ValueReader::U32(ref mut reader) => reader.read_field().map(Value::U32),
+      ValueReader::I32(ref mut reader) => reader.read_field().map(Value::I32),
+      ValueReader::U64(ref mut reader) => reader.read_field().map(Value::U64),
+      ValueReader::I64(ref mut reader) => reader.read_field().map(Value::I64),
+      ValueReader::F32(ref mut reader) => reader.read_field().map(Value::F32),
+      ValueReader::F64(ref mut reader) => reader.read_field().map(Value::F64),
+      ValueReader::Timestamp(ref mut reader) => reader.read_field().map(Value::Timestamp),
+      ValueReader::Array(ref mut reader) => reader.read_field().map(Value::Array),
+      ValueReader::String(ref mut reader) => reader.read_field().map(Value::String),
+      ValueReader::List(ref mut reader) => reader.read_field().map(Value::List),
+      ValueReader::Map(ref mut reader) => reader.read_field().map(Value::Map),
+      ValueReader::Group(ref mut reader) => reader.read_field().map(Value::Group),
+      ValueReader::Option(ref mut reader) => reader.read_field().map(|x|Value::Option(Box::new(x))),
+    }
+  }
+  fn advance_columns(&mut self) {
+    match self {
+      ValueReader::Bool(ref mut reader) => reader.advance_columns(),
+      ValueReader::U8(ref mut reader) => reader.advance_columns(),
+      ValueReader::I8(ref mut reader) => reader.advance_columns(),
+      ValueReader::U16(ref mut reader) => reader.advance_columns(),
+      ValueReader::I16(ref mut reader) => reader.advance_columns(),
+      ValueReader::U32(ref mut reader) => reader.advance_columns(),
+      ValueReader::I32(ref mut reader) => reader.advance_columns(),
+      ValueReader::U64(ref mut reader) => reader.advance_columns(),
+      ValueReader::I64(ref mut reader) => reader.advance_columns(),
+      ValueReader::F32(ref mut reader) => reader.advance_columns(),
+      ValueReader::F64(ref mut reader) => reader.advance_columns(),
+      ValueReader::Timestamp(ref mut reader) => reader.advance_columns(),
+      ValueReader::Array(ref mut reader) => reader.advance_columns(),
+      ValueReader::String(ref mut reader) => reader.advance_columns(),
+      ValueReader::List(ref mut reader) => reader.advance_columns(),
+      ValueReader::Map(ref mut reader) => reader.advance_columns(),
+      ValueReader::Group(ref mut reader) => reader.advance_columns(),
+      ValueReader::Option(ref mut reader) => reader.advance_columns(),
+    }
+  }
+  fn has_next(&self) -> bool {
+    match self {
+      ValueReader::Bool(ref reader) => reader.has_next(),
+      ValueReader::U8(ref reader) => reader.has_next(),
+      ValueReader::I8(ref reader) => reader.has_next(),
+      ValueReader::U16(ref reader) => reader.has_next(),
+      ValueReader::I16(ref reader) => reader.has_next(),
+      ValueReader::U32(ref reader) => reader.has_next(),
+      ValueReader::I32(ref reader) => reader.has_next(),
+      ValueReader::U64(ref reader) => reader.has_next(),
+      ValueReader::I64(ref reader) => reader.has_next(),
+      ValueReader::F32(ref reader) => reader.has_next(),
+      ValueReader::F64(ref reader) => reader.has_next(),
+      ValueReader::Timestamp(ref reader) => reader.has_next(),
+      ValueReader::Array(ref reader) => reader.has_next(),
+      ValueReader::String(ref reader) => reader.has_next(),
+      ValueReader::List(ref reader) => reader.has_next(),
+      ValueReader::Map(ref reader) => reader.has_next(),
+      ValueReader::Group(ref reader) => reader.has_next(),
+      ValueReader::Option(ref reader) => reader.has_next(),
+    }
+  }
+  fn current_def_level(&self) -> i16 {
+    match self {
+      ValueReader::Bool(ref reader) => reader.current_def_level(),
+      ValueReader::U8(ref reader) => reader.current_def_level(),
+      ValueReader::I8(ref reader) => reader.current_def_level(),
+      ValueReader::U16(ref reader) => reader.current_def_level(),
+      ValueReader::I16(ref reader) => reader.current_def_level(),
+      ValueReader::U32(ref reader) => reader.current_def_level(),
+      ValueReader::I32(ref reader) => reader.current_def_level(),
+      ValueReader::U64(ref reader) => reader.current_def_level(),
+      ValueReader::I64(ref reader) => reader.current_def_level(),
+      ValueReader::F32(ref reader) => reader.current_def_level(),
+      ValueReader::F64(ref reader) => reader.current_def_level(),
+      ValueReader::Timestamp(ref reader) => reader.current_def_level(),
+      ValueReader::Array(ref reader) => reader.current_def_level(),
+      ValueReader::String(ref reader) => reader.current_def_level(),
+      ValueReader::List(ref reader) => reader.current_def_level(),
+      ValueReader::Map(ref reader) => reader.current_def_level(),
+      ValueReader::Group(ref reader) => reader.current_def_level(),
+      ValueReader::Option(ref reader) => reader.current_def_level(),
+    }
+  }
+  fn current_rep_level(&self) -> i16 {
+    match self {
+      ValueReader::Bool(ref reader) => reader.current_rep_level(),
+      ValueReader::U8(ref reader) => reader.current_rep_level(),
+      ValueReader::I8(ref reader) => reader.current_rep_level(),
+      ValueReader::U16(ref reader) => reader.current_rep_level(),
+      ValueReader::I16(ref reader) => reader.current_rep_level(),
+      ValueReader::U32(ref reader) => reader.current_rep_level(),
+      ValueReader::I32(ref reader) => reader.current_rep_level(),
+      ValueReader::U64(ref reader) => reader.current_rep_level(),
+      ValueReader::I64(ref reader) => reader.current_rep_level(),
+      ValueReader::F32(ref reader) => reader.current_rep_level(),
+      ValueReader::F64(ref reader) => reader.current_rep_level(),
+      ValueReader::Timestamp(ref reader) => reader.current_rep_level(),
+      ValueReader::Array(ref reader) => reader.current_rep_level(),
+      ValueReader::String(ref reader) => reader.current_rep_level(),
+      ValueReader::List(ref reader) => reader.current_rep_level(),
+      ValueReader::Map(ref reader) => reader.current_rep_level(),
+      ValueReader::Group(ref reader) => reader.current_rep_level(),
+      ValueReader::Option(ref reader) => reader.current_rep_level(),
+    }
+  }
+}
+
+pub struct TupleReader<T>(pub(super) T);
+
+pub struct TryIntoReader<R: RRReader, T>(pub(super) R, pub(super) PhantomData<fn(T)>);
+impl<R: RRReader, T> RRReader for TryIntoReader<R, T> where R::Item: TryInto<T>, <R::Item as TryInto<T>>::Error: Error {
+  type Item = T;
+
+  fn read_field(&mut self) -> Result<Self::Item> {
+    self.0.read_field().and_then(|x|x.try_into().map_err(|err|ParquetError::General(err.description().to_owned())))
+  }
+  fn advance_columns(&mut self) {
+    self.0.advance_columns()
+  }
+  fn has_next(&self) -> bool {
+    self.0.has_next()
+  }
+  fn current_def_level(&self) -> i16 {
+    self.0.current_def_level()
+  }
+  fn current_rep_level(&self) -> i16 {
+    self.0.current_rep_level()
+  }
+}
+
+pub struct MapReader<R: RRReader, F>(pub(super) R, pub(super) F);
+impl<R: RRReader, F, T> RRReader for MapReader<R, F> where F: FnMut(R::Item) -> Result<T> {
+  type Item = T;
+
+  fn read_field(&mut self) -> Result<Self::Item> {
+    self.0.read_field().and_then(&mut self.1)
+  }
+  fn advance_columns(&mut self) {
+    self.0.advance_columns()
+  }
+  fn has_next(&self) -> bool {
+    self.0.has_next()
+  }
+  fn current_def_level(&self) -> i16 {
+    self.0.current_def_level()
+  }
+  fn current_rep_level(&self) -> i16 {
+    self.0.current_rep_level()
+  }
+}
+
+pub struct RootReader<R>(pub R);
+impl<R> RRReader for RootReader<R> where R: RRReader {
+  type Item = Root<R::Item>;
+
+  fn read_field(&mut self) -> Result<Self::Item> {
+    self.0.read_field().map(Root)
+  }
+  fn advance_columns(&mut self) {
+    self.0.advance_columns();
+  }
+  fn has_next(&self) -> bool {
+    self.0.has_next()
+  }
+  fn current_def_level(&self) -> i16 {
+    self.0.current_def_level()
+  }
+  fn current_rep_level(&self) -> i16 {
+    self.0.current_rep_level()
+  }
+}
+
 // /// Reader tree for record assembly
 // pub enum Reader {
 //   // Primitive reader with type information and triplet iterator