switch to ipv6, add monotonic_mapping tests

PSeitz · PSeitz · commit b2ca83a93cec · 2022-10-07T18:47:55.000+08:00
diff --git a/fastfield_codecs/src/monotonic_mapping.rs b/fastfield_codecs/src/monotonic_mapping.rs
@@ -202,14 +202,30 @@ impl MonotonicallyMappableToU64 for f64 {
 
 #[cfg(test)]
 mod tests {
+
     use super::*;
 
     #[test]
     fn strictly_monotonic_test() {
+        // identity mapping
+        test_round_trip(&StrictlyMonotonicMappingToInternal::<u64>::new(), 100u64);
+        // round trip to i64
+        test_round_trip(&StrictlyMonotonicMappingToInternal::<i64>::new(), 100u64);
+        // identity mapping
+        test_round_trip(&StrictlyMonotonicMappingToInternal::<u128>::new(), 100u128);
+
+        // base value to i64 round trip
+        let mapping = StrictlyMonotonicMappingToInternalBaseval::new(100);
+        test_round_trip::<_, _, u64>(&mapping, 100i64);
+        // base value and gcd to u64 round trip
         let mapping = StrictlyMonotonicMappingToInternalGCDBaseval::new(10, 100);
+        test_round_trip::<_, _, u64>(&mapping, 100u64);
+    }
 
-        let test_val = 100u64;
-        assert_eq!(mapping.mapping(test_val), 0);
+    fn test_round_trip<T: StrictlyMonotonicFn<K, L>, K: std::fmt::Debug + Eq + Copy, L>(
+        mapping: &T,
+        test_val: K,
+    ) {
         assert_eq!(mapping.inverse(mapping.mapping(test_val)), test_val);
     }
 }
diff --git a/fastfield_codecs/src/monotonic_mapping_u128.rs b/fastfield_codecs/src/monotonic_mapping_u128.rs
@@ -1,4 +1,4 @@
-use std::net::{IpAddr, Ipv6Addr};
+use std::net::Ipv6Addr;
 
 pub trait MonotonicallyMappableToU128: 'static + PartialOrd + Copy + Send + Sync {
     /// Converts a value to u128.
@@ -23,20 +23,16 @@ impl MonotonicallyMappableToU128 for u128 {
     }
 }
 
-impl MonotonicallyMappableToU128 for IpAddr {
+impl MonotonicallyMappableToU128 for Ipv6Addr {
     fn to_u128(self) -> u128 {
         ip_to_u128(self)
     }
 
     fn from_u128(val: u128) -> Self {
-        IpAddr::from(val.to_be_bytes())
+        Ipv6Addr::from(val.to_be_bytes())
     }
 }
 
-fn ip_to_u128(ip_addr: IpAddr) -> u128 {
-    let ip_addr_v6: Ipv6Addr = match ip_addr {
-        IpAddr::V4(v4) => v4.to_ipv6_mapped(),
-        IpAddr::V6(v6) => v6,
-    };
-    u128::from_be_bytes(ip_addr_v6.octets())
+fn ip_to_u128(ip_addr: Ipv6Addr) -> u128 {
+    u128::from_be_bytes(ip_addr.octets())
 }
diff --git a/src/fastfield/readers.rs b/src/fastfield/readers.rs
@@ -1,4 +1,4 @@
-use std::net::IpAddr;
+use std::net::Ipv6Addr;
 use std::sync::Arc;
 
 use fastfield_codecs::{open, open_u128, Column};
@@ -155,18 +155,21 @@ impl FastFieldReaders {
     pub fn ip_addr(&self, field: Field) -> crate::Result<Arc<dyn Column<Ipv6Addr>>> {
         self.check_type(field, FastType::U128, Cardinality::SingleValue)?;
         let bytes = self.fast_field_data(field, 0)?.read_bytes()?;
-        Ok(open_u128::<IpAddr>(bytes)?)
+        Ok(open_u128::<Ipv6Addr>(bytes)?)
     }
 
     /// Returns the `ip` fast field reader reader associated to `field`.
     ///
     /// If `field` is not a u128 fast field, this method returns an Error.
-    pub fn ip_addrs(&self, field: Field) -> crate::Result<MultiValuedU128FastFieldReader<IpAddr>> {
+    pub fn ip_addrs(
+        &self,
+        field: Field,
+    ) -> crate::Result<MultiValuedU128FastFieldReader<Ipv6Addr>> {
         self.check_type(field, FastType::U128, Cardinality::MultiValues)?;
         let idx_reader: Arc<dyn Column<u64>> = self.typed_fast_field_reader(field)?;
 
         let bytes = self.fast_field_data(field, 1)?.read_bytes()?;
-        let vals_reader = open_u128::<IpAddr>(bytes)?;
+        let vals_reader = open_u128::<Ipv6Addr>(bytes)?;
 
         Ok(MultiValuedU128FastFieldReader::open(
             idx_reader,
diff --git a/src/indexer/index_writer.rs b/src/indexer/index_writer.rs
@@ -803,7 +803,7 @@ impl Drop for IndexWriter {
 #[cfg(test)]
 mod tests {
     use std::collections::{HashMap, HashSet};
-    use std::net::IpAddr;
+    use std::net::Ipv6Addr;
 
     use fastfield_codecs::MonotonicallyMappableToU128;
     use proptest::prelude::*;
@@ -1655,7 +1655,7 @@ mod tests {
             match op {
                 IndexingOp::AddDoc { id } => {
                     let facet = Facet::from(&("/cola/".to_string() + &id.to_string()));
-                    let ip_from_id = IpAddr::from_u128(id as u128);
+                    let ip_from_id = Ipv6Addr::from_u128(id as u128);
 
                     if id % 3 == 0 {
                         // every 3rd doc has no ip field
@@ -1772,14 +1772,14 @@ mod tests {
         );
 
         // Load all ips addr
-        let ips: HashSet<IpAddr> = searcher
+        let ips: HashSet<Ipv6Addr> = searcher
             .segment_readers()
             .iter()
             .flat_map(|segment_reader| {
                 let ff_reader = segment_reader.fast_fields().ip_addr(ip_field).unwrap();
                 segment_reader.doc_ids_alive().flat_map(move |doc| {
                     let val = ff_reader.get_val(doc as u64);
-                    if val == IpAddr::from_u128(0) {
+                    if val == Ipv6Addr::from_u128(0) {
                         // TODO Fix null handling
                         None
                     } else {
@@ -1795,7 +1795,7 @@ mod tests {
                 if id % 3 == 0 {
                     None
                 } else {
-                    Some(IpAddr::from_u128(*id as u128))
+                    Some(Ipv6Addr::from_u128(*id as u128))
                 }
             })
             .collect::<HashSet<_>>();
@@ -1807,11 +1807,11 @@ mod tests {
                 if id % 3 == 0 {
                     None
                 } else {
-                    Some(IpAddr::from_u128(*id as u128))
+                    Some(Ipv6Addr::from_u128(*id as u128))
                 }
             })
             .collect::<HashSet<_>>();
-        let ips: HashSet<IpAddr> = searcher
+        let ips: HashSet<Ipv6Addr> = searcher
             .segment_readers()
             .iter()
             .flat_map(|segment_reader| {
diff --git a/src/schema/document.rs b/src/schema/document.rs
@@ -1,7 +1,7 @@
 use std::collections::{HashMap, HashSet};
 use std::io::{self, Read, Write};
 use std::mem;
-use std::net::IpAddr;
+use std::net::Ipv6Addr;
 
 use common::{BinarySerializable, VInt};
 
@@ -76,7 +76,9 @@ impl Document {
 
     /// Adding a facet to the document.
     pub fn add_facet<F>(&mut self, field: Field, path: F)
-    where Facet: From<F> {
+    where
+        Facet: From<F>,
+    {
         let facet = Facet::from(path);
         let value = Value::Facet(facet);
         self.add_field_value(field, value);
@@ -98,8 +100,8 @@ impl Document {
         self.add_field_value(field, value);
     }
 
-    /// Add a IP address field
-    pub fn add_ip_addr(&mut self, field: Field, value: IpAddrV6) {
+    /// Add a IP address field. Internally only Ipv6Addr is used.
+    pub fn add_ip_addr(&mut self, field: Field, value: Ipv6Addr) {
         self.add_field_value(field, value);
     }
 
diff --git a/src/schema/field_type.rs b/src/schema/field_type.rs
@@ -1,4 +1,4 @@
-use std::net::IpAddr;
+use std::net::{IpAddr, Ipv6Addr};
 use std::str::FromStr;
 
 use serde::{Deserialize, Serialize};
@@ -320,43 +320,50 @@ impl FieldType {
     /// target field is a `Str`, this method will return an Error.
     pub fn value_from_json(&self, json: JsonValue) -> Result<Value, ValueParsingError> {
         match json {
-            JsonValue::String(field_text) => match self {
-                FieldType::Date(_) => {
-                    let dt_with_fixed_tz =
-                        OffsetDateTime::parse(&field_text, &Rfc3339).map_err(|_err| {
-                            ValueParsingError::TypeError {
+            JsonValue::String(field_text) => {
+                match self {
+                    FieldType::Date(_) => {
+                        let dt_with_fixed_tz = OffsetDateTime::parse(&field_text, &Rfc3339)
+                            .map_err(|_err| ValueParsingError::TypeError {
                                 expected: "rfc3339 format",
                                 json: JsonValue::String(field_text),
+                            })?;
+                        Ok(DateTime::from_utc(dt_with_fixed_tz).into())
+                    }
+                    FieldType::Str(_) => Ok(Value::Str(field_text)),
+                    FieldType::U64(_) | FieldType::I64(_) | FieldType::F64(_) => {
+                        Err(ValueParsingError::TypeError {
+                            expected: "an integer",
+                            json: JsonValue::String(field_text),
+                        })
+                    }
+                    FieldType::Bool(_) => Err(ValueParsingError::TypeError {
+                        expected: "a boolean",
+                        json: JsonValue::String(field_text),
+                    }),
+                    FieldType::Facet(_) => Ok(Value::Facet(Facet::from(&field_text))),
+                    FieldType::Bytes(_) => base64::decode(&field_text)
+                        .map(Value::Bytes)
+                        .map_err(|_| ValueParsingError::InvalidBase64 { base64: field_text }),
+                    FieldType::JsonObject(_) => Err(ValueParsingError::TypeError {
+                        expected: "a json object",
+                        json: JsonValue::String(field_text),
+                    }),
+                    FieldType::IpAddr(_) => {
+                        let ip_addr: IpAddr = IpAddr::from_str(&field_text).map_err(|err| {
+                            ValueParsingError::ParseError {
+                                error: err.to_string(),
+                                json: JsonValue::String(field_text),
                             }
                         })?;
-                    Ok(DateTime::from_utc(dt_with_fixed_tz).into())
-                }
-                FieldType::Str(_) => Ok(Value::Str(field_text)),
-                FieldType::U64(_) | FieldType::I64(_) | FieldType::F64(_) => {
-                    Err(ValueParsingError::TypeError {
-                        expected: "an integer",
-                        json: JsonValue::String(field_text),
-                    })
+                        let ip_addr_v6: Ipv6Addr = match ip_addr {
+                            IpAddr::V4(v4) => v4.to_ipv6_mapped(),
+                            IpAddr::V6(v6) => v6,
+                        };
+                        Ok(Value::IpAddr(ip_addr_v6))
+                    }
                 }
-                FieldType::Bool(_) => Err(ValueParsingError::TypeError {
-                    expected: "a boolean",
-                    json: JsonValue::String(field_text),
-                }),
-                FieldType::Facet(_) => Ok(Value::Facet(Facet::from(&field_text))),
-                FieldType::Bytes(_) => base64::decode(&field_text)
-                    .map(Value::Bytes)
-                    .map_err(|_| ValueParsingError::InvalidBase64 { base64: field_text }),
-                FieldType::JsonObject(_) => Err(ValueParsingError::TypeError {
-                    expected: "a json object",
-                    json: JsonValue::String(field_text),
-                }),
-                FieldType::IpAddr(_) => Ok(Value::IpAddr(IpAddr::from_str(&field_text).map_err(
-                    |err| ValueParsingError::ParseError {
-                        error: err.to_string(),
-                        json: JsonValue::String(field_text),
-                    },
-                )?)),
-            },
+            }
             JsonValue::Number(field_val_num) => match self {
                 FieldType::I64(_) | FieldType::Date(_) => {
                     if let Some(field_val_i64) = field_val_num.as_i64() {
diff --git a/src/schema/value.rs b/src/schema/value.rs
@@ -1,5 +1,5 @@
 use std::fmt;
-use std::net::IpAddr;
+use std::net::Ipv6Addr;
 
 use serde::de::Visitor;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
@@ -33,15 +33,17 @@ pub enum Value {
     Bytes(Vec<u8>),
     /// Json object value.
     JsonObject(serde_json::Map<String, serde_json::Value>),
-    /// Ip Address value
-    IpAddr(IpAddr),
+    /// IpV6 Address. Internally there is no IpV4, it needs to be converted to `Ipv6Addr`.
+    IpAddr(Ipv6Addr),
 }
 
 impl Eq for Value {}
 
 impl Serialize for Value {
     fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where S: Serializer {
+    where
+        S: Serializer,
+    {
         match *self {
             Value::Str(ref v) => serializer.serialize_str(v),
             Value::PreTokStr(ref v) => v.serialize(serializer),
@@ -60,7 +62,9 @@ impl Serialize for Value {
 
 impl<'de> Deserialize<'de> for Value {
     fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-    where D: Deserializer<'de> {
+    where
+        D: Deserializer<'de>,
+    {
         struct ValueVisitor;
 
         impl<'de> Visitor<'de> for ValueVisitor {
@@ -208,7 +212,7 @@ impl Value {
 
     /// Returns the ip addr, provided the value is of the `Ip` type.
     /// (Returns None if the value is not of the `Ip` type)
-    pub fn as_ip_addr(&self) -> Option<IpAddr> {
+    pub fn as_ip_addr(&self) -> Option<Ipv6Addr> {
         if let Value::IpAddr(val) = self {
             Some(*val)
         } else {
@@ -223,8 +227,8 @@ impl From<String> for Value {
     }
 }
 
-impl From<IpAddr> for Value {
-    fn from(v: IpAddr) -> Value {
+impl From<Ipv6Addr> for Value {
+    fn from(v: Ipv6Addr) -> Value {
         Value::IpAddr(v)
     }
 }
@@ -308,7 +312,7 @@ impl From<serde_json::Value> for Value {
 
 mod binary_serialize {
     use std::io::{self, Read, Write};
-    use std::net::IpAddr;
+    use std::net::Ipv6Addr;
 
     use common::{f64_to_u64, u64_to_f64, BinarySerializable};
     use fastfield_codecs::MonotonicallyMappableToU128;
@@ -465,7 +469,7 @@ mod binary_serialize {
                 }
                 IP_CODE => {
                     let value = u128::deserialize(reader)?;
-                    Ok(Value::IpAddr(IpAddr::from_u128(value)))
+                    Ok(Value::IpAddr(Ipv6Addr::from_u128(value)))
                 }
 
                 _ => Err(io::Error::new(

Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,4 @@`
`1`		`-use std::net::{IpAddr, Ipv6Addr};`
	`1`	`+use std::net::Ipv6Addr;`
`2`	`2`
`3`	`3`	`pub trait MonotonicallyMappableToU128: 'static + PartialOrd + Copy + Send + Sync {`
`4`	`4`	`/// Converts a value to u128.`
`@@ -23,20 +23,16 @@ impl MonotonicallyMappableToU128 for u128 {`
`23`	`23`	`}`
`24`	`24`	`}`
`25`	`25`
`26`		`-impl MonotonicallyMappableToU128 for IpAddr {`
	`26`	`+impl MonotonicallyMappableToU128 for Ipv6Addr {`
`27`	`27`	`fn to_u128(self) -> u128 {`
`28`	`28`	`ip_to_u128(self)`
`29`	`29`	`}`
`30`	`30`
`31`	`31`	`fn from_u128(val: u128) -> Self {`
`32`		`- IpAddr::from(val.to_be_bytes())`
	`32`	`+ Ipv6Addr::from(val.to_be_bytes())`
`33`	`33`	`}`
`34`	`34`	`}`
`35`	`35`
`36`		`-fn ip_to_u128(ip_addr: IpAddr) -> u128 {`
`37`		`- let ip_addr_v6: Ipv6Addr = match ip_addr {`
`38`		`- IpAddr::V4(v4) => v4.to_ipv6_mapped(),`
`39`		`- IpAddr::V6(v6) => v6,`
`40`		`- };`
`41`		`- u128::from_be_bytes(ip_addr_v6.octets())`
	`36`	`+fn ip_to_u128(ip_addr: Ipv6Addr) -> u128 {`
	`37`	`+ u128::from_be_bytes(ip_addr.octets())`
`42`	`38`	`}`