minimal upgrade to PyO3 0.23 (ignoring deprecations)

davidhewitt · davidhewitt · commit cc14951d206c · 2024-11-13T19:13:07.000Z
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -29,7 +29,7 @@ rust-version = "1.75"
 [dependencies]
 # TODO it would be very nice to remove the "py-clone" feature as it can panic,
 # but needs a bit of work to make sure it's not used in the codebase
-pyo3 = { version = "0.22.5", features = ["generate-import-lib", "num-bigint", "py-clone"] }
+pyo3 = { git = "https://github.com/pyo3/pyo3", branch = "release-0.23", features = ["generate-import-lib", "num-bigint", "py-clone"] }
 regex = "1.11.1"
 strum = { version = "0.26.3", features = ["derive"] }
 strum_macros = "0.26.4"
@@ -46,7 +46,7 @@ base64 = "0.22.1"
 num-bigint = "0.4.6"
 python3-dll-a = "0.2.10"
 uuid = "1.11.0"
-jiter = { version = "0.7", features = ["python"] }
+jiter = { git = "https://github.com/pydantic/jiter", branch = "dh/pyo3-0.23", features = ["python"] }
 hex = "0.4.3"
 
 [lib]
@@ -74,12 +74,12 @@ debug = true
 strip = false
 
 [dev-dependencies]
-pyo3 = { version = "0.22.5", features = ["auto-initialize"] }
+pyo3 = { git = "https://github.com/pyo3/pyo3", branch = "release-0.23", features = ["auto-initialize"] }
 
 [build-dependencies]
 version_check = "0.9.5"
 # used where logic has to be version/distribution specific, e.g. pypy
-pyo3-build-config = { version = "0.22.0" }
+pyo3-build-config = { git = "https://github.com/pyo3/pyo3", branch = "release-0.23" }
 
 [lints.clippy]
 dbg_macro = "warn"
diff --git a/src/errors/validation_exception.rs b/src/errors/validation_exception.rs
@@ -274,7 +274,7 @@ impl ValidationError {
                 .iter()
                 .map(|error| PyLineError::try_from(&error))
                 .collect::<PyResult<Vec<PyLineError>>>()?,
-            InputType::try_from(input_type)?,
+            InputType::try_from(input_type)?.into_py(cls.py()),
             hide_input,
         ))
     }
diff --git a/src/input/datetime.rs b/src/input/datetime.rs
@@ -527,7 +527,7 @@ impl TzInfo {
     }
 
     #[allow(unused_variables)]
-    fn dst(&self, dt: &Bound<'_, PyAny>) -> Option<&PyDelta> {
+    fn dst(&self, dt: &Bound<'_, PyAny>) -> Option<&Bound<'_, PyDelta>> {
         None
     }
 
diff --git a/src/input/return_enums.rs b/src/input/return_enums.rs
@@ -269,18 +269,16 @@ pub(crate) fn iterate_mapping_items<'a, 'py>(
         .items()
         .map_err(|e| mapping_err(e, py, input))?
         .iter()
-        .map_err(|e| mapping_err(e, py, input))?
-        .map(move |item| match item {
-            Ok(item) => item.extract().map_err(|_| {
+        .map(move |item| {
+            item.extract().map_err(|_| {
                 ValError::new(
                     ErrorType::MappingType {
                         error: MAPPING_TUPLE_ERROR.into(),
                         context: None,
                     },
                     input,
                 )
-            }),
-            Err(e) => Err(mapping_err(e, py, input)),
+            })
         });
     Ok(iterator)
 }
diff --git a/src/lib.rs b/src/lib.rs
@@ -1,4 +1,5 @@
 #![cfg_attr(has_coverage_attribute, feature(coverage_attribute))]
+#![allow(deprecated)] // FIXME: just used during upgrading PyO3 to 0.23
 
 extern crate core;
 
diff --git a/src/lookup_key.rs b/src/lookup_key.rs
@@ -448,7 +448,7 @@ impl PathItem {
             None
         } else {
             // otherwise, blindly try getitem on v since no better logic is realistic
-            py_any.get_item(self).ok()
+            py_any.get_item(self.to_object(py_any.py())).ok()
         }
     }
 
diff --git a/src/serializers/extra.rs b/src/serializers/extra.rs
@@ -1,5 +1,5 @@
-use std::cell::RefCell;
 use std::fmt;
+use std::sync::Mutex;
 
 use pyo3::exceptions::{PyTypeError, PyValueError};
 use pyo3::intern;
@@ -366,18 +366,27 @@ impl From<bool> for WarningsMode {
     }
 }
 
-#[derive(Clone)]
 #[cfg_attr(debug_assertions, derive(Debug))]
 pub(crate) struct CollectWarnings {
     mode: WarningsMode,
-    warnings: RefCell<Option<Vec<String>>>,
+    // FIXME: mutex is to satisfy PyO3 0.23, we should be able to refactor this away
+    warnings: Mutex<Vec<String>>,
+}
+
+impl Clone for CollectWarnings {
+    fn clone(&self) -> Self {
+        Self {
+            mode: self.mode,
+            warnings: Mutex::new(self.warnings.lock().expect("lock poisoned").clone()),
+        }
+    }
 }
 
 impl CollectWarnings {
     pub(crate) fn new(mode: WarningsMode) -> Self {
         Self {
             mode,
-            warnings: RefCell::new(None),
+            warnings: Mutex::new(Vec::new()),
         }
     }
 
@@ -443,41 +452,46 @@ impl CollectWarnings {
     }
 
     fn add_warning(&self, message: String) {
-        let mut op_warnings = self.warnings.borrow_mut();
-        if let Some(ref mut warnings) = *op_warnings {
-            warnings.push(message);
-        } else {
-            *op_warnings = Some(vec![message]);
-        }
+        self.warnings.lock().expect("lock poisoned").push(message)
     }
 
     pub fn final_check(&self, py: Python) -> PyResult<()> {
         if self.mode == WarningsMode::None {
             return Ok(());
         }
-        match *self.warnings.borrow() {
-            Some(ref warnings) => {
-                let message = format!("Pydantic serializer warnings:\n  {}", warnings.join("\n  "));
-                if self.mode == WarningsMode::Warn {
-                    let user_warning_type = py.import_bound("builtins")?.getattr("UserWarning")?;
-                    PyErr::warn_bound(py, &user_warning_type, &message, 0)
-                } else {
-                    Err(PydanticSerializationError::new_err(message))
-                }
-            }
-            _ => Ok(()),
+        let warnings = self.warnings.lock().expect("lock poisoned");
+
+        if warnings.is_empty() {
+            return Ok(());
+        }
+
+        let message = format!("Pydantic serializer warnings:\n  {}", warnings.join("\n  "));
+        if self.mode == WarningsMode::Warn {
+            let user_warning_type = py.import_bound("builtins")?.getattr("UserWarning")?;
+            PyErr::warn_bound(py, &user_warning_type, &message, 0)
+        } else {
+            Err(PydanticSerializationError::new_err(message))
         }
     }
 }
 
-#[derive(Default, Clone)]
+#[derive(Default)]
 #[cfg_attr(debug_assertions, derive(Debug))]
 pub struct SerRecursionState {
-    guard: RefCell<RecursionState>,
+    // FIXME: mutex is to satisfy PyO3 0.23, we should be able to refactor this away
+    guard: Mutex<RecursionState>,
+}
+
+impl Clone for SerRecursionState {
+    fn clone(&self) -> Self {
+        Self {
+            guard: Mutex::new(self.guard.lock().expect("lock poisoned").clone()),
+        }
+    }
 }
 
 impl ContainsRecursionState for &'_ Extra<'_> {
     fn access_recursion_state<R>(&mut self, f: impl FnOnce(&mut RecursionState) -> R) -> R {
-        f(&mut self.rec_guard.guard.borrow_mut())
+        f(&mut self.rec_guard.guard.lock().expect("lock poisoned"))
     }
 }
diff --git a/src/serializers/filter.rs b/src/serializers/filter.rs
@@ -173,7 +173,9 @@ trait FilterLogic<T: Eq + Copy> {
                     next_exclude = Some(exc_value);
                 }
             } else if let Ok(exclude_set) = exclude.downcast::<PySet>() {
-                if exclude_set.contains(py_key)? || exclude_set.contains(intern!(exclude_set.py(), "__all__"))? {
+                if exclude_set.contains(py_key.to_object(exclude_set.py()))?
+                    || exclude_set.contains(intern!(exclude_set.py(), "__all__"))?
+                {
                     // index is in the exclude set, we return Ok(None) to omit this index
                     return Ok(None);
                 }
@@ -205,7 +207,9 @@ trait FilterLogic<T: Eq + Copy> {
                     return Ok(None);
                 }
             } else if let Ok(include_set) = include.downcast::<PySet>() {
-                if include_set.contains(py_key)? || include_set.contains(intern!(include_set.py(), "__all__"))? {
+                if include_set.contains(py_key.to_object(include_set.py()))?
+                    || include_set.contains(intern!(include_set.py(), "__all__"))?
+                {
                     return Ok(Some((None, next_exclude)));
                 } else if !self.explicit_include(int_key) {
                     // if the index is not in include, include exists, AND it's not in schema include,
@@ -332,7 +336,7 @@ fn merge_all_value<'py>(
     dict: &Bound<'py, PyDict>,
     py_key: impl ToPyObject + Copy,
 ) -> PyResult<Option<Bound<'py, PyAny>>> {
-    let op_item_value = dict.get_item(py_key)?;
+    let op_item_value = dict.get_item(py_key.to_object(dict.py()))?;
     let op_all_value = dict.get_item(intern!(dict.py(), "__all__"))?;
 
     match (op_item_value, op_all_value) {
diff --git a/src/tools.rs b/src/tools.rs
@@ -11,17 +11,17 @@ use crate::input::Int;
 use jiter::{cached_py_string, pystring_fast_new, StringCacheMode};
 
 pub trait SchemaDict<'py> {
-    fn get_as<T>(&self, key: &Bound<'_, PyString>) -> PyResult<Option<T>>
+    fn get_as<T>(&self, key: &Bound<'py, PyString>) -> PyResult<Option<T>>
     where
         T: FromPyObject<'py>;
 
-    fn get_as_req<T>(&self, key: &Bound<'_, PyString>) -> PyResult<T>
+    fn get_as_req<T>(&self, key: &Bound<'py, PyString>) -> PyResult<T>
     where
         T: FromPyObject<'py>;
 }
 
 impl<'py> SchemaDict<'py> for Bound<'py, PyDict> {
-    fn get_as<T>(&self, key: &Bound<'_, PyString>) -> PyResult<Option<T>>
+    fn get_as<T>(&self, key: &Bound<'py, PyString>) -> PyResult<Option<T>>
     where
         T: FromPyObject<'py>,
     {
@@ -31,7 +31,7 @@ impl<'py> SchemaDict<'py> for Bound<'py, PyDict> {
         }
     }
 
-    fn get_as_req<T>(&self, key: &Bound<'_, PyString>) -> PyResult<T>
+    fn get_as_req<T>(&self, key: &Bound<'py, PyString>) -> PyResult<T>
     where
         T: FromPyObject<'py>,
     {
@@ -43,7 +43,7 @@ impl<'py> SchemaDict<'py> for Bound<'py, PyDict> {
 }
 
 impl<'py> SchemaDict<'py> for Option<&Bound<'py, PyDict>> {
-    fn get_as<T>(&self, key: &Bound<'_, PyString>) -> PyResult<Option<T>>
+    fn get_as<T>(&self, key: &Bound<'py, PyString>) -> PyResult<Option<T>>
     where
         T: FromPyObject<'py>,
     {
@@ -54,7 +54,7 @@ impl<'py> SchemaDict<'py> for Option<&Bound<'py, PyDict>> {
     }
 
     #[cfg_attr(has_coverage_attribute, coverage(off))]
-    fn get_as_req<T>(&self, key: &Bound<'_, PyString>) -> PyResult<T>
+    fn get_as_req<T>(&self, key: &Bound<'py, PyString>) -> PyResult<T>
     where
         T: FromPyObject<'py>,
     {
diff --git a/src/validators/arguments.rs b/src/validators/arguments.rs
@@ -348,7 +348,8 @@ impl Validator for ArgumentsValidator {
                             },
                             VarKwargsMode::UnpackedTypedDict => {
                                 // Save to the remaining kwargs, we will validate as a single dict:
-                                remaining_kwargs.set_item(either_str.as_py_string(py, state.cache_str()), value)?;
+                                remaining_kwargs
+                                    .set_item(either_str.as_py_string(py, state.cache_str()), value.to_object(py))?;
                             }
                         }
                     }
diff --git a/src/validators/dataclass.rs b/src/validators/dataclass.rs
@@ -326,8 +326,10 @@ impl Validator for DataclassArgsValidator {
                                                 Err(err) => return Err(err),
                                             }
                                         } else {
-                                            output_dict
-                                                .set_item(either_str.as_py_string(py, state.cache_str()), value)?;
+                                            output_dict.set_item(
+                                                either_str.as_py_string(py, state.cache_str()),
+                                                value.to_object(py),
+                                            )?;
                                         }
                                     }
                                 }
diff --git a/src/validators/decimal.rs b/src/validators/decimal.rs
@@ -279,9 +279,9 @@ pub(crate) fn create_decimal<'py>(arg: &Bound<'py, PyAny>, input: impl ToErrorVa
 
 fn handle_decimal_new_error(input: impl ToErrorValue, error: PyErr, decimal_exception: Bound<'_, PyAny>) -> ValError {
     let py = decimal_exception.py();
-    if error.matches(py, decimal_exception) {
+    if error.matches(py, decimal_exception).unwrap_or(false) {
         ValError::new(ErrorTypeDefaults::DecimalParsing, input)
-    } else if error.matches(py, PyTypeError::type_object_bound(py)) {
+    } else if error.matches(py, PyTypeError::type_object_bound(py)).unwrap_or(false) {
         ValError::new(ErrorTypeDefaults::DecimalType, input)
     } else {
         ValError::InternalErr(error)

Original file line number	Diff line number	Diff line change
`@@ -274,7 +274,7 @@ impl ValidationError {`
`274`	`274`	`.iter()`
`275`	`275`	`.map(\|error\| PyLineError::try_from(&error))`
`276`	`276`	`.collect::<PyResult<Vec<PyLineError>>>()?,`
`277`		`- InputType::try_from(input_type)?,`
	`277`	`+ InputType::try_from(input_type)?.into_py(cls.py()),`
`278`	`278`	`hide_input,`
`279`	`279`	`))`
`280`	`280`	`}`
Original file line number	Diff line number	Diff line change
`@@ -527,7 +527,7 @@ impl TzInfo {`
`527`	`527`	`}`
`528`	`528`
`529`	`529`	`#[allow(unused_variables)]`
`530`		`- fn dst(&self, dt: &Bound<'_, PyAny>) -> Option<&PyDelta> {`
	`530`	`+ fn dst(&self, dt: &Bound<'_, PyAny>) -> Option<&Bound<'_, PyDelta>> {`
`531`	`531`	`None`
`532`	`532`	`}`
`533`	`533`
Original file line number	Diff line number	Diff line change
`@@ -1,4 +1,5 @@`
`1`	`1`	`#![cfg_attr(has_coverage_attribute, feature(coverage_attribute))]`
	`2`	`+#![allow(deprecated)] // FIXME: just used during upgrading PyO3 to 0.23`
`2`	`3`
`3`	`4`	`extern crate core;`
`4`	`5`
Original file line number	Diff line number	Diff line change
`@@ -448,7 +448,7 @@ impl PathItem {`
`448`	`448`	`None`
`449`	`449`	`} else {`
`450`	`450`	`// otherwise, blindly try getitem on v since no better logic is realistic`
`451`		`- py_any.get_item(self).ok()`
	`451`	`+ py_any.get_item(self.to_object(py_any.py())).ok()`
`452`	`452`	`}`
`453`	`453`	`}`
`454`	`454`
Original file line number	Diff line number	Diff line change
`@@ -11,17 +11,17 @@ use crate::input::Int;`
`11`	`11`	`use jiter::{cached_py_string, pystring_fast_new, StringCacheMode};`
`12`	`12`
`13`	`13`	`pub trait SchemaDict<'py> {`
`14`		`- fn get_as<T>(&self, key: &Bound<'_, PyString>) -> PyResult<Option<T>>`
	`14`	`+ fn get_as<T>(&self, key: &Bound<'py, PyString>) -> PyResult<Option<T>>`
`15`	`15`	`where`
`16`	`16`	`T: FromPyObject<'py>;`
`17`	`17`
`18`		`- fn get_as_req<T>(&self, key: &Bound<'_, PyString>) -> PyResult<T>`
	`18`	`+ fn get_as_req<T>(&self, key: &Bound<'py, PyString>) -> PyResult<T>`
`19`	`19`	`where`
`20`	`20`	`T: FromPyObject<'py>;`
`21`	`21`	`}`
`22`	`22`
`23`	`23`	`impl<'py> SchemaDict<'py> for Bound<'py, PyDict> {`
`24`		`- fn get_as<T>(&self, key: &Bound<'_, PyString>) -> PyResult<Option<T>>`
	`24`	`+ fn get_as<T>(&self, key: &Bound<'py, PyString>) -> PyResult<Option<T>>`
`25`	`25`	`where`
`26`	`26`	`T: FromPyObject<'py>,`
`27`	`27`	`{`
`@@ -31,7 +31,7 @@ impl<'py> SchemaDict<'py> for Bound<'py, PyDict> {`
`31`	`31`	`}`
`32`	`32`	`}`
`33`	`33`
`34`		`- fn get_as_req<T>(&self, key: &Bound<'_, PyString>) -> PyResult<T>`
	`34`	`+ fn get_as_req<T>(&self, key: &Bound<'py, PyString>) -> PyResult<T>`
`35`	`35`	`where`
`36`	`36`	`T: FromPyObject<'py>,`
`37`	`37`	`{`
`@@ -43,7 +43,7 @@ impl<'py> SchemaDict<'py> for Bound<'py, PyDict> {`
`43`	`43`	`}`
`44`	`44`
`45`	`45`	`impl<'py> SchemaDict<'py> for Option<&Bound<'py, PyDict>> {`
`46`		`- fn get_as<T>(&self, key: &Bound<'_, PyString>) -> PyResult<Option<T>>`
	`46`	`+ fn get_as<T>(&self, key: &Bound<'py, PyString>) -> PyResult<Option<T>>`
`47`	`47`	`where`
`48`	`48`	`T: FromPyObject<'py>,`
`49`	`49`	`{`
`@@ -54,7 +54,7 @@ impl<'py> SchemaDict<'py> for Option<&Bound<'py, PyDict>> {`
`54`	`54`	`}`
`55`	`55`
`56`	`56`	`#[cfg_attr(has_coverage_attribute, coverage(off))]`
`57`		`- fn get_as_req<T>(&self, key: &Bound<'_, PyString>) -> PyResult<T>`
	`57`	`+ fn get_as_req<T>(&self, key: &Bound<'py, PyString>) -> PyResult<T>`
`58`	`58`	`where`
`59`	`59`	`T: FromPyObject<'py>,`
`60`	`60`	`{`
Original file line number	Diff line number	Diff line change
`@@ -348,7 +348,8 @@ impl Validator for ArgumentsValidator {`
`348`	`348`	`},`
`349`	`349`	`VarKwargsMode::UnpackedTypedDict => {`
`350`	`350`	`// Save to the remaining kwargs, we will validate as a single dict:`
`351`		`- remaining_kwargs.set_item(either_str.as_py_string(py, state.cache_str()), value)?;`
	`351`	`+ remaining_kwargs`
	`352`	`+ .set_item(either_str.as_py_string(py, state.cache_str()), value.to_object(py))?;`
`352`	`353`	`}`
`353`	`354`	`}`
`354`	`355`	`}`
Original file line number	Diff line number	Diff line change
`@@ -326,8 +326,10 @@ impl Validator for DataclassArgsValidator {`
`326`	`326`	`Err(err) => return Err(err),`
`327`	`327`	`}`
`328`	`328`	`} else {`
`329`		`- output_dict`
`330`		`- .set_item(either_str.as_py_string(py, state.cache_str()), value)?;`
	`329`	`+ output_dict.set_item(`
	`330`	`+ either_str.as_py_string(py, state.cache_str()),`
	`331`	`+ value.to_object(py),`
	`332`	`+ )?;`
`331`	`333`	`}`
`332`	`334`	`}`
`333`	`335`	`}`