Skip to content

Store SyntaxError members in struct fields instead of the instance dict #8348

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
devyubin wants to merge 3 commits into
base: main
Choose a base branch
from
+231 −71
Draft

Store SyntaxError members in struct fields instead of the instance dict #8348

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
172292a
Store SyntaxError members in struct fields instead of class attributes
devyubin Jul 22, 2026
c0fccaa
Coerce SyntaxError location argument from any sequence
devyubin Jul 23, 2026
fc3a36f
Surface construction errors from new_exception instead of panicking
devyubin Jul 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion Lib/test/test_exceptions.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2527,7 +2527,6 @@ def test_incorrect_constructor(self):
args = ("bad.py", 1, 2, "abcdefg", 1)
self.assertRaises(TypeError, SyntaxError, "bad bad", args)

@unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 2 is not None
def test_syntax_error_memory_leak(self):
# gh-146250: memory leak with re-initialization of SyntaxError
e = SyntaxError("msg", ("file.py", 1, 2, "txt", 2, 3))
Expand Down
290 changes: 226 additions & 64 deletions crates/vm/src/exceptions.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ use crate::{
},
class::{PyClassImpl, StaticType},
convert::{IntoPyException, ToPyException, ToPyObject},
function::{ArgIterable, FuncArgs, IntoFuncArgs, PySetterValue},
function::{ArgIterable, FuncArgs, IntoFuncArgs},
py_io::{self, Write},
stdlib::sys,
suggestion::offer_suggestions,
Expand Down Expand Up @@ -1029,21 +1029,7 @@ impl ExceptionZoo {
excs.python_finalization_error
);

extend_exception!(PySyntaxError, ctx, excs.syntax_error, {
"msg" => ctx.new_static_getset(
"msg",
excs.syntax_error,
make_arg_getter(0),
syntax_error_set_msg,
),
// TODO: members
"filename" => ctx.none(),
"lineno" => ctx.none(),
"end_lineno" => ctx.none(),
"offset" => ctx.none(),
"end_offset" => ctx.none(),
"text" => ctx.none(),
});
extend_exception!(PySyntaxError, ctx, excs.syntax_error);
extend_exception!(PyIncompleteInputError, ctx, excs.incomplete_input_error);
extend_exception!(PyIndentationError, ctx, excs.indentation_error);
extend_exception!(PyTabError, ctx, excs.tab_error);
Expand Down Expand Up @@ -1082,20 +1068,6 @@ fn make_arg_getter(idx: usize) -> impl Fn(PyBaseExceptionRef) -> Option<PyObject
move |exc| exc.get_arg(idx)
}

fn syntax_error_set_msg(exc: PyBaseExceptionRef, value: PySetterValue, vm: &VirtualMachine) {
let mut args = exc.args.write();
let mut new_args = args.as_slice().to_vec();
// Ensure the message slot at index 0 always exists for SyntaxError.args.
if new_args.is_empty() {
new_args.push(vm.ctx.none());
}
match value {
PySetterValue::Assign(value) => new_args[0] = value,
PySetterValue::Delete => new_args[0] = vm.ctx.none(),
}
*args = PyTuple::new_ref(new_args, &vm.ctx);
}

#[cfg(feature = "serde")]
pub struct SerializeException<'vm, 's> {
vm: &'vm VirtualMachine,
Expand Down Expand Up @@ -2551,12 +2523,64 @@ pub(super) mod types {
#[repr(transparent)]
pub struct PyPythonFinalizationError(PyRuntimeError);

#[pyexception(name, base = PyException, ctx = "syntax_error")]
#[derive(Debug)]
#[repr(transparent)]
pub struct PySyntaxError(PyException);
#[pyexception(name, base = PyException, ctx = "syntax_error", traverse = "manual")]
#[repr(C)]
pub struct PySyntaxError {
base: PyException,
msg: PyAtomicRef<Option<PyObject>>,
filename: PyAtomicRef<Option<PyObject>>,
lineno: PyAtomicRef<Option<PyObject>>,
offset: PyAtomicRef<Option<PyObject>>,
text: PyAtomicRef<Option<PyObject>>,
end_lineno: PyAtomicRef<Option<PyObject>>,
end_offset: PyAtomicRef<Option<PyObject>>,
print_file_and_line: PyAtomicRef<Option<PyObject>>,
}

#[pyexception(with(Initializer))]
impl crate::class::PySubclass for PySyntaxError {
type Base = PyException;
fn as_base(&self) -> &Self::Base {
&self.base
}
}

impl core::fmt::Debug for PySyntaxError {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
f.debug_struct("PySyntaxError").finish_non_exhaustive()
}
}

unsafe impl Traverse for PySyntaxError {
fn traverse(&self, tracer_fn: &mut TraverseFn<'_>) {
self.base.0.traverse(tracer_fn);
if let Some(obj) = self.msg.deref() {
tracer_fn(obj);
}
if let Some(obj) = self.filename.deref() {
tracer_fn(obj);
}
if let Some(obj) = self.lineno.deref() {
tracer_fn(obj);
}
if let Some(obj) = self.offset.deref() {
tracer_fn(obj);
}
if let Some(obj) = self.text.deref() {
tracer_fn(obj);
}
if let Some(obj) = self.end_lineno.deref() {
tracer_fn(obj);
}
if let Some(obj) = self.end_offset.deref() {
tracer_fn(obj);
}
if let Some(obj) = self.print_file_and_line.deref() {
tracer_fn(obj);
}
}
}

#[pyexception(with(Constructor, Initializer))]
impl PySyntaxError {
#[pymethod]
fn __str__(zelf: &Py<PyBaseException>, vm: &VirtualMachine) -> PyStrRef {
Expand Down Expand Up @@ -2620,58 +2644,196 @@ pub(super) mod types {

vm.ctx.new_str(msg_with_location_info)
}

#[pygetset]
fn msg(&self) -> Option<PyObjectRef> {
self.msg.to_owned()
}

#[pygetset(setter)]
fn set_msg(&self, value: PySetterValue, vm: &VirtualMachine) {
let value = match value {
PySetterValue::Assign(v) => Some(v),
PySetterValue::Delete => None,
};
self.msg.swap_to_temporary_refs(value, vm);
}

#[pygetset]
fn filename(&self) -> Option<PyObjectRef> {
self.filename.to_owned()
}

#[pygetset(setter)]
fn set_filename(&self, value: PySetterValue, vm: &VirtualMachine) {
let value = match value {
PySetterValue::Assign(v) => Some(v),
PySetterValue::Delete => None,
};
self.filename.swap_to_temporary_refs(value, vm);
}

#[pygetset]
fn lineno(&self) -> Option<PyObjectRef> {
self.lineno.to_owned()
}

#[pygetset(setter)]
fn set_lineno(&self, value: PySetterValue, vm: &VirtualMachine) {
let value = match value {
PySetterValue::Assign(v) => Some(v),
PySetterValue::Delete => None,
};
self.lineno.swap_to_temporary_refs(value, vm);
}

#[pygetset]
fn offset(&self) -> Option<PyObjectRef> {
self.offset.to_owned()
}

#[pygetset(setter)]
fn set_offset(&self, value: PySetterValue, vm: &VirtualMachine) {
let value = match value {
PySetterValue::Assign(v) => Some(v),
PySetterValue::Delete => None,
};
self.offset.swap_to_temporary_refs(value, vm);
}

#[pygetset]
fn text(&self) -> Option<PyObjectRef> {
self.text.to_owned()
}

#[pygetset(setter)]
fn set_text(&self, value: PySetterValue, vm: &VirtualMachine) {
let value = match value {
PySetterValue::Assign(v) => Some(v),
PySetterValue::Delete => None,
};
self.text.swap_to_temporary_refs(value, vm);
}

#[pygetset]
fn end_lineno(&self) -> Option<PyObjectRef> {
self.end_lineno.to_owned()
}

#[pygetset(setter)]
fn set_end_lineno(&self, value: PySetterValue, vm: &VirtualMachine) {
let value = match value {
PySetterValue::Assign(v) => Some(v),
PySetterValue::Delete => None,
};
self.end_lineno.swap_to_temporary_refs(value, vm);
}

#[pygetset]
fn end_offset(&self) -> Option<PyObjectRef> {
self.end_offset.to_owned()
}

#[pygetset(setter)]
fn set_end_offset(&self, value: PySetterValue, vm: &VirtualMachine) {
let value = match value {
PySetterValue::Assign(v) => Some(v),
PySetterValue::Delete => None,
};
self.end_offset.swap_to_temporary_refs(value, vm);
}

#[pygetset]
fn print_file_and_line(&self) -> Option<PyObjectRef> {
self.print_file_and_line.to_owned()
}

#[pygetset(setter)]
fn set_print_file_and_line(&self, value: PySetterValue, vm: &VirtualMachine) {
let value = match value {
PySetterValue::Assign(v) => Some(v),
PySetterValue::Delete => None,
};
self.print_file_and_line.swap_to_temporary_refs(value, vm);
}
}

impl Constructor for PySyntaxError {
type Args = FuncArgs;

fn py_new(_cls: &Py<PyType>, args: FuncArgs, vm: &VirtualMachine) -> PyResult<Self> {
// msg must also be set here, not only in slot_init: second-level
// subclasses such as TabError never reach PySyntaxError::slot_init.
let msg = args.args.first().cloned();
let base_exception = PyBaseException::new(args.args, vm);
Ok(Self {
base: PyException(base_exception),
msg: msg.into(),

@kangdora kangdora Jul 24, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: setting msg here is redundant — slot_init sets it again — and diverges slightly from CPython, where __new__ leaves msg unset (SyntaxError.__new__(SyntaxError, "x").msg is None there, since only __init__ assigns it). msg: None.into() would match.

@devyubin devyubin Jul 26, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You're right about the CPython __new__ behavior, but msg: None.into() brings back a CI failure from earlier in this PR.

Since TabError (a second-level subclass) never reaches PySyntaxError::slot_init, TabError("error", …) renders as TabError: <no detail available> in tracebacks, which breaks test_doctest's test_syntax_error_with_note.

Setting it in py_new is a workaround until slot inheritance works properly for second-level subclasses.

filename: None.into(),
lineno: None.into(),
offset: None.into(),
text: None.into(),
end_lineno: None.into(),
end_offset: None.into(),
print_file_and_line: None.into(),
})
}
}

impl Initializer for PySyntaxError {
type Args = FuncArgs;

fn slot_init(zelf: PyObjectRef, args: FuncArgs, vm: &VirtualMachine) -> PyResult<()> {
let len = args.args.len();
let new_args = args;
let msg = args.args.first().cloned();
let location_arg = (args.args.len() == 2).then(|| args.args[1].clone());

zelf.set_attr("print_file_and_line", vm.ctx.none(), vm)?;
PyBaseException::slot_init(zelf.clone(), args, vm)?;
let exc: &Py<Self> = zelf.downcast_ref::<Self>().unwrap();

if len == 2
&& let Ok(location_tuple) = new_args.args[1]
.clone()
.downcast::<crate::builtins::PyTuple>()
{
let location_tup_len = location_tuple.len();
if let Some(msg) = msg {
exc.msg.swap_to_temporary_refs(Some(msg), vm);
}

match location_tup_len {
// SyntaxError(msg, (filename, lineno, offset, text[, end_lineno, end_offset]))
// The location argument is coerced from any sequence like CPython's
// PySequence_Tuple; a non-sequence raises TypeError.
if let Some(location_arg) = location_arg {
let location: Vec<PyObjectRef> = location_arg.try_to_value(vm)?;

match location.len() {
4 | 6 => {}
5 => {
return Err(vm.new_type_error(
"end_offset must be provided when end_lineno is provided",
));
}
_ => {
len => {
return Err(vm.new_type_error(format!(
"function takes exactly 4 or 6 arguments ({location_tup_len} given)"
"function takes exactly 4 or 6 arguments ({len} given)"
)));
}
}

for (i, &attr) in [
"filename",
"lineno",
"offset",
"text",
"end_lineno",
"end_offset",
]
.iter()
.enumerate()
{
if location_tup_len > i {
zelf.set_attr(attr, location_tuple[i].to_owned(), vm)?;
} else {
break;
}
exc.end_lineno.swap_to_temporary_refs(None, vm);
exc.end_offset.swap_to_temporary_refs(None, vm);

exc.filename
.swap_to_temporary_refs(Some(location[0].clone()), vm);
exc.lineno
.swap_to_temporary_refs(Some(location[1].clone()), vm);
exc.offset
.swap_to_temporary_refs(Some(location[2].clone()), vm);
exc.text
.swap_to_temporary_refs(Some(location[3].clone()), vm);
if location.len() == 6 {
exc.end_lineno
.swap_to_temporary_refs(Some(location[4].clone()), vm);
exc.end_offset
.swap_to_temporary_refs(Some(location[5].clone()), vm);
}
}

PyBaseException::slot_init(zelf, new_args, vm)
Ok(())
}

fn init(_zelf: PyRef<Self>, _args: Self::Args, _vm: &VirtualMachine) -> PyResult<()> {
Expand Down
11 changes: 5 additions & 6 deletions crates/vm/src/vm/vm_new.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -341,12 +341,11 @@ impl VirtualMachine {
/// [`vm.invoke_exception()`][Self::invoke_exception] or
/// [`exceptions::ExceptionCtor`][crate::exceptions::ExceptionCtor] instead.
pub fn new_exception(&self, exc_type: PyTypeRef, args: Vec<PyObjectRef>) -> PyBaseExceptionRef {
debug_assert_eq!(
exc_type.slots.basicsize,
core::mem::size_of::<PyBaseException>(),
"vm.new_exception() is only for exception types without additional payload. The given type '{}' is not allowed. Use vm.new_os_subtype_error() for OSError subtypes.",
exc_type.name()
);
if exc_type.slots.basicsize != core::mem::size_of::<PyBaseException>() {
// If constructing the exception raises (e.g. __init__ rejects the
// args), surface that exception instead of panicking.
return self.invoke_exception(&exc_type, args).unwrap_or_else(|e| e);

@youknowone youknowone Jul 26, 2026 •
edited
Loading

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

unwrap_or_else is wrong folding here.
The result of invoke_exception return error instance of given exc_type. But unwrapped error is not. it is error caused during the process of raising error.

in my opinion, new_exception must not call invoke_exception for a few reason
new_exception is fast exception creation path for types which doesn't require python invoke. the reason why this function can skip returning Err is based on the restriction.
If we allow to invoke exception initializer, the restriction goes broken. we have to keep this as thin and fast path.

Note: it doesn't mean i am justifying the name new_exception and invoke_exception. if anyone can suggest good names that shows this difference, welcome.

@kangdora kangdora Jul 27, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Agreed on keeping new_exception thin.

I prototyped a dedicated payload path that skips PyType::call, for the internal builders that know their type at compile time:

pub fn new_payload_exception<T>(&self, cls: PyTypeRef, args: FuncArgs) -> PyResult<PyRef<T>>
where
    T: Constructor<Args = FuncArgs> + Initializer,
{
    let payload = T::py_new(&cls, args.clone(), self)?;
    let exc = payload
        .into_ref_with_type_lazy_dict(self, cls)
        .expect("new_payload_exception: cls is not a matching subtype of T");
    T::slot_init(exc.as_object().to_owned(), args, self)?;
    Ok(exc)
}

I verified it by routing both StopIteration and OSError through it — behaves identically, vm tests pass, and it simplifies OSErrorBuilder. new_exception stays untouched, and invoke_exception still handles the runtime-typed / user-subclass path.

If you're on board with this direction, I'd open it as a separate draft to work on further.

@youknowone youknowone Jul 27, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sounds good, let's try it

@devyubin devyubin Jul 27, 2026

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for the work, @kangdora. I'll leave this as a draft until #8403 lands. After that, I'll rebase onto new_payload_exception, revert the new_exception change, and route the SyntaxError constructions through it while keeping the struct-field conversion.

}

PyBaseException::new(args, self)
.into_ref_with_type_lazy_dict(self, exc_type)
Expand Down
Loading