Rework Java-backed object layout: fixed offsets, fungible boxed values

Replaces the runtime allocator that re-derived each Java-backed Python
object's layout from version-sensitive CPython internals (reading private
PyLongObject fields on every single value access) with fixed, type-baked
offsets resolved once at type-creation time. The abstract/concrete type
split makes this offset a hard invariant rather than something callers
branch on or recompute.

Builds on that to eliminate per-instance Java-value storage entirely for
the boxed Long/Boolean/Character wrapper family: their JPClass* moves onto
the shared metaclass (redundant per instance otherwise), and their jvalue
is reconstructed on demand from the instance's own native representation --
boxing via a real JNI call only when a boxed wrapper's value actually
crosses back into Java. This also removes a version-gated digit-layout
workaround that existed solely to keep a trailing value at a fixed offset
inside a variable-length PyLongObject subtype, along with a related heap
corruption bug it caused on Python 3.8-3.11 when combined with an implicit
__dict__ slot. A per-leaf-class null-boxed singleton represents
JObject(None, cls), since a genuinely null Java reference has no native
Python representation to reconstruct from.

No user-visible API change. Boxed wrapper instances no longer retain
Java-side reference identity across repeated round-trips through Python --
this was never guaranteed for freshly-converted values either, so the
change is specifically about previously-received-then-passed-back boxed
objects.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

Author: Thrameos

doc/CHANGELOG.rst

@@ -7,6 +7,25 @@ Latest Changes:
 
 - **1.7.2.dev0**
 
+  - Reworked the internal object layout for Java-backed Python objects to use
+    fixed, type-baked offsets instead of a runtime allocator that re-derived
+    each object's layout from version-sensitive CPython internals on every
+    access. For the boxed `Long`/`Boolean`/`Character` wrapper types this
+    also removes their per-instance Java-value storage entirely (reconstructed
+    on demand instead), shrinking those instances and eliminating a
+    version-gated digit-layout workaround. No user-visible API change; boxed
+    wrapper instances no longer retain Java-side reference identity across
+    repeated round-trips through Python.
+
+  - Fixed heap corruption when boxing large `JLong`/`JInt`/`JShort`/`JByte`/`JBoolean`
+    values on Python 3.8-3.11, caused by a fixed-offset allocator layout
+    assumption colliding with CPython's own implicit `__dict__` slot for
+    variable-length int subclasses.
+
+  - Fixed a GC refcount-accounting bug in the internal Java-class metaclass
+    where `tp_traverse`/`tp_clear` did not chain to `type`'s own
+    implementation.
+
   - Fixed memory leak with int and float conversions. #1379
 
   - Fixed instablity in threading for method dispatch. #1366

native/common/include/jp_match.h

@@ -36,23 +36,41 @@ class JPMatch
 	JPMatch(JPJavaFrame *frame, PyObject *object);
 
 	/**
-	 * Get the Java slot associated with the Python object.
+	 * Get the JPClass associated with the Python object, if any.
 	 *
-	 * Thus uses caching.
+	 * Cached alongside getJValue() -- both are resolved together on first
+	 * use since some JPConversion::matches() implementations (e.g.
+	 * JPConversionUnbox) read the cached jvalue directly, relying on an
+	 * earlier matches() call on the same argument having already resolved
+	 * it.
 	 *
-	 * @return the Java slot or 0 if not available.
+	 * @return the class, or nullptr if not a Java value.
 	 */
-	JPValue *getJavaSlot();
+	JPClass *getJPClass();
+
+	/**
+	 * Get the jvalue associated with the Python object.
+	 *
+	 * Only meaningful once getJPClass() is non-null.
+	 */
+	jvalue getJValue();
 
 	jvalue convert();
 
+private:
+	void resolveSlot();
+
 public:
 	JPMatch::Type type;
 	JPConversion *conversion;
 	JPJavaFrame *frame;
 	PyObject *object;
-	JPValue *slot;
 	void *closure;
+
+private:
+	bool m_SlotResolved;
+	JPClass *m_SlotClass;
+	jvalue m_SlotValue;
 } ;
 
 class JPMethodMatch

native/common/include/jp_primitive_accessor.h

@@ -251,10 +251,9 @@ class JPConversionLongWiden : public JPConversion
 
 	jvalue convert(JPMatch &match) override
 	{
-		JPValue *value = match.getJavaSlot();
 		jvalue ret;
 		base_t::field(ret) = (typename base_t::type_t) (dynamic_cast<JPPrimitiveType*>(
-				value->getClass()))->getAsLong(value->getValue());
+				match.getJPClass()))->getAsLong(match.getJValue());
 		return ret;
 	}
 } ;
@@ -331,9 +330,8 @@ class JPConversionFloatWiden : public JPConversion
 
 	jvalue convert(JPMatch &match) override
 	{
-		JPValue *value = match.getJavaSlot();
 		jvalue ret;
-		base_t::field(ret) = (typename base_t::type_t) (dynamic_cast<JPPrimitiveType*>( value->getClass()))->getAsDouble(value->getValue());
+		base_t::field(ret) = (typename base_t::type_t) (dynamic_cast<JPPrimitiveType*>( match.getJPClass()))->getAsDouble(match.getJValue());
 		return ret;
 	}
 } ;

native/common/jp_booleantype.cpp

@@ -85,8 +85,7 @@ class JPConversionAsBooleanJBool : public JPConversionJavaValue
 	JPMatch::Type matches(JPClass *cls, JPMatch &match) override
 	{
 
-		JPValue *value = match.getJavaSlot();
-		if (value == nullptr)
+		if (match.getJPClass() == nullptr)
 			return match.type = JPMatch::_none;
 		match.type = JPMatch::_none;
 		// Implied conversion from boxed to primitive (JLS 5.1.8)

native/common/jp_boxedtype.cpp

@@ -116,16 +116,39 @@ JPPyObject JPBoxedType::convertToPythonObject(JPJavaFrame& frame, jvalue value,
 	}
 
 	JPPyObject wrapper = PyJPClass_create(frame, cls);
-	JPPyObject obj;
+	auto *wrapperType = (PyTypeObject*) wrapper.get();
 	JPContext *context = JPContext_global;
+
+	// Reconstructing families (Long/Boolean/Char) keep no per-instance Java
+	// value at all -- a real Java null needs a dedicated singleton instance
+	// instead, since there's no slot left on an ordinary instance to mark
+	// "this one's null". Built once per class and cached; Double/Float (no
+	// tp_jvalue yet) fall through to the ordinary path below, unchanged.
+	if (cast && value.l == nullptr && PyJPClass_GetJValueFn(wrapperType) != nullptr)
+	{
+		PyObject *nullBoxed = PyJPClass_GetNullBoxed(wrapperType);
+		if (nullBoxed == nullptr)
+		{
+			JPPyObject built;
+			if (this->getPrimitive() == context->_char)
+				built = JPPyObject::call(PyJPChar_Create(wrapperType, 0));
+			else
+				built = PyJPNumber_create(frame, wrapper, JPValue(cls, value));
+			PyJPClass_SetNullBoxed(wrapperType, built.get());
+			nullBoxed = built.get();
+		}
+		return JPPyObject::use(nullBoxed);
+	}
+
+	JPPyObject obj;
 	if (this->getPrimitive() == context->_char)
 	{
 		jchar value2 = 0;
 		// Not null get the char value
 		if (value.l != nullptr)
 			value2 = context->_char->getValueFromObject(frame, JPValue(this, value)).getValue().c;
 		// Create a char string object
-		obj = JPPyObject::call(PyJPChar_Create((PyTypeObject*) wrapper.get(), value2));
+		obj = JPPyObject::call(PyJPChar_Create(wrapperType, value2));
 	} else
 		obj = PyJPNumber_create(frame, wrapper, JPValue(cls, value));
 	PyJPValue_assignJavaSlot(frame, obj.get(), JPValue(cls, value));

native/common/jp_bytetype.cpp

@@ -58,8 +58,7 @@ class JPConversionJByte : public JPConversionJavaValue
 
 	JPMatch::Type matches(JPClass *cls, JPMatch &match) override
 	{
-		JPValue *value = match.getJavaSlot();
-		if (value == nullptr)
+		if (match.getJPClass() == nullptr)
 			return match.type = JPMatch::_none;
 		match.type = JPMatch::_none;
 		// Implied conversion from boxed to primitive (JLS 5.1.8)

native/common/jp_chartype.cpp

@@ -100,8 +100,7 @@ class JPConversionAsJChar : public JPConversionJavaValue
 
 	JPMatch::Type matches(JPClass *cls, JPMatch &match)  override
 	{
-		JPValue *value = match.getJavaSlot();
-		if (value == nullptr)
+		if (match.getJPClass() == nullptr)
 			return match.type = JPMatch::_none;
 		match.type = JPMatch::_none;
 

native/common/jp_class.cpp

@@ -376,9 +376,31 @@ JPPyObject JPClass::convertToPythonObject(JPJavaFrame& frame, jvalue value, bool
 	} else
 	{
 		PyTypeObject *type = ((PyTypeObject*) wrapper.get());
-		// Simple objects don't have a new or init function
-		PyObject *obj2 = type->tp_alloc(type, 0);
+		// Simple objects don't have a new or init function.  If this type is
+		// abstract (kept layout-trivial so it can be mixed into any foreign
+		// family -- see PyJPClass_FromSpecWithBases), redirect the actual
+		// allocation to its hidden concrete companion, exactly as
+		// PyJPObject_new does for the ordinary constructor path. offset is
+		// no longer usable to detect abstract-ness (see PyJPObject_new's
+		// comment) -- tp_concrete is the real signal.
+		PyTypeObject *allocType = PyJPClass_getConcrete(type);
+		if (allocType == nullptr)
+			allocType = type;
+		PyObject *obj2 = allocType->tp_alloc(allocType, 0);
 		JP_PY_CHECK();
+
+		if (allocType != type)
+		{
+			// Polymorph back to the canonical/abstract type, so
+			// type(instance) stays consistent with cls->getHost() and any
+			// other identity-sensitive code -- mirrors PyJPObject_new's own
+			// polymorph-back (see pyjp_object.cpp) and the legacy
+			// PyJPValue_alloc's Py_SET_TYPE trick.
+			Py_INCREF(type);
+			Py_SET_TYPE(obj2, type);
+			Py_DECREF(allocType);
+		}
+
 		obj = JPPyObject::claim(obj2);
 	}
 

native/common/jp_classhints.cpp

@@ -25,18 +25,35 @@
 #include "pyjp.h"
 
 JPMatch::JPMatch() : conversion(nullptr), frame(nullptr), object(nullptr),
-					 type(JPMatch::_none), slot((JPValue*) - 1), closure(nullptr)
+					 type(JPMatch::_none), closure(nullptr),
+					 m_SlotResolved(false), m_SlotClass(nullptr), m_SlotValue()
 {}
 
 JPMatch::JPMatch(JPJavaFrame *fr, PyObject *obj) : conversion(nullptr), frame(fr), object(obj),
-												   type(JPMatch::_none), slot((JPValue*) - 1), closure(nullptr)
+												   type(JPMatch::_none), closure(nullptr),
+												   m_SlotResolved(false), m_SlotClass(nullptr), m_SlotValue()
 {}
 
-JPValue *JPMatch::getJavaSlot()
+void JPMatch::resolveSlot()
 {
-	if (slot == (JPValue*) - 1)
-		return slot = PyJPValue_getJavaSlot(object);
-	return slot;
+	if (m_SlotResolved)
+		return;
+	m_SlotResolved = true;
+	m_SlotClass = PyJPValue_getJPClass(object);
+	if (m_SlotClass != nullptr && frame != nullptr)
+		m_SlotValue = PyJPValue_getJValue(*frame, object);
+}
+
+JPClass *JPMatch::getJPClass()
+{
+	resolveSlot();
+	return m_SlotClass;
+}
+
+jvalue JPMatch::getJValue()
+{
+	resolveSlot();
+	return m_SlotValue;
 }
 
 jvalue JPMatch::convert()
@@ -150,10 +167,10 @@ class JPPythonConversion : public JPConversion
 		JPClass *cls = ((JPClass*) match.closure);
 		JPPyObject args = JPPyTuple_Pack(cls->getHost(), match.object);
 		JPPyObject ret = JPPyObject::call(PyObject_Call(method_.get(), args.get(), nullptr));
-		JPValue *value = PyJPValue_getJavaSlot(ret.get());
-		if (value != nullptr)
+		JPClass *retCls = PyJPValue_getJPClass(ret.get());
+		if (retCls != nullptr)
 		{
-			jvalue v = value->getValue();
+			jvalue v = PyJPValue_getJValue(*match.frame, ret.get());
 			JP_TRACE("Value", v.l);
 			v.l = match.frame->NewLocalRef(v.l);
 			return v;
@@ -618,13 +635,10 @@ class JPConversionObject : public JPConversion
 	JPMatch::Type matches(JPClass *cls, JPMatch &match) override
 	{
 		JP_TRACE_IN("JPConversionObject::matches");
-		JPValue *value = match.getJavaSlot();
-		if (value == nullptr || match.frame == nullptr)
+		JPClass *oc = match.getJPClass();
+		if (oc == nullptr || match.frame == nullptr)
 			return match.type = JPMatch::_none;
 		match.conversion = this;
-		JPClass *oc = value->getClass();
-		if (oc == nullptr)
-			return match.type = JPMatch::_none;
 		if (oc == cls)
 		{
 			// hey, this is me! :)
@@ -660,17 +674,16 @@ class JPConversionObject : public JPConversion
 	jvalue convert(JPMatch &match) override
 	{
 		jvalue res;
-		JPValue* value = match.getJavaSlot();
-		res.l = match.frame->NewLocalRef(value->getValue().l);
+		res.l = match.frame->NewLocalRef(match.getJValue().l);
 		return res;
 	}
 } _objectConversion;
 
 JPMatch::Type JPConversionJavaValue::matches(JPClass *cls, JPMatch &match)
 {
 	JP_TRACE_IN("JPConversionJavaValue::matches");
-	JPValue *slot = match.getJavaSlot();
-	if (slot == nullptr || slot->getClass() != cls)
+	JPClass *oc = match.getJPClass();
+	if (oc == nullptr || oc != cls)
 		return match.type = JPMatch::_none;
 	match.conversion = this;
 	return match.type = JPMatch::_exact;
@@ -685,8 +698,7 @@ void JPConversionJavaValue::getInfo(JPClass *cls, JPConversionInfo &info)
 
 jvalue JPConversionJavaValue::convert(JPMatch &match)
 {
-	JPValue* value = match.getJavaSlot();
-	return *value;
+	return match.getJValue();
 }
 
 JPConversionJavaValue _javaValueConversion;
@@ -851,13 +863,13 @@ class JPConversionJavaObjectAny : public JPConversionBox
 	JPMatch::Type matches(JPClass *cls, JPMatch &match) override
 	{
 		JP_TRACE_IN("JPConversionJavaObjectAny::matches");
-		JPValue *value = match.getJavaSlot();
-		if (value == nullptr || match.frame == nullptr || value->getClass() == nullptr)
+		JPClass *oc = match.getJPClass();
+		if (oc == nullptr || match.frame == nullptr)
 			return match.type = JPMatch::_none;
 		match.conversion = this;
-		if (value->getClass()->isPrimitive())
+		if (oc->isPrimitive())
 			match.type = JPMatch::_implicit;
-		else if (value->getClass() == cls)
+		else if (oc == cls)
 			match.type = JPMatch::_exact;
 		else
 			match.type = JPMatch::_derived;
@@ -875,15 +887,15 @@ class JPConversionJavaObjectAny : public JPConversionBox
 	{
 		jvalue res;
 		JPJavaFrame *frame = match.frame;
-		JPValue *value = match.getJavaSlot();
-		if (!value->getClass()->isPrimitive())
+		JPClass *oc = match.getJPClass();
+		if (!oc->isPrimitive())
 		{
-			res.l = frame->NewLocalRef(value->getJavaObject());
+			res.l = frame->NewLocalRef(match.getJValue().l);
 			return res;
 		} else
 		{
 			// Okay we need to box it.
-			auto* type = dynamic_cast<JPPrimitiveType*> (value->getClass());
+			auto* type = dynamic_cast<JPPrimitiveType*> (oc);
 			match.closure = type->getBoxedClass(*frame);
 			res = JPConversionBox::convert(match);
 			return res;
@@ -899,15 +911,14 @@ class JPConversionJavaNumberAny : public JPConversionJavaObjectAny
 	{
 		JP_TRACE_IN("JPConversionJavaNumberAny::matches");
 		JPContext *context = JPContext_global;
-		JPValue *value = match.getJavaSlot();
+		JPClass *oc = match.getJPClass();
 		// This converter only works for number types, thus boolean and char
 		// are excluded.
-		if (value == nullptr || match.frame == nullptr || value->getClass() == nullptr
-				|| value->getClass() == context->_boolean
-				|| value->getClass() == context->_char)
+		if (oc == nullptr || match.frame == nullptr
+				|| oc == context->_boolean
+				|| oc == context->_char)
 			return match.type = JPMatch::_none;
 		match.conversion = this;
-		JPClass *oc = value->getClass();
 		// If it is the exact type, then it is exact
 		if (oc == cls)
 			return match.type = JPMatch::_exact;
@@ -937,9 +948,9 @@ class JPConversionUnbox : public JPConversion
 		JPContext *context = JPContext_global;
 		if (context == nullptr)
 			return match.type = JPMatch::_none;
-		JPValue *slot = match.slot;
+		JPClass *oc = match.getJPClass();
 		auto *pcls = dynamic_cast<JPPrimitiveType*>( cls);
-		if (slot->getClass() != pcls->getBoxedClass(*match.frame))
+		if (oc != pcls->getBoxedClass(*match.frame))
 			return match.type = JPMatch::_none;
 		match.conversion = this;
 		match.closure = cls;
@@ -956,9 +967,8 @@ class JPConversionUnbox : public JPConversion
 
 	jvalue convert(JPMatch &match) override
 	{
-		JPValue* value = match.getJavaSlot();
 		auto *cls = (JPClass*) match.closure;
-		return cls->getValueFromObject(*match.frame, *value);
+		return cls->getValueFromObject(*match.frame, JPValue(match.getJPClass(), match.getJValue()));
 	}
 } _unboxConversion;