MAINT: Manually insert DTypes into the existing table

numpy/_core/code_generators/cversions.txt

@@ -73,5 +73,5 @@
 0x00000011 = ca1aebdad799358149567d9d93cbca09
 
 # Version 18 (NumPy 2.0.0)
-0x00000012 = b6d09b252bc4550aa4cf67b2665c6f84
+0x00000012 = e1a344a47c4449f5fd0e3e5c7cd7a4ed
 

numpy/_core/code_generators/generate_numpy_api.py

@@ -31,94 +31,29 @@
         _NPY_VERSION_CONCAT_HELPER2(arg, PyArray_RUNTIME_VERSION)
     #define PyArray_RUNTIME_VERSION \
         _NPY_VERSION_CONCAT_HELPER(PY_ARRAY_UNIQUE_SYMBOL)
-    #define __dtype_api_table \
-        _NPY_VERSION_CONCAT_HELPER2(PY_ARRAY_UNIQUE_SYMBOL, \
-                                    __dtype_api_table)
-#else
-#define __dtype_api_table __dtype_api_table
 #endif
 
 #if defined(NO_IMPORT) || defined(NO_IMPORT_ARRAY)
 extern void **PyArray_API;
 extern int PyArray_RUNTIME_VERSION;
-extern void **__dtype_api_table;
 #else
 
-static void *__uninitialized_table[] = {NULL};
-
 #if defined(PY_ARRAY_UNIQUE_SYMBOL)
 void **PyArray_API;
-void **__dtype_api_table = __uninitialized_table;
 int PyArray_RUNTIME_VERSION;
 #else
 static void **PyArray_API = NULL;
 static int PyArray_RUNTIME_VERSION = 0;
-static void **__dtype_api_table = __uninitialized_table;
 #endif
 #endif
 
 %s
 
-#ifndef NPY_INTERNAL_BUILD
 /*
- * The type of the DType metaclass
+ * The DType classes are inconvenient for the Python generation so exposed
+ * manualy in the header below  (may be moved).
  */
-#define PyArrayDTypeMeta_Type (*(PyTypeObject *)__dtype_api_table[0])
-/*
- * NumPy's builtin DTypes:
- */
-#define PyArray_BoolDType (*(PyArray_DTypeMeta *)__dtype_api_table[1])
-/* Integers */
-#define PyArray_ByteDType (*(PyArray_DTypeMeta *)__dtype_api_table[2])
-#define PyArray_UByteDType (*(PyArray_DTypeMeta *)__dtype_api_table[3])
-#define PyArray_ShortDType (*(PyArray_DTypeMeta *)__dtype_api_table[4])
-#define PyArray_UShortDType (*(PyArray_DTypeMeta *)__dtype_api_table[5])
-#define PyArray_IntDType (*(PyArray_DTypeMeta *)__dtype_api_table[6])
-#define PyArray_UIntDType (*(PyArray_DTypeMeta *)__dtype_api_table[7])
-#define PyArray_LongDType (*(PyArray_DTypeMeta *)__dtype_api_table[8])
-#define PyArray_ULongDType (*(PyArray_DTypeMeta *)__dtype_api_table[9])
-#define PyArray_LongLongDType (*(PyArray_DTypeMeta *)__dtype_api_table[10])
-#define PyArray_ULongLongDType (*(PyArray_DTypeMeta *)__dtype_api_table[11])
-/* Integer aliases */
-#define PyArray_Int8DType (*(PyArray_DTypeMeta *)__dtype_api_table[12])
-#define PyArray_UInt8DType (*(PyArray_DTypeMeta *)__dtype_api_table[13])
-#define PyArray_Int16DType (*(PyArray_DTypeMeta *)__dtype_api_table[14])
-#define PyArray_UInt16DType (*(PyArray_DTypeMeta *)__dtype_api_table[15])
-#define PyArray_Int32DType (*(PyArray_DTypeMeta *)__dtype_api_table[16])
-#define PyArray_UInt32DType (*(PyArray_DTypeMeta *)__dtype_api_table[17])
-#define PyArray_Int64DType (*(PyArray_DTypeMeta *)__dtype_api_table[18])
-#define PyArray_UInt64DType (*(PyArray_DTypeMeta *)__dtype_api_table[19])
-#define PyArray_IntpDType (*(PyArray_DTypeMeta *)__dtype_api_table[20])
-#define PyArray_UIntpDType (*(PyArray_DTypeMeta *)__dtype_api_table[21])
-/* Floats */
-#define PyArray_HalfDType (*(PyArray_DTypeMeta *)__dtype_api_table[22])
-#define PyArray_FloatDType (*(PyArray_DTypeMeta *)__dtype_api_table[23])
-#define PyArray_DoubleDType (*(PyArray_DTypeMeta *)__dtype_api_table[24])
-#define PyArray_LongDoubleDType (*(PyArray_DTypeMeta *)__dtype_api_table[25])
-/* Complex */
-#define PyArray_CFloatDType (*(PyArray_DTypeMeta *)__dtype_api_table[26])
-#define PyArray_CDoubleDType (*(PyArray_DTypeMeta *)__dtype_api_table[27])
-#define PyArray_CLongDoubleDType (*(PyArray_DTypeMeta *)__dtype_api_table[28])
-/* String/Bytes */
-#define PyArray_BytesDType (*(PyArray_DTypeMeta *)__dtype_api_table[29])
-#define PyArray_UnicodeDType (*(PyArray_DTypeMeta *)__dtype_api_table[30])
-/* Datetime/Timedelta */
-#define PyArray_DatetimeDType (*(PyArray_DTypeMeta *)__dtype_api_table[31])
-#define PyArray_TimedeltaDType (*(PyArray_DTypeMeta *)__dtype_api_table[32])
-/* Object/Void */
-#define PyArray_ObjectDType (*(PyArray_DTypeMeta *)__dtype_api_table[33])
-#define PyArray_VoidDType (*(PyArray_DTypeMeta *)__dtype_api_table[34])
-/* Abstract */
-#define PyArray_PyIntAbstractDType \
-    (*(PyArray_DTypeMeta *)__dtype_api_table[35])
-#define PyArray_PyFloatAbstractDType \
-    (*(PyArray_DTypeMeta *)__dtype_api_table[36])
-#define PyArray_PyComplexAbstractDType \
-    (*(PyArray_DTypeMeta *)__dtype_api_table[37])
-#define PyArray_DefaultIntDType (*(PyArray_DTypeMeta *)__dtype_api_table[38])
-/* New non-legacy DTypes follow in the order they were added */
-#define PyArray_StringDType (*(PyArray_DTypeMeta *)__dtype_api_table[39])
-#endif /* NPY_INTERNAL_BUILD */
+#include "numpy/_public_dtype_api_table.h"
 
 #if !defined(NO_IMPORT_ARRAY) && !defined(NO_IMPORT)
 static int
@@ -136,27 +71,11 @@
   }
 
   PyObject *c_api = PyObject_GetAttrString(numpy, "_ARRAY_API");
-
+  Py_DECREF(numpy);
   if (c_api == NULL) {
-      Py_DECREF(numpy);
-      return -1;
-  }
-
-  PyObject *dtype_api = PyObject_CallMethod(numpy, "_get_dtype_api", NULL);
-
-  if (dtype_api == NULL) {
-      Py_DECREF(numpy);
-      return -1;
-  }
-  if (!PyCapsule_CheckExact(dtype_api)) {
-      PyErr_SetString(PyExc_RuntimeError, "dtype API is not PyCapsule "
-                      "object");
-      Py_DECREF(c_api);
       return -1;
   }
 
-  Py_DECREF(numpy);
-
   if (!PyCapsule_CheckExact(c_api)) {
       PyErr_SetString(PyExc_RuntimeError, "_ARRAY_API is not PyCapsule object");
       Py_DECREF(c_api);
@@ -169,14 +88,6 @@
       return -1;
   }
 
-  __dtype_api_table = (void **)PyCapsule_GetPointer(
-      dtype_api, "dtype_api_table");
-  Py_DECREF(dtype_api);
-  if (__dtype_api_table == NULL) {
-      __dtype_api_table = __uninitialized_table;
-      return -1;
-  }
-
   /*
    * On exceedingly few platforms these sizes may not match, in which case
    * We do not support older NumPy versions at all.

numpy/_core/code_generators/numpy_api.py

@@ -91,6 +91,10 @@ def get_annotations():
     'PyHalfArrType_Type':               (217,),
     'NpyIter_Type':                     (218,),
     # End 1.6 API
+    # NOTE: The Slots 320-360 are defined in `_experimental_dtype_api.h`
+    #       and filled explicitly outside the code generator as the metaclass
+    #       makes them tricky to expose.  (This may be refactored.)
+    # End 2.0 API
 }
 
 # define NPY_NUMUSERTYPES (*(int *)PyArray_API[6])
@@ -102,7 +106,7 @@ def get_annotations():
         1, 4, 40, 41, 65, 66, 67, 68, 81, 82, 83,
         103, 115, 117, 122, 163, 164, 171, 173, 197,
         201, 202, 208, 219, 278, 291, 293, 294, 295,
-        301],
+        301] + list(range(320, 361)),  # range reserves DType class slots
     'PyArray_GetNDArrayCVersion':           (0,),
     # Unused slot 40, was `PyArray_SetNumericOps`
     # Unused slot 41, was `PyArray_GetNumericOps`,
@@ -388,10 +392,11 @@ def get_annotations():
     'NpyString_acquire_allocators':                  (317, MinVersion("2.0")),
     'NpyString_release_allocator':                   (318, MinVersion("2.0")),
     'NpyString_release_allocators':                  (319, MinVersion("2.0")),
-    'PyArray_GetDefaultDescr':                       (320, MinVersion("2.0")),
-    'PyArrayInitDTypeMeta_FromSpec':                 (321, MinVersion("2.0")),
-    'PyArray_CommonDType':                           (322, MinVersion("2.0")),
-    'PyArray_PromoteDTypeSequence':                  (323, MinVersion("2.0")),
+    # Slots 320-360 reserved for DType classes (see comment in types)
+    'PyArray_GetDefaultDescr':                       (361, MinVersion("2.0")),
+    'PyArrayInitDTypeMeta_FromSpec':                 (362, MinVersion("2.0")),
+    'PyArray_CommonDType':                           (363, MinVersion("2.0")),
+    'PyArray_PromoteDTypeSequence':                  (364, MinVersion("2.0")),
     # End 2.0 API
 }
 

numpy/_core/include/meson.build

@@ -1,5 +1,6 @@
 installed_headers = [
   'numpy/_neighborhood_iterator_imp.h',
+  'numpy/_public_dtype_api_table.h',
   'numpy/arrayobject.h',
   'numpy/arrayscalars.h',
   'numpy/dtype_api.h',

numpy/_core/include/numpy/_public_dtype_api_table.h

@@ -0,0 +1,79 @@
+/*
+ * Public exposure of the DType Classes.  These are tricky to expose
+ * via the Python API, so they are exposed through this header for now.
+ *
+ * These definitions are only relevant for the public API and we reserve
+ * the slots 320-360 in the API table generation for this (currently).
+ */
+#ifndef NUMPY_CORE_INCLUDE_NUMPY__PUBLIC_DTYPE_API_TABLE_H_
+#define NUMPY_CORE_INCLUDE_NUMPY__PUBLIC_DTYPE_API_TABLE_H_
+
+#if !(defined(NPY_INTERNAL_BUILD) && NPY_INTERNAL_BUILD)
+
+/* All of these require NumPy 2.0 support */
+#if NPY_FEATURE_VERSION >= NPY_2_0_API_VERSION
+
+/*
+ * The type of the DType metaclass
+ */
+#define PyArrayDTypeMeta_Type (*(PyTypeObject *)(PyArray_API + 320)[0])
+/*
+ * NumPy's builtin DTypes:
+ */
+#define PyArray_BoolDType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[1])
+/* Integers */
+#define PyArray_ByteDType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[2])
+#define PyArray_UByteDType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[3])
+#define PyArray_ShortDType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[4])
+#define PyArray_UShortDType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[5])
+#define PyArray_IntDType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[6])
+#define PyArray_UIntDType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[7])
+#define PyArray_LongDType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[8])
+#define PyArray_ULongDType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[9])
+#define PyArray_LongLongDType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[10])
+#define PyArray_ULongLongDType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[11])
+/* Integer aliases */
+#define PyArray_Int8DType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[12])
+#define PyArray_UInt8DType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[13])
+#define PyArray_Int16DType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[14])
+#define PyArray_UInt16DType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[15])
+#define PyArray_Int32DType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[16])
+#define PyArray_UInt32DType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[17])
+#define PyArray_Int64DType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[18])
+#define PyArray_UInt64DType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[19])
+#define PyArray_IntpDType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[20])
+#define PyArray_UIntpDType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[21])
+/* Floats */
+#define PyArray_HalfDType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[22])
+#define PyArray_FloatDType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[23])
+#define PyArray_DoubleDType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[24])
+#define PyArray_LongDoubleDType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[25])
+/* Complex */
+#define PyArray_CFloatDType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[26])
+#define PyArray_CDoubleDType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[27])
+#define PyArray_CLongDoubleDType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[28])
+/* String/Bytes */
+#define PyArray_BytesDType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[29])
+#define PyArray_UnicodeDType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[30])
+/* Datetime/Timedelta */
+#define PyArray_DatetimeDType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[31])
+#define PyArray_TimedeltaDType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[32])
+/* Object/Void */
+#define PyArray_ObjectDType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[33])
+#define PyArray_VoidDType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[34])
+/* Abstract */
+#define PyArray_PyIntAbstractDType \
+    (*(PyArray_DTypeMeta *)(PyArray_API + 320)[35])
+#define PyArray_PyFloatAbstractDType \
+    (*(PyArray_DTypeMeta *)(PyArray_API + 320)[36])
+#define PyArray_PyComplexAbstractDType \
+    (*(PyArray_DTypeMeta *)(PyArray_API + 320)[37])
+#define PyArray_DefaultIntDType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[38])
+/* New non-legacy DTypes follow in the order they were added */
+#define PyArray_StringDType (*(PyArray_DTypeMeta *)(PyArray_API + 320)[39])
+/* NOTE: offset 40 is free, after that a new range will need to be used */
+
+#endif /* NPY_FEATURE_VERSION >= NPY_2_0_API_VERSION */
+
+#endif  /* NPY_INTERNAL_BUILD */
+#endif  /* NUMPY_CORE_INCLUDE_NUMPY__PUBLIC_DTYPE_API_TABLE_H_ */

numpy/_core/src/multiarray/multiarraymodule.c

@@ -75,7 +75,7 @@ NPY_NO_EXPORT int NPY_NUMUSERTYPES = 0;
 #include "stringdtype/dtype.h"
 
 #include "get_attr_string.h"
-#include "public_dtype_api.h"  /* _get_dtype_api */
+#include "public_dtype_api.h"  /* _fill_dtype_api */
 #include "textreading/readtext.h"  /* _readtext_from_file_object */
 
 #include "npy_dlpack.h"
@@ -4559,8 +4559,6 @@ static struct PyMethodDef array_module_methods[] = {
         METH_FASTCALL | METH_KEYWORDS, NULL},
     {"_get_castingimpl",  (PyCFunction)_get_castingimpl,
         METH_VARARGS | METH_KEYWORDS, NULL},
-    {"_get_dtype_api", (PyCFunction)_get_dtype_api,
-        METH_NOARGS, NULL},
     {"_load_from_filelike", (PyCFunction)_load_from_filelike,
         METH_FASTCALL | METH_KEYWORDS, NULL},
     /* from umath */
@@ -5036,23 +5034,6 @@ PyMODINIT_FUNC PyInit__multiarray_umath(void) {
         goto err;
     }
 
-    c_api = PyCapsule_New((void *)PyArray_API, NULL, NULL);
-    if (c_api == NULL) {
-        goto err;
-    }
-    PyDict_SetItemString(d, "_ARRAY_API", c_api);
-    Py_DECREF(c_api);
-
-    c_api = PyCapsule_New((void *)PyUFunc_API, NULL, NULL);
-    if (c_api == NULL) {
-        goto err;
-    }
-    PyDict_SetItemString(d, "_UFUNC_API", c_api);
-    Py_DECREF(c_api);
-    if (PyErr_Occurred()) {
-        goto err;
-    }
-
     /*
      * PyExc_Exception should catch all the standard errors that are
      * now raised instead of the string exception "multiarray.error"
@@ -5228,6 +5209,28 @@ PyMODINIT_FUNC PyInit__multiarray_umath(void) {
         goto err;
     }
 
+    /*
+     * Export the API tables
+     */
+    c_api = PyCapsule_New((void *)PyArray_API, NULL, NULL);
+    /* The dtype API is not auto-filled/generated via Python scripts: */
+    _fill_dtype_api(PyArray_API);
+    if (c_api == NULL) {
+        goto err;
+    }
+    PyDict_SetItemString(d, "_ARRAY_API", c_api);
+    Py_DECREF(c_api);
+
+    c_api = PyCapsule_New((void *)PyUFunc_API, NULL, NULL);
+    if (c_api == NULL) {
+        goto err;
+    }
+    PyDict_SetItemString(d, "_UFUNC_API", c_api);
+    Py_DECREF(c_api);
+    if (PyErr_Occurred()) {
+        goto err;
+    }
+
     return m;
 
  err:

numpy/_core/src/multiarray/public_dtype_api.c

@@ -119,10 +119,10 @@ PyArrayInitDTypeMeta_FromSpec(
 }
 
 
-NPY_NO_EXPORT PyObject *
-_get_dtype_api(PyObject *NPY_UNUSED(mod), PyObject *NPY_UNUSED(arg))
+void
+_fill_dtype_api(void *full_api_table[])
 {
-    static void *api_table[40] = {0};
+    void **api_table = full_api_table + 320;
 
     /* The type of the DType metaclass */
     api_table[0] = &PyArrayDTypeMeta_Type;
@@ -175,6 +175,4 @@ _get_dtype_api(PyObject *NPY_UNUSED(mod), PyObject *NPY_UNUSED(arg))
     api_table[38] = &PyArray_DefaultIntDType;
     /* Non-legacy DTypes that are built in to NumPy */
     api_table[39] = &PyArray_StringDType;
-
-    return PyCapsule_New(&api_table, "dtype_api_table", NULL);
 }

numpy/_core/src/multiarray/public_dtype_api.h

@@ -7,8 +7,8 @@
 #ifndef NUMPY_CORE_SRC_MULTIARRAY_EXPERIMENTAL_PUBLIC_DTYPE_API_H_
 #define NUMPY_CORE_SRC_MULTIARRAY_EXPERIMENTAL_PUBLIC_DTYPE_API_H_
 
-NPY_NO_EXPORT PyObject *
-_get_dtype_api(PyObject *mod, PyObject *arg);
+NPY_NO_EXPORT void
+_fill_dtype_api(void *numpy_api_table[]);
 
 
 #endif  /* NUMPY_CORE_SRC_MULTIARRAY_EXPERIMENTAL_PUBLIC_DTYPE_API_H_ */