ENH: Adopt new macOS Accelerate BLAS/LAPACK Interfaces, including ILP64

.github/workflows/macos.yml

@@ -98,3 +98,38 @@ jobs:
       run: |
         conda activate numpy-dev
         ccache -s
+
+  accelerate:
+    name: Accelerate ILP64
+    if: "github.repository == 'numpy/numpy'"
+    runs-on: macos-13
+    steps:
+    - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+      with:
+        submodules: recursive
+        fetch-depth: 0
+
+    - uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # v4.7.0
+      with:
+        python-version: '3.10'
+
+    - uses: maxim-lobanov/setup-xcode@9a697e2b393340c3cacd97468baa318e4c883d98 # v1.5.1
+      with:
+        xcode-version: '14.3'
+
+    - name: Install dependencies
+      run: |
+        pip install -r build_requirements.txt
+        pip install pytest pytest-xdist hypothesis
+
+    - name: Build NumPy against Accelerate (ILP64)
+      run: |
+        spin build -- -Dblas=accelerate -Dlapack=accelerate -Duse-ilp64=true
+
+    - name: Show meson-log.txt
+      if: always()
+      run: 'cat build/meson-logs/meson-log.txt'
+
+    - name: Test
+      run: |
+        spin test -j2

doc/release/upcoming_changes/24053.new_feature.rst

@@ -0,0 +1,5 @@
+Support for the updated Accelerate BLAS/LAPACK library, including ILP64 (64-bit
+integer) support, in macOS 13.3 has been added. This brings arm64 support, and
+significant performance improvements of up to 10x for commonly used linear
+algebra operations. When Accelerate is selected at build time, the 13.3+
+version will automatically be used if available.

numpy/core/src/common/npy_cblas.h

@@ -25,6 +25,21 @@ enum CBLAS_SIDE {CblasLeft=141, CblasRight=142};
 
 #define CBLAS_INDEX size_t  /* this may vary between platforms */
 
+#ifdef ACCELERATE_NEW_LAPACK
+    #if __MAC_OS_X_VERSION_MAX_ALLOWED < 130300
+        #ifdef HAVE_BLAS_ILP64
+            #error "Accelerate ILP64 support is only available with macOS 13.3 SDK or later"
+        #endif
+    #else
+        #define NO_APPEND_FORTRAN
+        #ifdef HAVE_BLAS_ILP64
+            #define BLAS_SYMBOL_SUFFIX $NEWLAPACK$ILP64
+        #else
+            #define BLAS_SYMBOL_SUFFIX $NEWLAPACK
+        #endif
+    #endif
+#endif
+
 #ifdef NO_APPEND_FORTRAN
 #define BLAS_FORTRAN_SUFFIX
 #else

numpy/distutils/system_info.py

@@ -47,6 +47,7 @@
     _numpy_info:Numeric
     _pkg_config_info:None
     accelerate_info:accelerate
+    accelerate_lapack_info:accelerate
     agg2_info:agg2
     amd_info:amd
     atlas_3_10_blas_info:atlas
@@ -534,6 +535,7 @@ def get_info(name, notfound_action=0):
           'lapack_ssl2': lapack_ssl2_info,      
           'blas_ssl2': blas_ssl2_info,          
           'accelerate': accelerate_info,      # use blas_opt instead
+          'accelerate_lapack': accelerate_lapack_info,
           'openblas64_': openblas64__info,
           'openblas64__lapack': openblas64__lapack_info,
           'openblas_ilp64': openblas_ilp64_info,
@@ -2015,14 +2017,17 @@ def _check_info(self, info):
 
 class lapack_ilp64_opt_info(lapack_opt_info, _ilp64_opt_info_mixin):
     notfounderror = LapackILP64NotFoundError
-    lapack_order = ['openblas64_', 'openblas_ilp64']
+    lapack_order = ['openblas64_', 'openblas_ilp64', 'accelerate']
     order_env_var_name = 'NPY_LAPACK_ILP64_ORDER'
 
     def _calc_info(self, name):
+        print('lapack_ilp64_opt_info._calc_info(name=%s)' % (name))
         info = get_info(name + '_lapack')
         if self._check_info(info):
             self.set_info(**info)
             return True
+        else:
+            print('%s_lapack does not exist' % (name))
         return False
 
 
@@ -2163,7 +2168,7 @@ def calc_info(self):
 
 class blas_ilp64_opt_info(blas_opt_info, _ilp64_opt_info_mixin):
     notfounderror = BlasILP64NotFoundError
-    blas_order = ['openblas64_', 'openblas_ilp64']
+    blas_order = ['openblas64_', 'openblas_ilp64', 'accelerate']
     order_env_var_name = 'NPY_BLAS_ILP64_ORDER'
 
     def _calc_info(self, name):
@@ -2625,13 +2630,27 @@ def calc_info(self):
                 link_args.extend(['-Wl,-framework', '-Wl,vecLib'])
 
             if args:
+                macros = [
+                    ('NO_ATLAS_INFO', 3),
+                    ('HAVE_CBLAS', None),
+                    ('ACCELERATE_NEW_LAPACK', None),
+                ]
+                if(os.getenv('NPY_USE_BLAS_ILP64', None)):
+                    print('Setting HAVE_BLAS_ILP64')
+                    macros += [
+                        ('HAVE_BLAS_ILP64', None),
+                        ('ACCELERATE_LAPACK_ILP64', None),
+                    ]
                 self.set_info(extra_compile_args=args,
                               extra_link_args=link_args,
-                              define_macros=[('NO_ATLAS_INFO', 3),
-                                             ('HAVE_CBLAS', None)])
+                              define_macros=macros)
 
         return
 
+class accelerate_lapack_info(accelerate_info):
+    def _calc_info(self):
+        return super()._calc_info()
+
 class blas_src_info(system_info):
     # BLAS_SRC is deprecated, please do not use this!
     # Build or install a BLAS library via your package manager or from

numpy/linalg/meson.build

@@ -1,10 +1,10 @@
 # Note that `python_xerbla.c` was excluded on Windows in setup.py;
 # unclear why and it seems needed, so unconditionally used here.
-lapack_lite_sources = [
-  'lapack_lite/python_xerbla.c',
-]
+python_xerbla_sources = ['lapack_lite/python_xerbla.c']
+
+lapack_lite_sources = []
 if not have_lapack
-  lapack_lite_sources += [
+  lapack_lite_sources = [
     'lapack_lite/f2c.c',
     'lapack_lite/f2c_c_lapack.c',
     'lapack_lite/f2c_d_lapack.c',
@@ -19,6 +19,7 @@ endif
 py.extension_module('lapack_lite',
   [
     'lapack_litemodule.c',
+    python_xerbla_sources,
     lapack_lite_sources,
   ],
   dependencies: [np_core_dep, blas_dep, lapack_dep],
@@ -29,6 +30,7 @@ py.extension_module('lapack_lite',
 py.extension_module('_umath_linalg',
   [
     'umath_linalg.cpp',
+    python_xerbla_sources,
     lapack_lite_sources,
   ],
   dependencies: [np_core_dep, blas_dep, lapack_dep],

numpy/meson.build

@@ -48,6 +48,13 @@ else
   ]
 endif
 
+macOS13_3_or_later = false
+if host_machine.system() == 'darwin'
+  r = run_command('xcrun', '-sdk', 'macosx', '--show-sdk-version', check: true)
+  sdkVersion = r.stdout().strip()
+
+  macOS13_3_or_later = sdkVersion.version_compare('>=13.3')
+endif
 
 # This is currently injected directly into CFLAGS/CXXFLAGS for wheel builds
 # (see cibuildwheel settings in pyproject.toml), but used by CI jobs already
@@ -81,6 +88,7 @@ endif
 # https://github.com/mesonbuild/meson/issues/2835
 blas_name = get_option('blas')
 lapack_name = get_option('lapack')
+
 # pkg-config uses a lower-case name while CMake uses a capitalized name, so try
 # that too to make the fallback detection with CMake work
 if blas_name == 'openblas'
@@ -90,6 +98,23 @@ if blas_name == 'openblas'
     _openblas_names = ['openblas', 'OpenBLAS']
   endif
   blas = dependency(_openblas_names, required: false)
+elif blas_name.to_lower() == 'accelerate'
+  # macOS 13.3+ has updated interfaces aligned with BLAS/LAPACK 3.9.1.  Use them if available.
+  if macOS13_3_or_later
+    accelerate_compile_args = ['-DACCELERATE_NEW_LAPACK']
+    if(use_ilp64)
+      accelerate_compile_args += '-DACCELERATE_LAPACK_ILP64'
+    endif
+    blas = declare_dependency(
+            compile_args: accelerate_compile_args,
+            dependencies: dependency('Accelerate')
+          )
+  else
+    if(use_ilp64)
+      error('macOS SDK 13.3+ is required for ILP64 support.')
+    endif
+    blas = dependency('Accelerate')
+  endif
 else
   blas = dependency(blas_name, required: false)
 endif
@@ -112,14 +137,22 @@ if have_blas
   #   `dependency('blas', modules: cblas)`
   # see https://github.com/mesonbuild/meson/pull/10921.
   have_cblas = false
-  if cc.links('''
+  if blas_name.to_lower() == 'accelerate'
+    _cblas_header = '<Accelerate/Accelerate.h>'
+  elif blas_name.to_lower().startswith('mkl')
+    _cblas_header = '<mkl_cblas.h>'
+  else
+    _cblas_header = '<cblas.h>'
+  endif
+  if cc.links(f'''
       #ifndef BLAS_SYMBOL_SUFFIX
       #  define BLAS_SYMBOL_SUFFIX
       #endif
       #define EXPAND(suffix) cblas_ddot ## suffix
       #define DDOT(suffix) EXPAND(suffix)
 
-      #include <cblas.h>
+      #include @_cblas_header@
+
       int main(int argc, const char *argv[])
       {
           double a[4] = {1,2,3,4};
@@ -178,9 +211,27 @@ else
 endif
 
 if lapack_name == 'openblas'
-  lapack_name = ['openblas', 'OpenBLAS']
+  lapack_dep = dependency(['openblas', 'OpenBLAS'], required: false)
+elif lapack_name.to_lower() == 'accelerate'
+  # macOS 13.3+ has updated interfaces aligned with BLAS/LAPACK 3.9.1.  Use them if available.
+  if macOS13_3_or_later
+    accelerate_compile_args = ['-DACCELERATE_NEW_LAPACK']
+    if(use_ilp64)
+      accelerate_compile_args += '-DACCELERATE_LAPACK_ILP64'
+    endif
+    lapack_dep = declare_dependency(
+               compile_args: accelerate_compile_args,
+               dependencies: dependency('Accelerate')
+             )
+  else
+    if(use_ilp64)
+      error('macOS SDK 13.3+ is required for ILP64 support.')
+    endif
+    lapack_dep = dependency('Accelerate')
+  endif
+else
+  lapack_dep = dependency(lapack_name, required: false)
 endif
-lapack_dep = dependency(lapack_name, required: false)
 have_lapack = lapack_dep.found()
 if not have_lapack and not allow_noblas
   error('No LAPACK library detected! Install one, or use the ' + \