buffer.c

#define NPY_NO_DEPRECATED_API NPY_API_VERSION
#define _MULTIARRAYMODULE

#define PY_SSIZE_T_CLEAN
#include <Python.h>
#include <structmember.h>

#include "numpy/arrayobject.h"
#include "numpy/arrayscalars.h"

#include "npy_config.h"

#include "npy_pycompat.h"

#include "npy_buffer.h"
#include "common.h"
#include "numpyos.h"
#include "arrayobject.h"
#include "scalartypes.h"

/*************************************************************************
 ****************   Implement Buffer Protocol ****************************
 *************************************************************************/

/*************************************************************************
 * PEP 3118 buffer protocol
 *
 * Implementing PEP 3118 is somewhat convoluted because of the desirata:
 *
 * - Don't add new members to ndarray or descr structs, to preserve binary
 *   compatibility. (Also, adding the items is actually not very useful,
 *   since mutability issues prevent an 1 to 1 relationship between arrays
 *   and buffer views.)
 *
 * - Don't use bf_releasebuffer, because it prevents PyArg_ParseTuple("s#", ...
 *   from working. Breaking this would cause several backward compatibility
 *   issues already on Python 2.6.
 *
 * - Behave correctly when array is reshaped in-place, or it's dtype is
 *   altered.
 *
 * The solution taken below is to manually track memory allocated for
 * Py_buffers.
 *************************************************************************/

/*
 * Format string translator
 *
 * Translate PyArray_Descr to a PEP 3118 format string.
 */

/* Fast string 'class' */
typedef struct {
    char *s;
    size_t allocated;
    size_t pos;
} _tmp_string_t;

#define INIT_SIZE   16

static int
_append_char(_tmp_string_t *s, char c)
{
    if (s->pos >= s->allocated) {
        char *p;
        size_t to_alloc = (s->allocated == 0) ? INIT_SIZE : (2 * s->allocated);

        p = PyObject_Realloc(s->s, to_alloc);
        if (p == NULL) {
            PyErr_SetString(PyExc_MemoryError, "memory allocation failed");
            return -1;
        }
        s->s = p;
        s->allocated = to_alloc;
    }
    s->s[s->pos] = c;
    ++s->pos;
    return 0;
}

static int
_append_str(_tmp_string_t *s, char const *p)
{
    for (; *p != '\0'; p++) {
        if (_append_char(s, *p) < 0) {
            return -1;
        }
    }
    return 0;
}

/*
 * Append a PEP3118-formatted field name, ":name:", to str
 */
static int
_append_field_name(_tmp_string_t *str, PyObject *name)
{
    int ret = -1;
    char *p;
    Py_ssize_t len;
    PyObject *tmp;
    /* FIXME: XXX -- should it use UTF-8 here? */
    tmp = PyUnicode_AsUTF8String(name);
    if (tmp == NULL || PyBytes_AsStringAndSize(tmp, &p, &len) < 0) {
        PyErr_Clear();
        PyErr_SetString(PyExc_ValueError, "invalid field name");
        goto fail;
    }
    if (_append_char(str, ':') < 0) {
        goto fail;
    }
    while (len > 0) {
        if (*p == ':') {
            PyErr_SetString(PyExc_ValueError,
                            "':' is not an allowed character in buffer "
                            "field names");
            goto fail;
        }
        if (_append_char(str, *p) < 0) {
            goto fail;
        }
        ++p;
        --len;
    }
    if (_append_char(str, ':') < 0) {
        goto fail;
    }
    ret = 0;
fail:
    Py_XDECREF(tmp);
    return ret;
}

/*
 * Return non-zero if a type is aligned in each item in the given array,
 * AND, the descr element size is a multiple of the alignment,
 * AND, the array data is positioned to alignment granularity.
 */
static NPY_INLINE int
_is_natively_aligned_at(PyArray_Descr *descr,
                        PyArrayObject *arr, Py_ssize_t offset)
{
    int k;

    if (NPY_LIKELY(descr == PyArray_DESCR(arr))) {
        /*
         * If the descriptor is the arrays descriptor we can assume the
         * array's alignment is correct.
         */
        assert(offset == 0);
        if (PyArray_ISALIGNED(arr)) {
            assert(descr->elsize % descr->alignment == 0);
            return 1;
        }
        return 0;
    }

    if ((Py_ssize_t)(PyArray_DATA(arr)) % descr->alignment != 0) {
        return 0;
    }

    if (offset % descr->alignment != 0) {
        return 0;
    }

    if (descr->elsize % descr->alignment) {
        return 0;
    }

    for (k = 0; k < PyArray_NDIM(arr); ++k) {
        if (PyArray_DIM(arr, k) > 1) {
            if (PyArray_STRIDE(arr, k) % descr->alignment != 0) {
                return 0;
            }
        }
    }

    return 1;
}

/*
 * Fill in str with an appropriate PEP 3118 format string, based on
 * descr. For structured dtypes, calls itself recursively. Each call extends
 * str at offset then updates offset, and uses  descr->byteorder, (and
 * possibly the byte order in obj) to determine the byte-order char.
 *
 * Returns 0 for success, -1 for failure
 */
static int
_buffer_format_string(PyArray_Descr *descr, _tmp_string_t *str,
                      PyObject* obj, Py_ssize_t *offset,
                      char *active_byteorder)
{
    int k;
    char _active_byteorder = '@';
    Py_ssize_t _offset = 0;

    if (active_byteorder == NULL) {
        active_byteorder = &_active_byteorder;
    }
    if (offset == NULL) {
        offset = &_offset;
    }

    if (descr->subarray) {
        PyObject *item, *subarray_tuple;
        Py_ssize_t total_count = 1;
        Py_ssize_t dim_size;
        Py_ssize_t old_offset;
        char buf[128];
        int ret;

        if (PyTuple_Check(descr->subarray->shape)) {
            subarray_tuple = descr->subarray->shape;
            Py_INCREF(subarray_tuple);
        }
        else {
            subarray_tuple = Py_BuildValue("(O)", descr->subarray->shape);
        }

        if (_append_char(str, '(') < 0) {
            ret = -1;
            goto subarray_fail;
        }
        for (k = 0; k < PyTuple_GET_SIZE(subarray_tuple); ++k) {
            if (k > 0) {
                if (_append_char(str, ',') < 0) {
                    ret = -1;
                    goto subarray_fail;
                }
            }
            item = PyTuple_GET_ITEM(subarray_tuple, k);
            dim_size = PyNumber_AsSsize_t(item, NULL);

            PyOS_snprintf(buf, sizeof(buf), "%ld", (long)dim_size);
            if (_append_str(str, buf) < 0) {
                ret = -1;
                goto subarray_fail;
            }
            total_count *= dim_size;
        }
        if (_append_char(str, ')') < 0) {
            ret = -1;
            goto subarray_fail;
        }

        old_offset = *offset;
        ret = _buffer_format_string(descr->subarray->base, str, obj, offset,
                                    active_byteorder);
        *offset = old_offset + (*offset - old_offset) * total_count;

    subarray_fail:
        Py_DECREF(subarray_tuple);
        return ret;
    }
    else if (PyDataType_HASFIELDS(descr)) {
        Py_ssize_t base_offset = *offset;

        if (_append_str(str, "T{") < 0) return -1;
        for (k = 0; k < PyTuple_GET_SIZE(descr->names); ++k) {
            PyObject *name, *item, *offset_obj;
            PyArray_Descr *child;
            Py_ssize_t new_offset;
            int ret;

            name = PyTuple_GET_ITEM(descr->names, k);
            item = PyDict_GetItem(descr->fields, name);

            child = (PyArray_Descr*)PyTuple_GetItem(item, 0);
            offset_obj = PyTuple_GetItem(item, 1);
            new_offset = PyLong_AsLong(offset_obj);
            if (error_converting(new_offset)) {
                return -1;
            }
            new_offset += base_offset;

            /* Insert padding manually */
            if (*offset > new_offset) {
                PyErr_SetString(
                    PyExc_ValueError,
                    "dtypes with overlapping or out-of-order fields are not "
                    "representable as buffers. Consider reordering the fields."
                );
                return -1;
            }
            while (*offset < new_offset) {
                if (_append_char(str, 'x') < 0) return -1;
                ++*offset;
            }

            /* Insert child item */
            ret = _buffer_format_string(child, str, obj, offset,
                                  active_byteorder);
            if (ret < 0) {
                return -1;
            }

            /* Insert field name */
            if (_append_field_name(str, name) < 0) return -1;
        }
        if (_append_char(str, '}') < 0) return -1;
    }
    else {
        int is_standard_size = 1;
        int is_natively_aligned;
        int is_native_only_type = (descr->type_num == NPY_LONGDOUBLE ||
                                   descr->type_num == NPY_CLONGDOUBLE);
        if (sizeof(npy_longlong) != 8) {
            is_native_only_type = is_native_only_type || (
                descr->type_num == NPY_LONGLONG ||
                descr->type_num == NPY_ULONGLONG);
        }

        if (PyArray_IsScalar(obj, Generic)) {
            /* scalars are always natively aligned */
            is_natively_aligned = 1;
        }
        else {
            is_natively_aligned = _is_natively_aligned_at(descr,
                                              (PyArrayObject*)obj, *offset);
        }

        *offset += descr->elsize;

        if (descr->byteorder == '=' && is_natively_aligned) {
            /* Prefer native types, to cater for Cython */
            is_standard_size = 0;
            if (*active_byteorder != '@') {
                if (_append_char(str, '@') < 0) return -1;
                *active_byteorder = '@';
            }
        }
        else if (descr->byteorder == '=' && is_native_only_type) {
            /* Data types that have no standard size */
            is_standard_size = 0;
            if (*active_byteorder != '^') {
                if (_append_char(str, '^') < 0) return -1;
                *active_byteorder = '^';
            }
        }
        else if (descr->byteorder == '<' || descr->byteorder == '>' ||
                 descr->byteorder == '=') {
            is_standard_size = 1;
            if (*active_byteorder != descr->byteorder) {
                if (_append_char(str, descr->byteorder) < 0) return -1;
                *active_byteorder = descr->byteorder;
            }

            if (is_native_only_type) {
                /*
                 * It's not possible to express native-only data types
                 * in non-native npy_byte orders
                 */
                PyErr_Format(PyExc_ValueError,
                             "cannot expose native-only dtype '%c' in "
                             "non-native byte order '%c' via buffer interface",
                             descr->type, descr->byteorder);
                return -1;
            }
        }

        switch (descr->type_num) {
        case NPY_BOOL:         if (_append_char(str, '?') < 0) return -1; break;
        case NPY_BYTE:         if (_append_char(str, 'b') < 0) return -1; break;
        case NPY_UBYTE:        if (_append_char(str, 'B') < 0) return -1; break;
        case NPY_SHORT:        if (_append_char(str, 'h') < 0) return -1; break;
        case NPY_USHORT:       if (_append_char(str, 'H') < 0) return -1; break;
        case NPY_INT:          if (_append_char(str, 'i') < 0) return -1; break;
        case NPY_UINT:         if (_append_char(str, 'I') < 0) return -1; break;
        case NPY_LONG:
            if (is_standard_size && (NPY_SIZEOF_LONG == 8)) {
                if (_append_char(str, 'q') < 0) return -1;
            }
            else {
                if (_append_char(str, 'l') < 0) return -1;
            }
            break;
        case NPY_ULONG:
            if (is_standard_size && (NPY_SIZEOF_LONG == 8)) {
                if (_append_char(str, 'Q') < 0) return -1;
            }
            else {
                if (_append_char(str, 'L') < 0) return -1;
            }
            break;
        case NPY_LONGLONG:     if (_append_char(str, 'q') < 0) return -1; break;
        case NPY_ULONGLONG:    if (_append_char(str, 'Q') < 0) return -1; break;
        case NPY_HALF:         if (_append_char(str, 'e') < 0) return -1; break;
        case NPY_FLOAT:        if (_append_char(str, 'f') < 0) return -1; break;
        case NPY_DOUBLE:       if (_append_char(str, 'd') < 0) return -1; break;
        case NPY_LONGDOUBLE:   if (_append_char(str, 'g') < 0) return -1; break;
        case NPY_CFLOAT:       if (_append_str(str, "Zf") < 0) return -1; break;
        case NPY_CDOUBLE:      if (_append_str(str, "Zd") < 0) return -1; break;
        case NPY_CLONGDOUBLE:  if (_append_str(str, "Zg") < 0) return -1; break;
        /* XXX NPY_DATETIME */
        /* XXX NPY_TIMEDELTA */
        case NPY_OBJECT:       if (_append_char(str, 'O') < 0) return -1; break;
        case NPY_STRING: {
            char buf[128];
            PyOS_snprintf(buf, sizeof(buf), "%ds", descr->elsize);
            if (_append_str(str, buf) < 0) return -1;
            break;
        }
        case NPY_UNICODE: {
            /* NumPy Unicode is always 4-byte */
            char buf[128];
            assert(descr->elsize % 4 == 0);
            PyOS_snprintf(buf, sizeof(buf), "%dw", descr->elsize / 4);
            if (_append_str(str, buf) < 0) return -1;
            break;
        }
        case NPY_VOID: {
            /* Insert padding bytes */
            char buf[128];
            PyOS_snprintf(buf, sizeof(buf), "%dx", descr->elsize);
            if (_append_str(str, buf) < 0) return -1;
            break;
        }
        default:
            PyErr_Format(PyExc_ValueError,
                         "cannot include dtype '%c' in a buffer",
                         descr->type);
            return -1;
        }
    }

    return 0;
}


/*
 * Information about all active buffers is stored as a linked list on
 * the ndarray. The initial pointer is currently tagged to have a chance of
 * detecting incompatible subclasses.
 *
 * Note: because for backward compatibility we cannot define bf_releasebuffer,
 * we must manually keep track of the additional data required by the buffers.
 */

/* Additional per-array data required for providing the buffer interface */
typedef struct _buffer_info_t_tag {
    char *format;
    int ndim;
    Py_ssize_t *strides;
    Py_ssize_t *shape;
    struct _buffer_info_t_tag *next;
} _buffer_info_t;


/* Fill in the info structure */
static _buffer_info_t*
_buffer_info_new(PyObject *obj, int flags)
{
    /*
     * Note that the buffer info is cached as PyLongObjects making them appear
     * like unreachable lost memory to valgrind.
     */
    _buffer_info_t *info;
    _tmp_string_t fmt = {NULL, 0, 0};
    int k;
    PyArray_Descr *descr = NULL;
    int err = 0;

    if (PyArray_IsScalar(obj, Void)) {
        info = PyObject_Malloc(sizeof(_buffer_info_t));
        if (info == NULL) {
            PyErr_NoMemory();
            goto fail;
        }
        info->ndim = 0;
        info->shape = NULL;
        info->strides = NULL;

        descr = PyArray_DescrFromScalar(obj);
        if (descr == NULL) {
            goto fail;
        }
    }
    else {
        assert(PyArray_Check(obj));
        PyArrayObject * arr = (PyArrayObject *)obj;

        info = PyObject_Malloc(sizeof(_buffer_info_t) +
                               sizeof(Py_ssize_t) * PyArray_NDIM(arr) * 2);
        if (info == NULL) {
            PyErr_NoMemory();
            goto fail;
        }
        /* Fill in shape and strides */
        info->ndim = PyArray_NDIM(arr);

        if (info->ndim == 0) {
            info->shape = NULL;
            info->strides = NULL;
        }
        else {
            info->shape = (npy_intp *)((char *)info + sizeof(_buffer_info_t));
            assert((size_t)info->shape % sizeof(npy_intp) == 0);
            info->strides = info->shape + PyArray_NDIM(arr);

#if NPY_RELAXED_STRIDES_CHECKING
            /*
             * When NPY_RELAXED_STRIDES_CHECKING is used, some buffer users
             * may expect a contiguous buffer to have well formatted strides
             * also when a dimension is 1, but we do not guarantee this
             * internally. Thus, recalculate strides for contiguous arrays.
             * (This is unnecessary, but has no effect in the case where
             * NPY_RELAXED_STRIDES CHECKING is disabled.)
             */
            int f_contiguous = (flags & PyBUF_F_CONTIGUOUS) == PyBUF_F_CONTIGUOUS;
            if (PyArray_IS_C_CONTIGUOUS(arr) && !(
                    f_contiguous && PyArray_IS_F_CONTIGUOUS(arr))) {
                Py_ssize_t sd = PyArray_ITEMSIZE(arr);
                for (k = info->ndim-1; k >= 0; --k) {
                    info->shape[k] = PyArray_DIMS(arr)[k];
                    info->strides[k] = sd;
                    sd *= info->shape[k];
                }
            }
            else if (PyArray_IS_F_CONTIGUOUS(arr)) {
                Py_ssize_t sd = PyArray_ITEMSIZE(arr);
                for (k = 0; k < info->ndim; ++k) {
                    info->shape[k] = PyArray_DIMS(arr)[k];
                    info->strides[k] = sd;
                    sd *= info->shape[k];
                }
            }
            else {
#else  /* NPY_RELAXED_STRIDES_CHECKING */
            /* We can always use the arrays strides directly */
            {
#endif

                for (k = 0; k < PyArray_NDIM(arr); ++k) {
                    info->shape[k] = PyArray_DIMS(arr)[k];
                    info->strides[k] = PyArray_STRIDES(arr)[k];
                }
            }
        }
        descr = PyArray_DESCR(arr);
        Py_INCREF(descr);
    }

    /* Fill in format */
    if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT) {
        err = _buffer_format_string(descr, &fmt, obj, NULL, NULL);
        Py_DECREF(descr);
        if (err != 0) {
            goto fail;
        }
        if (_append_char(&fmt, '\0') < 0) {
            goto fail;
        }
        info->format = fmt.s;
    }
    else {
        Py_DECREF(descr);
        info->format = NULL;
    }
    info->next = NULL;
    return info;

fail:
    PyObject_Free(fmt.s);
    PyObject_Free(info);
    return NULL;
}

/* Compare two info structures */
static Py_ssize_t
_buffer_info_cmp(_buffer_info_t *a, _buffer_info_t *b)
{
    Py_ssize_t c;
    int k;

    if (a->format != NULL && b->format != NULL) {
        c = strcmp(a->format, b->format);
        if (c != 0) return c;
    }
    c = a->ndim - b->ndim;
    if (c != 0) return c;

    for (k = 0; k < a->ndim; ++k) {
        c = a->shape[k] - b->shape[k];
        if (c != 0) return c;
        c = a->strides[k] - b->strides[k];
        if (c != 0) return c;
    }

    return 0;
}


/*
 * Tag the buffer info pointer by adding 2 (unless it is NULL to simplify
 * object initialization).
 * The linked list of buffer-infos was appended to the array struct in
 * NumPy 1.20. Tagging the pointer gives us a chance to raise/print
 * a useful error message instead of crashing hard if a C-subclass uses
 * the same field.
 */
static NPY_INLINE void *
buffer_info_tag(void *buffer_info)
{
    if (buffer_info == NULL) {
        return buffer_info;
    }
    else {
        return (void *)((uintptr_t)buffer_info + 3);
    }
}


static NPY_INLINE int
_buffer_info_untag(
        void *tagged_buffer_info, _buffer_info_t **buffer_info, PyObject *obj)
{
    if (tagged_buffer_info == NULL) {
        *buffer_info = NULL;
        return 0;
    }
    if (NPY_UNLIKELY(((uintptr_t)tagged_buffer_info & 0x7) != 3)) {
        PyErr_Format(PyExc_RuntimeError,
                "Object of type %S appears to be C subclassed NumPy array, "
                "void scalar, or allocated in a non-standard way."
                "NumPy reserves the right to change the size of these "
                "structures. Projects are required to take this into account "
                "by either recompiling against a specific NumPy version or "
                "padding the struct and enforcing a maximum NumPy version.",
                Py_TYPE(obj));
        return -1;
    }
    *buffer_info = (void *)((uintptr_t)tagged_buffer_info - 3);
    return 0;
}


/*
 * NOTE: for backward compatibility (esp. with PyArg_ParseTuple("s#", ...))
 * we do *not* define bf_releasebuffer at all.
 *
 * Instead, any extra data allocated with the buffer is released only in
 * array_dealloc.
 *
 * Ensuring that the buffer stays in place is taken care by refcounting;
 * ndarrays do not reallocate if there are references to them, and a buffer
 * view holds one reference.
 *
 * This is stored in the array's _buffer_info slot (currently as a void *).
 */
static void
_buffer_info_free_untagged(void *_buffer_info)
{
    _buffer_info_t *next = _buffer_info;
    while (next != NULL) {
        _buffer_info_t *curr = next;
        next = curr->next;
        if (curr->format) {
            PyObject_Free(curr->format);
        }
        /* Shape is allocated as part of info */
        PyObject_Free(curr);
    }
}


/*
 * Checks whether the pointer is tagged, and then frees the cache list.
 * (The tag check is only for transition due to changed structure size in 1.20)
 */
NPY_NO_EXPORT int
_buffer_info_free(void *buffer_info, PyObject *obj)
{
    _buffer_info_t *untagged_buffer_info;
    if (_buffer_info_untag(buffer_info, &untagged_buffer_info, obj) < 0) {
        return -1;
    }
    _buffer_info_free_untagged(untagged_buffer_info);
    return 0;
}


/*
 * Get the buffer info returning either the old one (passed in) or a new
 * buffer info which adds holds on to (and thus replaces) the old one.
 */
static _buffer_info_t*
_buffer_get_info(void **buffer_info_cache_ptr, PyObject *obj, int flags)
{
    _buffer_info_t *info = NULL;
    _buffer_info_t *stored_info;  /* First currently stored buffer info */

    if (_buffer_info_untag(*buffer_info_cache_ptr, &stored_info, obj) < 0) {
        return NULL;
    }
    _buffer_info_t *old_info = stored_info;

    /* Compute information (it would be nice to skip this in simple cases) */
    info = _buffer_info_new(obj, flags);
    if (info == NULL) {
        return NULL;
    }

    if (old_info != NULL && _buffer_info_cmp(info, old_info) != 0) {
        _buffer_info_t *next_info = old_info->next;
        old_info = NULL;  /* Can't use this one, but possibly next */

         if (info->ndim > 1 && next_info != NULL) {
             /*
              * Some arrays are C- and F-contiguous and if they have more
              * than one dimension, the buffer-info may differ between
              * the two due to RELAXED_STRIDES_CHECKING.
              * If we export both buffers, the first stored one may be
              * the one for the other contiguity, so check both.
              * This is generally very unlikely in all other cases, since
              * in all other cases the first one will match unless array
              * metadata was modified in-place (which is discouraged).
              */
             if (_buffer_info_cmp(info, next_info) == 0) {
                 old_info = next_info;
             }
         }
    }
    if (old_info != NULL) {
        /*
         * The two info->format are considered equal if one of them
         * has no format set (meaning the format is arbitrary and can
         * be modified). If the new info has a format, but we reuse
         * the old one, this transfers the ownership to the old one.
         */
        if (old_info->format == NULL) {
            old_info->format = info->format;
            info->format = NULL;
        }
        _buffer_info_free_untagged(info);
        info = old_info;
    }
    else {
        /* Insert new info as first item in the linked buffer-info list. */
        info->next = stored_info;
        *buffer_info_cache_ptr = buffer_info_tag(info);
    }

    return info;
}


/*
 * Retrieving buffers for ndarray
 */
static int
array_getbuffer(PyObject *obj, Py_buffer *view, int flags)
{
    PyArrayObject *self;
    _buffer_info_t *info = NULL;

    self = (PyArrayObject*)obj;

    /* Check whether we can provide the wanted properties */
    if ((flags & PyBUF_C_CONTIGUOUS) == PyBUF_C_CONTIGUOUS &&
            !PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)) {
        PyErr_SetString(PyExc_ValueError, "ndarray is not C-contiguous");
        goto fail;
    }
    if ((flags & PyBUF_F_CONTIGUOUS) == PyBUF_F_CONTIGUOUS &&
            !PyArray_CHKFLAGS(self, NPY_ARRAY_F_CONTIGUOUS)) {
        PyErr_SetString(PyExc_ValueError, "ndarray is not Fortran contiguous");
        goto fail;
    }
    if ((flags & PyBUF_ANY_CONTIGUOUS) == PyBUF_ANY_CONTIGUOUS
            && !PyArray_ISONESEGMENT(self)) {
        PyErr_SetString(PyExc_ValueError, "ndarray is not contiguous");
        goto fail;
    }
    if ((flags & PyBUF_STRIDES) != PyBUF_STRIDES &&
            !PyArray_CHKFLAGS(self, NPY_ARRAY_C_CONTIGUOUS)) {
        /* Non-strided N-dim buffers must be C-contiguous */
        PyErr_SetString(PyExc_ValueError, "ndarray is not C-contiguous");
        goto fail;
    }
    if ((flags & PyBUF_WRITEABLE) == PyBUF_WRITEABLE) {
        if (PyArray_FailUnlessWriteable(self, "buffer source array") < 0) {
            goto fail;
        }
    }

    if (view == NULL) {
        PyErr_SetString(PyExc_ValueError, "NULL view in getbuffer");
        goto fail;
    }

    /* Fill in information (and add it to _buffer_info if necessary) */
    info = _buffer_get_info(
            &((PyArrayObject_fields *)self)->_buffer_info, obj, flags);
    if (info == NULL) {
        goto fail;
    }

    view->buf = PyArray_DATA(self);
    view->suboffsets = NULL;
    view->itemsize = PyArray_ITEMSIZE(self);
    /*
     * If a read-only buffer is requested on a read-write array, we return a
     * read-write buffer as per buffer protocol.
     * We set a requested buffer to readonly also if the array will be readonly
     * after a deprecation. This jumps the deprecation, but avoiding the
     * warning is not convenient here. A warning is given if a writeable
     * buffer is requested since `PyArray_FailUnlessWriteable` is called above
     * (and clears the `NPY_ARRAY_WARN_ON_WRITE` flag).
     */
    view->readonly = (!PyArray_ISWRITEABLE(self) ||
                      PyArray_CHKFLAGS(self, NPY_ARRAY_WARN_ON_WRITE));
    view->internal = NULL;
    view->len = PyArray_NBYTES(self);
    if ((flags & PyBUF_FORMAT) == PyBUF_FORMAT) {
        view->format = info->format;
    } else {
        view->format = NULL;
    }
    if ((flags & PyBUF_ND) == PyBUF_ND) {
        view->ndim = info->ndim;
        view->shape = info->shape;
    }
    else {
        view->ndim = 0;
        view->shape = NULL;
    }
    if ((flags & PyBUF_STRIDES) == PyBUF_STRIDES) {
        view->strides = info->strides;
    }
    else {
        view->strides = NULL;
    }
    view->obj = (PyObject*)self;

    Py_INCREF(self);
    return 0;

fail:
    return -1;
}

/*
 * Retrieving buffers for void scalar (which can contain any complex types),
 * defined in buffer.c since it requires the complex format building logic.
 */
NPY_NO_EXPORT int
void_getbuffer(PyObject *self, Py_buffer *view, int flags)
{
    PyVoidScalarObject *scalar = (PyVoidScalarObject *)self;

    if (flags & PyBUF_WRITABLE) {
        PyErr_SetString(PyExc_BufferError, "scalar buffer is readonly");
        return -1;
    }

    view->ndim = 0;
    view->shape = NULL;
    view->strides = NULL;
    view->suboffsets = NULL;
    view->len = scalar->descr->elsize;
    view->itemsize = scalar->descr->elsize;
    view->readonly = 1;
    view->suboffsets = NULL;
    Py_INCREF(self);
    view->obj = self;
    view->buf = scalar->obval;

    if (((flags & PyBUF_FORMAT) != PyBUF_FORMAT)) {
        /* It is unnecessary to find the correct format */
        view->format = NULL;
        return 0;
    }

    /*
     * If a format is being exported, we need to use _buffer_get_info
     * to find the correct format.  This format must also be stored, since
     * at least in theory it can change (in practice it should never change).
     */
    _buffer_info_t *info = _buffer_get_info(&scalar->_buffer_info, self, flags);
    if (info == NULL) {
        Py_DECREF(self);
        return -1;
    }
    view->format = info->format;
    return 0;
}


/*************************************************************************/

NPY_NO_EXPORT PyBufferProcs array_as_buffer = {
    (getbufferproc)array_getbuffer,
    (releasebufferproc)0,
};


/*************************************************************************
 * Convert PEP 3118 format string to PyArray_Descr
 */

static int
_descriptor_from_pep3118_format_fast(char const *s, PyObject **result);

static int
_pep3118_letter_to_type(char letter, int native, int complex);

NPY_NO_EXPORT PyArray_Descr*
_descriptor_from_pep3118_format(char const *s)
{
    char *buf, *p;
    int in_name = 0;
    int obtained;
    PyObject *descr;
    PyObject *str;
    PyObject *_numpy_internal;

    if (s == NULL) {
        return PyArray_DescrNewFromType(NPY_BYTE);
    }

    /* Fast path */
    obtained = _descriptor_from_pep3118_format_fast(s, &descr);
    if (obtained) {
        return (PyArray_Descr*)descr;
    }

    /* Strip whitespace, except from field names */
    buf = malloc(strlen(s) + 1);
    if (buf == NULL) {
        PyErr_NoMemory();
        return NULL;
    }
    p = buf;
    while (*s != '\0') {
        if (*s == ':') {
            in_name = !in_name;
            *p = *s;
            p++;
        }
        else if (in_name || !NumPyOS_ascii_isspace(*s)) {
            *p = *s;
            p++;
        }
        s++;
    }
    *p = '\0';

    str = PyUnicode_FromStringAndSize(buf, strlen(buf));
    if (str == NULL) {
        free(buf);
        return NULL;
    }

    /* Convert */
    _numpy_internal = PyImport_ImportModule("numpy.core._internal");
    if (_numpy_internal == NULL) {
        Py_DECREF(str);
        free(buf);
        return NULL;
    }
    descr = PyObject_CallMethod(
        _numpy_internal, "_dtype_from_pep3118", "O", str);
    Py_DECREF(str);
    Py_DECREF(_numpy_internal);
    if (descr == NULL) {
        PyObject *exc, *val, *tb;
        PyErr_Fetch(&exc, &val, &tb);
        PyErr_Format(PyExc_ValueError,
                     "'%s' is not a valid PEP 3118 buffer format string", buf);
        npy_PyErr_ChainExceptionsCause(exc, val, tb);
        free(buf);
        return NULL;
    }
    if (!PyArray_DescrCheck(descr)) {
        PyErr_Format(PyExc_RuntimeError,
                     "internal error: numpy.core._internal._dtype_from_pep3118 "
                     "did not return a valid dtype, got %s", buf);
        Py_DECREF(descr);
        free(buf);
        return NULL;
    }
    free(buf);
    return (PyArray_Descr*)descr;
}

/*
 * Fast path for parsing buffer strings corresponding to simple types.
 *
 * Currently, this deals only with single-element data types.
 */

static int
_descriptor_from_pep3118_format_fast(char const *s, PyObject **result)
{
    PyArray_Descr *descr;

    int is_standard_size = 0;
    char byte_order = '=';
    int is_complex = 0;

    int type_num = NPY_BYTE;
    int item_seen = 0;

    for (; *s != '\0'; ++s) {
        is_complex = 0;
        switch (*s) {
        case '@':
        case '^':
            /* ^ means no alignment; doesn't matter for a single element */
            byte_order = '=';
            is_standard_size = 0;
            break;
        case '<':
            byte_order = '<';
            is_standard_size = 1;
            break;
        case '>':
        case '!':
            byte_order = '>';
            is_standard_size = 1;
            break;
        case '=':
            byte_order = '=';
            is_standard_size = 1;
            break;
        case 'Z':
            is_complex = 1;
            ++s;
        default:
            if (item_seen) {
                /* Not a single-element data type */
                return 0;
            }
            type_num = _pep3118_letter_to_type(*s, !is_standard_size,
                                               is_complex);
            if (type_num < 0) {
                /* Something unknown */
                return 0;
            }
            item_seen = 1;
            break;
        }
    }

    if (!item_seen) {
        return 0;
    }

    descr = PyArray_DescrFromType(type_num);
    if (descr == NULL) {
        return 0;
    }
    if (byte_order == '=') {
        *result = (PyObject*)descr;
    }
    else {
        *result = (PyObject*)PyArray_DescrNewByteorder(descr, byte_order);
        Py_DECREF(descr);
        if (result == NULL) {
            return 0;
        }
    }

    return 1;
}

static int
_pep3118_letter_to_type(char letter, int native, int complex)
{
    switch (letter)
    {
    case '?': return NPY_BOOL;
    case 'b': return NPY_BYTE;
    case 'B': return NPY_UBYTE;
    case 'h': return native ? NPY_SHORT : NPY_INT16;
    case 'H': return native ? NPY_USHORT : NPY_UINT16;
    case 'i': return native ? NPY_INT : NPY_INT32;
    case 'I': return native ? NPY_UINT : NPY_UINT32;
    case 'l': return native ? NPY_LONG : NPY_INT32;
    case 'L': return native ? NPY_ULONG : NPY_UINT32;
    case 'q': return native ? NPY_LONGLONG : NPY_INT64;
    case 'Q': return native ? NPY_ULONGLONG : NPY_UINT64;
    case 'e': return NPY_HALF;
    case 'f': return complex ? NPY_CFLOAT : NPY_FLOAT;
    case 'd': return complex ? NPY_CDOUBLE : NPY_DOUBLE;
    case 'g': return native ? (complex ? NPY_CLONGDOUBLE : NPY_LONGDOUBLE) : -1;
    default:
        /* Other unhandled cases */
        return -1;
    }
    return -1;
}