Function count_nonzero_u8

numpy/core/src/multiarray/item_selection.c:2456–2490

NPY_SIMD * Counts the number of non-zero values in a raw array. * The one loop process is shown below(take SSE2 with 128bits vector for example): * |------------16 lanes---------| *[vsum8] 255 255 255 ... 255 255 255 255 count_zero_bytes_u8: counting 255*16 elements * !! * |------------8 lanes---------| *[vsum16] 65535 65535 65535 ... 65535

 *                     (2*16-1)*16
*/
static inline NPY_GCC_OPT_3 npy_intp
count_nonzero_u8(const char *data, npy_intp bstride, npy_uintp len)
{
    npy_intp count = 0;
    if (bstride == 1) {
    #if NPY_SIMD
        npy_uintp len_m = len & -npyv_nlanes_u8;
        npy_uintp zcount = 0;
        for (const char *end = data + len_m; data < end;) {
            npyv_u16x2 vsum16 = count_zero_bytes_u16((const npy_uint8**)&data, (const npy_uint8*)end, NPY_MAX_UINT16);
            npyv_u32x2 sum_32_0 = npyv_expand_u32_u16(vsum16.val[0]);
            npyv_u32x2 sum_32_1 = npyv_expand_u32_u16(vsum16.val[1]);
            zcount += npyv_sum_u32(npyv_add_u32(
                    npyv_add_u32(sum_32_0.val[0], sum_32_0.val[1]),
                    npyv_add_u32(sum_32_1.val[0], sum_32_1.val[1])
            ));
        }
        len  -= len_m;
        count = len_m - zcount;
    #else
        if (!NPY_ALIGNMENT_REQUIRED || npy_is_aligned(data, sizeof(npy_uint64))) {
            int step = 6 * sizeof(npy_uint64);
            int left_bytes = len % step;
            for (const char *end = data + len; data < end - left_bytes; data += step) {
                 count += count_nonzero_bytes_384((const npy_uint64 *)data);
            }
            len = left_bytes;
        }
    #endif // NPY_SIMD
    }
    for (; len > 0; --len, data += bstride) {
        count += (*data != 0);
    }
    return count;
}

static inline NPY_GCC_OPT_3 npy_intp
count_nonzero_u16(const char *data, npy_intp bstride, npy_uintp len)