Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

MAINT Consistent cython types continued #25810

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Mar 20, 2023
Merged

MAINT Consistent cython types continued #25810

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c29b078
cln types
jeremiedbb Mar 10, 2023
a8ee529
Merge remote-tracking branch 'upstream/main' into pr/jeremiedbb/25810
jeremiedbb Mar 15, 2023
74a1baf
argkmin classmode as well
jeremiedbb Mar 15, 2023
fb95ed4
cln
jeremiedbb Mar 15, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions 10 sklearn/feature_extraction/_hashing_fast.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ from libcpp.vector cimport vector

cimport numpy as cnp
import numpy as np
from ..utils._typedefs cimport INT32TYPE_t, INT64TYPE_t
from ..utils._typedefs cimport int32_t, int64_t
from ..utils.murmurhash cimport murmurhash3_bytes_s32
from ..utils._vector_sentinel cimport vector_to_nd_array

Expand All @@ -24,17 +24,17 @@ def transform(raw_X, Py_ssize_t n_features, dtype,
For constructing a scipy.sparse.csr_matrix.

"""
cdef INT32TYPE_t h
cdef int32_t h
cdef double value

cdef vector[INT32TYPE_t] indices
cdef vector[INT64TYPE_t] indptr
cdef vector[int32_t] indices
cdef vector[int64_t] indptr
indptr.push_back(0)

# Since Python array does not understand Numpy dtypes, we grow the indices
# and values arrays ourselves. Use a Py_ssize_t capacity for safety.
cdef Py_ssize_t capacity = 8192 # arbitrary
cdef cnp.int64_t size = 0
cdef int64_t size = 0
cdef cnp.ndarray values = np.empty(capacity, dtype=dtype)

for x in raw_X:
Expand Down
19 changes: 8 additions & 11 deletions 19 sklearn/metrics/_pairwise_distances_reduction/_argkmin.pxd.tp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
cimport numpy as cnp
from ...utils._typedefs cimport ITYPE_t, DTYPE_t

cnp.import_array()
from ...utils._typedefs cimport intp_t, float64_t

{{for name_suffix in ['64', '32']}}

Expand All @@ -12,22 +9,22 @@ cdef class ArgKmin{{name_suffix}}(BaseDistancesReduction{{name_suffix}}):
"""float{{name_suffix}} implementation of the ArgKmin."""

cdef:
ITYPE_t k
intp_t k

ITYPE_t[:, ::1] argkmin_indices
DTYPE_t[:, ::1] argkmin_distances
intp_t[:, ::1] argkmin_indices
float64_t[:, ::1] argkmin_distances

# Used as array of pointers to private datastructures used in threads.
DTYPE_t ** heaps_r_distances_chunks
ITYPE_t ** heaps_indices_chunks
float64_t ** heaps_r_distances_chunks
intp_t ** heaps_indices_chunks


cdef class EuclideanArgKmin{{name_suffix}}(ArgKmin{{name_suffix}}):
"""EuclideanDistance-specialisation of ArgKmin{{name_suffix}}."""
cdef:
MiddleTermComputer{{name_suffix}} middle_term_computer
const DTYPE_t[::1] X_norm_squared
const DTYPE_t[::1] Y_norm_squared
const float64_t[::1] X_norm_squared
const float64_t[::1] Y_norm_squared

bint use_squared_distances

Expand Down
150 changes: 72 additions & 78 deletions 150 sklearn/metrics/_pairwise_distances_reduction/_argkmin.pyx.tp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,11 @@
cimport numpy as cnp

from libc.stdlib cimport free, malloc
from libc.float cimport DBL_MAX
from cython cimport final
from cython.parallel cimport parallel, prange

from ...utils._heap cimport heap_push
from ...utils._sorting cimport simultaneous_sort
from ...utils._typedefs cimport ITYPE_t, DTYPE_t
from ...utils._typedefs cimport intp_t, float64_t

import numpy as np
import warnings
Expand All @@ -16,10 +14,6 @@ from numbers import Integral
from scipy.sparse import issparse
from ...utils import check_array, check_scalar, _in_unstable_openblas_configuration
from ...utils.fixes import threadpool_limits
from ...utils._typedefs import ITYPE, DTYPE


cnp.import_array()

{{for name_suffix in ['64', '32']}}

Expand All @@ -41,7 +35,7 @@ cdef class ArgKmin{{name_suffix}}(BaseDistancesReduction{{name_suffix}}):
cls,
X,
Y,
ITYPE_t k,
intp_t k,
str metric="euclidean",
chunk_size=None,
dict metric_kwargs=None,
Expand Down Expand Up @@ -104,7 +98,7 @@ cdef class ArgKmin{{name_suffix}}(BaseDistancesReduction{{name_suffix}}):
DatasetsPair{{name_suffix}} datasets_pair,
chunk_size=None,
strategy=None,
ITYPE_t k=1,
intp_t k=1,
):
super().__init__(
datasets_pair=datasets_pair,
Expand All @@ -122,16 +116,16 @@ cdef class ArgKmin{{name_suffix}}(BaseDistancesReduction{{name_suffix}}):
# - when parallelizing on Y, the pointers of those heaps are referencing
# small heaps which are thread-wise-allocated and whose content will be
# merged with the main heaps'.
self.heaps_r_distances_chunks = <DTYPE_t **> malloc(
sizeof(DTYPE_t *) * self.chunks_n_threads
self.heaps_r_distances_chunks = <float64_t **> malloc(
sizeof(float64_t *) * self.chunks_n_threads
)
self.heaps_indices_chunks = <ITYPE_t **> malloc(
sizeof(ITYPE_t *) * self.chunks_n_threads
self.heaps_indices_chunks = <intp_t **> malloc(
sizeof(intp_t *) * self.chunks_n_threads
)

# Main heaps which will be returned as results by `ArgKmin{{name_suffix}}.compute`.
self.argkmin_indices = np.full((self.n_samples_X, self.k), 0, dtype=ITYPE)
self.argkmin_distances = np.full((self.n_samples_X, self.k), DBL_MAX, dtype=DTYPE)
self.argkmin_indices = np.full((self.n_samples_X, self.k), 0, dtype=np.intp)
self.argkmin_distances = np.full((self.n_samples_X, self.k), DBL_MAX, dtype=np.float64)

def __dealloc__(self):
if self.heaps_indices_chunks is not NULL:
Expand All @@ -142,18 +136,18 @@ cdef class ArgKmin{{name_suffix}}(BaseDistancesReduction{{name_suffix}}):

cdef void _compute_and_reduce_distances_on_chunks(
self,
ITYPE_t X_start,
ITYPE_t X_end,
ITYPE_t Y_start,
ITYPE_t Y_end,
ITYPE_t thread_num,
intp_t X_start,
intp_t X_end,
intp_t Y_start,
intp_t Y_end,
intp_t thread_num,
) noexcept nogil:
cdef:
ITYPE_t i, j
ITYPE_t n_samples_X = X_end - X_start
ITYPE_t n_samples_Y = Y_end - Y_start
DTYPE_t *heaps_r_distances = self.heaps_r_distances_chunks[thread_num]
ITYPE_t *heaps_indices = self.heaps_indices_chunks[thread_num]
intp_t i, j
intp_t n_samples_X = X_end - X_start
intp_t n_samples_Y = Y_end - Y_start
float64_t *heaps_r_distances = self.heaps_r_distances_chunks[thread_num]
intp_t *heaps_indices = self.heaps_indices_chunks[thread_num]

# Pushing the distances and their associated indices on a heap
# which by construction will keep track of the argkmin.
Expand All @@ -169,9 +163,9 @@ cdef class ArgKmin{{name_suffix}}(BaseDistancesReduction{{name_suffix}}):

cdef void _parallel_on_X_init_chunk(
self,
ITYPE_t thread_num,
ITYPE_t X_start,
ITYPE_t X_end,
intp_t thread_num,
intp_t X_start,
intp_t X_end,
) noexcept nogil:
# As this strategy is embarrassingly parallel, we can set each
# thread's heaps pointer to the proper position on the main heaps.
Expand All @@ -180,12 +174,12 @@ cdef class ArgKmin{{name_suffix}}(BaseDistancesReduction{{name_suffix}}):

cdef void _parallel_on_X_prange_iter_finalize(
self,
ITYPE_t thread_num,
ITYPE_t X_start,
ITYPE_t X_end,
intp_t thread_num,
intp_t X_start,
intp_t X_end,
) noexcept nogil:
cdef:
ITYPE_t idx
intp_t idx

# Sorting the main heaps portion associated to `X[X_start:X_end]`
# in ascending order w.r.t the distances.
Expand All @@ -201,8 +195,8 @@ cdef class ArgKmin{{name_suffix}}(BaseDistancesReduction{{name_suffix}}):
) noexcept nogil:
cdef:
# Maximum number of scalar elements (the last chunks can be smaller)
ITYPE_t heaps_size = self.X_n_samples_chunk * self.k
ITYPE_t thread_num
intp_t heaps_size = self.X_n_samples_chunk * self.k
intp_t thread_num

# The allocation is done in parallel for data locality purposes: this way
# the heaps used in each threads are allocated in pages which are closer
Expand All @@ -214,18 +208,18 @@ cdef class ArgKmin{{name_suffix}}(BaseDistancesReduction{{name_suffix}}):
# As chunks of X are shared across threads, so must their
# heaps. To solve this, each thread has its own heaps
# which are then synchronised back in the main ones.
self.heaps_r_distances_chunks[thread_num] = <DTYPE_t *> malloc(
heaps_size * sizeof(DTYPE_t)
self.heaps_r_distances_chunks[thread_num] = <float64_t *> malloc(
heaps_size * sizeof(float64_t)
)
self.heaps_indices_chunks[thread_num] = <ITYPE_t *> malloc(
heaps_size * sizeof(ITYPE_t)
self.heaps_indices_chunks[thread_num] = <intp_t *> malloc(
heaps_size * sizeof(intp_t)
)

cdef void _parallel_on_Y_parallel_init(
self,
ITYPE_t thread_num,
ITYPE_t X_start,
ITYPE_t X_end,
intp_t thread_num,
intp_t X_start,
intp_t X_end,
) noexcept nogil:
# Initialising heaps (memset can't be used here)
for idx in range(self.X_n_samples_chunk * self.k):
Expand All @@ -235,11 +229,11 @@ cdef class ArgKmin{{name_suffix}}(BaseDistancesReduction{{name_suffix}}):
@final
cdef void _parallel_on_Y_synchronize(
self,
ITYPE_t X_start,
ITYPE_t X_end,
intp_t X_start,
intp_t X_end,
) noexcept nogil:
cdef:
ITYPE_t idx, jdx, thread_num
intp_t idx, jdx, thread_num
with nogil, parallel(num_threads=self.effective_n_threads):
# Synchronising the thread heaps with the main heaps.
# This is done in parallel sample-wise (no need for locks).
Expand All @@ -263,7 +257,7 @@ cdef class ArgKmin{{name_suffix}}(BaseDistancesReduction{{name_suffix}}):
self,
) noexcept nogil:
cdef:
ITYPE_t idx, thread_num
intp_t idx, thread_num

with nogil, parallel(num_threads=self.chunks_n_threads):
# Deallocating temporary datastructures
Expand All @@ -283,8 +277,8 @@ cdef class ArgKmin{{name_suffix}}(BaseDistancesReduction{{name_suffix}}):

cdef void compute_exact_distances(self) noexcept nogil:
cdef:
ITYPE_t i, j
DTYPE_t[:, ::1] distances = self.argkmin_distances
intp_t i, j
float64_t[:, ::1] distances = self.argkmin_distances
for i in prange(self.n_samples_X, schedule='static', nogil=True,
num_threads=self.effective_n_threads):
for j in range(self.k):
Expand Down Expand Up @@ -319,7 +313,7 @@ cdef class EuclideanArgKmin{{name_suffix}}(ArgKmin{{name_suffix}}):
self,
X,
Y,
ITYPE_t k,
intp_t k,
bint use_squared_distances=False,
chunk_size=None,
strategy=None,
Expand All @@ -344,7 +338,7 @@ cdef class EuclideanArgKmin{{name_suffix}}(ArgKmin{{name_suffix}}):
k=k,
)
cdef:
ITYPE_t dist_middle_terms_chunks_size = self.Y_n_samples_chunk * self.X_n_samples_chunk
intp_t dist_middle_terms_chunks_size = self.Y_n_samples_chunk * self.X_n_samples_chunk

self.middle_term_computer = MiddleTermComputer{{name_suffix}}.get_for(
X,
Expand Down Expand Up @@ -396,29 +390,29 @@ cdef class EuclideanArgKmin{{name_suffix}}(ArgKmin{{name_suffix}}):
@final
cdef void _parallel_on_X_parallel_init(
self,
ITYPE_t thread_num,
intp_t thread_num,
) noexcept nogil:
ArgKmin{{name_suffix}}._parallel_on_X_parallel_init(self, thread_num)
self.middle_term_computer._parallel_on_X_parallel_init(thread_num)

@final
cdef void _parallel_on_X_init_chunk(
self,
ITYPE_t thread_num,
ITYPE_t X_start,
ITYPE_t X_end,
intp_t thread_num,
intp_t X_start,
intp_t X_end,
) noexcept nogil:
ArgKmin{{name_suffix}}._parallel_on_X_init_chunk(self, thread_num, X_start, X_end)
self.middle_term_computer._parallel_on_X_init_chunk(thread_num, X_start, X_end)

@final
cdef void _parallel_on_X_pre_compute_and_reduce_distances_on_chunks(
self,
ITYPE_t X_start,
ITYPE_t X_end,
ITYPE_t Y_start,
ITYPE_t Y_end,
ITYPE_t thread_num,
intp_t X_start,
intp_t X_end,
intp_t Y_start,
intp_t Y_end,
intp_t thread_num,
) noexcept nogil:
ArgKmin{{name_suffix}}._parallel_on_X_pre_compute_and_reduce_distances_on_chunks(
self,
Expand All @@ -440,21 +434,21 @@ cdef class EuclideanArgKmin{{name_suffix}}(ArgKmin{{name_suffix}}):
@final
cdef void _parallel_on_Y_parallel_init(
self,
ITYPE_t thread_num,
ITYPE_t X_start,
ITYPE_t X_end,
intp_t thread_num,
intp_t X_start,
intp_t X_end,
) noexcept nogil:
ArgKmin{{name_suffix}}._parallel_on_Y_parallel_init(self, thread_num, X_start, X_end)
self.middle_term_computer._parallel_on_Y_parallel_init(thread_num, X_start, X_end)

@final
cdef void _parallel_on_Y_pre_compute_and_reduce_distances_on_chunks(
self,
ITYPE_t X_start,
ITYPE_t X_end,
ITYPE_t Y_start,
ITYPE_t Y_end,
ITYPE_t thread_num,
intp_t X_start,
intp_t X_end,
intp_t Y_start,
intp_t Y_end,
intp_t thread_num,
) noexcept nogil:
ArgKmin{{name_suffix}}._parallel_on_Y_pre_compute_and_reduce_distances_on_chunks(
self,
Expand All @@ -469,22 +463,22 @@ cdef class EuclideanArgKmin{{name_suffix}}(ArgKmin{{name_suffix}}):
@final
cdef void _compute_and_reduce_distances_on_chunks(
self,
ITYPE_t X_start,
ITYPE_t X_end,
ITYPE_t Y_start,
ITYPE_t Y_end,
ITYPE_t thread_num,
intp_t X_start,
intp_t X_end,
intp_t Y_start,
intp_t Y_end,
intp_t thread_num,
) noexcept nogil:
cdef:
ITYPE_t i, j
DTYPE_t sqeuclidean_dist_i_j
ITYPE_t n_X = X_end - X_start
ITYPE_t n_Y = Y_end - Y_start
DTYPE_t * dist_middle_terms = self.middle_term_computer._compute_dist_middle_terms(
intp_t i, j
float64_t sqeuclidean_dist_i_j
intp_t n_X = X_end - X_start
intp_t n_Y = Y_end - Y_start
float64_t * dist_middle_terms = self.middle_term_computer._compute_dist_middle_terms(
X_start, X_end, Y_start, Y_end, thread_num
)
DTYPE_t * heaps_r_distances = self.heaps_r_distances_chunks[thread_num]
ITYPE_t * heaps_indices = self.heaps_indices_chunks[thread_num]
float64_t * heaps_r_distances = self.heaps_r_distances_chunks[thread_num]
intp_t * heaps_indices = self.heaps_indices_chunks[thread_num]

# Pushing the distance and their associated indices on heaps
# which keep tracks of the argkmin.
Expand Down
Loading
Morty Proxy This is a proxified and sanitized view of the page, visit original site.