Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit a2e0fe6

Browse filesBrowse files
guenthermiguenthermi
authored
refactor: change r precision calculation and documentation (#621)
1 parent 5c2915a commit a2e0fe6
Copy full SHA for a2e0fe6

3 files changed

+177-18Lines changed: 177 additions & 18 deletions

File tree

Expand file treeCollapse file tree
Open diff view settings
Filter options
Expand file treeCollapse file tree
Open diff view settings
Collapse file

‎docarray/math/evaluation.py‎

Copy file name to clipboardExpand all lines: docarray/math/evaluation.py
+18-7Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -12,32 +12,40 @@ def _check_k(k):
1212

1313

1414
def r_precision(binary_relevance: List[int], **kwargs) -> float:
15-
"""R Precision after all relevant documents have been retrieved
16-
Relevance is binary (nonzero is relevant).
15+
"""R-Precision determines the precision in the fist R documents, where R is the
16+
number of documents relevant to the query.
17+
18+
Relevance is considered binary by this function (nonzero is relevant).
19+
20+
Please note, that it is necessary to provide relevance scores for all documents,
21+
i.e., the calculated metric is wrong, if you apply it on the Top-K scores only.
1722
1823
.. seealso::
1924
https://en.wikipedia.org/wiki/Evaluation_measures_(information_retrieval)#R-precision
2025
2126
:param binary_relevance: binary relevancy in rank order
22-
:return: precision
27+
:return: R-Precision
2328
"""
2429
binary_relevance = np.array(binary_relevance) != 0
2530
z = binary_relevance.nonzero()[0]
2631
if not z.size:
2732
return 0.0
28-
return float(np.mean(binary_relevance[: z[-1] + 1]))
33+
return float(np.mean(binary_relevance[: z.size]))
2934

3035

3136
def precision_at_k(
3237
binary_relevance: List[int], k: Optional[int] = None, **kwargs
3338
) -> float:
3439
"""Precision @K.
40+
If `binary_relevance` is empty, 0.0 is returned.
3541
3642
:param binary_relevance: binary relevancy in rank order
3743
:param k: measured on top-k
3844
:return: precision @k
3945
"""
4046
_check_k(k)
47+
if len(binary_relevance) == 0:
48+
return 0.0
4149
binary_relevance = np.array(binary_relevance)[:k] != 0
4250
return float(np.mean(binary_relevance))
4351

@@ -147,9 +155,12 @@ def dcg_at_k(
147155
def ndcg_at_k(
148156
relevance: List[float], method: int = 0, k: Optional[int] = None, **kwargs
149157
):
150-
"""Score is normalized discounted cumulative gain (ndcg)
151-
Relevance is positive real values. Can use binary
152-
as the previous methods.
158+
"""Calculates a normalized discounted cumulative gain (ndcg).
159+
Relevance values can be positive real values. However, one can also use binary
160+
scores as in other evaluation methods.
161+
162+
Please note, that it is necessary to provide relevance scores for all documents,
163+
i.e., the calculated metric is wrong, if you apply it on the Top-K scores only.
153164
154165
Example from
155166
http://www.stanford.edu/class/cs276/handouts/EvaluationNew-handout-6-per.pdf
Collapse file

‎docs/fundamentals/documentarray/evaluation.md‎

Copy file name to clipboardExpand all lines: docs/fundamentals/documentarray/evaluation.md
+17-11Lines changed: 17 additions & 11 deletions
  • Display the source diff
  • Display the rich diff
Original file line numberDiff line numberDiff line change
@@ -10,17 +10,23 @@ The results are stored in `.evaluations` field of each Document.
1010

1111
DocArray provides some common metrics used in the information retrieval community that allows one to evaluate the nearest-neighbour matches. Different metric accepts different arguments as `kwargs`:
1212

13-
| Metric | Accept `kwargs` |
14-
|---------------------|------------------|
15-
| `r_precision` | None |
16-
| `average_precision` | None |
17-
| `reciprocal_rank` | None |
18-
| `precision_at_k` | `k` |
19-
| `hit_at_k` | `k` |
20-
| `recall_at_k` | `max_rel`, `k` |
21-
| `f1_score_at_k` | `max_rel`, `k` |
22-
| `dcg_at_k` | `method`, `k` |
23-
| `ndcg_at_k` | `method`, `k` |
13+
| Metric | Accept `kwargs` |
14+
|-----------------------------------------------------|------------------|
15+
| {meth}`~docarray.math.evaluation.r_precision` | None |
16+
| {meth}`~docarray.math.evaluation.average_precision` | None |
17+
| {meth}`~docarray.math.evaluation.reciprocal_rank` | None |
18+
| {meth}`~docarray.math.evaluation.precision_at_k` | `k` |
19+
| {meth}`~docarray.math.evaluation.hit_at_k` | `k` |
20+
| {meth}`~docarray.math.evaluation.recall_at_k` | `max_rel`, `k` |
21+
| {meth}`~docarray.math.evaluation.f1_score_at_k` | `max_rel`, `k` |
22+
| {meth}`~docarray.math.evaluation.dcg_at_k` | `method`, `k` |
23+
| {meth}`~docarray.math.evaluation.ndcg_at_k` | `method`, `k` |
24+
25+
```{danger}
26+
This metric scores might change if the `limit` attribute of the match function is set differently.
27+
28+
**Note:** Not all of these metrics can be applied to a Top-K result, i.e., `ndcg_at_k` and `r_precision` are calculated correctly only if the limit is set equal or higher than the number of documents in the `DocumentArray` provided to the match function.
29+
```
2430

2531

2632
For example, let's create a DocumentArray with random embeddings and matching it to itself:
Collapse file

‎tests/unit/math/test_evaluation_metrics.py‎

Copy file name to clipboard
+142Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
import pytest
2+
3+
from docarray.math.evaluation import (
4+
average_precision,
5+
dcg_at_k,
6+
f1_score_at_k,
7+
hit_at_k,
8+
ndcg_at_k,
9+
precision_at_k,
10+
r_precision,
11+
recall_at_k,
12+
reciprocal_rank,
13+
)
14+
15+
16+
@pytest.mark.parametrize(
17+
"binary_relevance, score",
18+
[
19+
([0, 1, 0, 0, 1, 1, 1], 0.25),
20+
([], 0),
21+
([1, 1, 1], 1),
22+
([0, 0], 0),
23+
],
24+
)
25+
def test_r_precision(binary_relevance, score):
26+
assert abs(r_precision(binary_relevance) - score) < 0.001
27+
28+
29+
@pytest.mark.parametrize(
30+
"binary_relevance, score, k",
31+
[
32+
([0, 1, 0, 0, 1, 1, 1], 4.0 / 7, None),
33+
([0, 1, 0, 0, 1, 1, 1], 0.5, 2),
34+
([], 0, None),
35+
([1, 1, 1], 1, None),
36+
([0, 0], 0, None),
37+
],
38+
)
39+
def test_precision_at_k(binary_relevance, score, k):
40+
assert abs(precision_at_k(binary_relevance, k=k) - score) < 0.001
41+
42+
43+
@pytest.mark.parametrize(
44+
"binary_relevance, score, k",
45+
[
46+
([0, 1, 0, 0, 1, 1, 1], 1, None),
47+
([0, 1, 0, 0, 1, 1, 1], 0, 1),
48+
([], 0, None),
49+
([1, 1, 1], 1, None),
50+
([0, 0], 0, None),
51+
],
52+
)
53+
def test_hit_at_k(binary_relevance, score, k):
54+
assert abs(hit_at_k(binary_relevance, k=k) - score) < 0.001
55+
56+
57+
@pytest.mark.parametrize(
58+
"binary_relevance, score",
59+
[
60+
([0, 1, 0, 0, 1, 1, 1], (1.0 / 2 + 2.0 / 5 + 3.0 / 6 + 4.0 / 7) / 4),
61+
([], 0),
62+
([1, 1, 1], 1),
63+
([0, 0], 0),
64+
],
65+
)
66+
def test_average_precision(binary_relevance, score):
67+
assert abs(average_precision(binary_relevance) - score) < 0.001
68+
69+
70+
@pytest.mark.parametrize(
71+
"binary_relevance, score",
72+
[
73+
([0, 1, 0, 0, 1, 1, 1], 0.5),
74+
([], 0),
75+
([1, 1, 1], 1.0),
76+
([0, 0], 0),
77+
],
78+
)
79+
def test_reciprocal_rank(binary_relevance, score):
80+
assert abs(reciprocal_rank(binary_relevance) - score) < 0.001
81+
82+
83+
@pytest.mark.parametrize(
84+
"binary_relevance, score, max_rel, k",
85+
[
86+
([0, 1, 0, 0, 1, 1, 1], 4.0 / 7, 7, None),
87+
([0, 1, 0, 0, 1, 1, 1], 1, 4, None),
88+
([0, 1, 0, 0, 1, 1, 1], 0.25, 4, 2),
89+
([], 0, 4, None),
90+
([1, 1, 1], 0.75, 4, None),
91+
([0, 0], 0, 4, None),
92+
],
93+
)
94+
def test_recall_at_k(binary_relevance, score, max_rel, k):
95+
calculated_score = recall_at_k(binary_relevance, max_rel=max_rel, k=k)
96+
assert abs(calculated_score - score) < 0.001
97+
98+
99+
@pytest.mark.parametrize(
100+
"binary_relevance, score, max_rel, k",
101+
[
102+
([0, 1, 0, 0, 1, 1, 1], 4.0 / 7, 7, None),
103+
([0, 1, 0, 0, 1, 1, 1], 2 / (1 / (4 / 7) + 1), 4, None),
104+
([0, 1, 0, 0, 1, 1, 1], 2 / (1 / 0.5 + 1 / 0.25), 4, 2),
105+
([], 0, 4, None),
106+
([1, 1, 1], 2 / (1 / 0.75 + 1), 4, None),
107+
([0, 0], 0, 4, None),
108+
],
109+
)
110+
def test_f1_score_at_k(binary_relevance, score, max_rel, k):
111+
calculated_score = f1_score_at_k(binary_relevance, max_rel=max_rel, k=k)
112+
assert abs(calculated_score - score) < 0.001
113+
114+
115+
@pytest.mark.parametrize(
116+
"binary_relevance, score, method, k",
117+
[
118+
([0, 1, 0, 0, 1, 1, 1], 2.1737, 0, None),
119+
([0, 1, 0, 0, 1, 1, 1], 1.7073, 1, None),
120+
([0, 1, 0, 0, 1, 1, 1], 1, 0, 4),
121+
([], 0, 0, None),
122+
([1, 1, 1], 2.6309, 0, None),
123+
([0, 0], 0, 0, None),
124+
],
125+
)
126+
def test_dcg_at_k(binary_relevance, score, method, k):
127+
assert abs(dcg_at_k(binary_relevance, method=method, k=k) - score) < 0.001
128+
129+
130+
@pytest.mark.parametrize(
131+
"binary_relevance, score, method, k",
132+
[
133+
([0, 1, 0, 0, 1, 1, 1], 0.6942, 0, None),
134+
([0, 1, 0, 0, 1, 1, 1], 0.6665, 1, None),
135+
([0, 1, 0, 0, 1, 1, 1], 0.3194, 0, 4),
136+
([], 0, 0, None),
137+
([1, 1, 1], 1, 0, None),
138+
([0, 0], 0, 0, None),
139+
],
140+
)
141+
def test_ndcg_at_k(binary_relevance, score, method, k):
142+
assert abs(ndcg_at_k(binary_relevance, method=method, k=k) - score) < 0.001

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.