Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

Appearance settings

Commit eae4495

Browse filesBrowse files
Joan FontanalsJohannesMessner
andauthored
fix: default column config should be DBConfig and not RuntimeConfig (#1648)
Signed-off-by: Joan Fontanals Martinez <joan.martinez@jina.ai> Signed-off-by: Johannes Messner <messnerjo@gmail.com> Co-authored-by: Johannes Messner <messnerjo@gmail.com>
1 parent 67a328f commit eae4495
Copy full SHA for eae4495

14 files changed

+127-107Lines changed: 127 additions & 107 deletions

File tree

Expand file treeCollapse file tree
Open diff view settings
Filter options
Expand file treeCollapse file tree
Open diff view settings
Collapse file

‎docarray/index/abstract.py‎

Copy file name to clipboardExpand all lines: docarray/index/abstract.py
+6-5Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -145,9 +145,6 @@ def build(self, *args, **kwargs) -> Any:
145145
@dataclass
146146
class DBConfig(ABC):
147147
index_name: Optional[str] = None
148-
149-
@dataclass
150-
class RuntimeConfig(ABC):
151148
# default configurations for every column type
152149
# a dictionary from a column type (DB specific) to a dictionary
153150
# of default configurations for that type
@@ -156,6 +153,10 @@ class RuntimeConfig(ABC):
156153
# Example: `default_column_config['VARCHAR'] = {'length': 255}`
157154
default_column_config: Dict[Type, Dict[str, Any]] = field(default_factory=dict)
158155

156+
@dataclass
157+
class RuntimeConfig(ABC):
158+
pass
159+
159160
@property
160161
def index_name(self):
161162
"""Return the name of the index in the database."""
@@ -896,14 +897,14 @@ def _create_single_column(self, field: 'ModelField', type_: Type) -> _ColumnInfo
896897
if 'col_type' in custom_config.keys():
897898
db_type = custom_config['col_type']
898899
custom_config.pop('col_type')
899-
if db_type not in self._runtime_config.default_column_config.keys():
900+
if db_type not in self._db_config.default_column_config.keys():
900901
raise ValueError(
901902
f'The given col_type is not a valid db type: {db_type}'
902903
)
903904
else:
904905
db_type = self.python_type_to_db_type(type_)
905906

906-
config = self._runtime_config.default_column_config[db_type].copy()
907+
config = self._db_config.default_column_config[db_type].copy()
907908
config.update(custom_config)
908909
# parse n_dim from parametrized tensor type
909910
if (
Collapse file

‎docarray/index/backends/elastic.py‎

Copy file name to clipboardExpand all lines: docarray/index/backends/elastic.py
+7-9Lines changed: 7 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,6 @@
3939

4040
ELASTIC_PY_VEC_TYPES: List[Any] = [list, tuple, np.ndarray, AbstractTensor]
4141

42-
4342
if TYPE_CHECKING:
4443
import tensorflow as tf # type: ignore
4544
import torch
@@ -57,7 +56,6 @@
5756
torch = import_library('torch', raise_error=False)
5857
tf = import_library('tensorflow', raise_error=False)
5958

60-
6159
if torch is not None:
6260
ELASTIC_PY_VEC_TYPES.append(torch.Tensor)
6361

@@ -254,13 +252,7 @@ class DBConfig(BaseDocIndex.DBConfig):
254252
es_config: Dict[str, Any] = field(default_factory=dict)
255253
index_settings: Dict[str, Any] = field(default_factory=dict)
256254
index_mappings: Dict[str, Any] = field(default_factory=dict)
257-
258-
@dataclass
259-
class RuntimeConfig(BaseDocIndex.RuntimeConfig):
260-
"""Dataclass that contains all "dynamic" configurations of ElasticDocIndex."""
261-
262255
default_column_config: Dict[Any, Dict[str, Any]] = field(default_factory=dict)
263-
chunk_size: int = 500
264256

265257
def __post_init__(self):
266258
self.default_column_config = {
@@ -323,6 +315,12 @@ def dense_vector_config(self):
323315

324316
return config
325317

318+
@dataclass
319+
class RuntimeConfig(BaseDocIndex.RuntimeConfig):
320+
"""Dataclass that contains all "dynamic" configurations of ElasticDocIndex."""
321+
322+
chunk_size: int = 500
323+
326324
###############################################
327325
# Implementation of abstract methods #
328326
###############################################
@@ -624,7 +622,7 @@ def _form_search_body(
624622
num_candidates: Optional[int] = None,
625623
) -> Dict[str, Any]:
626624
if not num_candidates:
627-
num_candidates = self._runtime_config.default_column_config['dense_vector'][
625+
num_candidates = self._db_config.default_column_config['dense_vector'][
628626
'num_candidates'
629627
]
630628
body = {
Collapse file

‎docarray/index/backends/elasticv7.py‎

Copy file name to clipboardExpand all lines: docarray/index/backends/elasticv7.py
+4-2Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -90,12 +90,14 @@ class DBConfig(ElasticDocIndex.DBConfig):
9090

9191
hosts: Union[str, List[str], None] = 'http://localhost:9200' # type: ignore
9292

93+
def dense_vector_config(self):
94+
return {'dims': 128}
95+
9396
@dataclass
9497
class RuntimeConfig(ElasticDocIndex.RuntimeConfig):
9598
"""Dataclass that contains all "dynamic" configurations of ElasticDocIndex."""
9699

97-
def dense_vector_config(self):
98-
return {'dims': 128}
100+
pass
99101

100102
###############################################
101103
# Implementation of abstract methods #
Collapse file

‎docarray/index/backends/hnswlib.py‎

Copy file name to clipboardExpand all lines: docarray/index/backends/hnswlib.py
+6-5Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -165,11 +165,6 @@ class DBConfig(BaseDocIndex.DBConfig):
165165
"""Dataclass that contains all "static" configurations of HnswDocumentIndex."""
166166

167167
work_dir: str = '.'
168-
169-
@dataclass
170-
class RuntimeConfig(BaseDocIndex.RuntimeConfig):
171-
"""Dataclass that contains all "dynamic" configurations of HnswDocumentIndex."""
172-
173168
default_column_config: Dict[Type, Dict[str, Any]] = field(
174169
default_factory=lambda: {
175170
np.ndarray: {
@@ -188,6 +183,12 @@ class RuntimeConfig(BaseDocIndex.RuntimeConfig):
188183
}
189184
)
190185

186+
@dataclass
187+
class RuntimeConfig(BaseDocIndex.RuntimeConfig):
188+
"""Dataclass that contains all "dynamic" configurations of HnswDocumentIndex."""
189+
190+
pass
191+
191192
###############################################
192193
# Implementation of abstract methods #
193194
###############################################
Collapse file

‎docarray/index/backends/in_memory.py‎

Copy file name to clipboardExpand all lines: docarray/index/backends/in_memory.py
+6-6Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -137,12 +137,6 @@ def build(self, *args, **kwargs) -> Any:
137137
class DBConfig(BaseDocIndex.DBConfig):
138138
"""Dataclass that contains all "static" configurations of InMemoryExactNNIndex."""
139139

140-
pass
141-
142-
@dataclass
143-
class RuntimeConfig(BaseDocIndex.RuntimeConfig):
144-
"""Dataclass that contains all "dynamic" configurations of InMemoryExactNNIndex."""
145-
146140
default_column_config: Dict[Type, Dict[str, Any]] = field(
147141
default_factory=lambda: defaultdict(
148142
dict,
@@ -152,6 +146,12 @@ class RuntimeConfig(BaseDocIndex.RuntimeConfig):
152146
)
153147
)
154148

149+
@dataclass
150+
class RuntimeConfig(BaseDocIndex.RuntimeConfig):
151+
"""Dataclass that contains all "dynamic" configurations of InMemoryExactNNIndex."""
152+
153+
pass
154+
155155
def index(self, docs: Union[BaseDoc, Sequence[BaseDoc]], **kwargs):
156156
"""index Documents into the index.
157157
Collapse file

‎docarray/index/backends/qdrant.py‎

Copy file name to clipboardExpand all lines: docarray/index/backends/qdrant.py
+6-5Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -241,11 +241,6 @@ class DBConfig(BaseDocIndex.DBConfig):
241241
optimizers_config: Optional[types.OptimizersConfigDiff] = None
242242
wal_config: Optional[types.WalConfigDiff] = None
243243
quantization_config: Optional[types.QuantizationConfig] = None
244-
245-
@dataclass
246-
class RuntimeConfig(BaseDocIndex.RuntimeConfig):
247-
"""Dataclass that contains all "dynamic" configurations of QdrantDocumentIndex."""
248-
249244
default_column_config: Dict[Type, Dict[str, Any]] = field(
250245
default_factory=lambda: {
251246
'id': {}, # type: ignore[dict-item]
@@ -254,6 +249,12 @@ class RuntimeConfig(BaseDocIndex.RuntimeConfig):
254249
}
255250
)
256251

252+
@dataclass
253+
class RuntimeConfig(BaseDocIndex.RuntimeConfig):
254+
"""Dataclass that contains all "dynamic" configurations of QdrantDocumentIndex."""
255+
256+
pass
257+
257258
def python_type_to_db_type(self, python_type: Type) -> Any:
258259
"""Map python type to database type.
259260
Takes any python type and returns the corresponding database column type.
Collapse file

‎docarray/index/backends/weaviate.py‎

Copy file name to clipboardExpand all lines: docarray/index/backends/weaviate.py
+4-5Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -245,11 +245,6 @@ class DBConfig(BaseDocIndex.DBConfig):
245245
scopes: List[str] = field(default_factory=lambda: ["offline_access"])
246246
auth_api_key: Optional[str] = None
247247
embedded_options: Optional[EmbeddedOptions] = None
248-
249-
@dataclass
250-
class RuntimeConfig(BaseDocIndex.RuntimeConfig):
251-
"""Dataclass that contains all "dynamic" configurations of WeaviateDocumentIndex."""
252-
253248
default_column_config: Dict[Any, Dict[str, Any]] = field(
254249
default_factory=lambda: {
255250
np.ndarray: {},
@@ -264,6 +259,10 @@ class RuntimeConfig(BaseDocIndex.RuntimeConfig):
264259
}
265260
)
266261

262+
@dataclass
263+
class RuntimeConfig(BaseDocIndex.RuntimeConfig):
264+
"""Dataclass that contains all "dynamic" configurations of WeaviateDocumentIndex."""
265+
267266
batch_config: Dict[str, Any] = field(
268267
default_factory=lambda: DEFAULT_BATCH_CONFIG
269268
)
Collapse file

‎docs/how_to/add_doc_index.md‎

Copy file name to clipboardExpand all lines: docs/how_to/add_doc_index.md
+13-12Lines changed: 13 additions & 12 deletions
  • Display the source diff
  • Display the rich diff
Original file line numberDiff line numberDiff line change
@@ -288,12 +288,12 @@ To define what can be stored in them, and what the default values are, you need
288288
```python
289289
@dataclass
290290
class DBConfig(BaseDocIndex.DBConfig):
291-
...
291+
default_column_config: Dict[Type, Dict[str, Any]] = ...
292292

293293

294294
@dataclass
295295
class RuntimeConfig(BaseDocIndex.RuntimeConfig):
296-
default_column_config: Dict[Type, Dict[str, Any]] = ...
296+
...
297297
```
298298

299299
!!! note
@@ -306,16 +306,8 @@ The `DBConfig` class defines the static configurations of your Document Index.
306306
These are configurations that are tied to the database (or library) running in the background, such as `host`, `port`, etc.
307307
Here you should put everything that the user cannot or should not change after initialization.
308308

309-
### The `RuntimeConfig` class
310-
311-
The `RuntimeConfig` class defines the dynamic configurations of your Document Index.
312-
These are configurations that can be changed at runtime, for example default behaviours such as batch sizes, consistency levels, etc.
313-
314-
It is a common pattern to allow such parameters both in the `RuntimeConfig`, where they will act as global defaults, and
315-
in specific methods (`index`, `find`, etc.), where they will act as local overrides.
316-
317309
!!! note
318-
Every `RuntimeConfig` needs to contain a `default_column_config` field.
310+
Every `DBConfig` needs to contain a `default_column_config` field.
319311
This is a dictionary that, for each possible column type in your database, defines a default configuration for that column type.
320312
This will automatically be passed to a `_ColumnInfo` whenever a user does not manually specify a configuration for that column.
321313

@@ -327,6 +319,15 @@ and for `varchar` columns you could define a `max_length` configuration.
327319

328320
It is probably best to see this in action, so you should check out the `HnswDocumentIndex` implementation.
329321

322+
### The `RuntimeConfig` class
323+
324+
The `RuntimeConfig` class defines the dynamic configurations of your Document Index.
325+
These are configurations that can be changed at runtime, for example default behaviours such as batch sizes, consistency levels, etc.
326+
327+
It is a common pattern to allow such parameters both in the `RuntimeConfig`, where they will act as global defaults, and
328+
in specific methods (`index`, `find`, etc.), where they will act as local overrides.
329+
330+
330331
## Implement abstract methods for indexing, searching, and deleting
331332

332333
After you've done the basic setup above, you can jump into the good stuff: implementing the actual indexing, searching, and deleting.
@@ -374,7 +375,7 @@ class MySchema(BaseDoc):
374375

375376
In this case, the `db_type` of `my_num` will be `'float64'` and the `db_type` of `my_text` will be `'varchar'`.
376377
Additional information regarding the `col_type`, such as `max_len` for `varchar` will be stored in the `_ColumnsInfo.config`.
377-
The given `col_type` has to be a valid `db_type`, meaning that has to be described in the index's `RuntimeConfig.default_column_config`.
378+
The given `col_type` has to be a valid `db_type`, meaning that has to be described in the index's `DBConfig.default_column_config`.
378379

379380
### The `_index()` method
380381

Collapse file

‎docs/user_guide/storing/docindex.md‎

Copy file name to clipboardExpand all lines: docs/user_guide/storing/docindex.md
+32-7Lines changed: 32 additions & 7 deletions
  • Display the source diff
  • Display the rich diff
Original file line numberDiff line numberDiff line change
@@ -445,26 +445,25 @@ You can customize every field in this configuration:
445445

446446
### Runtime configurations
447447

448-
_Runtime configurations_ are configurations that pertain to the entire database or table (as opposed to just a specific column),
449-
and that you can dynamically change at runtime.
448+
_Runtime configurations_ are configurations that relate to the way how an `instance` operates with respect to a specific
449+
database.
450450

451451

452452
This commonly includes:
453453
- default batch size for batching operations
454-
- default mapping from pythong types to database column types
455454
- default consistency level for various database operations
456455
- ...
457456

458457
For every backend, you can get the full list of configurations and their defaults:
459458

460459
```python
461-
from docarray.index import HnswDocumentIndex
460+
from docarray.index import ElasticDocIndex
462461

463462

464-
runtime_config = HnswDocumentIndex.RuntimeConfig()
463+
runtime_config = ElasticDocIndex.RuntimeConfig()
465464
print(runtime_config)
466465

467-
# > HnswDocumentIndex.RuntimeConfig(default_column_config={<class 'numpy.ndarray'>: {'dim': -1, 'index': True, 'space': 'l2', 'max_elements': 1024, 'ef_construction': 200, 'ef': 10, 'M': 16, 'allow_replace_deleted': True, 'num_threads': 1}, None: {}})
466+
# > ElasticDocIndex.RuntimeConfig(chunk_size=500)
468467
```
469468

470469
As you can see, `HnswDocumentIndex.RuntimeConfig` is a dataclass that contains only one configuration:
@@ -559,7 +558,33 @@ The `HnswDocumentIndex` above contains two columns which are configured differen
559558
- `tens` has a dimensionality of `100`, can take up to `12` elements, and uses the `cosine` similarity space
560559
- `tens_two` has a dimensionality of `10`, and uses the `ip` similarity space, and an `M` hyperparameter of 4
561560

562-
All configurations that are not explicitly set will be taken from the `default_column_config` of the `RuntimeConfig`.
561+
All configurations that are not explicitly set will be taken from the `default_column_config` of the `DBConfig`.
562+
You can modify these defaults in the following way:
563+
564+
```python
565+
import numpy as np
566+
from pydantic import Field
567+
568+
from docarray import BaseDoc
569+
from docarray.index import HnswDocumentIndex
570+
from docarray.typing import NdArray
571+
572+
573+
class Schema(BaseDoc):
574+
tens: NdArray[100] = Field(max_elements=12, space='cosine')
575+
tens_two: NdArray[10] = Field(M=4, space='ip')
576+
577+
578+
# create a DBConfig for your Document Index
579+
conf = HnswDocumentIndex.DBConfig(work_dir='/tmp/my_db')
580+
# update the default max_elements for np.ndarray columns
581+
conf.default_column_config.get(np.ndarray).update(max_elements=2048)
582+
# create Document Index
583+
# tens has a max_elements of 12, specified in the schema
584+
# tens_two has a max_elements of 2048, specified by the default in the DBConfig
585+
db = HnswDocumentIndex[Schema](conf)
586+
```
587+
563588

564589
For an explanation of the configurations that are tweaked in this example, see the `HnswDocumentIndex` [documentation](index_hnswlib.md).
565590

Collapse file

‎docs/user_guide/storing/index_elastic.md‎

Copy file name to clipboardExpand all lines: docs/user_guide/storing/index_elastic.md
+10-9Lines changed: 10 additions & 9 deletions
  • Display the source diff
  • Display the rich diff
Original file line numberDiff line numberDiff line change
@@ -491,19 +491,11 @@ The following configs can be set in `DBConfig`:
491491
| `index_name` | Elasticsearch index name, the name of Elasticsearch index object | None. Data will be stored in an index named after the Document type used as schema. |
492492
| `index_settings` | Other [index settings](https://www.elastic.co/guide/en/elasticsearch/reference/8.6/index-modules.html#index-modules-settings) in a Dict for creating the index | dict |
493493
| `index_mappings` | Other [index mappings](https://www.elastic.co/guide/en/elasticsearch/reference/8.6/mapping.html) in a Dict for creating the index | dict |
494+
| `default_column_config` | The default configurations for every column type. | dict |
494495

495496
You can pass any of the above as keyword arguments to the `__init__()` method or pass an entire configuration object.
496497
See [here](docindex.md#configuration-options#customize-configurations) for more information.
497498

498-
### RuntimeConfig
499-
500-
The `RuntimeConfig` dataclass of `ElasticDocIndex` consists of `default_column_config` and `chunk_size`. You can change `chunk_size` for batch operations:
501-
502-
```python
503-
doc_index = ElasticDocIndex[SimpleDoc]()
504-
doc_index.configure(ElasticDocIndex.RuntimeConfig(chunk_size=1000))
505-
```
506-
507499
`default_column_config` is the default configurations for every column type. Since there are many column types in Elasticsearch, you can also consider changing the column config when defining the schema.
508500

509501
```python
@@ -514,5 +506,14 @@ class SimpleDoc(BaseDoc):
514506
doc_index = ElasticDocIndex[SimpleDoc]()
515507
```
516508

509+
### RuntimeConfig
510+
511+
The `RuntimeConfig` dataclass of `ElasticDocIndex` consists of `chunk_size`. You can change `chunk_size` for batch operations:
512+
513+
```python
514+
doc_index = ElasticDocIndex[SimpleDoc]()
515+
doc_index.configure(ElasticDocIndex.RuntimeConfig(chunk_size=1000))
516+
```
517+
517518
You can pass the above as keyword arguments to the `configure()` method or pass an entire configuration object.
518519
See [here](docindex.md#configuration-options#customize-configurations) for more information.

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.