Showcase: yaml config management with secret env var placeholders #672
Description
There isn't a discussion board, so I thought to share it in an issue since it's a pretty nice :). With the setup below, you can actually create yamls with this structure with env var placeholders using pydantic basesettings:
mcp_servers:
- endpoint: "https://hello_world"
api_key: ${EXAMPLE_KEY}And the ${} are environment variables evaluated at runtime, when initializing the config.
You have to create a custom YamlConfigSettingSource, that does that:
import os
from functools import lru_cache
from pathlib import Path
from string import Template
from typing import Any
from pydantic import BaseModel, Field, SecretStr
from pydantic_settings import (
BaseSettings,
PydanticBaseSettingsSource,
SettingsConfigDict,
YamlConfigSettingsSource,
)
class EnvParsedYamlConfigSettingsSource(YamlConfigSettingsSource):
"""Yaml Settings Source that parses envVar references before loading.
ENV VARS need to be configured like this: ${ENV_VAR}
An example of a valid config.yaml would be:
```yaml
config:
api_key: ${OPENAI_KEY}
```
"""
def _read_file(self, file_path: Path) -> dict[str, Any]:
import json
data = super()._read_file(file_path)
data = json.dumps(data)
data = Template(data).substitute(os.environ)
return json.loads(data)Now we can proceed with creating an example Config:
class MCPServer(BaseModel):
"""MCP Server connection"""
endpoint: str
api_key: SecretStr
class MCPServerConfiguration(BaseSettings):
"""MCP server configuration"""
servers: list[MCPServer] | None = Field(alias="mcp_servers", default=None)
model_config = SettingsConfigDict(
extra="ignore",
yaml_file=os.environ.get(
"CONFIG_FILE_PATH",
Path(__file__).parent.parent / "config.yaml",
),
)
@classmethod
def settings_customise_sources( # noqa: D102
cls,
settings_cls: type[BaseSettings],
init_settings: PydanticBaseSettingsSource, # noqa: ARG003
env_settings: PydanticBaseSettingsSource, # noqa: ARG003
dotenv_settings: PydanticBaseSettingsSource, # noqa: ARG003
file_secret_settings: PydanticBaseSettingsSource, # noqa: ARG003
) -> tuple[PydanticBaseSettingsSource, ...]:
return (EnvParsedYamlConfigSettingsSource(settings_cls),)
class Config(BaseSettings):
"""Application settings"""
mcp: MCPServerConfiguration = MCPServerConfiguration() # type: ignore
# API Configuration
api_host: str = Field(default="0.0.0.0", description="API host")
api_port: int = Field(default=8000, description="API port")
class Config:
"""Config class for .env values"""
env_file = ".env"
env_file_encoding = "utf-8"
case_sensitive = True
extra = "ignore"
@lru_cache(1)
def get_config() -> Config:
"""Lazy static to get the config for the project"""
config = Config() # type: ignore[call-arg]`
return config # type: ignoreSo let's bring this together and run:
print(config)
print(config.mcp.servers[0].api_key.get_secret_value())export EXAMPLE_KEY='12345678' && python test.py, which will print out the following:
mcp=MCPServerConfiguration(servers=[MCPServer(endpoint='https://hello_world', api_key=SecretStr('**********'))]) api_host='0.0.0.0' api_port=8000
12345678