Add simpler BPE, and make previous BPE better (#870) #449
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Code tests Linux | |
| on: | |
| push: | |
| branches: [ main ] | |
| paths: | |
| - '**/*.py' | |
| - '**/*.ipynb' | |
| - '**/*.yaml' | |
| - '**/*.yml' | |
| - '**/*.sh' | |
| pull_request: | |
| branches: [ main ] | |
| paths: | |
| - '**/*.py' | |
| - '**/*.ipynb' | |
| - '**/*.yaml' | |
| - '**/*.yml' | |
| - '**/*.sh' | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| uv-tests: | |
| name: Code tests (Linux) | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python (uv) | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.13" | |
| - name: Install uv and dependencies | |
| shell: bash | |
| run: | | |
| curl -LsSf https://astral.sh/uv/install.sh | sh | |
| uv sync --dev # tests for backwards compatibility | |
| uv pip install -r ch05/07_gpt_to_llama/tests/test-requirements-extra.txt | |
| uv add pytest-ruff nbval | |
| - name: Test Selected Python Scripts (uv) | |
| shell: bash | |
| run: | | |
| source .venv/bin/activate | |
| pytest setup/02_installing-python-libraries/tests.py | |
| pytest ch04/01_main-chapter-code/tests.py | |
| pytest ch04/03_kv-cache/tests.py | |
| pytest ch05/01_main-chapter-code/tests.py | |
| pytest ch05/07_gpt_to_llama/tests/tests_rope_and_parts.py | |
| pytest ch05/07_gpt_to_llama/tests/test_llama32_nb.py | |
| pytest ch05/11_qwen3/tests/test_qwen3_nb.py | |
| pytest ch05/12_gemma3/tests/test_gemma3_nb.py | |
| pytest ch05/12_gemma3/tests/test_gemma3_kv_nb.py | |
| pytest ch06/01_main-chapter-code/tests.py | |
| - name: Validate Selected Jupyter Notebooks (uv) | |
| shell: bash | |
| run: | | |
| source .venv/bin/activate | |
| pytest --nbval ch02/01_main-chapter-code/dataloader.ipynb | |
| pytest --nbval ch03/01_main-chapter-code/multihead-attention.ipynb | |
| pytest --nbval ch02/04_bonus_dataloader-intuition/dataloader-intuition.ipynb | |
| - name: Test Selected Bonus Materials | |
| shell: bash | |
| run: | | |
| source .venv/bin/activate | |
| pytest ch02/05_bpe-from-scratch/tests.py | |
| - name: Test Selected Bonus Materials | |
| shell: bash | |
| run: | | |
| source .venv/bin/activate | |
| uv pip install transformers | |
| pytest pkg/llms_from_scratch/tests/ |