From e698ee408f1278d494b89e80ae7fa0c9cae4c7d8 Mon Sep 17 00:00:00 2001 From: Chad Phillips Date: Thu, 27 Mar 2025 12:25:19 -0400 Subject: [PATCH 01/22] add requesty provider plugin doc --- docs/plugins.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/plugins.rst b/docs/plugins.rst index 83ef91ce..5793e548 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -152,6 +152,8 @@ Supported providers https://github.com/llm-workflow-engine/lwe-plugin-provider-openai * **provider_openrouter:** Access to `OpenRouter `_ models https://github.com/llm-workflow-engine/lwe-plugin-provider-openrouter +* **provider_requesty:** Access to `Requesty `_ models + https://github.com/llm-workflow-engine/lwe-plugin-provider-requesty * **provider_vertexai:** Access to `Google Vertex AI `_ text/code models. https://github.com/llm-workflow-engine/lwe-plugin-provider-vertexai From ea3b2568e80ac058081bfbe65e4b74453903040a Mon Sep 17 00:00:00 2001 From: Chad Phillips Date: Sat, 5 Apr 2025 12:41:59 -0400 Subject: [PATCH 02/22] test Python 3.13 --- .github/workflows/python-app.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 2171af55..205a304d 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v3 From 51745c6feac9fc21095709850b8e9ea9416976c8 Mon Sep 17 00:00:00 2001 From: Chad Phillips Date: Sat, 5 Apr 2025 13:46:41 -0400 Subject: [PATCH 03/22] remove custom codeql config --- .github/workflows/codeql.yml | 83 ------------------------------------ 1 file changed, 83 deletions(-) delete mode 100644 .github/workflows/codeql.yml diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml deleted file mode 100644 index 0ac03351..00000000 --- a/.github/workflows/codeql.yml +++ /dev/null @@ -1,83 +0,0 @@ -# For most projects, this workflow file will not need changing; you simply need -# to commit it to your repository. -# -# You may wish to alter this file to override the set of languages analyzed, -# or to provide custom queries or build logic. -# -# ******** NOTE ******** -# We have attempted to detect the languages in your repository. Please check -# the `language` matrix defined below to confirm you have the correct set of -# supported CodeQL languages. -# -name: "CodeQL" - -on: - push: - branches: [ "main" ] - pull_request: - # The branches below must be a subset of the branches above - branches: [ "main" ] - schedule: - - cron: '24 0 * * 2' - -jobs: - analyze: - name: Analyze - # Runner size impacts CodeQL analysis time. To learn more, please see: - # - https://gh.io/recommended-hardware-resources-for-running-codeql - # - https://gh.io/supported-runners-and-hardware-resources - # - https://gh.io/using-larger-runners - # Consider using larger runners for possible analysis time improvements. - runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }} - timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }} - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: [ 'python' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby', 'swift' ] - # Use only 'java' to analyze code written in Java, Kotlin or both - # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both - # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support - - steps: - - name: Checkout repository - uses: actions/checkout@v3 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@v2 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - - # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs - # queries: security-extended,security-and-quality - - - # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). - # If this step fails, then you should remove it and run the build manually (see below) - - name: Autobuild - uses: github/codeql-action/autobuild@v2 - - # ℹī¸ Command-line programs to run using the OS shell. - # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun - - # If the Autobuild fails above, remove it and uncomment the following three lines. - # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. - - # - run: | - # echo "Run, Build Application using script" - # ./location_of_script_within_repo/buildscript.sh - - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 - with: - category: "/language:${{matrix.language}}" - From 1b821b869c248492f250ba650fa518081d8ae037 Mon Sep 17 00:00:00 2001 From: Chad Phillips Date: Sat, 5 Apr 2025 14:00:29 -0400 Subject: [PATCH 04/22] fix codeql badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 61860c4b..a226fce6 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ [![Test status](https://github.com/llm-workflow-engine/llm-workflow-engine/actions/workflows/python-app.yml/badge.svg)](https://github.com/llm-workflow-engine/llm-workflow-engine/actions/workflows/python-app.yml) -[![CodeQL status](https://github.com/llm-workflow-engine/llm-workflow-engine/actions/workflows/codeql.yml/badge.svg)](https://github.com/llm-workflow-engine/llm-workflow-engine/actions/workflows/codeql.yml) +[![CodeQL](https://github.com/llm-workflow-engine/llm-workflow-engine/actions/workflows/github-code-scanning/codeql/badge.svg)](https://github.com/llm-workflow-engine/llm-workflow-engine/actions/workflows/github-code-scanning/codeql)

lwe-logo-small From f5a24361e0e3d79efb71a81c209c3d8e9843614f Mon Sep 17 00:00:00 2001 From: Chad Phillips Date: Mon, 7 Apr 2025 19:41:05 -0400 Subject: [PATCH 05/22] workflow for auto-publish to PyPi --- .github/workflows/pypi.yml | 71 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 .github/workflows/pypi.yml diff --git a/.github/workflows/pypi.yml b/.github/workflows/pypi.yml new file mode 100644 index 00000000..594f161c --- /dev/null +++ b/.github/workflows/pypi.yml @@ -0,0 +1,71 @@ +name: Publish Python 🐍 distribution đŸ“Ļ to PyPI and TestPyPI + +on: push + +jobs: + build: + name: Build distribution đŸ“Ļ + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.x" + - name: Install pypa/build + run: >- + python3 -m + pip install + build + --user + - name: Build a binary wheel and a source tarball + run: python3 -m build + - name: Store the distribution packages + uses: actions/upload-artifact@v4 + with: + name: python-package-distributions + path: dist/ + + publish-to-pypi: + name: >- + Publish Python 🐍 distribution đŸ“Ļ to PyPI + if: startsWith(github.ref, 'refs/tags/') # only publish to PyPI on tag pushes + needs: + - build + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/llm-workflow-engine + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution đŸ“Ļ to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + publish-to-testpypi: + name: Publish Python 🐍 distribution đŸ“Ļ to TestPyPI + needs: + - build + runs-on: ubuntu-latest + environment: + name: testpypi + url: https://test.pypi.org/p/llm-workflow-engine + permissions: + id-token: write # IMPORTANT: mandatory for trusted publishing + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution đŸ“Ļ to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: https://test.pypi.org/legacy/ From 2430b98b7fbd79abb95ce40e165cba0d5609413b Mon Sep 17 00:00:00 2001 From: Chad Phillips Date: Mon, 7 Apr 2025 19:56:24 -0400 Subject: [PATCH 06/22] 0.22.8 --- COMMIT_LOG.md | 4 ++++ lwe/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/COMMIT_LOG.md b/COMMIT_LOG.md index 7da6096c..de279247 100644 --- a/COMMIT_LOG.md +++ b/COMMIT_LOG.md @@ -1,3 +1,7 @@ +### v0.22.8 - 04/07/2025 + +* **Mon Apr 07 2025:** workflow for auto-publish to PyPi + ### v0.22.7 - 03/19/2025 * **Wed Mar 19 2025:** support AIMessage objects in streaming diff --git a/lwe/version.py b/lwe/version.py index 12f825ca..4cd0c92c 100644 --- a/lwe/version.py +++ b/lwe/version.py @@ -1 +1 @@ -__version__ = "0.22.7" +__version__ = "0.22.8" From d315f6556a2320177cdee1f08f55a921d0d72bd2 Mon Sep 17 00:00:00 2001 From: Chad Phillips Date: Mon, 7 Apr 2025 20:22:19 -0400 Subject: [PATCH 07/22] switch to newer build system --- pyproject.toml | 89 ++++++++++++++++++++++++++++++++++++++++++++++++++ setup.py | 57 -------------------------------- 2 files changed, 89 insertions(+), 57 deletions(-) delete mode 100644 setup.py diff --git a/pyproject.toml b/pyproject.toml index aa4949aa..f5794aca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,2 +1,91 @@ +[build-system] +requires = ["setuptools>=42", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "llm-workflow-engine" +dynamic = ["version"] +description = "CLI tool and workflow manager for common LLMs" +readme = "README.md" +authors = [ + {name = "Mahmoud Mabrouk", email = "mahmoudmabrouk.mail@gmail.com"}, + {name = "Chad Phillips"} +] +requires-python = ">=3.9" +license = {text = "MIT License"} +classifiers = [ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", +] +dependencies = [ + "ansible>=8.0", + "ansible-core>=2.15", + "alembic", + "beautifulsoup4", + "docutils>=0.20.1", + "email-validator", + "Jinja2", + "langchain>=0.3.19,<0.4", + "langchain-core>=0.3.39,<0.4", + "langchain-community>=0.3.16,<0.4", + "langchain_openai>=0.3.3", + "names", + "numexpr>=2.8.4", + "openpyxl", + "pdfminer.six", + "prompt-toolkit", + "pymupdf4llm", + "pyperclip", + "python-frontmatter", + "PyYAML", + "rich", + "setuptools", + "sqlalchemy>=1.4.48", + "textract @ git+https://github.com/thehunmonkgroup/textract@61a25dcb6f1d3e747b6e58b716d058f3dfe3d662", + "tiktoken", +] + +[project.optional-dependencies] +dev = [ + "pytest", + "pytest-datadir", + "pip-tools", + "flake8", + "black", +] + +[project.urls] +Homepage = "https://github.com/llm-workflow-engine/llm-workflow-engine" +Repository = "https://github.com/llm-workflow-engine/llm-workflow-engine" + +[project.scripts] +lwe = "lwe.main:main" + +[project.entry-points."lwe_plugins"] + +[tool.setuptools] +packages = {find = {}} + +[tool.setuptools.package-data] +"lwe" = [ + "backends/api/schema/alembic.ini", + "backends/api/schema/alembic/*", + "backends/api/schema/alembic/**/*", + "examples/*", + "examples/**/*", + "tools/*", + "tools/**/*", + "presets/*", + "presets/**/*", + "templates/*", + "templates/**/*", + "workflows/*", + "workflows/**/*", +] + +[tool.setuptools.dynamic] +version = {attr = "lwe.version.__version__"} + [tool.black] line-length = 100 diff --git a/setup.py b/setup.py deleted file mode 100644 index 50e11356..00000000 --- a/setup.py +++ /dev/null @@ -1,57 +0,0 @@ -from setuptools import find_packages, setup -import re -from os import path - -FILE_DIR = path.dirname(path.abspath(path.realpath(__file__))) - -with open("README.md", "r", encoding="utf-8") as fh: - long_description = fh.read() - -with open("requirements.txt") as f: - install_requirement = f.readlines() - -with open(path.join(FILE_DIR, "lwe", "version.py")) as f: - version = re.match(r'^__version__ = "([\w\.]+)"$', f.read().strip())[1] - -setup( - name="llm-workflow-engine", - version=version, - author="Mahmoud Mabrouk, Chad Phillips", - author_email="mahmoudmabrouk.mail@gmail.com", - description="CLI tool and workflow manager for common LLMs, with a focus on OpenAI's models", - long_description=long_description, - long_description_content_type="text/markdown", - url="https://github.com/llm-workflow-engine/llm-workflow-engine", - packages=find_packages(), - package_data={ - "lwe": [ - "backends/api/schema/alembic.ini", - "backends/api/schema/alembic/*", - "backends/api/schema/alembic/**/*", - "examples/*", - "examples/**/*", - "tools/*", - "tools/**/*", - "presets/*", - "presets/**/*", - "templates/*", - "templates/**/*", - "workflows/*", - "workflows/**/*", - ], - }, - install_requires=install_requirement, - classifiers=[ - "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", - "Operating System :: OS Independent", - ], - python_requires=">=3.9", - entry_points={ - "console_scripts": [ - "lwe = lwe.main:main", - ], - "lwe_plugins": [], - }, - scripts=[], -) From d9eeb831900f312226f456f1b0d8d8d68b3c363e Mon Sep 17 00:00:00 2001 From: Chad Phillips Date: Mon, 7 Apr 2025 20:23:41 -0400 Subject: [PATCH 08/22] 0.22.9 --- COMMIT_LOG.md | 4 ++++ lwe/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/COMMIT_LOG.md b/COMMIT_LOG.md index de279247..65a0f931 100644 --- a/COMMIT_LOG.md +++ b/COMMIT_LOG.md @@ -1,3 +1,7 @@ +### v0.22.9 - 04/07/2025 + +* **Mon Apr 07 2025:** switch to newer build system + ### v0.22.8 - 04/07/2025 * **Mon Apr 07 2025:** workflow for auto-publish to PyPi diff --git a/lwe/version.py b/lwe/version.py index 4cd0c92c..5456ec47 100644 --- a/lwe/version.py +++ b/lwe/version.py @@ -1 +1 @@ -__version__ = "0.22.8" +__version__ = "0.22.9" From b34cbeca9799c322e8f6992fab02b22c8d816993 Mon Sep 17 00:00:00 2001 From: Chad Phillips Date: Mon, 7 Apr 2025 23:08:31 -0400 Subject: [PATCH 09/22] clean up license syntax --- pyproject.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f5794aca..0c361e0e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,10 +12,9 @@ authors = [ {name = "Chad Phillips"} ] requires-python = ">=3.9" -license = {text = "MIT License"} +license = "MIT" classifiers = [ "Programming Language :: Python :: 3", - "License :: OSI Approved :: MIT License", "Operating System :: OS Independent", ] dependencies = [ From 8282c0a70dacaa6f7b774270ed5a9e3284173b1d Mon Sep 17 00:00:00 2001 From: Chad Phillips Date: Mon, 7 Apr 2025 23:08:45 -0400 Subject: [PATCH 10/22] improve pypi publishing script --- scripts/release-pypi.sh | 41 ++++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/scripts/release-pypi.sh b/scripts/release-pypi.sh index 5bb04c09..c3950bf1 100755 --- a/scripts/release-pypi.sh +++ b/scripts/release-pypi.sh @@ -3,39 +3,54 @@ # Convenience script to handle preparing for PyPi release, and printing out the # commands to execute it. +clean() { + echo "Cleaning build environment" + rm -rfv dist/ build/ + find . -depth -name __pycache__ -type d -exec rm -rfv {} \; +} + execute() { - local update_build_release_packages="pip install --upgrade wheel build twine" - local clean="rm -rfv dist/ build/" + local update_build_release_packages="pip install --upgrade build twine validate-pyproject" + local validate="validate-pyproject pyproject.toml" local build="python -m build" + local check_dist="twine check dist/*" local test_pypi_upload="python -m twine upload --repository testpypi dist/*" local pypi_upload="python -m twine upload --skip-existing dist/*" + # Uncomment the following line if you want to sign releases + # local pypi_upload="python -m twine upload --skip-existing --sign dist/*" echo "Updating build and release packages with command:" echo " ${update_build_release_packages}" ${update_build_release_packages} if [ $? -eq 0 ]; then - echo "Cleaning build environment with command:" - echo " ${clean}" - ${clean} + echo "Validating pyproject.toml with command:" + echo " ${validate}" + ${validate} if [ $? -eq 0 ]; then echo "Building release with command:" echo " ${build}" ${build} if [ $? -eq 0 ]; then - echo "Build successful" - echo - echo "Test release with command:" - echo " ${test_pypi_upload}" - echo - echo "Release with command:" - echo " ${pypi_upload}" + echo "Checking built distribution with command:" + echo " ${check_dist}" + ${check_dist} + if [ $? -eq 0 ]; then + echo "Build successful and verified" + echo + echo "Test release with command:" + echo " ${test_pypi_upload}" + echo + echo "Release with command:" + echo " ${pypi_upload}" + fi fi fi fi } -if [ -d vit ] && [ -r setup.py ]; then +if [ -r pyproject.toml ]; then + clean execute else echo "ERROR: must run script from repository root" From 2d3ad92492b313b68e12254ade3dd79167c9a7e7 Mon Sep 17 00:00:00 2001 From: Chad Phillips Date: Mon, 7 Apr 2025 23:36:25 -0400 Subject: [PATCH 11/22] migrate from textract to kreuzberg --- docs/installation.rst | 5 +++-- docs/workflows.rst | 2 +- .../api/workflow/library/text_extractor.py | 19 ++++++++++--------- pyproject.toml | 2 +- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/docs/installation.rst b/docs/installation.rst index 673ac593..29612a07 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -16,9 +16,10 @@ Requirements To use this project, you need: -* Python 3.9 or later -* ``setuptools`` installed. You can install it using ``pip install setuptools``. Make sure that you have the last version of pip: ``pip install --upgrade pip`` +* Python 3.9 or later with pip installed. * A database backend (`SQLite `_ by default, any configurable in `SQLAlchemy `_ allowed). +* Optional: + * `Pandoc `_ for document extraction *other than* PDFs. ----------------------------------------------- From packages diff --git a/docs/workflows.rst b/docs/workflows.rst index 1557c5d6..21a70ddd 100644 --- a/docs/workflows.rst +++ b/docs/workflows.rst @@ -66,7 +66,7 @@ Example: retries: 10 delay: 3 -``text_extractor``: Provides an easy way to extract text content from many different file types. For supported arguments and return values, see the `text_extractor module documentation `_. +``text_extractor``: Provides an easy way to extract text content from many different file types. For supported arguments and return values, see the `text_extractor module documentation `_. *NOTE: PDF document extraction is supported natively, to extract other document types (.docx, .xlsx, etc.) you need to install* `pandoc `_ *and make sure it's available in your PATH.* Example: diff --git a/lwe/backends/api/workflow/library/text_extractor.py b/lwe/backends/api/workflow/library/text_extractor.py index e3208e10..647127e4 100644 --- a/lwe/backends/api/workflow/library/text_extractor.py +++ b/lwe/backends/api/workflow/library/text_extractor.py @@ -6,7 +6,7 @@ import tempfile from urllib.parse import urlparse -import textract +import kreuzberg import pymupdf4llm import fitz fitz.TOOLS.mupdf_display_errors(False) @@ -20,7 +20,7 @@ config.set("debug.log.enabled", True) log = Logger("text_extractor", config) -TEXTRACT_SUPPORTED_FILE_EXTENSIONS = [ +KREUZBERG_SUPPORTED_FILE_EXTENSIONS = [ # Microsoft Office formats ".docx", ".pptx", @@ -121,13 +121,13 @@ def extract_text_pymupdf(path): return pymupdf4llm.to_markdown(path) -def extract_text_textract(path): - return textract.process(path).decode("utf-8") +def extract_text_kreuzberg(path): + return kreuzberg.extract_file_sync(path) def get_file_extension(path, default_extension): file_extension = default_extension - for ext in set(TEXTRACT_SUPPORTED_FILE_EXTENSIONS + PYMUPDF_SUPPORTED_EXTENSIONS): + for ext in set(KREUZBERG_SUPPORTED_FILE_EXTENSIONS + PYMUPDF_SUPPORTED_EXTENSIONS): if path.lower().endswith(ext): file_extension = ext break @@ -184,12 +184,13 @@ def main(): message = f"Error extracting {path} content with pymupdf4llm: {str(e)}" log.error(message) module.fail_json(msg=message) - elif file_extension in TEXTRACT_SUPPORTED_FILE_EXTENSIONS: - log.debug(f"Extracting content from {path} with textract") + elif file_extension in KREUZBERG_SUPPORTED_FILE_EXTENSIONS: + log.debug(f"Extracting content from {path} with kreuzberg") try: - content = extract_text_textract(path) + extraction = extract_text_kreuzberg(path) + content = extraction.content except Exception as e: - message = f"Error extracting {file_extension} content with textract: {str(e)}" + message = f"Error extracting {file_extension} content with kreuzberg: {str(e)}" log.error(message) module.fail_json(msg=message) else: diff --git a/pyproject.toml b/pyproject.toml index 0c361e0e..b601337e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ dependencies = [ "docutils>=0.20.1", "email-validator", "Jinja2", + "kreuzberg", "langchain>=0.3.19,<0.4", "langchain-core>=0.3.39,<0.4", "langchain-community>=0.3.16,<0.4", @@ -41,7 +42,6 @@ dependencies = [ "rich", "setuptools", "sqlalchemy>=1.4.48", - "textract @ git+https://github.com/thehunmonkgroup/textract@61a25dcb6f1d3e747b6e58b716d058f3dfe3d662", "tiktoken", ] From f41498f606369071f45f9b5132e38289b75d0937 Mon Sep 17 00:00:00 2001 From: Chad Phillips Date: Tue, 8 Apr 2025 15:29:19 -0400 Subject: [PATCH 12/22] 0.22.9 --- COMMIT_LOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/COMMIT_LOG.md b/COMMIT_LOG.md index 65a0f931..e0a9b621 100644 --- a/COMMIT_LOG.md +++ b/COMMIT_LOG.md @@ -1,5 +1,6 @@ -### v0.22.9 - 04/07/2025 +### v0.22.9 - 04/08/2025 +* **Mon Apr 07 2025:** migrate from textract to kreuzberg * **Mon Apr 07 2025:** switch to newer build system ### v0.22.8 - 04/07/2025 From e1c8bf8fdbfac0b493cfeac5da2a655f7ed1d4e7 Mon Sep 17 00:00:00 2001 From: Chad Phillips Date: Thu, 10 Apr 2025 12:38:02 -0400 Subject: [PATCH 13/22] 0.22.9 commit log tweak --- COMMIT_LOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/COMMIT_LOG.md b/COMMIT_LOG.md index e0a9b621..e1d43cce 100644 --- a/COMMIT_LOG.md +++ b/COMMIT_LOG.md @@ -1,6 +1,6 @@ ### v0.22.9 - 04/08/2025 -* **Mon Apr 07 2025:** migrate from textract to kreuzberg +* **Mon Apr 07 2025:** migrate from textract to kreuzberg *(NOTE: with kreuzberg, pandoc is a required dependency for using workflows with non-PDF documents)* * **Mon Apr 07 2025:** switch to newer build system ### v0.22.8 - 04/07/2025 From e9d5c021a7b9d8901fe524d943e28eeb92422909 Mon Sep 17 00:00:00 2001 From: Chad Phillips Date: Mon, 14 Apr 2025 13:56:25 -0400 Subject: [PATCH 14/22] gpt-4.1 --- lwe/plugins/provider_chat_openai.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lwe/plugins/provider_chat_openai.py b/lwe/plugins/provider_chat_openai.py index f3ae1ac1..49c469b9 100644 --- a/lwe/plugins/provider_chat_openai.py +++ b/lwe/plugins/provider_chat_openai.py @@ -106,6 +106,12 @@ def static_models(self): "gpt-4o-mini-2024-07-18": { "max_tokens": 131072, }, + "gpt-4.1": { + "max_tokens": 1047576, + }, + "gpt-4.1-2025-04-14": { + "max_tokens": 1047576, + }, "gpt-4.5-preview": { "max_tokens": 131072, }, From 4aeffb0b6c210ab30b4a2fc46428b2f314406e0d Mon Sep 17 00:00:00 2001 From: Chad Phillips Date: Mon, 14 Apr 2025 13:58:12 -0400 Subject: [PATCH 15/22] 0.22.10 --- COMMIT_LOG.md | 4 ++++ lwe/version.py | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/COMMIT_LOG.md b/COMMIT_LOG.md index e1d43cce..13c3a792 100644 --- a/COMMIT_LOG.md +++ b/COMMIT_LOG.md @@ -1,3 +1,7 @@ +### v0.22.10 - 04/14/2025 + +* **Mon Apr 14 2025:** gpt-4.1 + ### v0.22.9 - 04/08/2025 * **Mon Apr 07 2025:** migrate from textract to kreuzberg *(NOTE: with kreuzberg, pandoc is a required dependency for using workflows with non-PDF documents)* diff --git a/lwe/version.py b/lwe/version.py index 5456ec47..f5c474ba 100644 --- a/lwe/version.py +++ b/lwe/version.py @@ -1 +1 @@ -__version__ = "0.22.9" +__version__ = "0.22.10" From c2b6c8e94a2c768c353f0f0400134284fbb43e66 Mon Sep 17 00:00:00 2001 From: Chad Phillips Date: Mon, 14 Apr 2025 14:14:06 -0400 Subject: [PATCH 16/22] bump doc requirements --- docs/requirements.txt | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index e221e961..7ba6cf97 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -2,31 +2,31 @@ # This file is autogenerated by pip-compile with Python 3.12 # by the following command: # -# pip-compile requirements.in +# pip-compile --output-file=- # alabaster==0.7.16 # via sphinx -babel==2.12.1 +babel==2.17.0 # via sphinx -certifi==2023.5.7 +certifi==2025.1.31 # via requests -charset-normalizer==3.2.0 +charset-normalizer==3.4.1 # via requests -docutils==0.20.1 +docutils==0.21.2 # via sphinx -idna==3.4 +idna==3.10 # via requests imagesize==1.4.1 # via sphinx -jinja2==3.1.2 +jinja2==3.1.6 # via sphinx -markupsafe==2.1.3 +markupsafe==3.0.2 # via jinja2 -packaging==23.1 +packaging==24.2 # via sphinx -pygments==2.15.1 +pygments==2.19.1 # via sphinx -requests==2.31.0 +requests==2.32.3 # via sphinx snowballstemmer==2.2.0 # via sphinx @@ -36,17 +36,17 @@ sphinx==7.3.7 # sphinx-copybutton sphinx-copybutton==0.5.2 # via -r requirements.in -sphinxcontrib-applehelp==1.0.4 +sphinxcontrib-applehelp==2.0.0 # via sphinx -sphinxcontrib-devhelp==1.0.2 +sphinxcontrib-devhelp==2.0.0 # via sphinx -sphinxcontrib-htmlhelp==2.0.1 +sphinxcontrib-htmlhelp==2.1.0 # via sphinx sphinxcontrib-jsmath==1.0.1 # via sphinx -sphinxcontrib-qthelp==1.0.3 +sphinxcontrib-qthelp==2.0.0 # via sphinx -sphinxcontrib-serializinghtml==1.1.10 +sphinxcontrib-serializinghtml==2.0.0 # via sphinx -urllib3==2.0.3 +urllib3==2.4.0 # via requests From 34e2f7a6b98fac7fb735bb54e3e4967499573514 Mon Sep 17 00:00:00 2001 From: Chad Phillips Date: Mon, 14 Apr 2025 14:14:17 -0400 Subject: [PATCH 17/22] update latest release install method --- docs/installation.rst | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/installation.rst b/docs/installation.rst index 29612a07..218de6e1 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -25,7 +25,13 @@ To use this project, you need: From packages ----------------------------------------------- -Install the latest version of this software directly from github with pip: +Install the latest release of this software via pip: + +.. code-block:: bash + + pip install llm-workflow-engine + +Install the 'bleeding edge' of this software directly from github with pip: .. code-block:: bash From 9af542dacb4987dd48f5e3c1003f5a48b1b537db Mon Sep 17 00:00:00 2001 From: Chad Phillips Date: Mon, 14 Apr 2025 14:57:00 -0400 Subject: [PATCH 18/22] gpt 4.1 mini/nano --- lwe/plugins/provider_chat_openai.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lwe/plugins/provider_chat_openai.py b/lwe/plugins/provider_chat_openai.py index 49c469b9..f1c69c29 100644 --- a/lwe/plugins/provider_chat_openai.py +++ b/lwe/plugins/provider_chat_openai.py @@ -112,6 +112,18 @@ def static_models(self): "gpt-4.1-2025-04-14": { "max_tokens": 1047576, }, + "gpt-4.1-mini": { + "max_tokens": 1047576, + }, + "gpt-4.1-mini-2025-04-14": { + "max_tokens": 1047576, + }, + "gpt-4.1-nano": { + "max_tokens": 1047576, + }, + "gpt-4.1-nano-2025-04-14": { + "max_tokens": 1047576, + }, "gpt-4.5-preview": { "max_tokens": 131072, }, From 485b32c8145c0046991fd4eb5a128913edb485b8 Mon Sep 17 00:00:00 2001 From: Chad Phillips Date: Wed, 16 Apr 2025 14:51:52 -0400 Subject: [PATCH 19/22] o3/o4-mini --- lwe/backends/api/request.py | 2 +- lwe/plugins/provider_chat_openai.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/lwe/backends/api/request.py b/lwe/backends/api/request.py index 7d3ea5d0..8c4abeaa 100644 --- a/lwe/backends/api/request.py +++ b/lwe/backends/api/request.py @@ -309,7 +309,7 @@ def strip_out_messages_over_max_tokens(self, messages, max_tokens): def is_openai_o_series(self): if self.provider.name == "provider_chat_openai": model_name = getattr(self.llm, self.provider.model_property_name) - if model_name.startswith("o1-") or model_name.startswith("o3-"): + if model_name.startswith("o1") or model_name.startswith("o3") or model_name.startswith("o4"): return True return False diff --git a/lwe/plugins/provider_chat_openai.py b/lwe/plugins/provider_chat_openai.py index f1c69c29..3fecd953 100644 --- a/lwe/plugins/provider_chat_openai.py +++ b/lwe/plugins/provider_chat_openai.py @@ -148,12 +148,31 @@ def static_models(self): "o1-2024-12-17": { "max_tokens": 204800, }, + # TODO: These are not chat models, how to support? + # "o1-pro": { + # "max_tokens": 204800, + # }, + # "o1-pro-2025-03-19": { + # "max_tokens": 204800, + # }, "o3-mini": { "max_tokens": 204800, }, "o3-mini-2025-01-31": { "max_tokens": 204800, }, + "o3": { + "max_tokens": 204800, + }, + "o3-2025-04-16": { + "max_tokens": 204800, + }, + "o4-mini": { + "max_tokens": 204800, + }, + "o4-mini-2025-04-16": { + "max_tokens": 204800, + }, } def prepare_messages_method(self): From 50a00681c90aadbcacb5e5a1964604689cb2af69 Mon Sep 17 00:00:00 2001 From: Chad Phillips Date: Wed, 16 Apr 2025 14:53:27 -0400 Subject: [PATCH 20/22] 0.22.11 --- COMMIT_LOG.md | 5 +++++ lwe/version.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/COMMIT_LOG.md b/COMMIT_LOG.md index 13c3a792..5f4d28bf 100644 --- a/COMMIT_LOG.md +++ b/COMMIT_LOG.md @@ -1,3 +1,8 @@ +### v0.22.11 - 04/16/2025 + +* **Wed Apr 16 2025:** o3/o4-mini +* **Mon Apr 14 2025:** gpt 4.1 mini/nano + ### v0.22.10 - 04/14/2025 * **Mon Apr 14 2025:** gpt-4.1 diff --git a/lwe/version.py b/lwe/version.py index f5c474ba..c98ace01 100644 --- a/lwe/version.py +++ b/lwe/version.py @@ -1 +1 @@ -__version__ = "0.22.10" +__version__ = "0.22.11" From be531743c048a6e1e59df0a533d3fea29dd6490e Mon Sep 17 00:00:00 2001 From: Chad Phillips Date: Sat, 3 May 2025 18:16:33 -0400 Subject: [PATCH 21/22] add doc for xAI provider plugin --- docs/plugins.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/plugins.rst b/docs/plugins.rst index 5793e548..2effedbe 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -146,6 +146,8 @@ Supported providers https://github.com/llm-workflow-engine/lwe-plugin-provider-chat-together * **provider_chat_vertexai:** Access to `Google Vertex AI `_ chat models. https://github.com/llm-workflow-engine/lwe-plugin-provider-chat-vertexai +* **provider_chat_xai:** Access to `xAI `_ chat models. + https://github.com/llm-workflow-engine/lwe-plugin-provider-chat-xai * **provider_huggingface_hub:** Access to `Hugging Face Hub `_ models https://github.com/llm-workflow-engine/lwe-plugin-provider-huggingface-hub * **provider_openai:** Access to non-chat `OpenAI `_ models (GPT-3, etc.) From 66c8d62dccc793fbf8884d5d57acb5a219207693 Mon Sep 17 00:00:00 2001 From: Chad Phillips Date: Thu, 8 May 2025 12:49:25 -0400 Subject: [PATCH 22/22] update AI21 plugin doc --- docs/plugins.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/plugins.rst b/docs/plugins.rst index 2effedbe..02188fe2 100644 --- a/docs/plugins.rst +++ b/docs/plugins.rst @@ -122,8 +122,8 @@ Supported providers **NOTE:** While these provider integrations are working, none have been well-tested yet. -* **provider_ai21:** Access to `AI21 `_ models - https://github.com/llm-workflow-engine/lwe-plugin-provider-ai21 +* **provider_ai21:** Access to `Chat AI21 `_ models + https://github.com/llm-workflow-engine/lwe-plugin-provider-chat-ai21 * **provider_azure_openai_chat:** Access to `Azure OpenAI `_ chat models https://github.com/llm-workflow-engine/lwe-plugin-provider-azure-openai-chat * **provider_chat_anthropic:** Access to `Anthropic `_ chat models