From 1c89081fc7f6b996639cec174d73e999eccf191d Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Tue, 7 Jun 2022 21:17:03 -0700 Subject: [PATCH 01/11] [stubsabot] proof of concept --- scripts/stubsabot.py | 191 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 191 insertions(+) create mode 100644 scripts/stubsabot.py diff --git a/scripts/stubsabot.py b/scripts/stubsabot.py new file mode 100644 index 000000000000..46387dbc28e9 --- /dev/null +++ b/scripts/stubsabot.py @@ -0,0 +1,191 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import asyncio +import datetime +import os +import re +import subprocess +import sys +import urllib.parse +from dataclasses import dataclass +from pathlib import Path + +import aiohttp +import packaging.specifiers +import packaging.version +import tomli +import tomlkit + + +@dataclass +class StubInfo: + distribution: str + version_spec: str + obsolete: bool + no_longer_updated: bool + + +def read_typeshed_stub_metadata(stub_path: Path) -> StubInfo: + with (stub_path / "METADATA.toml").open("rb") as f: + meta = tomli.load(f) + return StubInfo( + distribution=stub_path.name, + version_spec=meta["version"], + obsolete="obsolete_since" in meta, + no_longer_updated=meta.get("no_longer_updated", False), + ) + + +@dataclass +class PypiInfo: + distribution: str + version: packaging.version.Version + upload_date: datetime.datetime + + +async def fetch_pypi_info(distribution: str, session: aiohttp.ClientSession) -> PypiInfo: + url = f"https://pypi.org/pypi/{urllib.parse.quote(distribution)}/json" + async with session.get(url) as response: + response.raise_for_status() + j = await response.json() + version = j["info"]["version"] + date = datetime.datetime.fromisoformat(j["releases"][version][0]["upload_time"]) + return PypiInfo( + distribution=distribution, version=packaging.version.Version(version), upload_date=date + ) + + +@dataclass +class Update: + distribution: str + stub_path: Path + old_version_spec: str + new_version_spec: str + + +@dataclass +class NoUpdate: + distribution: str + reason: str + + +def _check_spec(updated_spec: str, version: packaging.version.Version) -> str: + assert version in packaging.specifiers.SpecifierSet( + "==" + updated_spec + ), f"{version} not in {updated_spec}" + return updated_spec + + +def get_updated_version_spec(spec: str, version: packaging.version.Version) -> str: + if not spec.endswith(".*"): + return _check_spec(version.base_version, version) + + specificity = spec.count(".") if spec.removesuffix(".*") else 0 + rounded_version = version.base_version.split(".")[:specificity] + rounded_version.extend(["0"] * (specificity - len(rounded_version))) + + return _check_spec(".".join(rounded_version) + ".*", version) + + +async def determine_action(stub_path: Path, session: aiohttp.ClientSession) -> Update | NoUpdate: + stub_info = read_typeshed_stub_metadata(stub_path) + if stub_info.obsolete: + return NoUpdate(stub_info.distribution, "obsolete") + if stub_info.no_longer_updated: + return NoUpdate(stub_info.distribution, "no longer updated") + + pypi_info = await fetch_pypi_info(stub_info.distribution, session) + spec = packaging.specifiers.SpecifierSet("==" + stub_info.version_spec) + if pypi_info.version in spec: + return NoUpdate(stub_info.distribution, "up to date") + + return Update( + distribution=stub_info.distribution, + stub_path=stub_path, + old_version_spec=stub_info.version_spec, + new_version_spec=get_updated_version_spec(stub_info.version_spec, pypi_info.version), + ) + + +def normalize(name: str) -> str: + # PEP 503 normalization + return re.sub(r"[-_.]+", "-", name).lower() + + +_repo_lock = asyncio.Lock() + +TYPESHED_OWNER = "python" +FORK_OWNER = "hauntsaninja" + + +async def suggest_typeshed_update( + update: Update, session: aiohttp.ClientSession, dry_run: bool +) -> None: + title = f"[stubsabot] Bump {update.distribution} to {update.new_version_spec}" + + # lock should be unnecessary, but can't hurt to enforce mutual exclusion + async with _repo_lock: + branch_name = f"stubsabot/{normalize(update.distribution)}" + subprocess.check_call(["git", "checkout", "-B", branch_name, "origin/master"]) + with open(update.stub_path / "METADATA.toml", "rb") as f: + meta = tomlkit.load(f) + meta["version"] = update.new_version_spec + with open(update.stub_path / "METADATA.toml", "w") as f: + tomlkit.dump(meta, f) + subprocess.check_call(["git", "commit", "--all", "-m", title]) + if dry_run: + return + subprocess.check_call(["git", "push", "origin", branch_name, "--force-with-lease"]) + + secret = os.environ["GITHUB_TOKEN"] + if secret.startswith("ghp"): + auth = f"token {secret}" + else: + auth = f"Bearer {secret}" + + async with session.post( + f"https://api.github.com/repos/{TYPESHED_OWNER}/typeshed/pulls", + json={"title": title, "head": f"{FORK_OWNER}:{branch_name}", "base": "master"}, + headers={"Accept": "application/vnd.github.v3+json", "Authorization": auth}, + ) as response: + body = await response.json() + if response.status == 422 and any( + "A pull request already exists" in e.get("message", "") for e in body.get("errors", []) + ): + # TODO: diff and update existing pull request + return + response.raise_for_status() + + +async def main() -> None: + assert sys.version_info >= (3, 9) + + parser = argparse.ArgumentParser() + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + + try: + conn = aiohttp.TCPConnector(limit_per_host=10) + async with aiohttp.ClientSession(connector=conn) as session: + tasks = [ + asyncio.create_task(determine_action(stubs_path, session)) + for stubs_path in Path("stubs").iterdir() + ] + for task in asyncio.as_completed(tasks): + update = await task + if isinstance(update, NoUpdate): + continue + if isinstance(update, Update): + await suggest_typeshed_update(update, session, dry_run=args.dry_run) + continue + raise AssertionError + finally: + # if you need to cleanup, try: + # git branch -D $(git branch --list 'stubsabot/*') + subprocess.check_call(["git", "checkout", "master"]) + + +if __name__ == "__main__": + asyncio.run(main()) From 2cc1c9b5c6a1b0e660da7a4b55f6ae5baabb9851 Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Thu, 9 Jun 2022 01:14:16 -0700 Subject: [PATCH 02/11] stubsabot updates --- scripts/stubsabot.py | 173 +++++++++++++++++++++++++++++++++---------- 1 file changed, 135 insertions(+), 38 deletions(-) diff --git a/scripts/stubsabot.py b/scripts/stubsabot.py index 46387dbc28e9..e50a222aec2c 100644 --- a/scripts/stubsabot.py +++ b/scripts/stubsabot.py @@ -4,13 +4,18 @@ import argparse import asyncio import datetime +import enum +import io import os import re import subprocess import sys +import tarfile import urllib.parse +import zipfile from dataclasses import dataclass from pathlib import Path +from typing import Any import aiohttp import packaging.specifiers @@ -19,6 +24,12 @@ import tomlkit +class ActionLevel(enum.IntEnum): + nothing = 0 # make no changes + local = 1 # make changes that affect local repo + everything = 2 # do everything, e.g. open PRs + + @dataclass class StubInfo: distribution: str @@ -43,6 +54,7 @@ class PypiInfo: distribution: str version: packaging.version.Version upload_date: datetime.datetime + release_to_download: dict[str, Any] async def fetch_pypi_info(distribution: str, session: aiohttp.ClientSession) -> PypiInfo: @@ -51,9 +63,14 @@ async def fetch_pypi_info(distribution: str, session: aiohttp.ClientSession) -> response.raise_for_status() j = await response.json() version = j["info"]["version"] - date = datetime.datetime.fromisoformat(j["releases"][version][0]["upload_time"]) + # prefer wheels, since it's what most users will get / it's pretty easy to mess up MANIFEST + release_to_download = sorted(j["releases"][version], key=lambda x: bool(x["packagetype"] == "bdist_wheel"))[-1] + date = datetime.datetime.fromisoformat(release_to_download["upload_time"]) return PypiInfo( - distribution=distribution, version=packaging.version.Version(version), upload_date=date + distribution=distribution, + version=packaging.version.Version(version), + upload_date=date, + release_to_download=release_to_download, ) @@ -64,17 +81,47 @@ class Update: old_version_spec: str new_version_spec: str + def __str__(self) -> str: + return f"Updating {self.distribution} from {self.old_version_spec!r} to {self.new_version_spec!r}" + + +@dataclass +class Obsolete: + distribution: str + stub_path: Path + obsolete_since_version: str + + def __str__(self) -> str: + return f"Marking {self.distribution} as obsolete since {self.obsolete_since_version!r}" + @dataclass class NoUpdate: distribution: str reason: str + def __str__(self) -> str: + return f"Skipping {self.distribution}: {self.reason}" + + +async def package_contains_py_typed(release_to_download: dict[str, Any], session: aiohttp.ClientSession) -> bool: + async with session.get(release_to_download["url"]) as response: + body = io.BytesIO(await response.read()) + + if release_to_download["packagetype"] == "bdist_wheel": + assert release_to_download["filename"].endswith(".whl") + with zipfile.ZipFile(body) as zf: + return any(Path(f).name == "py.typed" for f in zf.namelist()) + elif release_to_download["packagetype"] == "sdist": + assert release_to_download["filename"].endswith(".tar.gz") + with tarfile.open(fileobj=body, mode="r:gz") as zf: + return any(Path(f).name == "py.typed" for f in zf.getnames()) + else: + raise AssertionError + def _check_spec(updated_spec: str, version: packaging.version.Version) -> str: - assert version in packaging.specifiers.SpecifierSet( - "==" + updated_spec - ), f"{version} not in {updated_spec}" + assert version in packaging.specifiers.SpecifierSet("==" + updated_spec), f"{version} not in {updated_spec}" return updated_spec @@ -89,7 +136,7 @@ def get_updated_version_spec(spec: str, version: packaging.version.Version) -> s return _check_spec(".".join(rounded_version) + ".*", version) -async def determine_action(stub_path: Path, session: aiohttp.ClientSession) -> Update | NoUpdate: +async def determine_action(stub_path: Path, session: aiohttp.ClientSession) -> Update | NoUpdate | Obsolete: stub_info = read_typeshed_stub_metadata(stub_path) if stub_info.obsolete: return NoUpdate(stub_info.distribution, "obsolete") @@ -101,6 +148,9 @@ async def determine_action(stub_path: Path, session: aiohttp.ClientSession) -> U if pypi_info.version in spec: return NoUpdate(stub_info.distribution, "up to date") + if await package_contains_py_typed(pypi_info.release_to_download, session): + return Obsolete(stub_info.distribution, stub_path, obsolete_since_version=str(pypi_info.version)) + return Update( distribution=stub_info.distribution, stub_path=stub_path, @@ -109,23 +159,60 @@ async def determine_action(stub_path: Path, session: aiohttp.ClientSession) -> U ) +TYPESHED_OWNER = "python" +FORK_OWNER = "hauntsaninja" + + +async def create_or_update_pull_request(title: str, branch_name: str, session: aiohttp.ClientSession): + secret = os.environ["GITHUB_TOKEN"] + if secret.startswith("ghp"): + auth = f"token {secret}" + else: + auth = f"Bearer {secret}" + + async with session.post( + f"https://api.github.com/repos/{TYPESHED_OWNER}/typeshed/pulls", + json={"title": title, "head": f"{FORK_OWNER}:{branch_name}", "base": "master"}, + headers={"Accept": "application/vnd.github.v3+json", "Authorization": auth}, + ) as response: + body = await response.json() + if response.status == 422 and any( + "A pull request already exists" in e.get("message", "") for e in body.get("errors", []) + ): + # Find the existing PR + async with session.get( + f"https://api.github.com/repos/{TYPESHED_OWNER}/typeshed/pulls", + params={"state": "open", "head": f"{FORK_OWNER}:{branch_name}", "base": "master"}, + headers={"Accept": "application/vnd.github.v3+json", "Authorization": auth}, + ) as response: + response.raise_for_status() + body = await response.json() + assert len(body) >= 1 + pr_number = body[0]["number"] + # Update the PR's title + async with session.patch( + f"https://api.github.com/repos/{TYPESHED_OWNER}/typeshed/pulls/{pr_number}", + json={"title": title}, + headers={"Accept": "application/vnd.github.v3+json", "Authorization": auth}, + ) as response: + response.raise_for_status() + return + response.raise_for_status() + + def normalize(name: str) -> str: # PEP 503 normalization return re.sub(r"[-_.]+", "-", name).lower() +# lock should be unnecessary, but can't hurt to enforce mutual exclusion _repo_lock = asyncio.Lock() -TYPESHED_OWNER = "python" -FORK_OWNER = "hauntsaninja" - -async def suggest_typeshed_update( - update: Update, session: aiohttp.ClientSession, dry_run: bool -) -> None: +async def suggest_typeshed_update(update: Update, session: aiohttp.ClientSession, action_level: ActionLevel) -> None: + if action_level <= ActionLevel.nothing: + return title = f"[stubsabot] Bump {update.distribution} to {update.new_version_spec}" - - # lock should be unnecessary, but can't hurt to enforce mutual exclusion async with _repo_lock: branch_name = f"stubsabot/{normalize(update.distribution)}" subprocess.check_call(["git", "checkout", "-B", branch_name, "origin/master"]) @@ -135,56 +222,66 @@ async def suggest_typeshed_update( with open(update.stub_path / "METADATA.toml", "w") as f: tomlkit.dump(meta, f) subprocess.check_call(["git", "commit", "--all", "-m", title]) - if dry_run: + if action_level <= ActionLevel.local: return subprocess.check_call(["git", "push", "origin", branch_name, "--force-with-lease"]) - secret = os.environ["GITHUB_TOKEN"] - if secret.startswith("ghp"): - auth = f"token {secret}" - else: - auth = f"Bearer {secret}" + await create_or_update_pull_request(title, branch_name, session) - async with session.post( - f"https://api.github.com/repos/{TYPESHED_OWNER}/typeshed/pulls", - json={"title": title, "head": f"{FORK_OWNER}:{branch_name}", "base": "master"}, - headers={"Accept": "application/vnd.github.v3+json", "Authorization": auth}, - ) as response: - body = await response.json() - if response.status == 422 and any( - "A pull request already exists" in e.get("message", "") for e in body.get("errors", []) - ): - # TODO: diff and update existing pull request + +async def suggest_typeshed_obsolete(obsolete: Obsolete, session: aiohttp.ClientSession, action_level: ActionLevel) -> None: + if action_level <= ActionLevel.nothing: + return + title = f"[stubsabot] Mark {obsolete.distribution} as obsolete since {obsolete.obsolete_since_version}" + async with _repo_lock: + branch_name = f"stubsabot/{normalize(obsolete.distribution)}" + subprocess.check_call(["git", "checkout", "-B", branch_name, "origin/master"]) + with open(obsolete.stub_path / "METADATA.toml", "rb") as f: + meta = tomlkit.load(f) + meta["obsolete_since"] = obsolete.obsolete_since_version + with open(obsolete.stub_path / "METADATA.toml", "w") as f: + tomlkit.dump(meta, f) + subprocess.check_call(["git", "commit", "--all", "-m", title]) + if action_level <= ActionLevel.local: return - response.raise_for_status() + subprocess.check_call(["git", "push", "origin", branch_name, "--force-with-lease"]) + + await create_or_update_pull_request(title, branch_name, session) async def main() -> None: assert sys.version_info >= (3, 9) parser = argparse.ArgumentParser() - parser.add_argument("--dry-run", action="store_true") + parser.add_argument( + "--action-level", + type=lambda x: getattr(ActionLevel, x), # type: ignore[no-any-return] + default=ActionLevel.everything, + help="Limit actions performed to achieve dry runs for different levels of dryness", + ) args = parser.parse_args() try: conn = aiohttp.TCPConnector(limit_per_host=10) async with aiohttp.ClientSession(connector=conn) as session: - tasks = [ - asyncio.create_task(determine_action(stubs_path, session)) - for stubs_path in Path("stubs").iterdir() - ] + tasks = [asyncio.create_task(determine_action(stubs_path, session)) for stubs_path in Path("stubs").iterdir()] for task in asyncio.as_completed(tasks): update = await task + print(update) if isinstance(update, NoUpdate): continue if isinstance(update, Update): - await suggest_typeshed_update(update, session, dry_run=args.dry_run) + await suggest_typeshed_update(update, session, action_level=args.action_level) + continue + if isinstance(update, Obsolete): + await suggest_typeshed_obsolete(update, session, action_level=args.action_level) continue raise AssertionError finally: # if you need to cleanup, try: # git branch -D $(git branch --list 'stubsabot/*') - subprocess.check_call(["git", "checkout", "master"]) + if args.action_level >= ActionLevel.local: + subprocess.check_call(["git", "checkout", "master"]) if __name__ == "__main__": From c602858c759e5cc18d27ed69a8fde60fc99bacd4 Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Thu, 9 Jun 2022 13:57:10 -0700 Subject: [PATCH 03/11] improvements --- scripts/stubsabot.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/scripts/stubsabot.py b/scripts/stubsabot.py index e50a222aec2c..e1494e424a98 100644 --- a/scripts/stubsabot.py +++ b/scripts/stubsabot.py @@ -90,6 +90,7 @@ class Obsolete: distribution: str stub_path: Path obsolete_since_version: str + obsolete_since_date: datetime.datetime def __str__(self) -> str: return f"Marking {self.distribution} as obsolete since {self.obsolete_since_version!r}" @@ -149,7 +150,12 @@ async def determine_action(stub_path: Path, session: aiohttp.ClientSession) -> U return NoUpdate(stub_info.distribution, "up to date") if await package_contains_py_typed(pypi_info.release_to_download, session): - return Obsolete(stub_info.distribution, stub_path, obsolete_since_version=str(pypi_info.version)) + return Obsolete( + stub_info.distribution, + stub_path, + obsolete_since_version=str(pypi_info.version), + obsolete_since_date=pypi_info.upload_date, + ) return Update( distribution=stub_info.distribution, @@ -161,6 +167,7 @@ async def determine_action(stub_path: Path, session: aiohttp.ClientSession) -> U TYPESHED_OWNER = "python" FORK_OWNER = "hauntsaninja" +BRANCH_PREFIX = "stubsabot" async def create_or_update_pull_request(title: str, branch_name: str, session: aiohttp.ClientSession): @@ -214,7 +221,7 @@ async def suggest_typeshed_update(update: Update, session: aiohttp.ClientSession return title = f"[stubsabot] Bump {update.distribution} to {update.new_version_spec}" async with _repo_lock: - branch_name = f"stubsabot/{normalize(update.distribution)}" + branch_name = f"{BRANCH_PREFIX}/{normalize(update.distribution)}" subprocess.check_call(["git", "checkout", "-B", branch_name, "origin/master"]) with open(update.stub_path / "METADATA.toml", "rb") as f: meta = tomlkit.load(f) @@ -234,11 +241,13 @@ async def suggest_typeshed_obsolete(obsolete: Obsolete, session: aiohttp.ClientS return title = f"[stubsabot] Mark {obsolete.distribution} as obsolete since {obsolete.obsolete_since_version}" async with _repo_lock: - branch_name = f"stubsabot/{normalize(obsolete.distribution)}" + branch_name = f"{BRANCH_PREFIX}/{normalize(obsolete.distribution)}" subprocess.check_call(["git", "checkout", "-B", branch_name, "origin/master"]) with open(obsolete.stub_path / "METADATA.toml", "rb") as f: meta = tomlkit.load(f) - meta["obsolete_since"] = obsolete.obsolete_since_version + obs_string = tomlkit.string(obsolete.obsolete_since_version) + obs_string.comment(f"Released on {obsolete.obsolete_since_date.date().isoformat()}") + meta["obsolete_since"] = obs_string with open(obsolete.stub_path / "METADATA.toml", "w") as f: tomlkit.dump(meta, f) subprocess.check_call(["git", "commit", "--all", "-m", title]) From 701e6ad2577677be92b185d28bbb8b976b86760c Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Tue, 14 Jun 2022 16:39:10 -0700 Subject: [PATCH 04/11] more improvements --- scripts/stubsabot.py | 39 +++++++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/scripts/stubsabot.py b/scripts/stubsabot.py index e1494e424a98..170f2c027ec3 100644 --- a/scripts/stubsabot.py +++ b/scripts/stubsabot.py @@ -5,6 +5,7 @@ import asyncio import datetime import enum +import functools import io import os import re @@ -166,8 +167,15 @@ async def determine_action(stub_path: Path, session: aiohttp.ClientSession) -> U TYPESHED_OWNER = "python" -FORK_OWNER = "hauntsaninja" -BRANCH_PREFIX = "stubsabot" + + +@functools.lru_cache() +def get_origin_owner(): + output = subprocess.check_output(["git", "remote", "get-url", "origin"], text=True) + match = re.search(r"(git@github.com:|https://github.com/)(?P[^/]+)/(?P[^/]+).git", output) + assert match is not None + assert match.group("repo") == "typeshed" + return match.group("owner") async def create_or_update_pull_request(title: str, branch_name: str, session: aiohttp.ClientSession): @@ -177,9 +185,11 @@ async def create_or_update_pull_request(title: str, branch_name: str, session: a else: auth = f"Bearer {secret}" + fork_owner = get_origin_owner() + async with session.post( f"https://api.github.com/repos/{TYPESHED_OWNER}/typeshed/pulls", - json={"title": title, "head": f"{FORK_OWNER}:{branch_name}", "base": "master"}, + json={"title": title, "head": f"{fork_owner}:{branch_name}", "base": "master"}, headers={"Accept": "application/vnd.github.v3+json", "Authorization": auth}, ) as response: body = await response.json() @@ -189,7 +199,7 @@ async def create_or_update_pull_request(title: str, branch_name: str, session: a # Find the existing PR async with session.get( f"https://api.github.com/repos/{TYPESHED_OWNER}/typeshed/pulls", - params={"state": "open", "head": f"{FORK_OWNER}:{branch_name}", "base": "master"}, + params={"state": "open", "head": f"{fork_owner}:{branch_name}", "base": "master"}, headers={"Accept": "application/vnd.github.v3+json", "Authorization": auth}, ) as response: response.raise_for_status() @@ -215,6 +225,8 @@ def normalize(name: str) -> str: # lock should be unnecessary, but can't hurt to enforce mutual exclusion _repo_lock = asyncio.Lock() +BRANCH_PREFIX = "stubsabot" + async def suggest_typeshed_update(update: Update, session: aiohttp.ClientSession, action_level: ActionLevel) -> None: if action_level <= ActionLevel.nothing: @@ -268,17 +280,36 @@ async def main() -> None: default=ActionLevel.everything, help="Limit actions performed to achieve dry runs for different levels of dryness", ) + parser.add_argument( + "--action-count-limit", + type=int, + default=None, + help="Limit number of actions performed and the remainder are logged. Useful for testing", + ) args = parser.parse_args() + if args.action_level > ActionLevel.local: + if os.environ.get("GITHUB_TOKEN") is None: + raise ValueError("GITHUB_TOKEN environment variable must be set") + try: conn = aiohttp.TCPConnector(limit_per_host=10) async with aiohttp.ClientSession(connector=conn) as session: tasks = [asyncio.create_task(determine_action(stubs_path, session)) for stubs_path in Path("stubs").iterdir()] + + action_count = 0 for task in asyncio.as_completed(tasks): update = await task print(update) + if isinstance(update, NoUpdate): continue + + if args.action_count_limit is not None and action_count >= args.action_count_limit: + print("... but we've reached action count limit") + continue + action_count += 1 + if isinstance(update, Update): await suggest_typeshed_update(update, session, action_level=args.action_level) continue From d09808f468773f8198b08168e7ba8f2fb59a56d9 Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Tue, 14 Jun 2022 16:47:06 -0700 Subject: [PATCH 05/11] add body to PR --- scripts/stubsabot.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/scripts/stubsabot.py b/scripts/stubsabot.py index 170f2c027ec3..609b90362b32 100644 --- a/scripts/stubsabot.py +++ b/scripts/stubsabot.py @@ -178,7 +178,7 @@ def get_origin_owner(): return match.group("owner") -async def create_or_update_pull_request(title: str, branch_name: str, session: aiohttp.ClientSession): +async def create_or_update_pull_request(*, title: str, body: str, branch_name: str, session: aiohttp.ClientSession): secret = os.environ["GITHUB_TOKEN"] if secret.startswith("ghp"): auth = f"token {secret}" @@ -189,12 +189,12 @@ async def create_or_update_pull_request(title: str, branch_name: str, session: a async with session.post( f"https://api.github.com/repos/{TYPESHED_OWNER}/typeshed/pulls", - json={"title": title, "head": f"{fork_owner}:{branch_name}", "base": "master"}, + json={"title": title, "body": body, "head": f"{fork_owner}:{branch_name}", "base": "master"}, headers={"Accept": "application/vnd.github.v3+json", "Authorization": auth}, ) as response: - body = await response.json() + resp_json = await response.json() if response.status == 422 and any( - "A pull request already exists" in e.get("message", "") for e in body.get("errors", []) + "A pull request already exists" in e.get("message", "") for e in resp_json.get("errors", []) ): # Find the existing PR async with session.get( @@ -203,13 +203,13 @@ async def create_or_update_pull_request(title: str, branch_name: str, session: a headers={"Accept": "application/vnd.github.v3+json", "Authorization": auth}, ) as response: response.raise_for_status() - body = await response.json() - assert len(body) >= 1 - pr_number = body[0]["number"] - # Update the PR's title + resp_json = await response.json() + assert len(resp_json) >= 1 + pr_number = resp_json[0]["number"] + # Update the PR's title and body async with session.patch( f"https://api.github.com/repos/{TYPESHED_OWNER}/typeshed/pulls/{pr_number}", - json={"title": title}, + json={"title": title, "body": body}, headers={"Accept": "application/vnd.github.v3+json", "Authorization": auth}, ) as response: response.raise_for_status() @@ -245,7 +245,12 @@ async def suggest_typeshed_update(update: Update, session: aiohttp.ClientSession return subprocess.check_call(["git", "push", "origin", branch_name, "--force-with-lease"]) - await create_or_update_pull_request(title, branch_name, session) + body = """\ +If stubtest fails for this PR: +- Leave this PR open (as a reminder, and to prevent stubsabot from opening another PR) +- Fix stubtest failures in another PR, then close this PR +""" + await create_or_update_pull_request(title=title, body=body, branch_name=branch_name, session=session) async def suggest_typeshed_obsolete(obsolete: Obsolete, session: aiohttp.ClientSession, action_level: ActionLevel) -> None: @@ -267,7 +272,7 @@ async def suggest_typeshed_obsolete(obsolete: Obsolete, session: aiohttp.ClientS return subprocess.check_call(["git", "push", "origin", branch_name, "--force-with-lease"]) - await create_or_update_pull_request(title, branch_name, session) + await create_or_update_pull_request(title=title, body="", branch_name=branch_name, session=session) async def main() -> None: From d189ccdad40b10aa94aaa94c30b91cc27fe702ac Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Tue, 14 Jun 2022 23:24:30 -0700 Subject: [PATCH 06/11] fix stubsabot for gdb --- scripts/stubsabot.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/scripts/stubsabot.py b/scripts/stubsabot.py index 609b90362b32..2e3fd6fd1e0c 100644 --- a/scripts/stubsabot.py +++ b/scripts/stubsabot.py @@ -297,10 +297,16 @@ async def main() -> None: if os.environ.get("GITHUB_TOKEN") is None: raise ValueError("GITHUB_TOKEN environment variable must be set") + denylist = {"gdb"} # gdb is not a pypi distribution + try: conn = aiohttp.TCPConnector(limit_per_host=10) async with aiohttp.ClientSession(connector=conn) as session: - tasks = [asyncio.create_task(determine_action(stubs_path, session)) for stubs_path in Path("stubs").iterdir()] + tasks = [ + asyncio.create_task(determine_action(stubs_path, session)) + for stubs_path in Path("stubs").iterdir() + if stubs_path.name not in denylist + ] action_count = 0 for task in asyncio.as_completed(tasks): From 89f7950b8669a1aaa5a00e13c1b8801eeba0c190 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Fri, 17 Jun 2022 00:21:32 -0700 Subject: [PATCH 07/11] Update scripts/stubsabot.py Co-authored-by: Alex Waygood --- scripts/stubsabot.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/scripts/stubsabot.py b/scripts/stubsabot.py index 2e3fd6fd1e0c..cf7f6f409b4c 100644 --- a/scripts/stubsabot.py +++ b/scripts/stubsabot.py @@ -25,10 +25,16 @@ import tomlkit -class ActionLevel(enum.IntEnum): - nothing = 0 # make no changes - local = 1 # make changes that affect local repo - everything = 2 # do everything, e.g. open PRs +class ActionLevel(IntEnum): + def __new__(cls, value: int, doc: str): + member = int.__new__(cls, value) + member._value_ = value + member.__doc__ = doc + return member + + nothing = 0, "make no changes" + local = 1, "make changes that affect local repo" + everything = 2, "do everything, e.g. open PRs" @dataclass From 9a5ecd1b1be91d29380f7da85ca26aa233dc72b6 Mon Sep 17 00:00:00 2001 From: Shantanu <12621235+hauntsaninja@users.noreply.github.com> Date: Fri, 17 Jun 2022 00:22:20 -0700 Subject: [PATCH 08/11] Update scripts/stubsabot.py Co-authored-by: Alex Waygood --- scripts/stubsabot.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/stubsabot.py b/scripts/stubsabot.py index cf7f6f409b4c..d1935e2ba84c 100644 --- a/scripts/stubsabot.py +++ b/scripts/stubsabot.py @@ -116,16 +116,17 @@ async def package_contains_py_typed(release_to_download: dict[str, Any], session async with session.get(release_to_download["url"]) as response: body = io.BytesIO(await response.read()) - if release_to_download["packagetype"] == "bdist_wheel": + packagetype = release_to_download["packagetype"] + if packagetype == "bdist_wheel": assert release_to_download["filename"].endswith(".whl") with zipfile.ZipFile(body) as zf: return any(Path(f).name == "py.typed" for f in zf.namelist()) - elif release_to_download["packagetype"] == "sdist": + elif packagetype == "sdist": assert release_to_download["filename"].endswith(".tar.gz") with tarfile.open(fileobj=body, mode="r:gz") as zf: return any(Path(f).name == "py.typed" for f in zf.getnames()) else: - raise AssertionError + raise AssertionError(f"Unknown package type: {packagetype}") def _check_spec(updated_spec: str, version: packaging.version.Version) -> str: From 1606c29f3478e9284777112fb1251fb56c83334b Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Fri, 17 Jun 2022 21:34:50 -0700 Subject: [PATCH 09/11] fix lint --- scripts/stubsabot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/stubsabot.py b/scripts/stubsabot.py index d1935e2ba84c..bb5a4ef924c1 100644 --- a/scripts/stubsabot.py +++ b/scripts/stubsabot.py @@ -25,7 +25,7 @@ import tomlkit -class ActionLevel(IntEnum): +class ActionLevel(enum.IntEnum): def __new__(cls, value: int, doc: str): member = int.__new__(cls, value) member._value_ = value From 3b165616db98fd920190b3f671375cc994f4aa44 Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Fri, 17 Jun 2022 21:45:48 -0700 Subject: [PATCH 10/11] docstring --- scripts/stubsabot.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/scripts/stubsabot.py b/scripts/stubsabot.py index bb5a4ef924c1..efef92827dfe 100644 --- a/scripts/stubsabot.py +++ b/scripts/stubsabot.py @@ -135,6 +135,18 @@ def _check_spec(updated_spec: str, version: packaging.version.Version) -> str: def get_updated_version_spec(spec: str, version: packaging.version.Version) -> str: + """ + Given the old specifier and an updated version, returns an updated specifier that has the + specificity of the old specifier, but matches the updated version. + + For example: + spec="1", version="1.2.3" -> "1.2.3" + spec="1.0.1", version="1.2.3" -> "1.2.3" + spec="1.*", version="1.2.3" -> "1.*" + spec="1.*", version="2.3.4" -> "2.*" + spec="1.1.*", version="1.2.3" -> "1.2.*" + spec="1.1.1.*", version="1.2.3" -> "1.2.3.*" + """ if not spec.endswith(".*"): return _check_spec(version.base_version, version) From 1291454925a0e4218af67867f7b58b0285001eac Mon Sep 17 00:00:00 2001 From: hauntsaninja <> Date: Fri, 17 Jun 2022 21:49:29 -0700 Subject: [PATCH 11/11] link to pypi documentation for release --- scripts/stubsabot.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/stubsabot.py b/scripts/stubsabot.py index efef92827dfe..9c36fbdf5c9b 100644 --- a/scripts/stubsabot.py +++ b/scripts/stubsabot.py @@ -61,6 +61,8 @@ class PypiInfo: distribution: str version: packaging.version.Version upload_date: datetime.datetime + # https://warehouse.pypa.io/api-reference/json.html#get--pypi--project_name--json + # Corresponds to a single entry from `releases` for the given version release_to_download: dict[str, Any]