From b04b01365f98a4880724a8088efabaaa222b9c61 Mon Sep 17 00:00:00 2001 From: Douglas Coburn Date: Fri, 31 May 2024 18:08:06 -0400 Subject: [PATCH 1/6] Added fixes for missing commit-sha option. Added option to export SBOM file. Added node, npm, and yarn to Dockerfile --- Dockerfile | 4 +-- pyproject.toml | 2 +- socketsecurity/core/__init__.py | 64 ++++++++++++++++++++++++++++++--- socketsecurity/core/github.py | 30 ++++++++++------ socketsecurity/core/gitlab.py | 32 ++++++++++------- socketsecurity/socketcli.py | 35 ++++++++++++++++-- test.py | 35 ++++++++++++++++++ 7 files changed, 169 insertions(+), 33 deletions(-) create mode 100644 test.py diff --git a/Dockerfile b/Dockerfile index 248810d..33025d2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,5 +2,5 @@ FROM python:3-alpine LABEL org.opencontainers.image.authors="socket.dev" RUN apk update \ - && apk add --no-cache git -RUN pip install socketsecurity \ No newline at end of file + && apk add --no-cache git nodejs npm yarn +RUN pip install socketsecurity --upgrade \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 471d801..388d66b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "socketsecurity" -version = "0.0.67" +version = "0.0.72" requires-python = ">= 3.9" dependencies = [ 'requests', diff --git a/socketsecurity/core/__init__.py b/socketsecurity/core/__init__.py index 4bdbe97..23ae98a 100644 --- a/socketsecurity/core/__init__.py +++ b/socketsecurity/core/__init__.py @@ -25,7 +25,7 @@ __author__ = 'socket.dev' -__version__ = '0.0.67' +__version__ = '0.0.72' __all__ = [ "Core", "log", @@ -246,6 +246,55 @@ def get_security_policy() -> dict: } return org_rules + # @staticmethod + # def get_supported_file_types() -> dict: + # path = "report/supported" + + @staticmethod + def get_manifest_files(package: Package, packages: dict) -> str: + if package.direct: + manifests = [] + for manifest_item in package.manifestFiles: + manifest = manifest_item["file"] + manifests.append(manifest) + manifest_files = ";".join(manifests) + else: + manifests = [] + for top_id in package.topLevelAncestors: + top_package: Package + top_package = packages[top_id] + for manifest_item in top_package.manifestFiles: + manifest = manifest_item["file"] + new_string = f"{package.name}@{package.version}({manifest})" + manifests.append(new_string) + manifest_files = ";".join(manifests) + return manifest_files + + @staticmethod + def create_sbom_output(diff: Diff) -> list: + sbom = [] + for package_id in diff.packages: + package: Package + package = diff.packages[package_id] + manifest_files = Core.get_manifest_files(package, diff.packages) + item = { + "id": package.id, + "license": package.license, + "license_text": package.license_text, + "manifestFiles": manifest_files, + "score": package.score, + "size": package.size, + "ecosystem": package.type, + "alerts": package.alerts, + "direct": package.direct, + "name": package.name, + "version": package.version, + "author": package.author, + "url": package.url + } + sbom.append(item) + return sbom + @staticmethod def find_files(path: str) -> list: """ @@ -314,8 +363,8 @@ def find_files(path: str) -> list: "requirements.frozen": { "pattern": "requirements.frozen" }, - "setup.py.old": { - "pattern": "setup.py.old" + "setup.py": { + "pattern": "setup.py" } }, "golang": { @@ -325,6 +374,11 @@ def find_files(path: str) -> list: "go.sum": { "pattern": "go.sum" } + }, + "java": { + "pom.xml": { + "pattern": "pom.xml" + } } } all_files = [] @@ -469,12 +523,12 @@ def compare_sboms(new_scan: list, head_scan: list) -> Diff: for package_id in new_packages: purl, package = Core.create_purl(package_id, new_packages) - if package_id not in head_packages: + if package_id not in head_packages and package.direct: diff.new_packages.append(purl) new_scan_alerts = Core.create_issue_alerts(package, new_scan_alerts, new_packages) for package_id in head_packages: purl, package = Core.create_purl(package_id, head_packages) - if package_id not in new_packages: + if package_id not in new_packages and package.direct: diff.removed_packages.append(purl) head_scan_alerts = Core.create_issue_alerts(package, head_scan_alerts, head_packages) diff.new_alerts = Core.compare_issue_alerts(new_scan_alerts, head_scan_alerts, diff.new_alerts) diff --git a/socketsecurity/core/github.py b/socketsecurity/core/github.py index d2dc249..cafbed4 100644 --- a/socketsecurity/core/github.py +++ b/socketsecurity/core/github.py @@ -163,19 +163,27 @@ def check_event_type() -> str: return event_type @staticmethod - def add_socket_comments(security_comment: str, overview_comment: str, comments: dict) -> None: + def add_socket_comments( + security_comment: str, + overview_comment: str, + comments: dict, + new_security_comment: bool = True, + new_overview_comment: bool = True + ) -> None: existing_overview_comment = comments.get("overview") existing_security_comment = comments.get("security") - if existing_overview_comment is not None: - existing_overview_comment: GithubComment - Github.update_comment(overview_comment, str(existing_overview_comment.id)) - else: - Github.post_comment(overview_comment) - if existing_security_comment is not None: - existing_security_comment: GithubComment - Github.update_comment(security_comment, str(existing_security_comment.id)) - else: - Github.post_comment(security_comment) + if new_overview_comment: + if existing_overview_comment is not None: + existing_overview_comment: GithubComment + Github.update_comment(overview_comment, str(existing_overview_comment.id)) + else: + Github.post_comment(overview_comment) + if new_security_comment: + if existing_security_comment is not None: + existing_security_comment: GithubComment + Github.update_comment(security_comment, str(existing_security_comment.id)) + else: + Github.post_comment(security_comment) @staticmethod def post_comment(body: str) -> None: diff --git a/socketsecurity/core/gitlab.py b/socketsecurity/core/gitlab.py index ccaf590..09cae98 100644 --- a/socketsecurity/core/gitlab.py +++ b/socketsecurity/core/gitlab.py @@ -141,7 +141,7 @@ def __init__(self): self.api_token = gitlab_token self.project_id = ci_merge_request_project_id if self.api_token is None: - print("Unable to get gitlab API Token from GH_API_TOKEN") + print("Unable to get gitlab API Token from GITLAB_TOKEN") sys.exit(2) @staticmethod @@ -159,19 +159,27 @@ def check_event_type() -> str: return event_type @staticmethod - def add_socket_comments(security_comment: str, overview_comment: str, comments: dict) -> None: + def add_socket_comments( + security_comment: str, + overview_comment: str, + comments: dict, + new_security_comment: bool = True, + new_overview_comment: bool = True + ) -> None: existing_overview_comment = comments.get("overview") existing_security_comment = comments.get("security") - if existing_overview_comment is not None: - existing_overview_comment: GitlabComment - Gitlab.update_comment(overview_comment, str(existing_overview_comment.id)) - else: - Gitlab.post_comment(overview_comment) - if existing_security_comment is not None: - existing_security_comment: GitlabComment - Gitlab.update_comment(security_comment, str(existing_security_comment.id)) - else: - Gitlab.post_comment(security_comment) + if new_overview_comment: + if existing_overview_comment is not None: + existing_overview_comment: GitlabComment + Gitlab.update_comment(overview_comment, str(existing_overview_comment.id)) + else: + Gitlab.post_comment(overview_comment) + if new_security_comment: + if existing_security_comment is not None: + existing_security_comment: GitlabComment + Gitlab.update_comment(security_comment, str(existing_security_comment.id)) + else: + Gitlab.post_comment(security_comment) @staticmethod def post_comment(body: str) -> None: diff --git a/socketsecurity/socketcli.py b/socketsecurity/socketcli.py index c5660d2..becf6f9 100644 --- a/socketsecurity/socketcli.py +++ b/socketsecurity/socketcli.py @@ -69,6 +69,20 @@ required=False ) +parser.add_argument( + '--sbom-file', + default=None, + help='If soecified save the SBOM details to the specified file', + required=False +) + +parser.add_argument( + '--commit-sha', + default="", + help='Optional git commit sha', + required=False +) + parser.add_argument( '--generate-license', default=False, @@ -115,6 +129,8 @@ def cli(): pr_number = arguments.pr_number target_path = arguments.target_path scm_type = arguments.scm + commit_sha = arguments.commit_sha + sbom_file = arguments.sbom_file license_mode = arguments.generate_license license_file = f"{repo}" if branch is not None: @@ -137,6 +153,7 @@ def cli(): scm = Gitlab() if scm is not None: default_branch = scm.is_default_branch + base_api_url = os.getenv("BASE_API_URL") or None core = Core(token=api_token, request_timeout=6000, base_api_url=base_api_url) set_as_pending_head = False @@ -146,7 +163,7 @@ def cli(): repo=repo, branch=branch, commit_message=commit_message, - commit_hash="", + commit_hash=commit_sha, pull_request=pr_number, committers=committer, make_default_branch=default_branch, @@ -166,7 +183,19 @@ def cli(): diff.new_alerts = scm.remove_alerts(comments, diff.new_alerts) overview_comment = Messages.dependency_overview_template(diff) security_comment = Messages.security_comment_template(diff) - scm.add_socket_comments(security_comment, overview_comment, comments) + new_security_comment = True + new_overview_comment = True + if len(diff.new_alerts) == 0: + new_security_comment = False + if len(diff.new_packages) == 0 and diff.removed_packages == 0: + new_overview_comment = False + scm.add_socket_comments( + security_comment, + overview_comment, + comments, + new_security_comment, + new_overview_comment + ) output_console_comments(diff) else: log.info("API Mode") @@ -190,6 +219,8 @@ def cli(): } all_packages[package_id] = output core.save_file(license_file, json.dumps(all_packages)) + if diff is not None and sbom_file is not None: + core.save_file(sbom_file, json.dumps(core.create_sbom_output(diff))) if __name__ == '__main__': diff --git a/test.py b/test.py new file mode 100644 index 0000000..23517cc --- /dev/null +++ b/test.py @@ -0,0 +1,35 @@ +import requests +import base64 +from urllib.parse import urlencode +import time +api_token = "sktsec_b1q-gQvQXJMR0ZtfzPgmQx_PZNRhxEsxJUoOMK7rYhzM_api" +token = f"{api_token}:" +encoded_token = base64.b64encode(token.encode()).decode('ascii') + +url = 'https://api.socket.dev/v0/orgs/socketdev-demo/full-scans' +params = { + 'repo': 'new-local-test', + 'branch': 'test' +} +full_url = f"{url}?{urlencode(params)}" +headers = { + 'Accept': 'application/json', + 'Authorization': f'Basic {encoded_token}' +} +key = 'requirements.txt' +files = [] +file = ( + key, + ( + key, + open(key, 'rb') + ) +) +files.append(file) + +resp = requests.post(full_url, headers=headers, files=files) + +print(resp.status_code) +print(resp.text) + +time.sleep(10) \ No newline at end of file From 96c5f39947d10e1059e58e4f16f5b7e0c346aa5d Mon Sep 17 00:00:00 2001 From: Douglas Date: Mon, 3 Jun 2024 14:10:05 -0700 Subject: [PATCH 2/6] Added fixes for missing commit-sha option. Added option to export SBOM file. Added node, npm, and yarn to Dockerfile (#1) --- Dockerfile | 4 +-- pyproject.toml | 2 +- socketsecurity/core/__init__.py | 64 ++++++++++++++++++++++++++++++--- socketsecurity/core/github.py | 30 ++++++++++------ socketsecurity/core/gitlab.py | 32 ++++++++++------- socketsecurity/socketcli.py | 35 ++++++++++++++++-- test.py | 35 ++++++++++++++++++ 7 files changed, 169 insertions(+), 33 deletions(-) create mode 100644 test.py diff --git a/Dockerfile b/Dockerfile index 248810d..33025d2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,5 +2,5 @@ FROM python:3-alpine LABEL org.opencontainers.image.authors="socket.dev" RUN apk update \ - && apk add --no-cache git -RUN pip install socketsecurity \ No newline at end of file + && apk add --no-cache git nodejs npm yarn +RUN pip install socketsecurity --upgrade \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 471d801..388d66b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "socketsecurity" -version = "0.0.67" +version = "0.0.72" requires-python = ">= 3.9" dependencies = [ 'requests', diff --git a/socketsecurity/core/__init__.py b/socketsecurity/core/__init__.py index 4bdbe97..23ae98a 100644 --- a/socketsecurity/core/__init__.py +++ b/socketsecurity/core/__init__.py @@ -25,7 +25,7 @@ __author__ = 'socket.dev' -__version__ = '0.0.67' +__version__ = '0.0.72' __all__ = [ "Core", "log", @@ -246,6 +246,55 @@ def get_security_policy() -> dict: } return org_rules + # @staticmethod + # def get_supported_file_types() -> dict: + # path = "report/supported" + + @staticmethod + def get_manifest_files(package: Package, packages: dict) -> str: + if package.direct: + manifests = [] + for manifest_item in package.manifestFiles: + manifest = manifest_item["file"] + manifests.append(manifest) + manifest_files = ";".join(manifests) + else: + manifests = [] + for top_id in package.topLevelAncestors: + top_package: Package + top_package = packages[top_id] + for manifest_item in top_package.manifestFiles: + manifest = manifest_item["file"] + new_string = f"{package.name}@{package.version}({manifest})" + manifests.append(new_string) + manifest_files = ";".join(manifests) + return manifest_files + + @staticmethod + def create_sbom_output(diff: Diff) -> list: + sbom = [] + for package_id in diff.packages: + package: Package + package = diff.packages[package_id] + manifest_files = Core.get_manifest_files(package, diff.packages) + item = { + "id": package.id, + "license": package.license, + "license_text": package.license_text, + "manifestFiles": manifest_files, + "score": package.score, + "size": package.size, + "ecosystem": package.type, + "alerts": package.alerts, + "direct": package.direct, + "name": package.name, + "version": package.version, + "author": package.author, + "url": package.url + } + sbom.append(item) + return sbom + @staticmethod def find_files(path: str) -> list: """ @@ -314,8 +363,8 @@ def find_files(path: str) -> list: "requirements.frozen": { "pattern": "requirements.frozen" }, - "setup.py.old": { - "pattern": "setup.py.old" + "setup.py": { + "pattern": "setup.py" } }, "golang": { @@ -325,6 +374,11 @@ def find_files(path: str) -> list: "go.sum": { "pattern": "go.sum" } + }, + "java": { + "pom.xml": { + "pattern": "pom.xml" + } } } all_files = [] @@ -469,12 +523,12 @@ def compare_sboms(new_scan: list, head_scan: list) -> Diff: for package_id in new_packages: purl, package = Core.create_purl(package_id, new_packages) - if package_id not in head_packages: + if package_id not in head_packages and package.direct: diff.new_packages.append(purl) new_scan_alerts = Core.create_issue_alerts(package, new_scan_alerts, new_packages) for package_id in head_packages: purl, package = Core.create_purl(package_id, head_packages) - if package_id not in new_packages: + if package_id not in new_packages and package.direct: diff.removed_packages.append(purl) head_scan_alerts = Core.create_issue_alerts(package, head_scan_alerts, head_packages) diff.new_alerts = Core.compare_issue_alerts(new_scan_alerts, head_scan_alerts, diff.new_alerts) diff --git a/socketsecurity/core/github.py b/socketsecurity/core/github.py index d2dc249..cafbed4 100644 --- a/socketsecurity/core/github.py +++ b/socketsecurity/core/github.py @@ -163,19 +163,27 @@ def check_event_type() -> str: return event_type @staticmethod - def add_socket_comments(security_comment: str, overview_comment: str, comments: dict) -> None: + def add_socket_comments( + security_comment: str, + overview_comment: str, + comments: dict, + new_security_comment: bool = True, + new_overview_comment: bool = True + ) -> None: existing_overview_comment = comments.get("overview") existing_security_comment = comments.get("security") - if existing_overview_comment is not None: - existing_overview_comment: GithubComment - Github.update_comment(overview_comment, str(existing_overview_comment.id)) - else: - Github.post_comment(overview_comment) - if existing_security_comment is not None: - existing_security_comment: GithubComment - Github.update_comment(security_comment, str(existing_security_comment.id)) - else: - Github.post_comment(security_comment) + if new_overview_comment: + if existing_overview_comment is not None: + existing_overview_comment: GithubComment + Github.update_comment(overview_comment, str(existing_overview_comment.id)) + else: + Github.post_comment(overview_comment) + if new_security_comment: + if existing_security_comment is not None: + existing_security_comment: GithubComment + Github.update_comment(security_comment, str(existing_security_comment.id)) + else: + Github.post_comment(security_comment) @staticmethod def post_comment(body: str) -> None: diff --git a/socketsecurity/core/gitlab.py b/socketsecurity/core/gitlab.py index ccaf590..09cae98 100644 --- a/socketsecurity/core/gitlab.py +++ b/socketsecurity/core/gitlab.py @@ -141,7 +141,7 @@ def __init__(self): self.api_token = gitlab_token self.project_id = ci_merge_request_project_id if self.api_token is None: - print("Unable to get gitlab API Token from GH_API_TOKEN") + print("Unable to get gitlab API Token from GITLAB_TOKEN") sys.exit(2) @staticmethod @@ -159,19 +159,27 @@ def check_event_type() -> str: return event_type @staticmethod - def add_socket_comments(security_comment: str, overview_comment: str, comments: dict) -> None: + def add_socket_comments( + security_comment: str, + overview_comment: str, + comments: dict, + new_security_comment: bool = True, + new_overview_comment: bool = True + ) -> None: existing_overview_comment = comments.get("overview") existing_security_comment = comments.get("security") - if existing_overview_comment is not None: - existing_overview_comment: GitlabComment - Gitlab.update_comment(overview_comment, str(existing_overview_comment.id)) - else: - Gitlab.post_comment(overview_comment) - if existing_security_comment is not None: - existing_security_comment: GitlabComment - Gitlab.update_comment(security_comment, str(existing_security_comment.id)) - else: - Gitlab.post_comment(security_comment) + if new_overview_comment: + if existing_overview_comment is not None: + existing_overview_comment: GitlabComment + Gitlab.update_comment(overview_comment, str(existing_overview_comment.id)) + else: + Gitlab.post_comment(overview_comment) + if new_security_comment: + if existing_security_comment is not None: + existing_security_comment: GitlabComment + Gitlab.update_comment(security_comment, str(existing_security_comment.id)) + else: + Gitlab.post_comment(security_comment) @staticmethod def post_comment(body: str) -> None: diff --git a/socketsecurity/socketcli.py b/socketsecurity/socketcli.py index c5660d2..becf6f9 100644 --- a/socketsecurity/socketcli.py +++ b/socketsecurity/socketcli.py @@ -69,6 +69,20 @@ required=False ) +parser.add_argument( + '--sbom-file', + default=None, + help='If soecified save the SBOM details to the specified file', + required=False +) + +parser.add_argument( + '--commit-sha', + default="", + help='Optional git commit sha', + required=False +) + parser.add_argument( '--generate-license', default=False, @@ -115,6 +129,8 @@ def cli(): pr_number = arguments.pr_number target_path = arguments.target_path scm_type = arguments.scm + commit_sha = arguments.commit_sha + sbom_file = arguments.sbom_file license_mode = arguments.generate_license license_file = f"{repo}" if branch is not None: @@ -137,6 +153,7 @@ def cli(): scm = Gitlab() if scm is not None: default_branch = scm.is_default_branch + base_api_url = os.getenv("BASE_API_URL") or None core = Core(token=api_token, request_timeout=6000, base_api_url=base_api_url) set_as_pending_head = False @@ -146,7 +163,7 @@ def cli(): repo=repo, branch=branch, commit_message=commit_message, - commit_hash="", + commit_hash=commit_sha, pull_request=pr_number, committers=committer, make_default_branch=default_branch, @@ -166,7 +183,19 @@ def cli(): diff.new_alerts = scm.remove_alerts(comments, diff.new_alerts) overview_comment = Messages.dependency_overview_template(diff) security_comment = Messages.security_comment_template(diff) - scm.add_socket_comments(security_comment, overview_comment, comments) + new_security_comment = True + new_overview_comment = True + if len(diff.new_alerts) == 0: + new_security_comment = False + if len(diff.new_packages) == 0 and diff.removed_packages == 0: + new_overview_comment = False + scm.add_socket_comments( + security_comment, + overview_comment, + comments, + new_security_comment, + new_overview_comment + ) output_console_comments(diff) else: log.info("API Mode") @@ -190,6 +219,8 @@ def cli(): } all_packages[package_id] = output core.save_file(license_file, json.dumps(all_packages)) + if diff is not None and sbom_file is not None: + core.save_file(sbom_file, json.dumps(core.create_sbom_output(diff))) if __name__ == '__main__': diff --git a/test.py b/test.py new file mode 100644 index 0000000..23517cc --- /dev/null +++ b/test.py @@ -0,0 +1,35 @@ +import requests +import base64 +from urllib.parse import urlencode +import time +api_token = "sktsec_b1q-gQvQXJMR0ZtfzPgmQx_PZNRhxEsxJUoOMK7rYhzM_api" +token = f"{api_token}:" +encoded_token = base64.b64encode(token.encode()).decode('ascii') + +url = 'https://api.socket.dev/v0/orgs/socketdev-demo/full-scans' +params = { + 'repo': 'new-local-test', + 'branch': 'test' +} +full_url = f"{url}?{urlencode(params)}" +headers = { + 'Accept': 'application/json', + 'Authorization': f'Basic {encoded_token}' +} +key = 'requirements.txt' +files = [] +file = ( + key, + ( + key, + open(key, 'rb') + ) +) +files.append(file) + +resp = requests.post(full_url, headers=headers, files=files) + +print(resp.status_code) +print(resp.text) + +time.sleep(10) \ No newline at end of file From 3f6789bc962852682944d019ebeaf717c7ecb6c8 Mon Sep 17 00:00:00 2001 From: Douglas Date: Tue, 4 Jun 2024 19:26:06 -0700 Subject: [PATCH 3/6] Doug/fix gitlab main flow (#2) * Added fixes for missing commit-sha option. Added option to export SBOM file. Added node, npm, and yarn to Dockerfile * Fixed how to check for the gitlab workflow --- pyproject.toml | 2 +- socketsecurity/core/__init__.py | 2 +- socketsecurity/core/gitlab.py | 2 +- socketsecurity/socketcli.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 388d66b..accca17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "socketsecurity" -version = "0.0.72" +version = "0.0.74" requires-python = ">= 3.9" dependencies = [ 'requests', diff --git a/socketsecurity/core/__init__.py b/socketsecurity/core/__init__.py index 23ae98a..f6c1ebb 100644 --- a/socketsecurity/core/__init__.py +++ b/socketsecurity/core/__init__.py @@ -25,7 +25,7 @@ __author__ = 'socket.dev' -__version__ = '0.0.72' +__version__ = '0.0.74' __all__ = [ "Core", "log", diff --git a/socketsecurity/core/gitlab.py b/socketsecurity/core/gitlab.py index 09cae98..78019bf 100644 --- a/socketsecurity/core/gitlab.py +++ b/socketsecurity/core/gitlab.py @@ -147,7 +147,7 @@ def __init__(self): @staticmethod def check_event_type() -> str: if ci_pipeline_source.lower() == "push" or ci_pipeline_source.lower() == 'merge_request_event': - if ci_merge_request_iid is None or ci_merge_request_iid == "": + if ci_merge_request_iid is None or ci_merge_request_iid == "" or str(ci_merge_request_iid) == "0": event_type = "main" else: event_type = "diff" diff --git a/socketsecurity/socketcli.py b/socketsecurity/socketcli.py index becf6f9..3619248 100644 --- a/socketsecurity/socketcli.py +++ b/socketsecurity/socketcli.py @@ -39,7 +39,7 @@ ) parser.add_argument( '--pr_number', - default=0, + default="0", help='The pr or build number', required=False ) From 42584b8969429380ac54a32eeb7f141c06fcc344 Mon Sep 17 00:00:00 2001 From: Douglas Coburn Date: Wed, 12 Jun 2024 15:51:22 -0700 Subject: [PATCH 4/6] Added in fix for unknown issue types and updated .gitignore --- .gitignore | 3 ++- pyproject.toml | 2 +- socketsecurity/core/__init__.py | 7 +++++-- test.py | 35 --------------------------------- 4 files changed, 8 insertions(+), 39 deletions(-) delete mode 100644 test.py diff --git a/.gitignore b/.gitignore index 3fbc794..23f720f 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,5 @@ scripts/*.py markdown_overview_temp.md markdown_security_temp.md .DS_Store -*.pyc \ No newline at end of file +*.pyc +test.py \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index accca17..20da55a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "socketsecurity" -version = "0.0.74" +version = "0.0.76" requires-python = ">= 3.9" dependencies = [ 'requests', diff --git a/socketsecurity/core/__init__.py b/socketsecurity/core/__init__.py index f6c1ebb..d79e0fb 100644 --- a/socketsecurity/core/__init__.py +++ b/socketsecurity/core/__init__.py @@ -25,7 +25,7 @@ __author__ = 'socket.dev' -__version__ = '0.0.74' +__version__ = '0.0.76' __all__ = [ "Core", "log", @@ -644,7 +644,10 @@ def create_issue_alerts(package: Package, alerts: dict, packages: dict) -> dict: """ for item in package.alerts: alert = Alert(**item) - props = getattr(all_issues, alert.type) + try: + props = getattr(all_issues, alert.type) + except AttributeError: + props = None if props is not None: description = props.description title = props.title diff --git a/test.py b/test.py deleted file mode 100644 index 23517cc..0000000 --- a/test.py +++ /dev/null @@ -1,35 +0,0 @@ -import requests -import base64 -from urllib.parse import urlencode -import time -api_token = "sktsec_b1q-gQvQXJMR0ZtfzPgmQx_PZNRhxEsxJUoOMK7rYhzM_api" -token = f"{api_token}:" -encoded_token = base64.b64encode(token.encode()).decode('ascii') - -url = 'https://api.socket.dev/v0/orgs/socketdev-demo/full-scans' -params = { - 'repo': 'new-local-test', - 'branch': 'test' -} -full_url = f"{url}?{urlencode(params)}" -headers = { - 'Accept': 'application/json', - 'Authorization': f'Basic {encoded_token}' -} -key = 'requirements.txt' -files = [] -file = ( - key, - ( - key, - open(key, 'rb') - ) -) -files.append(file) - -resp = requests.post(full_url, headers=headers, files=files) - -print(resp.status_code) -print(resp.text) - -time.sleep(10) \ No newline at end of file From 878330ff57dbd3b92de97bc50a14dd401cd4d544 Mon Sep 17 00:00:00 2001 From: Douglas Coburn Date: Wed, 12 Jun 2024 15:53:44 -0700 Subject: [PATCH 5/6] Added handler for keyboard interrupt and unexpected errors. Unexpected errors will have exit code 3, keyboard interrupt will be 2, unhealthy report due to Security policy 1, and no issues exit code 0 --- socketsecurity/socketcli.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/socketsecurity/socketcli.py b/socketsecurity/socketcli.py index 3619248..b6caddb 100644 --- a/socketsecurity/socketcli.py +++ b/socketsecurity/socketcli.py @@ -105,6 +105,13 @@ default=False ) +parser.add_argument( + '--enable-json', + help='Enable json output of results instead of table formatted', + action='store_true', + default=False +) + def output_console_comments(diff_report) -> None: console_security_comment = Messages.create_console_security_alert_table(diff_report) @@ -117,6 +124,18 @@ def output_console_comments(diff_report) -> None: def cli(): + try: + main_code() + except KeyboardInterrupt: + log.info("Keyboard Interrupt detected, exiting") + sys.exit(0) + except Exception as error: + log.error("Unexpected error when running the cli") + log.error(error) + sys.exit(2) + + +def main_code(): arguments = parser.parse_args() debug = arguments.enable_debug if debug: @@ -132,6 +151,7 @@ def cli(): commit_sha = arguments.commit_sha sbom_file = arguments.sbom_file license_mode = arguments.generate_license + enable_json = arguments.enable_json license_file = f"{repo}" if branch is not None: license_file += f"_{branch}" From 01406392ac60be38cbfc9609e0c36ade072ba753 Mon Sep 17 00:00:00 2001 From: Douglas Coburn Date: Wed, 12 Jun 2024 16:21:08 -0700 Subject: [PATCH 6/6] Added support for JSON output for the CLI with --enable-json --- socketsecurity/core/__init__.py | 4 ++-- socketsecurity/core/classes.py | 4 ++++ socketsecurity/core/messages.py | 40 +++++++++++++++++---------------- socketsecurity/socketcli.py | 21 +++++++++++++---- 4 files changed, 44 insertions(+), 25 deletions(-) diff --git a/socketsecurity/core/__init__.py b/socketsecurity/core/__init__.py index d79e0fb..290dbf8 100644 --- a/socketsecurity/core/__init__.py +++ b/socketsecurity/core/__init__.py @@ -489,7 +489,7 @@ def create_new_diff(path: str, params: FullScanParams, workspace: str) -> Diff: head_full_scan = Core.get_sbom_data(head_full_scan_id) head_end = time.time() total_head_time = head_end - head_start - print(f"Total time to get head full-scan {total_head_time: .2f}") + log.info(f"Total time to get head full-scan {total_head_time: .2f}") except APIResourceNotFound: head_full_scan = [] if files is not None and len(files) > 0: @@ -498,7 +498,7 @@ def create_new_diff(path: str, params: FullScanParams, workspace: str) -> Diff: new_full_scan.packages = Core.create_sbom_dict(new_full_scan.sbom_artifacts) new_scan_end = time.time() total_new_time = new_scan_end - new_scan_start - print(f"Total time to get new full-scan {total_new_time: .2f}") + log.info(f"Total time to get new full-scan {total_new_time: .2f}") diff_report = Core.compare_sboms(new_full_scan.sbom_artifacts, head_full_scan) diff_report.packages = new_full_scan.packages else: diff --git a/socketsecurity/core/classes.py b/socketsecurity/core/classes.py index 30b735a..c1fd0c3 100644 --- a/socketsecurity/core/classes.py +++ b/socketsecurity/core/classes.py @@ -145,6 +145,8 @@ class Issue: suggestion: str introduced_by: list manifests: str + url: str + purl: str def __init__(self, **kwargs): if kwargs: @@ -157,6 +159,8 @@ def __init__(self, **kwargs): self.introduced_by = [] if not hasattr(self, "manifests"): self.manifests = "" + self.url = f"https://socket.dev/{self.pkg_type}/{self.pkg_name}/overview/{self.pkg_version}" + self.purl = f"{self.pkg_type}/{self.pkg_name}@{self.pkg_version}" def __str__(self): return json.dumps(self.__dict__) diff --git a/socketsecurity/core/messages.py b/socketsecurity/core/messages.py index 4aa4290..1984236 100644 --- a/socketsecurity/core/messages.py +++ b/socketsecurity/core/messages.py @@ -1,3 +1,5 @@ +import json + from mdutils import MdUtils from socketsecurity.core.classes import Diff, Purl, Issue from prettytable import PrettyTable @@ -5,6 +7,22 @@ class Messages: + @staticmethod + def create_security_comment_json(diff: Diff) -> dict: + if len(diff.new_alerts) == 0: + scan_failed = False + else: + scan_failed = True + output = { + "scan_failed": scan_failed, + "new_alerts": [] + } + for alert in diff.new_alerts: + alert: Issue + output["new_alerts"].append(json.loads(str(alert))) + return output + + @staticmethod def security_comment_template(diff: Diff) -> str: """ @@ -124,14 +142,13 @@ def create_security_alert_table(diff: Diff, md: MdUtils) -> (MdUtils, list, dict alert.description, alert.suggestion ] - package_url, purl = Messages.create_package_link(alert) - ignore = f"`SocketSecurity ignore {purl}`" + ignore = f"`SocketSecurity ignore {alert.purl}`" if ignore not in ignore_commands: ignore_commands.append(ignore) manifest_str, sources = Messages.create_sources(alert, "console") row = [ alert.title, - package_url, + alert.url, ", ".join(sources), manifest_str ] @@ -247,20 +264,6 @@ def create_purl_link(details: Purl) -> str: package_url = f"[{purl}](https://socket.dev/{details.ecosystem}/{details.name}/overview/{details.version})" return package_url - @staticmethod - def create_package_link(details: Issue) -> (str, str): - """ - Creates the package link for the Security Comment Template - :param details: Purl - Details about the package needed to create the URLs - :return: - """ - purl = f"{details.pkg_name}@{details.pkg_version}" - package_url = ( - f"[{purl}]" - f"(https://socket.dev/{details.pkg_type}/{details.pkg_name}/overview/{details.pkg_version})" - ) - return package_url, purl - @staticmethod def create_console_security_alert_table(diff: Diff) -> PrettyTable: """ @@ -278,11 +281,10 @@ def create_console_security_alert_table(diff: Diff) -> PrettyTable: ) for alert in diff.new_alerts: alert: Issue - package_url, purl = Messages.create_package_link(alert) manifest_str, sources = Messages.create_sources(alert, "console") row = [ alert.title, - package_url, + alert.url, ", ".join(sources), manifest_str ] diff --git a/socketsecurity/socketcli.py b/socketsecurity/socketcli.py index b6caddb..4186210 100644 --- a/socketsecurity/socketcli.py +++ b/socketsecurity/socketcli.py @@ -123,16 +123,23 @@ def output_console_comments(diff_report) -> None: log.info("No New Security issues detected by Socket Security") +def output_console_json(diff_report) -> None: + console_security_comment = Messages.create_security_comment_json(diff_report) + print(json.dumps(console_security_comment)) + if len(diff_report.new_alerts) > 0: + sys.exit(1) + + def cli(): try: main_code() except KeyboardInterrupt: log.info("Keyboard Interrupt detected, exiting") - sys.exit(0) + sys.exit(2) except Exception as error: log.error("Unexpected error when running the cli") log.error(error) - sys.exit(2) + sys.exit(3) def main_code(): @@ -216,12 +223,18 @@ def main_code(): new_security_comment, new_overview_comment ) - output_console_comments(diff) + if enable_json: + output_console_json(diff) + else: + output_console_comments(diff) else: log.info("API Mode") diff: Diff diff = core.create_new_diff(target_path, params, workspace=target_path) - output_console_comments(diff) + if enable_json: + output_console_json(diff) + else: + output_console_comments(diff) if diff is not None and license_mode: all_packages = {} for package_id in diff.packages: