Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Shared: Remove the language-specific model generator scripts #19452

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
May 13, 2025
Merged

Shared: Remove the language-specific model generator scripts #19452

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c6d95ce
Shared: Remove the language-specific model generator scripts
paldepind May 2, 2025
14bdc1a
Shared: Minor tweaks to model generator script
paldepind May 13, 2025
014e7dc
Shared: Remove change note for internal MaD generator
paldepind May 13, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion 2 .github/workflows/mad_modelDiff.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ jobs:
DATABASE=$2
cd codeql-$QL_VARIANT
SHORTNAME=`basename $DATABASE`
python java/ql/src/utils/modelgenerator/GenerateFlowModel.py --with-summaries --with-sinks $DATABASE $SHORTNAME/$QL_VARIANT
python misc/scripts/models-as-data/generate_mad.py --language java --with-summaries --with-sinks $DATABASE $SHORTNAME/$QL_VARIANT
Copy link
Preview

Copilot AI May 2, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The CI invocation omits the --with-neutrals flag, but RegenerateModels.py includes it. Add --with-neutrals here to ensure neutral models are generated and compared in the diff job.

Suggested change
python misc/scripts/models-as-data/generate_mad.py --language java --with-summaries --with-sinks $DATABASE $SHORTNAME/$QL_VARIANT
python misc/scripts/models-as-data/generate_mad.py --language java --with-summaries --with-sinks --with-neutrals $DATABASE $SHORTNAME/$QL_VARIANT

Copilot uses AI. Check for mistakes.

Positive FeedbackNegative Feedback
mkdir -p $MODELS/$SHORTNAME
mv java/ql/lib/ext/generated/$SHORTNAME/$QL_VARIANT $MODELS/$SHORTNAME
cd ..
Expand Down
15 changes: 0 additions & 15 deletions 15 cpp/ql/src/utils/modelgenerator/GenerateFlowModel.py
View file
Open in desktop

This file was deleted.

15 changes: 0 additions & 15 deletions 15 csharp/ql/src/utils/modelgenerator/GenerateFlowModel.py
View file
Open in desktop

This file was deleted.

15 changes: 0 additions & 15 deletions 15 java/ql/src/utils/modelgenerator/GenerateFlowModel.py
View file
Open in desktop

This file was deleted.

4 changes: 2 additions & 2 deletions 4 java/ql/src/utils/modelgenerator/RegenerateModels.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,8 @@ def regenerateModel(lgtmSlug, extractedDb):
sys.exit(1)
modelFile = lgtmSlugToModelFile[lgtmSlug]
codeQlRoot = findGitRoot()
subprocess.check_call([codeQlRoot + "/java/ql/src/utils/modelgenerator/GenerateFlowModel.py",
"--with-summaries", "--with-sinks", "--with-neutrals",
subprocess.check_call([codeQlRoot + "/misc/scripts/models-as-data/generate_mad.py",
"--language", "java", "--with-summaries", "--with-sinks", "--with-neutrals",
extractedDb, modelFile])
print("Regenerated " + modelFile)
shutil.rmtree(tmpDir)
Expand Down
59 changes: 35 additions & 24 deletions 59 ...pts/models-as-data/generate_flow_model.py → misc/scripts/models-as-data/generate_mad.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
#!/usr/bin/python3

import helpers
import json
import os
import os.path
import shlex
import subprocess
import sys
import tempfile
Expand All @@ -27,24 +25,13 @@ def parseData(data):

return rows

class Generator:
def __init__ (self, language):
self.language = language
self.generateSinks = False
self.generateSources = False
self.generateSummaries = False
self.generateNeutrals = False
self.generateTypeBasedSummaries = False
self.dryRun = False
self.dirname = "modelgenerator"


def printHelp(self):
print(f"""Usage:
python3 GenerateFlowModel.py <library-database> [DIR] [--with-sinks] [--with-sources] [--with-summaries] [--with-neutrals] [--with-typebased-summaries] [--dry-run]
def printHelp():
print(f"""Usage:
python3 generate_mad.py <library-database> [DIR] --language LANGUAGE [--with-sinks] [--with-sources] [--with-summaries] [--with-neutrals] [--with-typebased-summaries] [--dry-run]

This generates summary, source, sink and neutral models for the code in the database.
The files will be placed in `{self.language}/ql/lib/ext/generated/DIR`
The files will be placed in `LANGUAGE/ql/lib/ext/generated/DIR`

Which models are generated is controlled by the flags:
--with-sinks
Expand All @@ -57,14 +44,25 @@ def printHelp(self):
--dry-run: Only run the queries, but don't write to file.

Example invocations:
$ python3 GenerateFlowModel.py /tmp/dbs/my_library_db
$ python3 GenerateFlowModel.py /tmp/dbs/my_library_db --with-sinks
$ python3 GenerateFlowModel.py /tmp/dbs/my_library_db --with-sinks my_directory
$ python3 generate_mad.py /tmp/dbs/my_library_db
$ python3 generate_mad.py /tmp/dbs/my_library_db --with-sinks
$ python3 generate_mad.py /tmp/dbs/my_library_db --with-sinks my_directory


Requirements: `codeql` should appear on your path.
""")

class Generator:
def __init__(self, language):
self.language = language
self.generateSinks = False
self.generateSources = False
self.generateSummaries = False
self.generateNeutrals = False
self.generateTypeBasedSummaries = False
self.dryRun = False
self.dirname = "modelgenerator"


def setenvironment(self, database, folder):
self.codeQlRoot = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode("utf-8").strip()
Expand All @@ -76,12 +74,22 @@ def setenvironment(self, database, folder):


@staticmethod
def make(language):
generator = Generator(language)
def make():
# Create a generator instance based on command line arguments.
if any(s == "--help" for s in sys.argv):
generator.printHelp()
printHelp()
sys.exit(0)

if "--language" in sys.argv:
language = sys.argv[sys.argv.index("--language") + 1]
sys.argv.remove("--language")
sys.argv.remove(language)
else:
printHelp()
sys.exit(0)

generator = Generator(language=language)

if "--with-sinks" in sys.argv:
sys.argv.remove("--with-sinks")
generator.generateSinks = True
Expand Down Expand Up @@ -115,7 +123,7 @@ def make(language):

n = len(sys.argv)
if n < 2:
generator.printHelp()
printHelp()
sys.exit(1)
elif n == 2:
generator.setenvironment(sys.argv[1], "")
Expand Down Expand Up @@ -204,3 +212,6 @@ def run(self):

if self.generateTypeBasedSummaries:
self.save(typeBasedContent, ".typebased.model.yml")

if __name__ == '__main__':
Generator.make().run()
15 changes: 0 additions & 15 deletions 15 rust/ql/src/utils/modelgenerator/GenerateFlowModel.py
View file
Open in desktop

This file was deleted.

Loading
Morty Proxy This is a proxified and sanitized view of the page, visit original site.