Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit d456108

Browse filesBrowse files
colesburycolesbury
authored
Support building free-threaded CPython (#319)
* Support building free-threaded CPython Add support for Python's free threading build mode where the global interpreter lock is disabled. The packages are marked using a suffix on the architecture, like 'x64-freethreaded' or 'arm64-freethreaded'. * Match '-freethreaded' in arch * Use type 'string' instead of 'str' * On Linux, only delete Python installations with the same architecture. This matches the macOS behavior and allows users to install both the free-threading and default builds at the same time.
1 parent e550a75 commit d456108
Copy full SHA for d456108

14 files changed

+176
-39
lines changed

‎.github/workflows/build-python-packages.yml

Copy file name to clipboardExpand all lines: .github/workflows/build-python-packages.yml
+41-23Lines changed: 41 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,11 @@ on:
1212
required: true
1313
type: boolean
1414
default: false
15+
THREADING_BUILD_MODES:
16+
description: 'CPython threading build modes'
17+
required: true
18+
type: string
19+
default: 'default,freethreaded'
1520
PLATFORMS:
1621
description: 'Platforms for execution in "os" or "os_arch" format (arch is "x64" by default)'
1722
required: true
@@ -40,32 +45,42 @@ jobs:
4045
id: generate-matrix
4146
run: |
4247
[String[]]$configurations = "${{ inputs.platforms || 'ubuntu-20.04,ubuntu-22.04,ubuntu-22.04_arm64,ubuntu-24.04,ubuntu-24.04_arm64,macos-13,macos-14_arm64,windows-2019_x64,windows-2019_x86,windows-2019_arm64' }}".Split(",").Trim()
48+
[String[]]$buildModes = "${{ inputs.threading_build_modes || 'default' }}".Split(",").Trim()
4349
$matrix = @()
4450
4551
foreach ($configuration in $configurations) {
46-
$parts = $configuration.Split("_")
47-
$os = $parts[0]
48-
$arch = if ($parts[1]) {$parts[1]} else {"x64"}
49-
switch -wildcard ($os) {
50-
"*ubuntu*" { $platform = $os.Replace("ubuntu","linux")}
51-
"*macos*" { $platform = 'darwin' }
52-
"*windows*" { $platform = 'win32' }
53-
}
54-
55-
if ($configuration -eq "ubuntu-22.04_arm64") {
56-
$os = "setup-actions-ubuntu-arm64-2-core"
57-
}
58-
elseif ($configuration -eq "ubuntu-24.04_arm64") {
59-
$os = "setup-actions-ubuntu24-arm64-2-core"
60-
}
61-
elseif ($configuration -eq "windows-2019_arm64") {
62-
$os = "setup-actions-windows-arm64-4-core"
63-
}
64-
65-
$matrix += @{
66-
'platform' = $platform
67-
'os' = $os
68-
'arch' = $arch
52+
foreach ($buildMode in $buildModes) {
53+
$parts = $configuration.Split("_")
54+
$os = $parts[0]
55+
$arch = if ($parts[1]) {$parts[1]} else {"x64"}
56+
switch -wildcard ($os) {
57+
"*ubuntu*" { $platform = $os.Replace("ubuntu","linux")}
58+
"*macos*" { $platform = 'darwin' }
59+
"*windows*" { $platform = 'win32' }
60+
}
61+
62+
if ($configuration -eq "ubuntu-22.04_arm64") {
63+
$os = "setup-actions-ubuntu-arm64-2-core"
64+
}
65+
elseif ($configuration -eq "ubuntu-24.04_arm64") {
66+
$os = "setup-actions-ubuntu24-arm64-2-core"
67+
}
68+
elseif ($configuration -eq "windows-2019_arm64") {
69+
$os = "setup-actions-windows-arm64-4-core"
70+
}
71+
72+
if ($buildMode -eq "freethreaded") {
73+
if ([semver]"${{ inputs.VERSION }}" -lt [semver]"3.13.0") {
74+
continue;
75+
}
76+
$arch += "-freethreaded"
77+
}
78+
79+
$matrix += @{
80+
'platform' = $platform
81+
'os' = $os
82+
'arch' = $arch
83+
}
6984
}
7085
}
7186
echo "matrix=$($matrix | ConvertTo-Json -Compress -AsArray)" >> $env:GITHUB_OUTPUT
@@ -205,6 +220,9 @@ jobs:
205220
python-version: ${{ env.VERSION }}
206221
architecture: ${{ matrix.arch }}
207222

223+
- name: Python version
224+
run: python -VVV
225+
208226
- name: Verbose sysconfig dump
209227
if: runner.os == 'Linux' || runner.os == 'macOS'
210228
run: python ./sources/python-config-output.py

‎builders/macos-python-builder.psm1

Copy file name to clipboardExpand all lines: builders/macos-python-builder.psm1
+33Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,37 @@ class macOSPythonBuilder : NixPythonBuilder {
151151
return $pkgLocation
152152
}
153153

154+
[string] GetFrameworkName() {
155+
<#
156+
.SYNOPSIS
157+
Get the Python installation Package name.
158+
#>
159+
160+
if ($this.IsFreeThreaded()) {
161+
return "PythonT.framework"
162+
} else {
163+
return "Python.framework"
164+
}
165+
}
166+
167+
[string] GetPkgChoices() {
168+
<#
169+
.SYNOPSIS
170+
Reads the configuration XML file for the Python installer
171+
#>
172+
173+
$config = if ($this.IsFreeThreaded()) { "freethreaded" } else { "default" }
174+
$choicesFile = Join-Path $PSScriptRoot "../config/macos-pkg-choices-$($config).xml"
175+
$choicesTemplate = Get-Content -Path $choicesFile -Raw
176+
177+
$variablesToReplace = @{
178+
"{{__VERSION_MAJOR_MINOR__}}" = "$($this.Version.Major).$($this.Version.Minor)";
179+
}
180+
181+
$variablesToReplace.keys | ForEach-Object { $choicesTemplate = $choicesTemplate.Replace($_, $variablesToReplace[$_]) }
182+
return $choicesTemplate
183+
}
184+
154185
[void] CreateInstallationScriptPkg() {
155186
<#
156187
.SYNOPSIS
@@ -165,6 +196,8 @@ class macOSPythonBuilder : NixPythonBuilder {
165196
"{{__VERSION_FULL__}}" = $this.Version;
166197
"{{__PKG_NAME__}}" = $this.GetPkgName();
167198
"{{__ARCH__}}" = $this.Architecture;
199+
"{{__FRAMEWORK_NAME__}}" = $this.GetFrameworkName();
200+
"{{__PKG_CHOICES__}}" = $this.GetPkgChoices();
168201
}
169202

170203
$variablesToReplace.keys | ForEach-Object { $installationTemplateContent = $installationTemplateContent.Replace($_, $variablesToReplace[$_]) }

‎builders/nix-python-builder.psm1

Copy file name to clipboardExpand all lines: builders/nix-python-builder.psm1
+1-1Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ class NixPythonBuilder : PythonBuilder {
115115
Write-Debug "make Python $($this.Version)-$($this.Architecture) $($this.Platform)"
116116
$buildOutputLocation = New-Item -Path $this.WorkFolderLocation -Name "build_output.txt" -ItemType File
117117

118-
Execute-Command -Command "make 2>&1 | tee $buildOutputLocation" -ErrorAction Continue
118+
Execute-Command -Command "make 2>&1 | tee $buildOutputLocation" -ErrorAction Continue
119119
Execute-Command -Command "make install" -ErrorAction Continue
120120

121121
Write-Debug "Done; Make log location: $buildOutputLocation"

‎builders/python-builder.psm1

Copy file name to clipboardExpand all lines: builders/python-builder.psm1
+18Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,24 @@ class PythonBuilder {
9494
return "$($this.Version.Major).$($this.Version.Minor).$($this.Version.Patch)"
9595
}
9696

97+
[string] GetHardwareArchitecture() {
98+
<#
99+
.SYNOPSIS
100+
The hardware architecture (x64, arm64) without any Python free threading suffix.
101+
#>
102+
103+
return $this.Architecture.Replace("-freethreaded", "")
104+
}
105+
106+
[bool] IsFreeThreaded() {
107+
<#
108+
.SYNOPSIS
109+
Check if Python version is free threaded.
110+
#>
111+
112+
return $this.Architecture.EndsWith("-freethreaded")
113+
}
114+
97115
[void] PreparePythonToolcacheLocation() {
98116
<#
99117
.SYNOPSIS

‎builders/ubuntu-python-builder.psm1

Copy file name to clipboardExpand all lines: builders/ubuntu-python-builder.psm1
+8Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,14 @@ class UbuntuPythonBuilder : NixPythonBuilder {
3737
$configureString += " --enable-shared"
3838
$configureString += " --enable-optimizations"
3939

40+
if ($this.IsFreeThreaded()) {
41+
if ($this.Version -lt "3.13.0") {
42+
Write-Host "Python versions lower than 3.13.0 do not support free threading"
43+
exit 1
44+
}
45+
$configureString += " --disable-gil"
46+
}
47+
4048
### Compile with support of loadable sqlite extensions.
4149
### Link to documentation (https://docs.python.org/3/library/sqlite3.html#sqlite3.Connection.enable_load_extension)
4250
$configureString += " --enable-loadable-sqlite-extensions"

‎builders/win-python-builder.psm1

Copy file name to clipboardExpand all lines: builders/win-python-builder.psm1
+3-2Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,13 +54,13 @@ class WinPythonBuilder : PythonBuilder {
5454
#>
5555

5656
$ArchitectureExtension = ""
57-
if ($this.Architecture -eq "x64") {
57+
if ($this.GetHardwareArchitecture() -eq "x64") {
5858
if ($this.Version -ge "3.5") {
5959
$ArchitectureExtension = "-amd64"
6060
} else {
6161
$ArchitectureExtension = ".amd64"
6262
}
63-
}elseif ($this.Architecture -eq "arm64") {
63+
} elseif ($this.GetHardwareArchitecture() -eq "arm64") {
6464
$ArchitectureExtension = "-arm64"
6565
}
6666

@@ -113,6 +113,7 @@ class WinPythonBuilder : PythonBuilder {
113113

114114
$variablesToReplace = @{
115115
"{{__ARCHITECTURE__}}" = $this.Architecture;
116+
"{{__HARDWARE_ARCHITECTURE__}}" = $this.GetHardwareArchitecture();
116117
"{{__VERSION__}}" = $this.Version;
117118
"{{__PYTHON_EXEC_NAME__}}" = $pythonExecName
118119
}

‎config/macos-pkg-choices-default.xml

Copy file name to clipboard
+8Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
3+
<plist version="1.0">
4+
<array>
5+
<dict>
6+
</dict>
7+
</array>
8+
</plist>

‎config/macos-pkg-choices-freethreaded.xml

Copy file name to clipboard
+14Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
3+
<plist version="1.0">
4+
<array>
5+
<dict>
6+
<key>attributeSetting</key>
7+
<integer>1</integer>
8+
<key>choiceAttribute</key>
9+
<string>selected</string>
10+
<key>choiceIdentifier</key>
11+
<string>org.python.Python.PythonTFramework-{{__VERSION_MAJOR_MINOR__}}</string>
12+
</dict>
13+
</array>
14+
</plist>

‎config/python-manifest-config.json

Copy file name to clipboardExpand all lines: config/python-manifest-config.json
+1-1Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"regex": "python-\\d+\\.\\d+\\.\\d+-(\\w+\\.\\d+)?-?(\\w+)-(\\d+\\.\\d+)?-?((x|arm)\\d+)",
2+
"regex": "python-\\d+\\.\\d+\\.\\d+-(\\w+\\.\\d+)?-?(\\w+)-(\\d+\\.\\d+)?-?((x|arm)\\d+(-freethreaded)?)",
33
"groups": {
44
"arch": 4,
55
"platform": 2,

‎installers/macos-pkg-setup-template.sh

Copy file name to clipboardExpand all lines: installers/macos-pkg-setup-template.sh
+17-3Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,11 @@ set -e
22

33
PYTHON_FULL_VERSION="{{__VERSION_FULL__}}"
44
PYTHON_PKG_NAME="{{__PKG_NAME__}}"
5+
PYTHON_FRAMEWORK_NAME="{{__FRAMEWORK_NAME__}}"
6+
PYTHON_PKG_CHOICES=$(cat << 'EOF'
7+
{{__PKG_CHOICES__}}
8+
EOF
9+
)
510
ARCH="{{__ARCH__}}"
611
MAJOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 1)
712
MINOR_VERSION=$(echo $PYTHON_FULL_VERSION | cut -d '.' -f 2)
@@ -20,7 +25,7 @@ fi
2025
PYTHON_TOOLCACHE_PATH=$TOOLCACHE_ROOT/Python
2126
PYTHON_TOOLCACHE_VERSION_PATH=$PYTHON_TOOLCACHE_PATH/$PYTHON_FULL_VERSION
2227
PYTHON_TOOLCACHE_VERSION_ARCH_PATH=$PYTHON_TOOLCACHE_VERSION_PATH/$ARCH
23-
PYTHON_FRAMEWORK_PATH="/Library/Frameworks/Python.framework/Versions/${MAJOR_VERSION}.${MINOR_VERSION}"
28+
PYTHON_FRAMEWORK_PATH="/Library/Frameworks/${PYTHON_FRAMEWORK_NAME}/Versions/${MAJOR_VERSION}.${MINOR_VERSION}"
2429
PYTHON_APPLICATION_PATH="/Applications/Python ${MAJOR_VERSION}.${MINOR_VERSION}"
2530

2631
echo "Check if Python hostedtoolcache folder exist..."
@@ -38,8 +43,11 @@ else
3843
done
3944
fi
4045

46+
PYTHON_PKG_CHOICES_FILES=$(mktemp)
47+
echo "$PYTHON_PKG_CHOICES" > $PYTHON_PKG_CHOICES_FILES
48+
4149
echo "Install Python binaries from prebuilt package"
42-
sudo installer -pkg $PYTHON_PKG_NAME -target /
50+
sudo installer -pkg $PYTHON_PKG_NAME -applyChoiceChangesXML $PYTHON_PKG_CHOICES_FILES -target /
4351

4452
echo "Create hostedtoolcach symlinks (Required for the backward compatibility)"
4553
echo "Create Python $PYTHON_FULL_VERSION folder"
@@ -53,7 +61,9 @@ ln -s "${PYTHON_FRAMEWORK_PATH}/lib" lib
5361

5462
echo "Create additional symlinks (Required for the UsePythonVersion Azure Pipelines task and the setup-python GitHub Action)"
5563
ln -s ./bin/$PYTHON_MAJOR_DOT_MINOR python
64+
chmod +x python
5665

66+
# Note that bin is a symlink so referencing .. from bin will not work as expected
5767
cd bin/
5868

5969
# This symlink already exists if Python version with the same major.minor version is installed,
@@ -62,11 +72,15 @@ if [ ! -f $PYTHON_MAJOR_MINOR ]; then
6272
ln -s $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR
6373
fi
6474

75+
if [ ! -f $PYTHON_MAJOR ]; then
76+
ln -s $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR
77+
fi
78+
6579
if [ ! -f python ]; then
6680
ln -s $PYTHON_MAJOR_DOT_MINOR python
6781
fi
6882

69-
chmod +x ../python $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR python
83+
chmod +x $PYTHON_MAJOR $PYTHON_MAJOR_DOT_MINOR $PYTHON_MAJOR_MINOR python
7084

7185
echo "Upgrading pip..."
7286
export PIP_ROOT_USER_ACTION=ignore

‎installers/nix-setup-template.sh

Copy file name to clipboardExpand all lines: installers/nix-setup-template.sh
+3-3Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,9 @@ echo "Check if Python hostedtoolcache folder exist..."
2424
if [ ! -d $PYTHON_TOOLCACHE_PATH ]; then
2525
echo "Creating Python hostedtoolcache folder..."
2626
mkdir -p $PYTHON_TOOLCACHE_PATH
27-
elif [ -d $PYTHON_TOOLCACHE_VERSION_PATH ]; then
28-
echo "Deleting Python $PYTHON_FULL_VERSION"
29-
rm -rf $PYTHON_TOOLCACHE_VERSION_PATH
27+
elif [ -d $PYTHON_TOOLCACHE_VERSION_ARCH_PATH ]; then
28+
echo "Deleting Python $PYTHON_FULL_VERSION ($ARCH)"
29+
rm -rf $PYTHON_TOOLCACHE_VERSION_ARCH_PATH
3030
fi
3131

3232
echo "Create Python $PYTHON_FULL_VERSION folder"

‎installers/win-setup-template.ps1

Copy file name to clipboardExpand all lines: installers/win-setup-template.ps1
+18-3Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
[String] $Architecture = "{{__ARCHITECTURE__}}"
2+
[String] $HardwareArchitecture = "{{__HARDWARE_ARCHITECTURE__}}"
23
[String] $Version = "{{__VERSION__}}"
34
[String] $PythonExecName = "{{__PYTHON_EXEC_NAME__}}"
45

@@ -25,7 +26,7 @@ function Remove-RegistryEntries {
2526
[Parameter(Mandatory)][Int32] $MinorVersion
2627
)
2728

28-
$versionFilter = Get-RegistryVersionFilter -Architecture $Architecture -MajorVersion $MajorVersion -MinorVersion $MinorVersion
29+
$versionFilter = Get-RegistryVersionFilter -Architecture $HardwareArchitecture -MajorVersion $MajorVersion -MinorVersion $MinorVersion
2930

3031
$regPath = "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\CurrentVersion\Installer\UserData\S-1-5-18\Products"
3132
if (Test-Path -Path Registry::$regPath) {
@@ -61,13 +62,15 @@ function Remove-RegistryEntries {
6162
function Get-ExecParams {
6263
param(
6364
[Parameter(Mandatory)][Boolean] $IsMSI,
65+
[Parameter(Mandatory)][Boolean] $IsFreeThreaded,
6466
[Parameter(Mandatory)][String] $PythonArchPath
6567
)
6668

6769
if ($IsMSI) {
6870
"TARGETDIR=$PythonArchPath ALLUSERS=1"
6971
} else {
70-
"DefaultAllUsersTargetDir=$PythonArchPath InstallAllUsers=1"
72+
$Include_freethreaded = if ($IsFreeThreaded) { "Include_freethreaded=1" } else { "" }
73+
"DefaultAllUsersTargetDir=$PythonArchPath InstallAllUsers=1 $Include_freethreaded"
7174
}
7275
}
7376

@@ -81,6 +84,7 @@ $PythonVersionPath = Join-Path -Path $PythonToolcachePath -ChildPath $Version
8184
$PythonArchPath = Join-Path -Path $PythonVersionPath -ChildPath $Architecture
8285

8386
$IsMSI = $PythonExecName -match "msi"
87+
$IsFreeThreaded = $Architecture -match "-freethreaded"
8488

8589
$MajorVersion = $Version.Split('.')[0]
8690
$MinorVersion = $Version.Split('.')[1]
@@ -120,13 +124,24 @@ Write-Host "Copy Python binaries to $PythonArchPath"
120124
Copy-Item -Path ./$PythonExecName -Destination $PythonArchPath | Out-Null
121125

122126
Write-Host "Install Python $Version in $PythonToolcachePath..."
123-
$ExecParams = Get-ExecParams -IsMSI $IsMSI -PythonArchPath $PythonArchPath
127+
$ExecParams = Get-ExecParams -IsMSI $IsMSI -IsFreeThreaded $IsFreeThreaded -PythonArchPath $PythonArchPath
124128

125129
cmd.exe /c "cd $PythonArchPath && call $PythonExecName $ExecParams /quiet"
126130
if ($LASTEXITCODE -ne 0) {
127131
Throw "Error happened during Python installation"
128132
}
129133

134+
# print out all files in $PythonArchPath
135+
Write-Host "Files in $PythonArchPath"
136+
$files = Get-ChildItem -Path $PythonArchPath -File -Recurse
137+
Write-Output $files
138+
139+
if ($IsFreeThreaded) {
140+
# Delete python.exe and create a symlink to free-threaded exe
141+
Remove-Item -Path "$PythonArchPath\python.exe" -Force
142+
New-Item -Path "$PythonArchPath\python.exe" -ItemType SymbolicLink -Value "$PythonArchPath\python${MajorVersion}.${MinorVersion}t.exe"
143+
}
144+
130145
Write-Host "Create `python3` symlink"
131146
if ($MajorVersion -ne "2") {
132147
New-Item -Path "$PythonArchPath\python3.exe" -ItemType SymbolicLink -Value "$PythonArchPath\python.exe"

‎tests/python-tests.ps1

Copy file name to clipboardExpand all lines: tests/python-tests.ps1
+3-1Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ param (
77
$Architecture
88
)
99

10+
$HardwareArchitecture = $Architecture -replace "-freethreaded", ""
11+
1012
Import-Module (Join-Path $PSScriptRoot "../helpers/pester-extensions.psm1")
1113
Import-Module (Join-Path $PSScriptRoot "../helpers/common-helpers.psm1")
1214
Import-Module (Join-Path $PSScriptRoot "../builders/python-version.psm1")
@@ -58,7 +60,7 @@ Describe "Tests" {
5860
# }
5961
# }
6062

61-
if (($Version -ge "3.2.0") -and ($Version -lt "3.11.0") -and (($Platform -ne "darwin") -or ($Architecture -ne "arm64"))) {
63+
if (($Version -ge "3.2.0") -and ($Version -lt "3.11.0") -and (($Platform -ne "darwin") -or ($HardwareArchitecture -ne "arm64"))) {
6264
It "Check if sqlite3 module is installed" {
6365
"python ./sources/python-sqlite3.py" | Should -ReturnZeroExitCode
6466
}

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.