Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit c68e7fb

Browse filesBrowse files
abetlenabetlen
committed
fix: pull all gh releases for self-hosted python index
1 parent e251a0b commit c68e7fb
Copy full SHA for c68e7fb

File tree

Expand file treeCollapse file tree

3 files changed

+133
-39
lines changed
Filter options
Expand file treeCollapse file tree

3 files changed

+133
-39
lines changed

‎.github/workflows/generate-index-from-release.yaml

Copy file name to clipboardExpand all lines: .github/workflows/generate-index-from-release.yaml
+3Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,10 @@ jobs:
3535
- name: Setup Pages
3636
uses: actions/configure-pages@v5
3737
- name: Build
38+
env:
39+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
3840
run: |
41+
./scripts/get-releases.sh
3942
./scripts/releases-to-pep-503.sh index/whl/cpu '^[v]?[0-9]+\.[0-9]+\.[0-9]+$'
4043
./scripts/releases-to-pep-503.sh index/whl/cu121 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu121$'
4144
./scripts/releases-to-pep-503.sh index/whl/cu122 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu122$'

‎scripts/get-releases.sh

Copy file name to clipboard
+47Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
#!/bin/bash
2+
3+
# Function to get all releases
4+
get_all_releases() {
5+
local page=1
6+
local per_page=100
7+
local releases=""
8+
local new_releases
9+
10+
# Prepare headers
11+
local headers=(-H "Accept: application/vnd.github.v3+json")
12+
if [ -n "$GITHUB_TOKEN" ]; then
13+
headers+=(-H "Authorization: Bearer $GITHUB_TOKEN")
14+
fi
15+
16+
while true; do
17+
response=$(curl -s "${headers[@]}" \
18+
"https://api.github.com/repos/abetlen/llama-cpp-python/releases?page=$page&per_page=$per_page")
19+
20+
# Check if the response is valid JSON
21+
if ! echo "$response" | jq empty > /dev/null 2>&1; then
22+
echo "Error: Invalid response from GitHub API" >&2
23+
echo "Response: $response" >&2
24+
return 1
25+
fi
26+
27+
new_releases=$(echo "$response" | jq -r '.[].tag_name')
28+
if [ -z "$new_releases" ]; then
29+
break
30+
fi
31+
releases="$releases $new_releases"
32+
((page++))
33+
done
34+
35+
echo $releases
36+
}
37+
38+
# Get all releases and save to file
39+
releases=$(get_all_releases)
40+
if [ $? -ne 0 ]; then
41+
echo "Failed to fetch releases. Please check your internet connection and try again later." >&2
42+
exit 1
43+
fi
44+
45+
echo "$releases" | tr ' ' '\n' > all_releases.txt
46+
47+
echo "All releases have been saved to all_releases.txt"

‎scripts/releases-to-pep-503.sh

Copy file name to clipboard
+83-39Lines changed: 83 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1,60 +1,104 @@
11
#!/bin/bash
22

3+
# Enable exit on error
4+
set -e
5+
6+
# Function for logging
7+
log_error() {
8+
echo "ERROR: $1" >&2
9+
}
10+
11+
log_info() {
12+
echo "INFO: $1"
13+
}
14+
315
# Get output directory or default to index/whl/cpu
416
output_dir=${1:-"index/whl/cpu"}
517

6-
# Create output directory
7-
mkdir -p $output_dir
8-
9-
# Change to output directory
10-
pushd $output_dir
18+
# Get pattern from second arg or default to valid python package version pattern
19+
pattern=${2:-"^[v]?[0-9]+\.[0-9]+\.[0-9]+$"}
1120

12-
# Create an index html file
13-
echo "<!DOCTYPE html>" > index.html
14-
echo "<html>" >> index.html
15-
echo " <head></head>" >> index.html
16-
echo " <body>" >> index.html
17-
echo " <a href=\"llama-cpp-python/\">llama-cpp-python</a>" >> index.html
18-
echo " <br>" >> index.html
19-
echo " </body>" >> index.html
20-
echo "</html>" >> index.html
21-
echo "" >> index.html
21+
# Get the current directory (where the script is run from)
22+
current_dir="$(pwd)"
2223

23-
# Create llama-cpp-python directory
24-
mkdir -p llama-cpp-python
24+
# Check if all_releases.txt exists
25+
if [ ! -f "$current_dir/all_releases.txt" ]; then
26+
log_error "all_releases.txt not found in the current directory."
27+
exit 1
28+
fi
2529

26-
# Change to llama-cpp-python directory
27-
pushd llama-cpp-python
30+
# Create output directory
31+
mkdir -p "$output_dir"
2832

2933
# Create an index html file
30-
echo "<!DOCTYPE html>" > index.html
31-
echo "<html>" >> index.html
32-
echo " <body>" >> index.html
33-
echo " <h1>Links for llama-cpp-python</h1>" >> index.html
34+
cat << EOF > "$output_dir/index.html"
35+
<!DOCTYPE html>
36+
<html>
37+
<head></head>
38+
<body>
39+
<a href="llama-cpp-python/">llama-cpp-python</a>
40+
<br>
41+
</body>
42+
</html>
3443
35-
# Get all releases
36-
releases=$(curl -s https://api.github.com/repos/abetlen/llama-cpp-python/releases | jq -r .[].tag_name)
44+
EOF
3745

38-
# Get pattern from second arg or default to valid python package version pattern
39-
pattern=${2:-"^[v]?[0-9]+\.[0-9]+\.[0-9]+$"}
46+
# Create llama-cpp-python directory
47+
mkdir -p "$output_dir/llama-cpp-python"
48+
49+
# Create an index html file in llama-cpp-python directory
50+
cat << EOF > "$output_dir/llama-cpp-python/index.html"
51+
<!DOCTYPE html>
52+
<html>
53+
<body>
54+
<h1>Links for llama-cpp-python</h1>
55+
EOF
4056

4157
# Filter releases by pattern
42-
releases=$(echo $releases | tr ' ' '\n' | grep -E $pattern)
58+
releases=$(grep -E "$pattern" "$current_dir/all_releases.txt")
59+
60+
# Prepare curl headers
61+
headers=('--header' 'Accept: application/vnd.github.v3+json')
62+
if [ -n "$GITHUB_TOKEN" ]; then
63+
headers+=('--header' "authorization: Bearer $GITHUB_TOKEN")
64+
fi
65+
headers+=('--header' 'content-type: application/json')
4366

4467
# For each release, get all assets
4568
for release in $releases; do
46-
assets=$(curl -s https://api.github.com/repos/abetlen/llama-cpp-python/releases/tags/$release | jq -r .assets)
69+
log_info "Processing release: $release"
70+
response=$(curl -s "${headers[@]}" \
71+
"https://api.github.com/repos/abetlen/llama-cpp-python/releases/tags/$release")
72+
73+
if [ -z "$response" ]; then
74+
log_error "Empty response from GitHub API for release $release"
75+
continue
76+
fi
77+
78+
if ! echo "$response" | jq -e '.assets' > /dev/null 2>&1; then
79+
log_error "Invalid or unexpected response from GitHub API for release $release"
80+
log_error "Response: $response"
81+
continue
82+
fi
83+
4784
# Get release version from release ie v0.1.0-cu121 -> v0.1.0
48-
release_version=$(echo $release | grep -oE "^[v]?[0-9]+\.[0-9]+\.[0-9]+")
49-
echo " <h2>$release_version</h2>" >> index.html
50-
for asset in $(echo $assets | jq -r .[].browser_download_url); do
51-
if [[ $asset == *".whl" ]]; then
52-
echo " <a href=\"$asset\">$asset</a>" >> index.html
53-
echo " <br>" >> index.html
54-
fi
85+
release_version=$(echo "$release" | grep -oE "^[v]?[0-9]+\.[0-9]+\.[0-9]+")
86+
echo " <h2>$release_version</h2>" >> "$output_dir/llama-cpp-python/index.html"
87+
88+
wheel_urls=$(echo "$response" | jq -r '.assets[] | select(.name | endswith(".whl")) | .browser_download_url')
89+
if [ -z "$wheel_urls" ]; then
90+
log_error "No wheel files found for release $release"
91+
continue
92+
fi
93+
94+
echo "$wheel_urls" | while read -r asset; do
95+
echo " <a href=\"$asset\">$asset</a>" >> "$output_dir/llama-cpp-python/index.html"
96+
echo " <br>" >> "$output_dir/llama-cpp-python/index.html"
5597
done
5698
done
5799

58-
echo " </body>" >> index.html
59-
echo "</html>" >> index.html
60-
echo "" >> index.html
100+
echo " </body>" >> "$output_dir/llama-cpp-python/index.html"
101+
echo "</html>" >> "$output_dir/llama-cpp-python/index.html"
102+
echo "" >> "$output_dir/llama-cpp-python/index.html"
103+
104+
log_info "Index generation complete. Output directory: $output_dir"

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.