Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 04414e2

Browse filesBrowse files
MatteoPologrutoMatteoPologruto
authored
Check if a signed URL is specified and use it download tools (#953)
* Wrap v2 tools install function inside tools.Download * Download tools defaulting to the replace behaviour * Improve archive renamer and fix failing tests * Find the correct tool and system when `version=latest` is specified * Reintroduce caching option when downloading tools
1 parent b02967e commit 04414e2
Copy full SHA for 04414e2

File tree

Expand file treeCollapse file tree

6 files changed

+142
-211
lines changed
Filter options
Expand file treeCollapse file tree

6 files changed

+142
-211
lines changed

‎tools/download.go

Copy file name to clipboardExpand all lines: tools/download.go
+13-157Lines changed: 13 additions & 157 deletions
Original file line numberDiff line numberDiff line change
@@ -16,43 +16,18 @@
1616
package tools
1717

1818
import (
19-
"bytes"
2019
"context"
21-
"crypto/sha256"
22-
"encoding/hex"
23-
"encoding/json"
2420
"errors"
25-
"fmt"
26-
"io"
27-
"net/http"
2821
"os"
2922
"os/exec"
3023
"path/filepath"
3124
"runtime"
3225

26+
"github.com/arduino/arduino-create-agent/gen/tools"
27+
"github.com/arduino/arduino-create-agent/utilities"
3328
"github.com/arduino/arduino-create-agent/v2/pkgs"
34-
"github.com/arduino/go-paths-helper"
35-
"github.com/blang/semver"
36-
"github.com/codeclysm/extract/v3"
3729
)
3830

39-
// public vars to allow override in the tests
40-
var (
41-
OS = runtime.GOOS
42-
Arch = runtime.GOARCH
43-
)
44-
45-
func pathExists(path string) bool {
46-
_, err := os.Stat(path)
47-
if err == nil {
48-
return true
49-
}
50-
if os.IsNotExist(err) {
51-
return false
52-
}
53-
return true
54-
}
55-
5631
// Download will parse the index at the indexURL for the tool to download.
5732
// It will extract it in a folder in .arduino-create, and it will update the
5833
// Installed map.
@@ -70,97 +45,21 @@ func pathExists(path string) bool {
7045
// if it already exists.
7146
func (t *Tools) Download(pack, name, version, behaviour string) error {
7247

73-
body, err := t.index.Read()
74-
if err != nil {
75-
return err
76-
}
77-
78-
var data pkgs.Index
79-
json.Unmarshal(body, &data)
80-
81-
// Find the tool by name
82-
correctTool, correctSystem := findTool(pack, name, version, data)
83-
84-
if correctTool.Name == "" || correctSystem.URL == "" {
85-
t.logger("We couldn't find a tool with the name " + name + " and version " + version + " packaged by " + pack)
86-
return nil
87-
}
88-
89-
key := correctTool.Name + "-" + correctTool.Version
90-
91-
// Check if it already exists
92-
if behaviour == "keep" {
93-
location, ok := t.getMapValue(key)
94-
if ok && pathExists(location) {
95-
// overwrite the default tool with this one
96-
t.setMapValue(correctTool.Name, location)
97-
t.logger("The tool is already present on the system")
98-
return t.writeMap()
99-
}
100-
}
101-
102-
// Download the tool
103-
t.logger("Downloading tool " + name + " from " + correctSystem.URL)
104-
resp, err := http.Get(correctSystem.URL)
48+
tool := pkgs.New(t.index, t.directory.String(), behaviour)
49+
_, err := tool.Install(context.Background(), &tools.ToolPayload{Name: name, Version: version, Packager: pack})
10550
if err != nil {
10651
return err
10752
}
108-
defer resp.Body.Close()
109-
110-
// Read the body
111-
body, err = io.ReadAll(resp.Body)
112-
if err != nil {
113-
return err
114-
}
115-
116-
// Checksum
117-
checksum := sha256.Sum256(body)
118-
checkSumString := "SHA-256:" + hex.EncodeToString(checksum[:sha256.Size])
119-
120-
if checkSumString != correctSystem.Checksum {
121-
return errors.New("checksum doesn't match")
122-
}
123-
124-
tempPath := paths.TempDir()
125-
// Create a temporary dir to extract package
126-
if err := tempPath.MkdirAll(); err != nil {
127-
return fmt.Errorf("creating temp dir for extraction: %s", err)
128-
}
129-
tempDir, err := tempPath.MkTempDir("package-")
130-
if err != nil {
131-
return fmt.Errorf("creating temp dir for extraction: %s", err)
132-
}
133-
defer tempDir.RemoveAll()
13453

135-
t.logger("Unpacking tool " + name)
136-
ctx := context.Background()
137-
reader := bytes.NewReader(body)
138-
// Extract into temp directory
139-
if err := extract.Archive(ctx, reader, tempDir.String(), nil); err != nil {
140-
return fmt.Errorf("extracting archive: %s", err)
141-
}
142-
143-
location := t.directory.Join(pack, correctTool.Name, correctTool.Version)
144-
err = location.RemoveAll()
54+
path := filepath.Join(pack, name, version)
55+
safePath, err := utilities.SafeJoin(t.directory.String(), path)
14556
if err != nil {
14657
return err
14758
}
14859

149-
// Check package content and find package root dir
150-
root, err := findPackageRoot(tempDir)
151-
if err != nil {
152-
return fmt.Errorf("searching package root dir: %s", err)
153-
}
154-
155-
if err := root.Rename(location); err != nil {
156-
if err := root.CopyDirTo(location); err != nil {
157-
return fmt.Errorf("moving extracted archive to destination dir: %s", err)
158-
}
159-
}
160-
16160
// if the tool contains a post_install script, run it: it means it is a tool that needs to install drivers
16261
// AFAIK this is only the case for the windows-driver tool
163-
err = t.installDrivers(location.String())
62+
err = t.installDrivers(safePath)
16463
if err != nil {
16564
return err
16665
}
@@ -169,63 +68,20 @@ func (t *Tools) Download(pack, name, version, behaviour string) error {
16968
t.logger("Ensure that the files are executable")
17069

17170
// Update the tool map
172-
t.logger("Updating map with location " + location.String())
173-
174-
t.setMapValue(name, location.String())
175-
t.setMapValue(name+"-"+correctTool.Version, location.String())
176-
return t.writeMap()
177-
}
71+
t.logger("Updating map with location " + safePath)
17872

179-
func findPackageRoot(parent *paths.Path) (*paths.Path, error) {
180-
files, err := parent.ReadDir()
181-
if err != nil {
182-
return nil, fmt.Errorf("reading package root dir: %s", err)
183-
}
184-
files.FilterOutPrefix("__MACOSX")
73+
t.setMapValue(name, safePath)
74+
t.setMapValue(name+"-"+version, safePath)
18575

186-
// if there is only one dir, it is the root dir
187-
if len(files) == 1 && files[0].IsDir() {
188-
return files[0], nil
189-
}
190-
return parent, nil
191-
}
192-
193-
func findTool(pack, name, version string, data pkgs.Index) (pkgs.Tool, pkgs.System) {
194-
var correctTool pkgs.Tool
195-
correctTool.Version = "0.0"
196-
197-
for _, p := range data.Packages {
198-
if p.Name != pack {
199-
continue
200-
}
201-
for _, t := range p.Tools {
202-
if version != "latest" {
203-
if t.Name == name && t.Version == version {
204-
correctTool = t
205-
}
206-
} else {
207-
// Find latest
208-
v1, _ := semver.Make(t.Version)
209-
v2, _ := semver.Make(correctTool.Version)
210-
if t.Name == name && v1.Compare(v2) > 0 {
211-
correctTool = t
212-
}
213-
}
214-
}
215-
}
216-
217-
// Find the url based on system
218-
correctSystem := correctTool.GetFlavourCompatibleWith(OS, Arch)
219-
220-
return correctTool, correctSystem
76+
return nil
22177
}
22278

22379
func (t *Tools) installDrivers(location string) error {
22480
OkPressed := 6
22581
extension := ".bat"
22682
// add .\ to force locality
22783
preamble := ".\\"
228-
if OS != "windows" {
84+
if runtime.GOOS != "windows" {
22985
extension = ".sh"
23086
// add ./ to force locality
23187
preamble = "./"
@@ -237,7 +93,7 @@ func (t *Tools) installDrivers(location string) error {
23793
os.Chdir(location)
23894
t.logger(preamble + "post_install" + extension)
23995
oscmd := exec.Command(preamble + "post_install" + extension)
240-
if OS != "linux" {
96+
if runtime.GOOS != "linux" {
24197
// spawning a shell could be the only way to let the user type his password
24298
TellCommandNotToSpawnShell(oscmd)
24399
}

‎tools/download_test.go

Copy file name to clipboardExpand all lines: tools/download_test.go
+13-11Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -42,8 +42,8 @@ func TestDownloadCorrectPlatform(t *testing.T) {
4242
{"linux", "arm", "arm-linux-gnueabihf"},
4343
}
4444
defer func() {
45-
OS = runtime.GOOS // restore `runtime.OS`
46-
Arch = runtime.GOARCH // restore `runtime.ARCH`
45+
pkgs.OS = runtime.GOOS // restore `runtime.OS`
46+
pkgs.Arch = runtime.GOARCH // restore `runtime.ARCH`
4747
}()
4848
testIndex := paths.New("testdata", "test_tool_index.json")
4949
buf, err := testIndex.ReadFile()
@@ -54,10 +54,11 @@ func TestDownloadCorrectPlatform(t *testing.T) {
5454
require.NoError(t, err)
5555
for _, tc := range testCases {
5656
t.Run(tc.hostOS+tc.hostArch, func(t *testing.T) {
57-
OS = tc.hostOS // override `runtime.OS` for testing purposes
58-
Arch = tc.hostArch // override `runtime.ARCH` for testing purposes
57+
pkgs.OS = tc.hostOS // override `runtime.OS` for testing purposes
58+
pkgs.Arch = tc.hostArch // override `runtime.ARCH` for testing purposes
5959
// Find the tool by name
60-
correctTool, correctSystem := findTool("arduino-test", "arduino-fwuploader", "2.2.2", data)
60+
correctTool, correctSystem, found := pkgs.FindTool("arduino-test", "arduino-fwuploader", "2.2.2", data)
61+
require.True(t, found)
6162
require.NotNil(t, correctTool)
6263
require.NotNil(t, correctSystem)
6364
require.Equal(t, correctTool.Name, "arduino-fwuploader")
@@ -78,8 +79,8 @@ func TestDownloadFallbackPlatform(t *testing.T) {
7879
{"windows", "amd64", "i686-mingw32"},
7980
}
8081
defer func() {
81-
OS = runtime.GOOS // restore `runtime.OS`
82-
Arch = runtime.GOARCH // restore `runtime.ARCH`
82+
pkgs.OS = runtime.GOOS // restore `runtime.OS`
83+
pkgs.Arch = runtime.GOARCH // restore `runtime.ARCH`
8384
}()
8485
testIndex := paths.New("testdata", "test_tool_index.json")
8586
buf, err := testIndex.ReadFile()
@@ -90,10 +91,11 @@ func TestDownloadFallbackPlatform(t *testing.T) {
9091
require.NoError(t, err)
9192
for _, tc := range testCases {
9293
t.Run(tc.hostOS+tc.hostArch, func(t *testing.T) {
93-
OS = tc.hostOS // override `runtime.OS` for testing purposes
94-
Arch = tc.hostArch // override `runtime.ARCH` for testing purposes
94+
pkgs.OS = tc.hostOS // override `runtime.OS` for testing purposes
95+
pkgs.Arch = tc.hostArch // override `runtime.ARCH` for testing purposes
9596
// Find the tool by name
96-
correctTool, correctSystem := findTool("arduino-test", "arduino-fwuploader", "2.2.0", data)
97+
correctTool, correctSystem, found := pkgs.FindTool("arduino-test", "arduino-fwuploader", "2.2.0", data)
98+
require.True(t, found)
9799
require.NotNil(t, correctTool)
98100
require.NotNil(t, correctSystem)
99101
require.Equal(t, correctTool.Name, "arduino-fwuploader")
@@ -145,7 +147,7 @@ func TestDownload(t *testing.T) {
145147
if filePath.IsDir() {
146148
require.DirExists(t, filePath.String())
147149
} else {
148-
if OS == "windows" {
150+
if runtime.GOOS == "windows" {
149151
require.FileExists(t, filePath.String()+".exe")
150152
} else {
151153
require.FileExists(t, filePath.String())

‎tools/tools.go

Copy file name to clipboardExpand all lines: tools/tools.go
-12Lines changed: 0 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -78,18 +78,6 @@ func (t *Tools) getMapValue(key string) (string, bool) {
7878
return value, ok
7979
}
8080

81-
// writeMap() writes installed map to the json file "installed.json"
82-
func (t *Tools) writeMap() error {
83-
t.mutex.RLock()
84-
defer t.mutex.RUnlock()
85-
b, err := json.Marshal(t.installed)
86-
if err != nil {
87-
return err
88-
}
89-
filePath := t.directory.Join("installed.json")
90-
return filePath.WriteFile(b)
91-
}
92-
9381
// readMap() reads the installed map from json file "installed.json"
9482
func (t *Tools) readMap() error {
9583
t.mutex.Lock()

‎v2/http.go

Copy file name to clipboardExpand all lines: v2/http.go
+1-1Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ func Server(directory string, index *index.Resource) http.Handler {
4040
logAdapter := LogAdapter{Logger: logger}
4141

4242
// Mount tools
43-
toolsSvc := pkgs.New(index, directory)
43+
toolsSvc := pkgs.New(index, directory, "replace")
4444
toolsEndpoints := toolssvc.NewEndpoints(toolsSvc)
4545
toolsServer := toolssvr.New(toolsEndpoints, mux, CustomRequestDecoder, goahttp.ResponseEncoder, errorHandler(logger), nil)
4646
toolssvr.Mount(mux, toolsServer)

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.