Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 53a6f25

Browse filesBrowse files
BikappaBikappa
authored
feat: purge build cache (#2033)
1 parent c222ad0 commit 53a6f25
Copy full SHA for 53a6f25

File tree

Expand file treeCollapse file tree

17 files changed

+333
-33
lines changed
Filter options
Expand file treeCollapse file tree

17 files changed

+333
-33
lines changed

‎.gitignore

Copy file name to clipboardExpand all lines: .gitignore
+3Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,3 +28,6 @@ venv
2828
/docsgen/arduino-cli.exe
2929
/docs/rpc/*.md
3030
/docs/commands/*.md
31+
32+
# Delve debugger binary file
33+
__debug_bin

‎arduino/sketch/sketch.go

Copy file name to clipboardExpand all lines: arduino/sketch/sketch.go
+1-1Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -294,7 +294,7 @@ func CheckForPdeFiles(sketch *paths.Path) []*paths.Path {
294294
// DefaultBuildPath generates the default build directory for a given sketch.
295295
// The build path is in a temporary directory and is unique for each sketch.
296296
func (s *Sketch) DefaultBuildPath() *paths.Path {
297-
return paths.TempDir().Join("arduino", "sketch-"+s.Hash())
297+
return paths.TempDir().Join("arduino", "sketches", s.Hash())
298298
}
299299

300300
// Hash generate a unique hash for the given sketch.

‎arduino/sketch/sketch_test.go

Copy file name to clipboardExpand all lines: arduino/sketch/sketch_test.go
+1-1Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -286,7 +286,7 @@ func TestNewSketchFolderSymlink(t *testing.T) {
286286
}
287287

288288
func TestGenBuildPath(t *testing.T) {
289-
want := paths.TempDir().Join("arduino", "sketch-ACBD18DB4CC2F85CEDEF654FCCC4A4D8")
289+
want := paths.TempDir().Join("arduino", "sketches", "ACBD18DB4CC2F85CEDEF654FCCC4A4D8")
290290
assert.True(t, (&Sketch{FullPath: paths.New("foo")}).DefaultBuildPath().EquivalentTo(want))
291291
assert.Equal(t, "ACBD18DB4CC2F85CEDEF654FCCC4A4D8", (&Sketch{FullPath: paths.New("foo")}).Hash())
292292
}

‎buildcache/build_cache.go

Copy file name to clipboard
+113Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
// This file is part of arduino-cli.
2+
//
3+
// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
4+
//
5+
// This software is released under the GNU General Public License version 3,
6+
// which covers the main part of arduino-cli.
7+
// The terms of this license can be found at:
8+
// https://www.gnu.org/licenses/gpl-3.0.en.html
9+
//
10+
// You can be released from the requirements of the above licenses by purchasing
11+
// a commercial license. Buying such a license is mandatory if you want to
12+
// modify or otherwise use the software for commercial activities involving the
13+
// Arduino software without disclosing the source code of your own applications.
14+
// To purchase a commercial license, send an email to license@arduino.cc.
15+
16+
package buildcache
17+
18+
import (
19+
"time"
20+
21+
"github.com/arduino/go-paths-helper"
22+
"github.com/pkg/errors"
23+
"github.com/sirupsen/logrus"
24+
)
25+
26+
const (
27+
createDirErrCode = 1
28+
fileWriteErrCode = 2
29+
)
30+
31+
type cacheError struct {
32+
Code int
33+
wrappedErr error
34+
}
35+
36+
func (e cacheError) Error() string {
37+
return e.wrappedErr.Error()
38+
}
39+
40+
func (e cacheError) Unwrap() error {
41+
return e.wrappedErr
42+
}
43+
44+
func (e cacheError) Is(target error) bool {
45+
te, ok := target.(cacheError)
46+
return ok && te.Code == e.Code
47+
}
48+
49+
var (
50+
// CreateDirErr error occurred when creating the cache directory
51+
CreateDirErr = cacheError{Code: createDirErrCode}
52+
// FileWriteErr error occurred when writing the placeholder file
53+
FileWriteErr = cacheError{Code: fileWriteErrCode}
54+
)
55+
56+
const lastUsedFileName = ".last-used"
57+
58+
// BuildCache represents a cache of built files (sketches and cores), it's designed
59+
// to work on directories. Given a directory as "base" it handles direct subdirectories as
60+
// keys
61+
type BuildCache struct {
62+
baseDir *paths.Path
63+
}
64+
65+
// GetOrCreate retrieves or creates the cache directory at the given path
66+
// If the cache already exists the lifetime of the cache is extended.
67+
func (bc *BuildCache) GetOrCreate(key string) (*paths.Path, error) {
68+
keyDir := bc.baseDir.Join(key)
69+
if err := keyDir.MkdirAll(); err != nil {
70+
return nil, cacheError{createDirErrCode, err}
71+
}
72+
73+
if err := keyDir.Join(lastUsedFileName).WriteFile([]byte{}); err != nil {
74+
return nil, cacheError{fileWriteErrCode, err}
75+
}
76+
return keyDir, nil
77+
}
78+
79+
// Purge removes all cache directories within baseDir that have expired
80+
// To know how long ago a directory has been last used
81+
// it checks into the .last-used file.
82+
func (bc *BuildCache) Purge(ttl time.Duration) {
83+
files, err := bc.baseDir.ReadDir()
84+
if err != nil {
85+
return
86+
}
87+
for _, file := range files {
88+
if file.IsDir() {
89+
removeIfExpired(file, ttl)
90+
}
91+
}
92+
}
93+
94+
// New instantiates a build cache
95+
func New(baseDir *paths.Path) *BuildCache {
96+
return &BuildCache{baseDir}
97+
}
98+
99+
func removeIfExpired(dir *paths.Path, ttl time.Duration) {
100+
fileInfo, err := dir.Join(lastUsedFileName).Stat()
101+
if err != nil {
102+
return
103+
}
104+
lifeExpectancy := ttl - time.Since(fileInfo.ModTime())
105+
if lifeExpectancy > 0 {
106+
return
107+
}
108+
logrus.Tracef(`Purging cache directory "%s". Expired by %s`, dir, lifeExpectancy.Abs())
109+
err = dir.RemoveAll()
110+
if err != nil {
111+
logrus.Tracef(`Error while pruning cache directory "%s": %s`, dir, errors.WithStack(err))
112+
}
113+
}

‎buildcache/build_cache_test.go

Copy file name to clipboard
+78Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
// This file is part of arduino-cli.
2+
//
3+
// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
4+
//
5+
// This software is released under the GNU General Public License version 3,
6+
// which covers the main part of arduino-cli.
7+
// The terms of this license can be found at:
8+
// https://www.gnu.org/licenses/gpl-3.0.en.html
9+
//
10+
// You can be released from the requirements of the above licenses by purchasing
11+
// a commercial license. Buying such a license is mandatory if you want to
12+
// modify or otherwise use the software for commercial activities involving the
13+
// Arduino software without disclosing the source code of your own applications.
14+
// To purchase a commercial license, send an email to license@arduino.cc.
15+
16+
package buildcache
17+
18+
import (
19+
"testing"
20+
"time"
21+
22+
"github.com/arduino/go-paths-helper"
23+
"github.com/stretchr/testify/require"
24+
)
25+
26+
func Test_UpdateLastUsedFileNotExisting(t *testing.T) {
27+
testBuildDir := paths.New(t.TempDir(), "sketches", "xxx")
28+
require.NoError(t, testBuildDir.MkdirAll())
29+
timeBeforeUpdating := time.Unix(0, 0)
30+
requireCorrectUpdate(t, testBuildDir, timeBeforeUpdating)
31+
}
32+
33+
func Test_UpdateLastUsedFileExisting(t *testing.T) {
34+
testBuildDir := paths.New(t.TempDir(), "sketches", "xxx")
35+
require.NoError(t, testBuildDir.MkdirAll())
36+
37+
// create the file
38+
preExistingFile := testBuildDir.Join(lastUsedFileName)
39+
require.NoError(t, preExistingFile.WriteFile([]byte{}))
40+
timeBeforeUpdating := time.Now().Add(-time.Second)
41+
preExistingFile.Chtimes(time.Now(), timeBeforeUpdating)
42+
requireCorrectUpdate(t, testBuildDir, timeBeforeUpdating)
43+
}
44+
45+
func requireCorrectUpdate(t *testing.T, dir *paths.Path, prevModTime time.Time) {
46+
_, err := New(dir.Parent()).GetOrCreate(dir.Base())
47+
require.NoError(t, err)
48+
expectedFile := dir.Join(lastUsedFileName)
49+
fileInfo, err := expectedFile.Stat()
50+
require.Nil(t, err)
51+
require.Greater(t, fileInfo.ModTime(), prevModTime)
52+
}
53+
54+
func TestPurge(t *testing.T) {
55+
ttl := time.Minute
56+
57+
dirToPurge := paths.New(t.TempDir(), "root")
58+
59+
lastUsedTimesByDirPath := map[*paths.Path]time.Time{
60+
(dirToPurge.Join("old")): time.Now().Add(-ttl - time.Hour),
61+
(dirToPurge.Join("fresh")): time.Now().Add(-ttl + time.Minute),
62+
}
63+
64+
// create the metadata files
65+
for dirPath, lastUsedTime := range lastUsedTimesByDirPath {
66+
require.NoError(t, dirPath.MkdirAll())
67+
infoFilePath := dirPath.Join(lastUsedFileName).Canonical()
68+
require.NoError(t, infoFilePath.WriteFile([]byte{}))
69+
// make sure access time does not matter
70+
accesstime := time.Now()
71+
require.NoError(t, infoFilePath.Chtimes(accesstime, lastUsedTime))
72+
}
73+
74+
New(dirToPurge).Purge(ttl)
75+
76+
require.False(t, dirToPurge.Join("old").Exist())
77+
require.True(t, dirToPurge.Join("fresh").Exist())
78+
}

‎commands/compile/compile.go

Copy file name to clipboardExpand all lines: commands/compile/compile.go
+29-1Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,11 @@ import (
2727
"github.com/arduino/arduino-cli/arduino/cores"
2828
"github.com/arduino/arduino-cli/arduino/cores/packagemanager"
2929
"github.com/arduino/arduino-cli/arduino/sketch"
30+
"github.com/arduino/arduino-cli/buildcache"
3031
"github.com/arduino/arduino-cli/commands"
3132
"github.com/arduino/arduino-cli/configuration"
3233
"github.com/arduino/arduino-cli/i18n"
34+
"github.com/arduino/arduino-cli/inventory"
3335
"github.com/arduino/arduino-cli/legacy/builder"
3436
"github.com/arduino/arduino-cli/legacy/builder/types"
3537
rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
@@ -135,6 +137,11 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
135137
if err = builderCtx.BuildPath.MkdirAll(); err != nil {
136138
return nil, &arduino.PermissionDeniedError{Message: tr("Cannot create build directory"), Cause: err}
137139
}
140+
141+
buildcache.New(builderCtx.BuildPath.Parent()).GetOrCreate(builderCtx.BuildPath.Base())
142+
// cache is purged after compilation to not remove entries that might be required
143+
defer maybePurgeBuildCache()
144+
138145
builderCtx.CompilationDatabase = bldr.NewCompilationDatabase(
139146
builderCtx.BuildPath.Join("compile_commands.json"),
140147
)
@@ -153,7 +160,7 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
153160
builderCtx.CustomBuildProperties = append(req.GetBuildProperties(), securityKeysOverride...)
154161

155162
if req.GetBuildCachePath() == "" {
156-
builderCtx.CoreBuildCachePath = paths.TempDir().Join("arduino", "core-cache")
163+
builderCtx.CoreBuildCachePath = paths.TempDir().Join("arduino", "cores")
157164
} else {
158165
buildCachePath, err := paths.New(req.GetBuildCachePath()).Abs()
159166
if err != nil {
@@ -287,3 +294,24 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
287294

288295
return r, nil
289296
}
297+
298+
// maybePurgeBuildCache runs the build files cache purge if the policy conditions are met.
299+
func maybePurgeBuildCache() {
300+
301+
compilationsBeforePurge := configuration.Settings.GetUint("build_cache.compilations_before_purge")
302+
// 0 means never purge
303+
if compilationsBeforePurge == 0 {
304+
return
305+
}
306+
compilationSinceLastPurge := inventory.Store.GetUint("build_cache.compilation_count_since_last_purge")
307+
compilationSinceLastPurge++
308+
inventory.Store.Set("build_cache.compilation_count_since_last_purge", compilationSinceLastPurge)
309+
defer inventory.WriteStore()
310+
if compilationsBeforePurge == 0 || compilationSinceLastPurge < compilationsBeforePurge {
311+
return
312+
}
313+
inventory.Store.Set("build_cache.compilation_count_since_last_purge", 0)
314+
cacheTTL := configuration.Settings.GetDuration("build_cache.ttl").Abs()
315+
buildcache.New(paths.TempDir().Join("arduino", "cores")).Purge(cacheTTL)
316+
buildcache.New(paths.TempDir().Join("arduino", "sketches")).Purge(cacheTTL)
317+
}

‎configuration/defaults.go

Copy file name to clipboardExpand all lines: configuration/defaults.go
+3Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ package configuration
1818
import (
1919
"path/filepath"
2020
"strings"
21+
"time"
2122

2223
"github.com/spf13/viper"
2324
)
@@ -41,6 +42,8 @@ func SetDefaults(settings *viper.Viper) {
4142

4243
// Sketch compilation
4344
settings.SetDefault("sketch.always_export_binaries", false)
45+
settings.SetDefault("build_cache.ttl", time.Hour*24*30)
46+
settings.SetDefault("build_cache.compilations_before_purge", 10)
4447

4548
// daemon settings
4649
settings.SetDefault("daemon.port", "50051")

‎docs/configuration.md

Copy file name to clipboardExpand all lines: docs/configuration.md
+6Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,12 @@
3333
to the sketch folder. This is the equivalent of using the [`--export-binaries`][arduino-cli compile options] flag.
3434
- `updater` - configuration options related to Arduino CLI updates
3535
- `enable_notification` - set to `false` to disable notifications of new Arduino CLI releases, defaults to `true`
36+
- `build_cache` configuration options related to the compilation cache
37+
- `compilations_before_purge` - interval, in number of compilations, at which the cache is purged, defaults to `10`.
38+
When `0` the cache is never purged.
39+
- `ttl` - cache expiration time of build folders. If the cache is hit by a compilation the corresponding build files
40+
lifetime is renewed. The value format must be a valid input for
41+
[time.ParseDuration()](https://pkg.go.dev/time#ParseDuration), defaults to `720h` (30 days).
3642

3743
## Configuration methods
3844

‎internal/integrationtest/arduino-cli.go

Copy file name to clipboardExpand all lines: internal/integrationtest/arduino-cli.go
+1Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,7 @@ func NewArduinoCliWithinEnvironment(env *Environment, config *ArduinoCLIConfig)
112112
"ARDUINO_DATA_DIR": cli.dataDir.String(),
113113
"ARDUINO_DOWNLOADS_DIR": cli.stagingDir.String(),
114114
"ARDUINO_SKETCHBOOK_DIR": cli.sketchbookDir.String(),
115+
"ARDUINO_BUILD_CACHE_COMPILATIONS_BEFORE_PURGE": "0",
115116
}
116117
env.RegisterCleanUpCallback(cli.CleanUp)
117118
return cli

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.