Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 5d44cb3

Browse filesBrowse files
author
Luca Bianconi
committed
feat: purge cache after expiration time
1 parent 271d241 commit 5d44cb3
Copy full SHA for 5d44cb3

File tree

Expand file treeCollapse file tree

17 files changed

+289
-35
lines changed
Filter options
Expand file treeCollapse file tree

17 files changed

+289
-35
lines changed

‎.gitignore

Copy file name to clipboardExpand all lines: .gitignore
+3Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,3 +28,6 @@ venv
2828
/docsgen/arduino-cli.exe
2929
/docs/rpc/*.md
3030
/docs/commands/*.md
31+
32+
# Delve debugger binary file
33+
__debug_bin

‎arduino/sketch/sketch.go

Copy file name to clipboardExpand all lines: arduino/sketch/sketch.go
+6-1Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -302,5 +302,10 @@ func GenBuildPath(sketchPath *paths.Path) *paths.Path {
302302
}
303303
md5SumBytes := md5.Sum([]byte(path))
304304
md5Sum := strings.ToUpper(hex.EncodeToString(md5SumBytes[:]))
305-
return paths.TempDir().Join("arduino", "sketch-"+md5Sum)
305+
306+
return getSketchesCacheDir().Join(md5Sum)
307+
}
308+
309+
func getSketchesCacheDir() *paths.Path {
310+
return paths.TempDir().Join("arduino", "sketches").Canonical()
306311
}

‎arduino/sketch/sketch_test.go

Copy file name to clipboardExpand all lines: arduino/sketch/sketch_test.go
+2-2Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -286,10 +286,10 @@ func TestNewSketchFolderSymlink(t *testing.T) {
286286
}
287287

288288
func TestGenBuildPath(t *testing.T) {
289-
want := paths.TempDir().Join("arduino", "sketch-ACBD18DB4CC2F85CEDEF654FCCC4A4D8")
289+
want := paths.TempDir().Join("arduino", "sketches", "ACBD18DB4CC2F85CEDEF654FCCC4A4D8")
290290
assert.True(t, GenBuildPath(paths.New("foo")).EquivalentTo(want))
291291

292-
want = paths.TempDir().Join("arduino", "sketch-D41D8CD98F00B204E9800998ECF8427E")
292+
want = paths.TempDir().Join("arduino", "sketches", "D41D8CD98F00B204E9800998ECF8427E")
293293
assert.True(t, GenBuildPath(nil).EquivalentTo(want))
294294
}
295295

‎buildcache/build_cache.go

Copy file name to clipboard
+69Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
// This file is part of arduino-cli.
2+
//
3+
// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
4+
//
5+
// This software is released under the GNU General Public License version 3,
6+
// which covers the main part of arduino-cli.
7+
// The terms of this license can be found at:
8+
// https://www.gnu.org/licenses/gpl-3.0.en.html
9+
//
10+
// You can be released from the requirements of the above licenses by purchasing
11+
// a commercial license. Buying such a license is mandatory if you want to
12+
// modify or otherwise use the software for commercial activities involving the
13+
// Arduino software without disclosing the source code of your own applications.
14+
// To purchase a commercial license, send an email to license@arduino.cc.
15+
16+
package buildcache
17+
18+
import (
19+
"time"
20+
21+
"github.com/arduino/go-paths-helper"
22+
"github.com/pkg/errors"
23+
"github.com/sirupsen/logrus"
24+
)
25+
26+
const lastUsedFileName = ".last-used"
27+
28+
// GetOrCreate retrieves or creates the cache directory at the given path
29+
// If the cache already exists the lifetime of the cache is extended.
30+
func GetOrCreate(dir *paths.Path) error {
31+
if !dir.Exist() {
32+
if err := dir.MkdirAll(); err != nil {
33+
return err
34+
}
35+
}
36+
37+
return dir.Join(lastUsedFileName).WriteFile([]byte{})
38+
}
39+
40+
// Purge removes all cache directories within baseDir that have expired
41+
// To know how long ago a directory has been last used
42+
// it checks into the .last-used file.
43+
func Purge(baseDir *paths.Path, ttl time.Duration) {
44+
files, err := baseDir.ReadDir()
45+
if err != nil {
46+
return
47+
}
48+
for _, file := range files {
49+
if file.IsDir() {
50+
removeIfExpired(file, ttl)
51+
}
52+
}
53+
}
54+
55+
func removeIfExpired(dir *paths.Path, ttl time.Duration) {
56+
fileInfo, err := dir.Join().Stat()
57+
if err != nil {
58+
return
59+
}
60+
lifeExpectancy := ttl - time.Since(fileInfo.ModTime())
61+
if lifeExpectancy > 0 {
62+
return
63+
}
64+
logrus.Tracef(`Purging cache directory "%s". Expired by %s`, dir, lifeExpectancy.Abs())
65+
err = dir.RemoveAll()
66+
if err != nil {
67+
logrus.Tracef(`Error while pruning cache directory "%s": %s`, dir, errors.WithStack(err))
68+
}
69+
}

‎buildcache/build_cache_test.go

Copy file name to clipboard
+79Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
// This file is part of arduino-cli.
2+
//
3+
// Copyright 2020 ARDUINO SA (http://www.arduino.cc/)
4+
//
5+
// This software is released under the GNU General Public License version 3,
6+
// which covers the main part of arduino-cli.
7+
// The terms of this license can be found at:
8+
// https://www.gnu.org/licenses/gpl-3.0.en.html
9+
//
10+
// You can be released from the requirements of the above licenses by purchasing
11+
// a commercial license. Buying such a license is mandatory if you want to
12+
// modify or otherwise use the software for commercial activities involving the
13+
// Arduino software without disclosing the source code of your own applications.
14+
// To purchase a commercial license, send an email to license@arduino.cc.
15+
16+
package buildcache
17+
18+
import (
19+
"testing"
20+
"time"
21+
22+
"github.com/arduino/go-paths-helper"
23+
"github.com/stretchr/testify/require"
24+
)
25+
26+
func Test_UpdateLastUsedFileNotExisting(t *testing.T) {
27+
testBuildDir := paths.New(t.TempDir(), "sketches", "xxx")
28+
require.NoError(t, testBuildDir.MkdirAll())
29+
timeBeforeUpdating := time.Unix(0, 0)
30+
requireCorrectUpdate(t, testBuildDir, timeBeforeUpdating)
31+
}
32+
33+
func Test_UpdateLastUsedFileExisting(t *testing.T) {
34+
testBuildDir := paths.New(t.TempDir(), "sketches", "xxx")
35+
require.NoError(t, testBuildDir.MkdirAll())
36+
37+
// create the file
38+
preExistingFile := testBuildDir.Join(lastUsedFileName)
39+
require.NoError(t, preExistingFile.WriteFile([]byte{}))
40+
timeBeforeUpdating := time.Now().Add(-time.Second)
41+
preExistingFile.Chtimes(time.Now(), timeBeforeUpdating)
42+
requireCorrectUpdate(t, testBuildDir, timeBeforeUpdating)
43+
}
44+
45+
func requireCorrectUpdate(t *testing.T, dir *paths.Path, prevModTime time.Time) {
46+
require.NoError(t, GetOrCreate(dir))
47+
expectedFile := dir.Join(lastUsedFileName)
48+
fileInfo, err := expectedFile.Stat()
49+
require.Nil(t, err)
50+
require.Greater(t, fileInfo.ModTime(), prevModTime)
51+
}
52+
53+
func TestPurge(t *testing.T) {
54+
ttl := time.Minute
55+
56+
dirToPurge := paths.New(t.TempDir(), "root")
57+
58+
lastUsedTimesByDirPath := map[*paths.Path]time.Time{
59+
(dirToPurge.Join("old")): time.Now().Add(-ttl - time.Hour),
60+
(dirToPurge.Join("fresh")): time.Now().Add(-ttl + time.Minute),
61+
}
62+
63+
// create the metadata files
64+
for dirPath, lastUsedTime := range lastUsedTimesByDirPath {
65+
require.NoError(t, dirPath.MkdirAll())
66+
infoFilePath := dirPath.Join(lastUsedFileName).Canonical()
67+
require.NoError(t, infoFilePath.WriteFile([]byte{}))
68+
// make sure access time does not matter
69+
accesstime := time.Now()
70+
require.NoError(t, infoFilePath.Chtimes(accesstime, lastUsedTime))
71+
}
72+
73+
Purge(dirToPurge, ttl)
74+
75+
files, err := dirToPurge.Join("fresh").Stat()
76+
require.Nil(t, err)
77+
require.True(t, files.IsDir())
78+
require.True(t, dirToPurge.Exist())
79+
}

‎commands/compile/compile.go

Copy file name to clipboardExpand all lines: commands/compile/compile.go
+29-2Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,11 @@ import (
2727
"github.com/arduino/arduino-cli/arduino/cores"
2828
"github.com/arduino/arduino-cli/arduino/cores/packagemanager"
2929
"github.com/arduino/arduino-cli/arduino/sketch"
30+
"github.com/arduino/arduino-cli/buildcache"
3031
"github.com/arduino/arduino-cli/commands"
3132
"github.com/arduino/arduino-cli/configuration"
3233
"github.com/arduino/arduino-cli/i18n"
34+
"github.com/arduino/arduino-cli/inventory"
3335
"github.com/arduino/arduino-cli/legacy/builder"
3436
"github.com/arduino/arduino-cli/legacy/builder/types"
3537
rpc "github.com/arduino/arduino-cli/rpc/cc/arduino/cli/commands/v1"
@@ -135,6 +137,11 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
135137
if err = builderCtx.BuildPath.MkdirAll(); err != nil {
136138
return nil, &arduino.PermissionDeniedError{Message: tr("Cannot create build directory"), Cause: err}
137139
}
140+
141+
buildcache.GetOrCreate(builderCtx.BuildPath)
142+
// cache is purged after compilation to not remove entries that might be required
143+
defer maybePurgeBuildCache()
144+
138145
builderCtx.CompilationDatabase = bldr.NewCompilationDatabase(
139146
builderCtx.BuildPath.Join("compile_commands.json"),
140147
)
@@ -143,8 +150,7 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
143150

144151
// Optimize for debug
145152
builderCtx.OptimizeForDebug = req.GetOptimizeForDebug()
146-
147-
builderCtx.CoreBuildCachePath = paths.TempDir().Join("arduino", "core-cache")
153+
builderCtx.CoreBuildCachePath = paths.TempDir().Join("arduino", "cores")
148154

149155
builderCtx.Jobs = int(req.GetJobs())
150156

@@ -284,3 +290,24 @@ func Compile(ctx context.Context, req *rpc.CompileRequest, outStream, errStream
284290

285291
return r, nil
286292
}
293+
294+
// maybePurgeBuildCache runs the build files cache purge if the policy conditions are met.
295+
func maybePurgeBuildCache() {
296+
297+
compilationsBeforePurge := configuration.Settings.GetUint("build_cache.compilations_before_purge")
298+
// 0 means never purge
299+
if compilationsBeforePurge == 0 {
300+
return
301+
}
302+
compilationSinceLastPurge := inventory.Store.GetUint("build_cache.compilation_count_since_last_purge")
303+
compilationSinceLastPurge++
304+
inventory.Store.Set("build_cache.compilation_count_since_last_purge", compilationSinceLastPurge)
305+
defer inventory.WriteStore()
306+
if compilationsBeforePurge == 0 || compilationSinceLastPurge < compilationsBeforePurge {
307+
return
308+
}
309+
inventory.Store.Set("build_cache.compilation_count_since_last_purge", 0)
310+
cacheTTL := configuration.Settings.GetDuration("build_cache.ttl").Abs()
311+
buildcache.Purge(paths.TempDir().Join("arduino", "cores"), cacheTTL)
312+
buildcache.Purge(paths.TempDir().Join("arduino", "sketches"), cacheTTL)
313+
}

‎configuration/defaults.go

Copy file name to clipboardExpand all lines: configuration/defaults.go
+3Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ package configuration
1818
import (
1919
"path/filepath"
2020
"strings"
21+
"time"
2122

2223
"github.com/spf13/viper"
2324
)
@@ -41,6 +42,8 @@ func SetDefaults(settings *viper.Viper) {
4142

4243
// Sketch compilation
4344
settings.SetDefault("sketch.always_export_binaries", false)
45+
settings.SetDefault("build_cache.ttl", time.Hour*24*30)
46+
settings.SetDefault("build_cache.compilations_before_purge", 10)
4447

4548
// daemon settings
4649
settings.SetDefault("daemon.port", "50051")

‎docs/configuration.md

Copy file name to clipboardExpand all lines: docs/configuration.md
+6Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,12 @@
3333
to the sketch folder. This is the equivalent of using the [`--export-binaries`][arduino-cli compile options] flag.
3434
- `updater` - configuration options related to Arduino CLI updates
3535
- `enable_notification` - set to `false` to disable notifications of new Arduino CLI releases, defaults to `true`
36+
- `build_cache` configuration options related to the compilation cache
37+
- `compilations_before_purge` - interval, in number of compilations, at which the cache is purged, defaults to `10`.
38+
When `0` the cache is never purged.
39+
- `ttl` - cache expiration time of build folders. If the cache is hit by a compilation the corresponding build files
40+
lifetime is renewed. The value format must be a valid input for
41+
[time.ParseDuration()](https://pkg.go.dev/time#ParseDuration), defaults to `720h` (30 days).
3642

3743
## Configuration methods
3844

‎internal/integrationtest/arduino-cli.go

Copy file name to clipboardExpand all lines: internal/integrationtest/arduino-cli.go
+1Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,7 @@ func NewArduinoCliWithinEnvironment(env *Environment, config *ArduinoCLIConfig)
112112
"ARDUINO_DATA_DIR": cli.dataDir.String(),
113113
"ARDUINO_DOWNLOADS_DIR": cli.stagingDir.String(),
114114
"ARDUINO_SKETCHBOOK_DIR": cli.sketchbookDir.String(),
115+
"ARDUINO_BUILD_CACHE_COMPILATIONS_BEFORE_PURGE": "0",
115116
}
116117
env.RegisterCleanUpCallback(cli.CleanUp)
117118
return cli

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.