Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

Appearance settings

Commit f408819

Browse filesBrowse files
committed
Parallelize library discovery phase in compile
1 parent 076158e commit f408819
Copy full SHA for f408819

File tree

2 files changed

+60
-9
lines changed
Filter options

2 files changed

+60
-9
lines changed

‎internal/arduino/builder/internal/detector/cache.go

Copy file name to clipboardExpand all lines: internal/arduino/builder/internal/detector/cache.go
+15-5Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ import (
1919
"encoding/json"
2020
"fmt"
2121

22+
"github.com/arduino/arduino-cli/internal/arduino/builder/internal/runner"
2223
"github.com/arduino/go-paths-helper"
2324
)
2425

@@ -28,17 +29,18 @@ type detectorCache struct {
2829
}
2930

3031
type detectorCacheEntry struct {
31-
AddedIncludePath *paths.Path `json:"added_include_path,omitempty"`
32-
Compile *sourceFile `json:"compile,omitempty"`
33-
MissingIncludeH *string `json:"missing_include_h,omitempty"`
32+
AddedIncludePath *paths.Path `json:"added_include_path,omitempty"`
33+
Compile *sourceFile `json:"compile,omitempty"`
34+
CompileTask *runner.Task `json:"compile_task,omitempty"`
35+
MissingIncludeH *string `json:"missing_include_h,omitempty"`
3436
}
3537

3638
func (e *detectorCacheEntry) String() string {
3739
if e.AddedIncludePath != nil {
3840
return "Added include path: " + e.AddedIncludePath.String()
3941
}
40-
if e.Compile != nil {
41-
return "Compiling: " + e.Compile.String()
42+
if e.Compile != nil && e.CompileTask != nil {
43+
return "Compiling: " + e.Compile.String() + " / " + e.CompileTask.String()
4244
}
4345
if e.MissingIncludeH != nil {
4446
if *e.MissingIncludeH == "" {
@@ -109,6 +111,14 @@ func (c *detectorCache) Peek() *detectorCacheEntry {
109111
return nil
110112
}
111113

114+
// EntriesAhead returns the entries that are ahead of the current cache position.
115+
func (c *detectorCache) EntriesAhead() []*detectorCacheEntry {
116+
if c.curr < len(c.entries) {
117+
return c.entries[c.curr:]
118+
}
119+
return nil
120+
}
121+
112122
// Save writes the current cache to the given file.
113123
func (c *detectorCache) Save(cacheFile *paths.Path) error {
114124
// Cut off the cache if it is not fully consumed

‎internal/arduino/builder/internal/detector/detector.go

Copy file name to clipboardExpand all lines: internal/arduino/builder/internal/detector/detector.go
+45-4Lines changed: 45 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ type SketchLibrariesDetector struct {
5858
includeFolders paths.PathList
5959
logger *logger.BuilderLogger
6060
diagnosticStore *diagnostics.Store
61+
preRunner *runner.Runner
6162
}
6263

6364
// NewSketchLibrariesDetector todo
@@ -236,6 +237,18 @@ func (l *SketchLibrariesDetector) findIncludes(
236237
l.logger.Warn(i18n.Tr("Failed to load library discovery cache: %[1]s", err))
237238
}
238239

240+
// Pre-run cache entries
241+
l.preRunner = runner.New(ctx)
242+
for _, entry := range l.cache.EntriesAhead() {
243+
if entry.Compile != nil && entry.CompileTask != nil {
244+
upToDate, _ := entry.Compile.ObjFileIsUpToDate()
245+
if !upToDate {
246+
l.preRunner.Enqueue(entry.CompileTask)
247+
}
248+
}
249+
}
250+
defer l.preRunner.Cancel()
251+
239252
l.addIncludeFolder(buildCorePath)
240253
if buildVariantPath != nil {
241254
l.addIncludeFolder(buildVariantPath)
@@ -263,6 +276,15 @@ func (l *SketchLibrariesDetector) findIncludes(
263276
cachePath.Remove()
264277
return err
265278
}
279+
280+
// Create a new pre-runner if the previous one was cancelled
281+
if l.preRunner == nil {
282+
l.preRunner = runner.New(ctx)
283+
// Push in the remainder of the queue
284+
for _, sourceFile := range *sourceFileQueue {
285+
l.preRunner.Enqueue(l.gccPreprocessTask(sourceFile, buildProperties))
286+
}
287+
}
266288
}
267289

268290
// Finalize the cache
@@ -326,9 +348,9 @@ func (l *SketchLibrariesDetector) findMissingIncludesInCompilationUnit(
326348

327349
first := true
328350
for {
329-
l.cache.Expect(&detectorCacheEntry{Compile: sourceFile})
330-
331351
preprocTask := l.gccPreprocessTask(sourceFile, buildProperties)
352+
l.cache.Expect(&detectorCacheEntry{Compile: sourceFile, CompileTask: preprocTask})
353+
332354
var preprocErr error
333355
var preprocResult *runner.Result
334356

@@ -340,8 +362,27 @@ func (l *SketchLibrariesDetector) findMissingIncludesInCompilationUnit(
340362
}
341363
first = false
342364
} else {
343-
preprocResult = preprocTask.Run(ctx)
344-
preprocErr = preprocResult.Error
365+
if l.preRunner != nil {
366+
if r := l.preRunner.Results(preprocTask); r != nil {
367+
preprocResult = r
368+
preprocErr = preprocResult.Error
369+
}
370+
}
371+
if preprocResult == nil {
372+
// The pre-runner missed this task, maybe the cache is outdated
373+
// or maybe the source code changed.
374+
375+
// Stop the pre-runner
376+
if l.preRunner != nil {
377+
preRunner := l.preRunner
378+
l.preRunner = nil
379+
go preRunner.Cancel()
380+
}
381+
382+
// Run the actual preprocessor
383+
preprocResult = preprocTask.Run(ctx)
384+
preprocErr = preprocResult.Error
385+
}
345386
if l.logger.Verbose() {
346387
l.logger.WriteStdout(preprocResult.Stdout)
347388
}

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.