Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

Appearance settings

Improved compile speed by running multi-threaded library discovery. #2625

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 28 commits into
base: master
Choose a base branch
Loading
from
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
31949e4
Removed unused field/parameter
cmaglie May 30, 2024
f0a1626
Moved Result structure into his own package
cmaglie May 30, 2024
0990aed
Moving sourceFile object in his own file
cmaglie May 31, 2024
dd46217
Moved uniqueSourceFileQueue in his own file
cmaglie May 31, 2024
a6de50a
Moved includeCache in his own file and made it a field of detector
cmaglie May 31, 2024
bc5f659
Fixed comment
cmaglie May 31, 2024
7f0ee88
Renamed variable (typo)
cmaglie Jun 5, 2024
89a16d1
Simplified handling of de-duplication of cache hit messages
cmaglie May 31, 2024
cc84072
Implemented a better include-cache
cmaglie May 31, 2024
c35baeb
Remove the old, no longer used, includeCache
cmaglie May 31, 2024
d077ae9
Simplified error reporting in library detection
cmaglie May 31, 2024
59accf4
Remove useless targetFilePath variable
cmaglie Jun 1, 2024
fe6273d
Slight improvement of removeBuildFromSketchFiles
cmaglie Jun 2, 2024
1ed2385
Rename variables for clarity
cmaglie Jun 2, 2024
623e23e
Removed hardcoded build.warn_data_percentage in build.options file
cmaglie Jun 2, 2024
885395e
Renamed variables for clarity
cmaglie Jun 2, 2024
bc6a931
Renamed variables for clarity
cmaglie Jun 2, 2024
e28be78
Pre-compute sourceFile fields, and save the in the includes.cache
cmaglie Jun 3, 2024
079a3a4
Added ObjFileIsUpToDate method to sourceFile
cmaglie Jun 3, 2024
917a2ae
Implemented parallel task runner
cmaglie Jun 4, 2024
7f22355
Simplify use of properties.SplitQuotedString
cmaglie Jun 4, 2024
47c5915
Use runner.Task in GCC preprocessor
cmaglie Jun 5, 2024
637c98f
Parallelize library discovery phase in compile
cmaglie Jun 5, 2024
e805906
The number of jobs in library detection now follows --jobs flag
cmaglie Jun 11, 2024
3246db0
Reordered properties construction for clarity
cmaglie Jun 13, 2024
2bf113b
Reordered compileFileWithRecipe for clarity
cmaglie Jun 13, 2024
a9cf210
Added integration test
cmaglie Jun 12, 2024
8913485
fix: libraries are recompiled if the list of include paths changes
cmaglie Jun 13, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Pre-compute sourceFile fields, and save the in the includes.cache
  • Loading branch information
cmaglie committed Feb 13, 2025
commit e28be78e25f549786b9f04d311c857c9b8e21353
10 changes: 5 additions & 5 deletions 10 internal/arduino/builder/internal/detector/cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,17 +28,17 @@ type detectorCache struct {
}

type detectorCacheEntry struct {
AddedIncludePath *paths.Path `json:"added_include_path,omitempty"`
CompilingSourcePath *paths.Path `json:"compiling_source_path,omitempty"`
MissingIncludeH *string `json:"missing_include_h,omitempty"`
AddedIncludePath *paths.Path `json:"added_include_path,omitempty"`
Compile *sourceFile `json:"compile,omitempty"`
MissingIncludeH *string `json:"missing_include_h,omitempty"`
}

func (e *detectorCacheEntry) String() string {
if e.AddedIncludePath != nil {
return "Added include path: " + e.AddedIncludePath.String()
}
if e.CompilingSourcePath != nil {
return "Compiling source path: " + e.CompilingSourcePath.String()
if e.Compile != nil {
return "Compiling: " + e.Compile.String()
}
if e.MissingIncludeH != nil {
if *e.MissingIncludeH == "" {
Expand Down
10 changes: 5 additions & 5 deletions 10 internal/arduino/builder/internal/detector/detector.go
Original file line number Diff line number Diff line change
Expand Up @@ -292,9 +292,9 @@ func (l *SketchLibrariesDetector) findMissingIncludesInCompilationUnit(
platformArch string,
) error {
sourceFile := sourceFileQueue.Pop()
sourcePath := sourceFile.SourcePath()
depPath := sourceFile.DepfilePath()
objPath := sourceFile.ObjectPath()
sourcePath := sourceFile.SourcePath
depPath := sourceFile.DepfilePath
objPath := sourceFile.ObjectPath

// TODO: This should perhaps also compare against the
// include.cache file timestamp. Now, it only checks if the file
Expand All @@ -315,14 +315,14 @@ func (l *SketchLibrariesDetector) findMissingIncludesInCompilationUnit(

first := true
for {
l.cache.Expect(&detectorCacheEntry{CompilingSourcePath: sourcePath})
l.cache.Expect(&detectorCacheEntry{Compile: sourceFile})

// Libraries may require the "utility" directory to be added to the include
// search path, but only for the source code of the library, so we temporary
// copy the current search path list and add the library' utility directory
// if needed.
includeFolders := l.includeFolders
if extraInclude := sourceFile.ExtraIncludePath(); extraInclude != nil {
if extraInclude := sourceFile.ExtraIncludePath; extraInclude != nil {
includeFolders = append(includeFolders, extraInclude)
}

Expand Down
79 changes: 32 additions & 47 deletions 79 internal/arduino/builder/internal/detector/source_file.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,85 +16,70 @@
package detector

import (
"fmt"
"slices"

"github.com/arduino/go-paths-helper"
)

type sourceFile struct {
// Path to the source file within the sketch/library root folder
relativePath *paths.Path
// SourcePath is the path to the source file
SourcePath *paths.Path `json:"source_path"`

// ObjectPath is the path to the object file that will be generated
ObjectPath *paths.Path `json:"object_path"`

// DepfilePath is the path to the dependency file that will be generated
DepfilePath *paths.Path `json:"depfile_path"`

// ExtraIncludePath contains an extra include path that must be
// used to compile this source file.
// This is mainly used for source files that comes from old-style libraries
// (Arduino IDE <1.5) requiring an extra include path to the "utility" folder.
extraIncludePath *paths.Path

// The source root for the given origin, where its source files
// can be found. Prepending this to SourceFile.RelativePath will give
// the full path to that source file.
sourceRoot *paths.Path
ExtraIncludePath *paths.Path `json:"extra_include_path,omitempty"`
}

// The build root for the given origin, where build products will
// be placed. Any directories inside SourceFile.RelativePath will be
// appended here.
buildRoot *paths.Path
func (f *sourceFile) String() string {
return fmt.Sprintf("SourcePath:%s SourceRoot:%s BuildRoot:%s ExtraInclude:%s",
f.SourcePath, f.ObjectPath, f.DepfilePath, f.ExtraIncludePath)
}

// Equals fixdoc
// Equals checks if a sourceFile is equal to another.
func (f *sourceFile) Equals(g *sourceFile) bool {
return f.relativePath.EqualsTo(g.relativePath) &&
f.buildRoot.EqualsTo(g.buildRoot) &&
f.sourceRoot.EqualsTo(g.sourceRoot)
return f.SourcePath.EqualsTo(g.SourcePath) &&
f.ObjectPath.EqualsTo(g.ObjectPath) &&
f.DepfilePath.EqualsTo(g.DepfilePath) &&
((f.ExtraIncludePath == nil && g.ExtraIncludePath == nil) ||
(f.ExtraIncludePath != nil && g.ExtraIncludePath != nil && f.ExtraIncludePath.EqualsTo(g.ExtraIncludePath)))
}

// makeSourceFile create a sourceFile object for the given source file path.
// The given sourceFilePath can be absolute, or relative within the sourceRoot root folder.
func makeSourceFile(sourceRoot, buildRoot, sourceFilePath *paths.Path, extraIncludePath ...*paths.Path) (*sourceFile, error) {
res := &sourceFile{
buildRoot: buildRoot,
sourceRoot: sourceRoot,
}

if len(extraIncludePath) > 1 {
func makeSourceFile(sourceRoot, buildRoot, sourceFilePath *paths.Path, extraIncludePaths ...*paths.Path) (*sourceFile, error) {
if len(extraIncludePaths) > 1 {
panic("only one extra include path allowed")
}
if len(extraIncludePath) > 0 {
res.extraIncludePath = extraIncludePath[0]
var extraIncludePath *paths.Path
if len(extraIncludePaths) > 0 {
extraIncludePath = extraIncludePaths[0]
}

if sourceFilePath.IsAbs() {
var err error
sourceFilePath, err = res.sourceRoot.RelTo(sourceFilePath)
sourceFilePath, err = sourceRoot.RelTo(sourceFilePath)
if err != nil {
return nil, err
}
}
res.relativePath = sourceFilePath
res := &sourceFile{
SourcePath: sourceRoot.JoinPath(sourceFilePath),
ObjectPath: buildRoot.Join(sourceFilePath.String() + ".o"),
DepfilePath: buildRoot.Join(sourceFilePath.String() + ".d"),
ExtraIncludePath: extraIncludePath,
}
return res, nil
}

// ExtraIncludePath returns the extra include path required to build the source file.
func (f *sourceFile) ExtraIncludePath() *paths.Path {
return f.extraIncludePath
}

// SourcePath return the full path to the source file.
func (f *sourceFile) SourcePath() *paths.Path {
return f.sourceRoot.JoinPath(f.relativePath)
}

// ObjectPath return the full path to the object file.
func (f *sourceFile) ObjectPath() *paths.Path {
return f.buildRoot.Join(f.relativePath.String() + ".o")
}

// DepfilePath return the full path to the dependency file.
func (f *sourceFile) DepfilePath() *paths.Path {
return f.buildRoot.Join(f.relativePath.String() + ".d")
}

// uniqueSourceFileQueue is a queue of source files that does not allow duplicates.
type uniqueSourceFileQueue []*sourceFile

Expand Down
Morty Proxy This is a proxified and sanitized view of the page, visit original site.