Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 3eeae4d

Browse filesBrowse files
author
Arslan Khabutdinov
committed
Reduce llvm-gsymutil memory usage
1 parent 86ba681 commit 3eeae4d
Copy full SHA for 3eeae4d

File tree

Expand file treeCollapse file tree

5 files changed

+143
-115
lines changed
Filter options
Expand file treeCollapse file tree

5 files changed

+143
-115
lines changed

‎llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h

Copy file name to clipboardExpand all lines: llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h
+5Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,7 @@ class DWARFContext : public DIContext {
103103
std::unique_ptr<DWARFDebugMacro>
104104
parseMacroOrMacinfo(MacroSecType SectionType);
105105

106+
virtual void doWorkThreadSafely(function_ref<void()> Work) = 0;
106107
};
107108
friend class DWARFContextState;
108109

@@ -491,6 +492,10 @@ class DWARFContext : public DIContext {
491492
/// manually only for DWARF5.
492493
void setParseCUTUIndexManually(bool PCUTU) { ParseCUTUIndexManually = PCUTU; }
493494

495+
void doWorkThreadSafely(function_ref<void()> Work) {
496+
State->doWorkThreadSafely(Work);
497+
}
498+
494499
private:
495500
void addLocalsForDie(DWARFCompileUnit *CU, DWARFDie Subprogram, DWARFDie Die,
496501
std::vector<DILocal> &Result);

‎llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h

Copy file name to clipboardExpand all lines: llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h
+3-3Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -566,6 +566,9 @@ class DWARFUnit {
566566

567567
Error tryExtractDIEsIfNeeded(bool CUDieOnly);
568568

569+
/// clearDIEs - Clear parsed DIEs to keep memory usage low.
570+
void clearDIEs(bool KeepCUDie, bool KeepDWODies = false);
571+
569572
private:
570573
/// Size in bytes of the .debug_info data associated with this compile unit.
571574
size_t getDebugInfoSize() const {
@@ -581,9 +584,6 @@ class DWARFUnit {
581584
void extractDIEsToVector(bool AppendCUDie, bool AppendNonCUDIEs,
582585
std::vector<DWARFDebugInfoEntry> &DIEs) const;
583586

584-
/// clearDIEs - Clear parsed DIEs to keep memory usage low.
585-
void clearDIEs(bool KeepCUDie);
586-
587587
/// parseDWO - Parses .dwo file for current compile unit. Returns true if
588588
/// it was actually constructed.
589589
/// The \p AlternativeLocation specifies an alternative location to get

‎llvm/lib/DebugInfo/DWARF/DWARFContext.cpp

Copy file name to clipboardExpand all lines: llvm/lib/DebugInfo/DWARF/DWARFContext.cpp
+6-1Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -622,7 +622,7 @@ class ThreadUnsafeDWARFContextState : public DWARFContext::DWARFContextState {
622622
return getNormalTypeUnitMap();
623623
}
624624

625-
625+
void doWorkThreadSafely(function_ref<void()> Work) override { Work(); }
626626
};
627627

628628
class ThreadSafeState : public ThreadUnsafeDWARFContextState {
@@ -738,6 +738,11 @@ class ThreadSafeState : public ThreadUnsafeDWARFContextState {
738738
std::unique_lock<std::recursive_mutex> LockGuard(Mutex);
739739
return ThreadUnsafeDWARFContextState::getTypeUnitMap(IsDWO);
740740
}
741+
742+
void doWorkThreadSafely(function_ref<void()> Work) override {
743+
std::unique_lock<std::recursive_mutex> LockGuard(Mutex);
744+
ThreadUnsafeDWARFContextState::doWorkThreadSafely(Work);
745+
}
741746
};
742747
} // namespace
743748

‎llvm/lib/DebugInfo/DWARF/DWARFUnit.cpp

Copy file name to clipboardExpand all lines: llvm/lib/DebugInfo/DWARF/DWARFUnit.cpp
+116-105Lines changed: 116 additions & 105 deletions
Original file line numberDiff line numberDiff line change
@@ -496,108 +496,114 @@ void DWARFUnit::extractDIEsIfNeeded(bool CUDieOnly) {
496496
}
497497

498498
Error DWARFUnit::tryExtractDIEsIfNeeded(bool CUDieOnly) {
499-
if ((CUDieOnly && !DieArray.empty()) ||
500-
DieArray.size() > 1)
501-
return Error::success(); // Already parsed.
502-
503-
bool HasCUDie = !DieArray.empty();
504-
extractDIEsToVector(!HasCUDie, !CUDieOnly, DieArray);
505-
506-
if (DieArray.empty())
507-
return Error::success();
508-
509-
// If CU DIE was just parsed, copy several attribute values from it.
510-
if (HasCUDie)
511-
return Error::success();
512-
513-
DWARFDie UnitDie(this, &DieArray[0]);
514-
if (std::optional<uint64_t> DWOId =
515-
toUnsigned(UnitDie.find(DW_AT_GNU_dwo_id)))
516-
Header.setDWOId(*DWOId);
517-
if (!IsDWO) {
518-
assert(AddrOffsetSectionBase == std::nullopt);
519-
assert(RangeSectionBase == 0);
520-
assert(LocSectionBase == 0);
521-
AddrOffsetSectionBase = toSectionOffset(UnitDie.find(DW_AT_addr_base));
522-
if (!AddrOffsetSectionBase)
523-
AddrOffsetSectionBase =
524-
toSectionOffset(UnitDie.find(DW_AT_GNU_addr_base));
525-
RangeSectionBase = toSectionOffset(UnitDie.find(DW_AT_rnglists_base), 0);
526-
LocSectionBase = toSectionOffset(UnitDie.find(DW_AT_loclists_base), 0);
527-
}
499+
Error e = Error::success();
500+
Context.doWorkThreadSafely([&] {
501+
if ((CUDieOnly && !DieArray.empty()) ||
502+
DieArray.size() > 1)
503+
return; // Already parsed.
528504

529-
// In general, in DWARF v5 and beyond we derive the start of the unit's
530-
// contribution to the string offsets table from the unit DIE's
531-
// DW_AT_str_offsets_base attribute. Split DWARF units do not use this
532-
// attribute, so we assume that there is a contribution to the string
533-
// offsets table starting at offset 0 of the debug_str_offsets.dwo section.
534-
// In both cases we need to determine the format of the contribution,
535-
// which may differ from the unit's format.
536-
DWARFDataExtractor DA(Context.getDWARFObj(), StringOffsetSection,
537-
IsLittleEndian, 0);
538-
if (IsDWO || getVersion() >= 5) {
539-
auto StringOffsetOrError =
540-
IsDWO ? determineStringOffsetsTableContributionDWO(DA)
541-
: determineStringOffsetsTableContribution(DA);
542-
if (!StringOffsetOrError)
543-
return createStringError(errc::invalid_argument,
544-
"invalid reference to or invalid content in "
545-
".debug_str_offsets[.dwo]: " +
546-
toString(StringOffsetOrError.takeError()));
547-
548-
StringOffsetsTableContribution = *StringOffsetOrError;
549-
}
505+
bool HasCUDie = !DieArray.empty();
506+
extractDIEsToVector(!HasCUDie, !CUDieOnly, DieArray);
507+
508+
if (DieArray.empty())
509+
return;
510+
511+
// If CU DIE was just parsed, copy several attribute values from it.
512+
if (HasCUDie)
513+
return;
514+
515+
DWARFDie UnitDie(this, &DieArray[0]);
516+
if (std::optional<uint64_t> DWOId =
517+
toUnsigned(UnitDie.find(DW_AT_GNU_dwo_id)))
518+
Header.setDWOId(*DWOId);
519+
if (!IsDWO) {
520+
assert(AddrOffsetSectionBase == std::nullopt);
521+
assert(RangeSectionBase == 0);
522+
assert(LocSectionBase == 0);
523+
AddrOffsetSectionBase = toSectionOffset(UnitDie.find(DW_AT_addr_base));
524+
if (!AddrOffsetSectionBase)
525+
AddrOffsetSectionBase =
526+
toSectionOffset(UnitDie.find(DW_AT_GNU_addr_base));
527+
RangeSectionBase = toSectionOffset(UnitDie.find(DW_AT_rnglists_base), 0);
528+
LocSectionBase = toSectionOffset(UnitDie.find(DW_AT_loclists_base), 0);
529+
}
530+
531+
// In general, in DWARF v5 and beyond we derive the start of the unit's
532+
// contribution to the string offsets table from the unit DIE's
533+
// DW_AT_str_offsets_base attribute. Split DWARF units do not use this
534+
// attribute, so we assume that there is a contribution to the string
535+
// offsets table starting at offset 0 of the debug_str_offsets.dwo section.
536+
// In both cases we need to determine the format of the contribution,
537+
// which may differ from the unit's format.
538+
DWARFDataExtractor DA(Context.getDWARFObj(), StringOffsetSection,
539+
IsLittleEndian, 0);
540+
if (IsDWO || getVersion() >= 5) {
541+
auto StringOffsetOrError =
542+
IsDWO ? determineStringOffsetsTableContributionDWO(DA)
543+
: determineStringOffsetsTableContribution(DA);
544+
if (!StringOffsetOrError) {
545+
e = createStringError(errc::invalid_argument,
546+
"invalid reference to or invalid content in "
547+
".debug_str_offsets[.dwo]: " +
548+
toString(StringOffsetOrError.takeError()));
549+
return;
550+
}
551+
552+
StringOffsetsTableContribution = *StringOffsetOrError;
553+
}
554+
555+
// DWARF v5 uses the .debug_rnglists and .debug_rnglists.dwo sections to
556+
// describe address ranges.
557+
if (getVersion() >= 5) {
558+
// In case of DWP, the base offset from the index has to be added.
559+
if (IsDWO) {
560+
uint64_t ContributionBaseOffset = 0;
561+
if (auto *IndexEntry = Header.getIndexEntry())
562+
if (auto *Contrib = IndexEntry->getContribution(DW_SECT_RNGLISTS))
563+
ContributionBaseOffset = Contrib->getOffset();
564+
setRangesSection(
565+
&Context.getDWARFObj().getRnglistsDWOSection(),
566+
ContributionBaseOffset +
567+
DWARFListTableHeader::getHeaderSize(Header.getFormat()));
568+
} else
569+
setRangesSection(&Context.getDWARFObj().getRnglistsSection(),
570+
toSectionOffset(UnitDie.find(DW_AT_rnglists_base),
571+
DWARFListTableHeader::getHeaderSize(
572+
Header.getFormat())));
573+
}
550574

551-
// DWARF v5 uses the .debug_rnglists and .debug_rnglists.dwo sections to
552-
// describe address ranges.
553-
if (getVersion() >= 5) {
554-
// In case of DWP, the base offset from the index has to be added.
555575
if (IsDWO) {
556-
uint64_t ContributionBaseOffset = 0;
576+
// If we are reading a package file, we need to adjust the location list
577+
// data based on the index entries.
578+
StringRef Data = Header.getVersion() >= 5
579+
? Context.getDWARFObj().getLoclistsDWOSection().Data
580+
: Context.getDWARFObj().getLocDWOSection().Data;
557581
if (auto *IndexEntry = Header.getIndexEntry())
558-
if (auto *Contrib = IndexEntry->getContribution(DW_SECT_RNGLISTS))
559-
ContributionBaseOffset = Contrib->getOffset();
560-
setRangesSection(
561-
&Context.getDWARFObj().getRnglistsDWOSection(),
562-
ContributionBaseOffset +
563-
DWARFListTableHeader::getHeaderSize(Header.getFormat()));
564-
} else
565-
setRangesSection(&Context.getDWARFObj().getRnglistsSection(),
566-
toSectionOffset(UnitDie.find(DW_AT_rnglists_base),
567-
DWARFListTableHeader::getHeaderSize(
568-
Header.getFormat())));
569-
}
582+
if (const auto *C = IndexEntry->getContribution(
583+
Header.getVersion() >= 5 ? DW_SECT_LOCLISTS : DW_SECT_EXT_LOC))
584+
Data = Data.substr(C->getOffset(), C->getLength());
585+
586+
DWARFDataExtractor DWARFData(Data, IsLittleEndian, getAddressByteSize());
587+
LocTable =
588+
std::make_unique<DWARFDebugLoclists>(DWARFData, Header.getVersion());
589+
LocSectionBase = DWARFListTableHeader::getHeaderSize(Header.getFormat());
590+
} else if (getVersion() >= 5) {
591+
LocTable = std::make_unique<DWARFDebugLoclists>(
592+
DWARFDataExtractor(Context.getDWARFObj(),
593+
Context.getDWARFObj().getLoclistsSection(),
594+
IsLittleEndian, getAddressByteSize()),
595+
getVersion());
596+
} else {
597+
LocTable = std::make_unique<DWARFDebugLoc>(DWARFDataExtractor(
598+
Context.getDWARFObj(), Context.getDWARFObj().getLocSection(),
599+
IsLittleEndian, getAddressByteSize()));
600+
}
570601

571-
if (IsDWO) {
572-
// If we are reading a package file, we need to adjust the location list
573-
// data based on the index entries.
574-
StringRef Data = Header.getVersion() >= 5
575-
? Context.getDWARFObj().getLoclistsDWOSection().Data
576-
: Context.getDWARFObj().getLocDWOSection().Data;
577-
if (auto *IndexEntry = Header.getIndexEntry())
578-
if (const auto *C = IndexEntry->getContribution(
579-
Header.getVersion() >= 5 ? DW_SECT_LOCLISTS : DW_SECT_EXT_LOC))
580-
Data = Data.substr(C->getOffset(), C->getLength());
581-
582-
DWARFDataExtractor DWARFData(Data, IsLittleEndian, getAddressByteSize());
583-
LocTable =
584-
std::make_unique<DWARFDebugLoclists>(DWARFData, Header.getVersion());
585-
LocSectionBase = DWARFListTableHeader::getHeaderSize(Header.getFormat());
586-
} else if (getVersion() >= 5) {
587-
LocTable = std::make_unique<DWARFDebugLoclists>(
588-
DWARFDataExtractor(Context.getDWARFObj(),
589-
Context.getDWARFObj().getLoclistsSection(),
590-
IsLittleEndian, getAddressByteSize()),
591-
getVersion());
592-
} else {
593-
LocTable = std::make_unique<DWARFDebugLoc>(DWARFDataExtractor(
594-
Context.getDWARFObj(), Context.getDWARFObj().getLocSection(),
595-
IsLittleEndian, getAddressByteSize()));
596-
}
602+
// Don't fall back to DW_AT_GNU_ranges_base: it should be ignored for
603+
// skeleton CU DIE, so that DWARF users not aware of it are not broken.
604+
});
597605

598-
// Don't fall back to DW_AT_GNU_ranges_base: it should be ignored for
599-
// skeleton CU DIE, so that DWARF users not aware of it are not broken.
600-
return Error::success();
606+
return e;
601607
}
602608

603609
bool DWARFUnit::parseDWO(StringRef DWOAlternativeLocation) {
@@ -652,15 +658,20 @@ bool DWARFUnit::parseDWO(StringRef DWOAlternativeLocation) {
652658
return true;
653659
}
654660

655-
void DWARFUnit::clearDIEs(bool KeepCUDie) {
656-
// Do not use resize() + shrink_to_fit() to free memory occupied by dies.
657-
// shrink_to_fit() is a *non-binding* request to reduce capacity() to size().
658-
// It depends on the implementation whether the request is fulfilled.
659-
// Create a new vector with a small capacity and assign it to the DieArray to
660-
// have previous contents freed.
661-
DieArray = (KeepCUDie && !DieArray.empty())
662-
? std::vector<DWARFDebugInfoEntry>({DieArray[0]})
663-
: std::vector<DWARFDebugInfoEntry>();
661+
void DWARFUnit::clearDIEs(bool KeepCUDie, bool KeepDWODies) {
662+
Context.doWorkThreadSafely([&] {
663+
if (!KeepDWODies && DWO) {
664+
DWO->clearDIEs(KeepCUDie, KeepDWODies);
665+
}
666+
// Do not use resize() + shrink_to_fit() to free memory occupied by dies.
667+
// shrink_to_fit() is a *non-binding* request to reduce capacity() to size().
668+
// It depends on the implementation whether the request is fulfilled.
669+
// Create a new vector with a small capacity and assign it to the DieArray to
670+
// have previous contents freed.
671+
DieArray = (KeepCUDie && !DieArray.empty())
672+
? std::vector<DWARFDebugInfoEntry>({DieArray[0]})
673+
: std::vector<DWARFDebugInfoEntry>();
674+
});
664675
}
665676

666677
Expected<DWARFAddressRangesVector>

‎llvm/lib/DebugInfo/GSYM/DwarfTransformer.cpp

Copy file name to clipboardExpand all lines: llvm/lib/DebugInfo/GSYM/DwarfTransformer.cpp
+13-6Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -656,6 +656,11 @@ Error DwarfTransformer::convert(uint32_t NumThreads, OutputAggregator &Out) {
656656
DWARFDie Die = getDie(*CU);
657657
CUInfo CUI(DICtx, dyn_cast<DWARFCompileUnit>(CU.get()));
658658
handleDie(Out, CUI, Die);
659+
// Release the line table, once we're done.
660+
DICtx.clearLineTableForUnit(CU.get());
661+
// Free any DIEs that were allocated by the DWARF parser.
662+
// If/when they're needed by other CU's, they'll be recreated.
663+
CU->clearDIEs(/*KeepCUDie=*/false, /*KeepDWODIEs=*/false);
659664
}
660665
} else {
661666
// LLVM Dwarf parser is not thread-safe and we need to parse all DWARF up
@@ -668,24 +673,23 @@ Error DwarfTransformer::convert(uint32_t NumThreads, OutputAggregator &Out) {
668673
for (const auto &CU : DICtx.compile_units())
669674
CU->getAbbreviations();
670675

671-
// Now parse all DIEs in case we have cross compile unit references in a
672-
// thread pool.
673676
DefaultThreadPool pool(hardware_concurrency(NumThreads));
674-
for (const auto &CU : DICtx.compile_units())
675-
pool.async([&CU]() { CU->getUnitDIE(false /*CUDieOnly*/); });
676-
pool.wait();
677677

678678
// Now convert all DWARF to GSYM in a thread pool.
679679
std::mutex LogMutex;
680680
for (const auto &CU : DICtx.compile_units()) {
681681
DWARFDie Die = getDie(*CU);
682682
if (Die) {
683683
CUInfo CUI(DICtx, dyn_cast<DWARFCompileUnit>(CU.get()));
684-
pool.async([this, CUI, &LogMutex, &Out, Die]() mutable {
684+
pool.async([this, CUI, &CU, &LogMutex, &Out, Die]() mutable {
685685
std::string storage;
686686
raw_string_ostream StrStream(storage);
687687
OutputAggregator ThreadOut(Out.GetOS() ? &StrStream : nullptr);
688688
handleDie(ThreadOut, CUI, Die);
689+
DICtx.clearLineTableForUnit(CU.get());
690+
// Free any DIEs that were allocated by the DWARF parser.
691+
// If/when they're needed by other CU's, they'll be recreated.
692+
CU->clearDIEs(/*KeepCUDie=*/false, /*KeepDWODIEs=*/false);
689693
// Print ThreadLogStorage lines into an actual stream under a lock
690694
std::lock_guard<std::mutex> guard(LogMutex);
691695
if (Out.GetOS()) {
@@ -697,6 +701,9 @@ Error DwarfTransformer::convert(uint32_t NumThreads, OutputAggregator &Out) {
697701
}
698702
pool.wait();
699703
}
704+
// Now get rid of all the DIEs that may have been recreated
705+
for (const auto &CU : DICtx.compile_units())
706+
CU->clearDIEs(/*KeepCUDie=*/false, /*KeepDWODIEs=*/false);
700707
size_t FunctionsAddedCount = Gsym.getNumFunctionInfos() - NumBefore;
701708
Out << "Loaded " << FunctionsAddedCount << " functions from DWARF.\n";
702709
return Error::success();

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.