Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit fd7f28c

Browse filesBrowse files
author
Arslan Khabutdinov
committed
Reduce llvm-gsymutil memory usage
1 parent 86ba681 commit fd7f28c
Copy full SHA for fd7f28c

File tree

5 files changed

+141
-113
lines changed
Filter options

5 files changed

+141
-113
lines changed

‎llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h

Copy file name to clipboardExpand all lines: llvm/include/llvm/DebugInfo/DWARF/DWARFContext.h
+5Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,7 @@ class DWARFContext : public DIContext {
103103
std::unique_ptr<DWARFDebugMacro>
104104
parseMacroOrMacinfo(MacroSecType SectionType);
105105

106+
virtual Error doWorkThreadSafely(function_ref<Error()> Work) = 0;
106107
};
107108
friend class DWARFContextState;
108109

@@ -491,6 +492,10 @@ class DWARFContext : public DIContext {
491492
/// manually only for DWARF5.
492493
void setParseCUTUIndexManually(bool PCUTU) { ParseCUTUIndexManually = PCUTU; }
493494

495+
Error doWorkThreadSafely(function_ref<Error()> Work) {
496+
return State->doWorkThreadSafely(Work);
497+
}
498+
494499
private:
495500
void addLocalsForDie(DWARFCompileUnit *CU, DWARFDie Subprogram, DWARFDie Die,
496501
std::vector<DILocal> &Result);

‎llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h

Copy file name to clipboardExpand all lines: llvm/include/llvm/DebugInfo/DWARF/DWARFUnit.h
+3-3Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -566,6 +566,9 @@ class DWARFUnit {
566566

567567
Error tryExtractDIEsIfNeeded(bool CUDieOnly);
568568

569+
/// clearDIEs - Clear parsed DIEs to keep memory usage low.
570+
void clearDIEs(bool KeepCUDie, bool KeepDWODies = false);
571+
569572
private:
570573
/// Size in bytes of the .debug_info data associated with this compile unit.
571574
size_t getDebugInfoSize() const {
@@ -581,9 +584,6 @@ class DWARFUnit {
581584
void extractDIEsToVector(bool AppendCUDie, bool AppendNonCUDIEs,
582585
std::vector<DWARFDebugInfoEntry> &DIEs) const;
583586

584-
/// clearDIEs - Clear parsed DIEs to keep memory usage low.
585-
void clearDIEs(bool KeepCUDie);
586-
587587
/// parseDWO - Parses .dwo file for current compile unit. Returns true if
588588
/// it was actually constructed.
589589
/// The \p AlternativeLocation specifies an alternative location to get

‎llvm/lib/DebugInfo/DWARF/DWARFContext.cpp

Copy file name to clipboardExpand all lines: llvm/lib/DebugInfo/DWARF/DWARFContext.cpp
+8-1Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -622,7 +622,9 @@ class ThreadUnsafeDWARFContextState : public DWARFContext::DWARFContextState {
622622
return getNormalTypeUnitMap();
623623
}
624624

625-
625+
Error doWorkThreadSafely(function_ref<Error()> Work) override {
626+
return Work();
627+
}
626628
};
627629

628630
class ThreadSafeState : public ThreadUnsafeDWARFContextState {
@@ -738,6 +740,11 @@ class ThreadSafeState : public ThreadUnsafeDWARFContextState {
738740
std::unique_lock<std::recursive_mutex> LockGuard(Mutex);
739741
return ThreadUnsafeDWARFContextState::getTypeUnitMap(IsDWO);
740742
}
743+
744+
Error doWorkThreadSafely(function_ref<Error()> Work) override {
745+
std::unique_lock<std::recursive_mutex> LockGuard(Mutex);
746+
return ThreadUnsafeDWARFContextState::doWorkThreadSafely(Work);
747+
}
741748
};
742749
} // namespace
743750

‎llvm/lib/DebugInfo/DWARF/DWARFUnit.cpp

Copy file name to clipboardExpand all lines: llvm/lib/DebugInfo/DWARF/DWARFUnit.cpp
+112-103Lines changed: 112 additions & 103 deletions
Original file line numberDiff line numberDiff line change
@@ -496,108 +496,111 @@ void DWARFUnit::extractDIEsIfNeeded(bool CUDieOnly) {
496496
}
497497

498498
Error DWARFUnit::tryExtractDIEsIfNeeded(bool CUDieOnly) {
499-
if ((CUDieOnly && !DieArray.empty()) ||
500-
DieArray.size() > 1)
501-
return Error::success(); // Already parsed.
502-
503-
bool HasCUDie = !DieArray.empty();
504-
extractDIEsToVector(!HasCUDie, !CUDieOnly, DieArray);
505-
506-
if (DieArray.empty())
507-
return Error::success();
499+
return Context.doWorkThreadSafely([&]() -> Error {
500+
if ((CUDieOnly && !DieArray.empty()) || DieArray.size() > 1)
501+
return Error::success(); // Already parsed.
502+
503+
bool HasCUDie = !DieArray.empty();
504+
extractDIEsToVector(!HasCUDie, !CUDieOnly, DieArray);
505+
506+
if (DieArray.empty())
507+
return Error::success();
508+
509+
// If CU DIE was just parsed, copy several attribute values from it.
510+
if (HasCUDie)
511+
return Error::success();
512+
513+
DWARFDie UnitDie(this, &DieArray[0]);
514+
if (std::optional<uint64_t> DWOId =
515+
toUnsigned(UnitDie.find(DW_AT_GNU_dwo_id)))
516+
Header.setDWOId(*DWOId);
517+
if (!IsDWO) {
518+
assert(AddrOffsetSectionBase == std::nullopt);
519+
assert(RangeSectionBase == 0);
520+
assert(LocSectionBase == 0);
521+
AddrOffsetSectionBase = toSectionOffset(UnitDie.find(DW_AT_addr_base));
522+
if (!AddrOffsetSectionBase)
523+
AddrOffsetSectionBase =
524+
toSectionOffset(UnitDie.find(DW_AT_GNU_addr_base));
525+
RangeSectionBase = toSectionOffset(UnitDie.find(DW_AT_rnglists_base), 0);
526+
LocSectionBase = toSectionOffset(UnitDie.find(DW_AT_loclists_base), 0);
527+
}
508528

509-
// If CU DIE was just parsed, copy several attribute values from it.
510-
if (HasCUDie)
511-
return Error::success();
529+
// In general, in DWARF v5 and beyond we derive the start of the unit's
530+
// contribution to the string offsets table from the unit DIE's
531+
// DW_AT_str_offsets_base attribute. Split DWARF units do not use this
532+
// attribute, so we assume that there is a contribution to the string
533+
// offsets table starting at offset 0 of the debug_str_offsets.dwo section.
534+
// In both cases we need to determine the format of the contribution,
535+
// which may differ from the unit's format.
536+
DWARFDataExtractor DA(Context.getDWARFObj(), StringOffsetSection,
537+
IsLittleEndian, 0);
538+
if (IsDWO || getVersion() >= 5) {
539+
auto StringOffsetOrError =
540+
IsDWO ? determineStringOffsetsTableContributionDWO(DA)
541+
: determineStringOffsetsTableContribution(DA);
542+
if (!StringOffsetOrError) {
543+
return createStringError(errc::invalid_argument,
544+
"invalid reference to or invalid content in "
545+
".debug_str_offsets[.dwo]: " +
546+
toString(StringOffsetOrError.takeError()));
547+
}
512548

513-
DWARFDie UnitDie(this, &DieArray[0]);
514-
if (std::optional<uint64_t> DWOId =
515-
toUnsigned(UnitDie.find(DW_AT_GNU_dwo_id)))
516-
Header.setDWOId(*DWOId);
517-
if (!IsDWO) {
518-
assert(AddrOffsetSectionBase == std::nullopt);
519-
assert(RangeSectionBase == 0);
520-
assert(LocSectionBase == 0);
521-
AddrOffsetSectionBase = toSectionOffset(UnitDie.find(DW_AT_addr_base));
522-
if (!AddrOffsetSectionBase)
523-
AddrOffsetSectionBase =
524-
toSectionOffset(UnitDie.find(DW_AT_GNU_addr_base));
525-
RangeSectionBase = toSectionOffset(UnitDie.find(DW_AT_rnglists_base), 0);
526-
LocSectionBase = toSectionOffset(UnitDie.find(DW_AT_loclists_base), 0);
527-
}
549+
StringOffsetsTableContribution = *StringOffsetOrError;
550+
}
528551

529-
// In general, in DWARF v5 and beyond we derive the start of the unit's
530-
// contribution to the string offsets table from the unit DIE's
531-
// DW_AT_str_offsets_base attribute. Split DWARF units do not use this
532-
// attribute, so we assume that there is a contribution to the string
533-
// offsets table starting at offset 0 of the debug_str_offsets.dwo section.
534-
// In both cases we need to determine the format of the contribution,
535-
// which may differ from the unit's format.
536-
DWARFDataExtractor DA(Context.getDWARFObj(), StringOffsetSection,
537-
IsLittleEndian, 0);
538-
if (IsDWO || getVersion() >= 5) {
539-
auto StringOffsetOrError =
540-
IsDWO ? determineStringOffsetsTableContributionDWO(DA)
541-
: determineStringOffsetsTableContribution(DA);
542-
if (!StringOffsetOrError)
543-
return createStringError(errc::invalid_argument,
544-
"invalid reference to or invalid content in "
545-
".debug_str_offsets[.dwo]: " +
546-
toString(StringOffsetOrError.takeError()));
547-
548-
StringOffsetsTableContribution = *StringOffsetOrError;
549-
}
552+
// DWARF v5 uses the .debug_rnglists and .debug_rnglists.dwo sections to
553+
// describe address ranges.
554+
if (getVersion() >= 5) {
555+
// In case of DWP, the base offset from the index has to be added.
556+
if (IsDWO) {
557+
uint64_t ContributionBaseOffset = 0;
558+
if (auto *IndexEntry = Header.getIndexEntry())
559+
if (auto *Contrib = IndexEntry->getContribution(DW_SECT_RNGLISTS))
560+
ContributionBaseOffset = Contrib->getOffset();
561+
setRangesSection(
562+
&Context.getDWARFObj().getRnglistsDWOSection(),
563+
ContributionBaseOffset +
564+
DWARFListTableHeader::getHeaderSize(Header.getFormat()));
565+
} else
566+
setRangesSection(&Context.getDWARFObj().getRnglistsSection(),
567+
toSectionOffset(UnitDie.find(DW_AT_rnglists_base),
568+
DWARFListTableHeader::getHeaderSize(
569+
Header.getFormat())));
570+
}
550571

551-
// DWARF v5 uses the .debug_rnglists and .debug_rnglists.dwo sections to
552-
// describe address ranges.
553-
if (getVersion() >= 5) {
554-
// In case of DWP, the base offset from the index has to be added.
555572
if (IsDWO) {
556-
uint64_t ContributionBaseOffset = 0;
573+
// If we are reading a package file, we need to adjust the location list
574+
// data based on the index entries.
575+
StringRef Data = Header.getVersion() >= 5
576+
? Context.getDWARFObj().getLoclistsDWOSection().Data
577+
: Context.getDWARFObj().getLocDWOSection().Data;
557578
if (auto *IndexEntry = Header.getIndexEntry())
558-
if (auto *Contrib = IndexEntry->getContribution(DW_SECT_RNGLISTS))
559-
ContributionBaseOffset = Contrib->getOffset();
560-
setRangesSection(
561-
&Context.getDWARFObj().getRnglistsDWOSection(),
562-
ContributionBaseOffset +
563-
DWARFListTableHeader::getHeaderSize(Header.getFormat()));
564-
} else
565-
setRangesSection(&Context.getDWARFObj().getRnglistsSection(),
566-
toSectionOffset(UnitDie.find(DW_AT_rnglists_base),
567-
DWARFListTableHeader::getHeaderSize(
568-
Header.getFormat())));
569-
}
579+
if (const auto *C = IndexEntry->getContribution(
580+
Header.getVersion() >= 5 ? DW_SECT_LOCLISTS : DW_SECT_EXT_LOC))
581+
Data = Data.substr(C->getOffset(), C->getLength());
582+
583+
DWARFDataExtractor DWARFData(Data, IsLittleEndian, getAddressByteSize());
584+
LocTable =
585+
std::make_unique<DWARFDebugLoclists>(DWARFData, Header.getVersion());
586+
LocSectionBase = DWARFListTableHeader::getHeaderSize(Header.getFormat());
587+
} else if (getVersion() >= 5) {
588+
LocTable = std::make_unique<DWARFDebugLoclists>(
589+
DWARFDataExtractor(Context.getDWARFObj(),
590+
Context.getDWARFObj().getLoclistsSection(),
591+
IsLittleEndian, getAddressByteSize()),
592+
getVersion());
593+
} else {
594+
LocTable = std::make_unique<DWARFDebugLoc>(DWARFDataExtractor(
595+
Context.getDWARFObj(), Context.getDWARFObj().getLocSection(),
596+
IsLittleEndian, getAddressByteSize()));
597+
}
570598

571-
if (IsDWO) {
572-
// If we are reading a package file, we need to adjust the location list
573-
// data based on the index entries.
574-
StringRef Data = Header.getVersion() >= 5
575-
? Context.getDWARFObj().getLoclistsDWOSection().Data
576-
: Context.getDWARFObj().getLocDWOSection().Data;
577-
if (auto *IndexEntry = Header.getIndexEntry())
578-
if (const auto *C = IndexEntry->getContribution(
579-
Header.getVersion() >= 5 ? DW_SECT_LOCLISTS : DW_SECT_EXT_LOC))
580-
Data = Data.substr(C->getOffset(), C->getLength());
581-
582-
DWARFDataExtractor DWARFData(Data, IsLittleEndian, getAddressByteSize());
583-
LocTable =
584-
std::make_unique<DWARFDebugLoclists>(DWARFData, Header.getVersion());
585-
LocSectionBase = DWARFListTableHeader::getHeaderSize(Header.getFormat());
586-
} else if (getVersion() >= 5) {
587-
LocTable = std::make_unique<DWARFDebugLoclists>(
588-
DWARFDataExtractor(Context.getDWARFObj(),
589-
Context.getDWARFObj().getLoclistsSection(),
590-
IsLittleEndian, getAddressByteSize()),
591-
getVersion());
592-
} else {
593-
LocTable = std::make_unique<DWARFDebugLoc>(DWARFDataExtractor(
594-
Context.getDWARFObj(), Context.getDWARFObj().getLocSection(),
595-
IsLittleEndian, getAddressByteSize()));
596-
}
599+
// Don't fall back to DW_AT_GNU_ranges_base: it should be ignored for
600+
// skeleton CU DIE, so that DWARF users not aware of it are not broken.
597601

598-
// Don't fall back to DW_AT_GNU_ranges_base: it should be ignored for
599-
// skeleton CU DIE, so that DWARF users not aware of it are not broken.
600-
return Error::success();
602+
return Error::success();
603+
});
601604
}
602605

603606
bool DWARFUnit::parseDWO(StringRef DWOAlternativeLocation) {
@@ -652,15 +655,21 @@ bool DWARFUnit::parseDWO(StringRef DWOAlternativeLocation) {
652655
return true;
653656
}
654657

655-
void DWARFUnit::clearDIEs(bool KeepCUDie) {
656-
// Do not use resize() + shrink_to_fit() to free memory occupied by dies.
657-
// shrink_to_fit() is a *non-binding* request to reduce capacity() to size().
658-
// It depends on the implementation whether the request is fulfilled.
659-
// Create a new vector with a small capacity and assign it to the DieArray to
660-
// have previous contents freed.
661-
DieArray = (KeepCUDie && !DieArray.empty())
662-
? std::vector<DWARFDebugInfoEntry>({DieArray[0]})
663-
: std::vector<DWARFDebugInfoEntry>();
658+
void DWARFUnit::clearDIEs(bool KeepCUDie, bool KeepDWODies) {
659+
assert(!Context.doWorkThreadSafely([&] {
660+
if (!KeepDWODies && DWO) {
661+
DWO->clearDIEs(KeepCUDie, KeepDWODies);
662+
}
663+
// Do not use resize() + shrink_to_fit() to free memory occupied by dies.
664+
// shrink_to_fit() is a *non-binding* request to reduce capacity() to
665+
// size(). It depends on the implementation whether the request is
666+
// fulfilled. Create a new vector with a small capacity and assign it to the
667+
// DieArray to have previous contents freed.
668+
DieArray = (KeepCUDie && !DieArray.empty())
669+
? std::vector<DWARFDebugInfoEntry>({DieArray[0]})
670+
: std::vector<DWARFDebugInfoEntry>();
671+
return Error::success();
672+
}));
664673
}
665674

666675
Expected<DWARFAddressRangesVector>

‎llvm/lib/DebugInfo/GSYM/DwarfTransformer.cpp

Copy file name to clipboardExpand all lines: llvm/lib/DebugInfo/GSYM/DwarfTransformer.cpp
+13-6Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -656,6 +656,11 @@ Error DwarfTransformer::convert(uint32_t NumThreads, OutputAggregator &Out) {
656656
DWARFDie Die = getDie(*CU);
657657
CUInfo CUI(DICtx, dyn_cast<DWARFCompileUnit>(CU.get()));
658658
handleDie(Out, CUI, Die);
659+
// Release the line table, once we're done.
660+
DICtx.clearLineTableForUnit(CU.get());
661+
// Free any DIEs that were allocated by the DWARF parser.
662+
// If/when they're needed by other CU's, they'll be recreated.
663+
CU->clearDIEs(/*KeepCUDie=*/false, /*KeepDWODIEs=*/false);
659664
}
660665
} else {
661666
// LLVM Dwarf parser is not thread-safe and we need to parse all DWARF up
@@ -668,24 +673,23 @@ Error DwarfTransformer::convert(uint32_t NumThreads, OutputAggregator &Out) {
668673
for (const auto &CU : DICtx.compile_units())
669674
CU->getAbbreviations();
670675

671-
// Now parse all DIEs in case we have cross compile unit references in a
672-
// thread pool.
673676
DefaultThreadPool pool(hardware_concurrency(NumThreads));
674-
for (const auto &CU : DICtx.compile_units())
675-
pool.async([&CU]() { CU->getUnitDIE(false /*CUDieOnly*/); });
676-
pool.wait();
677677

678678
// Now convert all DWARF to GSYM in a thread pool.
679679
std::mutex LogMutex;
680680
for (const auto &CU : DICtx.compile_units()) {
681681
DWARFDie Die = getDie(*CU);
682682
if (Die) {
683683
CUInfo CUI(DICtx, dyn_cast<DWARFCompileUnit>(CU.get()));
684-
pool.async([this, CUI, &LogMutex, &Out, Die]() mutable {
684+
pool.async([this, CUI, &CU, &LogMutex, &Out, Die]() mutable {
685685
std::string storage;
686686
raw_string_ostream StrStream(storage);
687687
OutputAggregator ThreadOut(Out.GetOS() ? &StrStream : nullptr);
688688
handleDie(ThreadOut, CUI, Die);
689+
DICtx.clearLineTableForUnit(CU.get());
690+
// Free any DIEs that were allocated by the DWARF parser.
691+
// If/when they're needed by other CU's, they'll be recreated.
692+
CU->clearDIEs(/*KeepCUDie=*/false, /*KeepDWODIEs=*/false);
689693
// Print ThreadLogStorage lines into an actual stream under a lock
690694
std::lock_guard<std::mutex> guard(LogMutex);
691695
if (Out.GetOS()) {
@@ -697,6 +701,9 @@ Error DwarfTransformer::convert(uint32_t NumThreads, OutputAggregator &Out) {
697701
}
698702
pool.wait();
699703
}
704+
// Now get rid of all the DIEs that may have been recreated
705+
for (const auto &CU : DICtx.compile_units())
706+
CU->clearDIEs(/*KeepCUDie=*/false, /*KeepDWODIEs=*/false);
700707
size_t FunctionsAddedCount = Gsym.getNumFunctionInfos() - NumBefore;
701708
Out << "Loaded " << FunctionsAddedCount << " functions from DWARF.\n";
702709
return Error::success();

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.