From 94aaca75a3af64cd20e2848fb93de090519af135 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Sun, 27 Jul 2025 12:11:56 -0700 Subject: [PATCH 1/2] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20in?= =?UTF-8?q?itial=20version?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Created using spr 1.3.5-bogner --- llvm/include/llvm/MC/MCObjectStreamer.h | 10 ++ llvm/include/llvm/MC/MCSection.h | 40 +---- llvm/lib/MC/MCMachOStreamer.cpp | 5 +- llvm/lib/MC/MCObjectStreamer.cpp | 137 ++++++++++++++---- llvm/lib/MC/MCWin64EH.cpp | 2 + llvm/lib/MC/MachObjectWriter.cpp | 2 +- .../Mips/MCTargetDesc/MipsTargetStreamer.cpp | 6 + 7 files changed, 134 insertions(+), 68 deletions(-) diff --git a/llvm/include/llvm/MC/MCObjectStreamer.h b/llvm/include/llvm/MC/MCObjectStreamer.h index bbbee15abf700..b6e24816d94db 100644 --- a/llvm/include/llvm/MC/MCObjectStreamer.h +++ b/llvm/include/llvm/MC/MCObjectStreamer.h @@ -52,6 +52,10 @@ class MCObjectStreamer : public MCStreamer { DenseMap> pendingAssignments; + SmallVector, 0> FragStorage; + // Available bytes in the current block for trailing data or new fragments. + size_t FragSpace = 0; + void emitInstToData(const MCInst &Inst, const MCSubtargetInfo &); void emitCFIStartProcImpl(MCDwarfFrameInfo &Frame) override; void emitCFIEndProcImpl(MCDwarfFrameInfo &Frame) override; @@ -84,9 +88,15 @@ class MCObjectStreamer : public MCStreamer { // Add a fragment with a variable-size tail and start a new empty fragment. void insert(MCFragment *F); + char *getCurFragEnd() const { + return reinterpret_cast(CurFrag + 1) + CurFrag->getFixedSize(); + } + MCFragment *allocFragSpace(size_t Headroom); // Add a new fragment to the current section without a variable-size tail. void newFragment(); + void ensureHeadroom(size_t Headroom); + void appendContents(ArrayRef Contents); void appendContents(size_t Num, char Elt); void addFixup(const MCExpr *Value, MCFixupKind Kind); diff --git a/llvm/include/llvm/MC/MCSection.h b/llvm/include/llvm/MC/MCSection.h index 7989310e5a8f2..1579fa6646c3c 100644 --- a/llvm/include/llvm/MC/MCSection.h +++ b/llvm/include/llvm/MC/MCSection.h @@ -93,8 +93,7 @@ class MCFragment { // Track content and fixups for the fixed-size part as fragments are // appended to the section. The content remains immutable, except when // modified by applyFixup. - uint32_t ContentStart = 0; - uint32_t ContentEnd = 0; + uint32_t FixedSize = 0; uint32_t FixupStart = 0; uint32_t FixupEnd = 0; @@ -205,18 +204,6 @@ class MCFragment { //== Content-related functions manage parent's storage using ContentStart and // ContentSize. - // Get a SmallVector reference. The caller should call doneAppending to update - // `ContentEnd`. - SmallVectorImpl &getContentsForAppending(); - void doneAppending(); - void appendContents(ArrayRef Contents) { - getContentsForAppending().append(Contents.begin(), Contents.end()); - doneAppending(); - } - void appendContents(size_t Num, char Elt) { - getContentsForAppending().append(Num, Elt); - doneAppending(); - } MutableArrayRef getContents(); ArrayRef getContents() const; @@ -225,10 +212,10 @@ class MCFragment { MutableArrayRef getVarContents(); ArrayRef getVarContents() const; - size_t getFixedSize() const { return ContentEnd - ContentStart; } + size_t getFixedSize() const { return FixedSize; } size_t getVarSize() const { return VarContentEnd - VarContentStart; } size_t getSize() const { - return ContentEnd - ContentStart + (VarContentEnd - VarContentStart); + return FixedSize + (VarContentEnd - VarContentStart); } //== Fixup-related functions manage parent's storage using FixupStart and @@ -651,28 +638,11 @@ class LLVM_ABI MCSection { bool isBssSection() const { return IsBss; } }; -inline SmallVectorImpl &MCFragment::getContentsForAppending() { - SmallVectorImpl &S = getParent()->ContentStorage; - if (LLVM_UNLIKELY(ContentEnd != S.size())) { - // Move the elements to the end. Reserve space to avoid invalidating - // S.begin()+I for `append`. - auto Size = ContentEnd - ContentStart; - auto I = std::exchange(ContentStart, S.size()); - S.reserve(S.size() + Size); - S.append(S.begin() + I, S.begin() + I + Size); - } - return S; -} -inline void MCFragment::doneAppending() { - ContentEnd = getParent()->ContentStorage.size(); -} inline MutableArrayRef MCFragment::getContents() { - return MutableArrayRef(getParent()->ContentStorage) - .slice(ContentStart, ContentEnd - ContentStart); + return {reinterpret_cast(this + 1), FixedSize}; } inline ArrayRef MCFragment::getContents() const { - return ArrayRef(getParent()->ContentStorage) - .slice(ContentStart, ContentEnd - ContentStart); + return {reinterpret_cast(this + 1), FixedSize}; } inline MutableArrayRef MCFragment::getVarContents() { diff --git a/llvm/lib/MC/MCMachOStreamer.cpp b/llvm/lib/MC/MCMachOStreamer.cpp index 107466912d090..1c5abd6ddacc7 100644 --- a/llvm/lib/MC/MCMachOStreamer.cpp +++ b/llvm/lib/MC/MCMachOStreamer.cpp @@ -484,7 +484,8 @@ void MCMachOStreamer::finalizeCGProfile() { // For each entry, reserve space for 2 32-bit indices and a 64-bit count. size_t SectionBytes = W.getCGProfile().size() * (2 * sizeof(uint32_t) + sizeof(uint64_t)); - (*CGProfileSection->begin()).appendContents(SectionBytes, 0); + (*CGProfileSection->begin()) + .setVarContents(std::vector(SectionBytes, 0)); } MCStreamer *llvm::createMachOStreamer(MCContext &Context, @@ -520,5 +521,5 @@ void MCMachOStreamer::createAddrSigSection() { // (instead of emitting a zero-sized section) so these relocations are // technically valid, even though we don't expect these relocations to // actually be applied by the linker. - Frag->appendContents(8, 0); + Frag->setVarContents(std::vector(8, 0)); } diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp index 9c7b05bd9a45a..28293501fb8e6 100644 --- a/llvm/lib/MC/MCObjectStreamer.cpp +++ b/llvm/lib/MC/MCObjectStreamer.cpp @@ -46,23 +46,84 @@ MCAssembler *MCObjectStreamer::getAssemblerPtr() { return nullptr; } +constexpr size_t FragChunkSize = 16384; +// Ensure the new fragment can at least at least a few bytes. +constexpr size_t NewFragHeadroom = 8; + +static_assert(NewFragHeadroom >= alignof(MCFragment)); +static_assert(FragChunkSize >= sizeof(MCFragment) + NewFragHeadroom); + +MCFragment *MCObjectStreamer::allocFragSpace(size_t Headroom) { + auto Size = std::max(FragChunkSize, sizeof(MCFragment) + Headroom); + FragSpace = Size - sizeof(MCFragment); + auto Chunk = std::unique_ptr(new char[Size]); + auto *F = reinterpret_cast(Chunk.get()); + FragStorage.push_back(std::move(Chunk)); + return F; +} + void MCObjectStreamer::newFragment() { - addFragment(getContext().allocFragment()); + MCFragment *F; + if (LLVM_LIKELY(sizeof(MCFragment) + NewFragHeadroom <= FragSpace)) { + auto End = size_t(getCurFragEnd()); + F = reinterpret_cast( + alignToPowerOf2(End, alignof(MCFragment))); + FragSpace -= size_t(F) - End + sizeof(MCFragment); + } else { + F = allocFragSpace(0); + } + new (F) MCFragment(); + addFragment(F); +} + +void MCObjectStreamer::ensureHeadroom(size_t Headroom) { + if (Headroom <= FragSpace) + return; + auto *F = allocFragSpace(Headroom); + new (F) MCFragment(); + addFragment(F); } -void MCObjectStreamer::insert(MCFragment *F) { - assert(F->getKind() != MCFragment::FT_Data && +void MCObjectStreamer::insert(MCFragment *Frag) { + assert(Frag->getKind() != MCFragment::FT_Data && "F should have a variable-size tail"); + // Allocate an empty fragment, potentially reusing the space associated with + // CurFrag. + MCFragment *F; + if (LLVM_LIKELY(sizeof(MCFragment) + NewFragHeadroom <= FragSpace)) { + auto End = size_t(getCurFragEnd()); + F = reinterpret_cast( + alignToPowerOf2(End, alignof(MCFragment))); + FragSpace -= size_t(F) - End + sizeof(MCFragment); + } else { + F = allocFragSpace(0); + } + + // Add Frag, which destroys CurFrag and the associated space. + addFragment(Frag); + new (F) MCFragment(); + // Add the empty fragment, which restores CurFrag and the associated space. addFragment(F); - newFragment(); +} + +void MCObjectStreamer::appendContents(ArrayRef Contents) { + ensureHeadroom(Contents.size()); + assert(FragSpace >= Contents.size()); + llvm::copy(Contents, getCurFragEnd()); + CurFrag->FixedSize += Contents.size(); + FragSpace -= Contents.size(); } void MCObjectStreamer::appendContents(size_t Num, char Elt) { - CurFrag->appendContents(Num, Elt); + ensureHeadroom(Num); + MutableArrayRef Data(getCurFragEnd(), Num); + llvm::fill(Data, Elt); + CurFrag->FixedSize += Num; + FragSpace -= Num; } void MCObjectStreamer::addFixup(const MCExpr *Value, MCFixupKind Kind) { - CurFrag->addFixup(MCFixup::create(CurFrag->getFixedSize(), Value, Kind)); + CurFrag->addFixup(MCFixup::create(getCurFragSize(), Value, Kind)); } // As a compile-time optimization, avoid allocating and evaluating an MCExpr @@ -111,6 +172,8 @@ void MCObjectStreamer::reset() { } EmitEHFrame = true; EmitDebugFrame = false; + FragStorage.clear(); + FragSpace = 0; MCStreamer::reset(); } @@ -139,7 +202,6 @@ void MCObjectStreamer::emitCFISections(bool EH, bool Debug, bool SFrame) { void MCObjectStreamer::emitValueImpl(const MCExpr *Value, unsigned Size, SMLoc Loc) { MCStreamer::emitValueImpl(Value, Size, Loc); - MCFragment *DF = getCurrentFragment(); MCDwarfLineEntry::make(this, getCurrentSectionOnly()); @@ -154,9 +216,9 @@ void MCObjectStreamer::emitValueImpl(const MCExpr *Value, unsigned Size, emitIntValue(AbsValue, Size); return; } - DF->addFixup(MCFixup::create(DF->getContents().size(), Value, - MCFixup::getDataKindForSize(Size))); - DF->appendContents(Size, 0); + ensureHeadroom(Size); + addFixup(Value, MCFixup::getDataKindForSize(Size)); + appendContents(Size, 0); } MCSymbol *MCObjectStreamer::emitCFILabel() { @@ -190,7 +252,7 @@ void MCObjectStreamer::emitLabel(MCSymbol *Symbol, SMLoc Loc) { // section. MCFragment *F = CurFrag; Symbol->setFragment(F); - Symbol->setOffset(F->getContents().size()); + Symbol->setOffset(F->getFixedSize()); emitPendingAssignments(Symbol); } @@ -256,6 +318,21 @@ void MCObjectStreamer::changeSection(MCSection *Section, uint32_t Subsection) { F0 = CurFrag; } + // CurFrag will be changed. To ensure that FragSpace remains connected with + // CurFrag, allocate an empty fragment and add it to the fragment list. + // (Subsections[I].second.Tail is disconnected with FragSpace.) + MCFragment *F; + if (LLVM_LIKELY(sizeof(MCFragment) + NewFragHeadroom <= FragSpace)) { + auto End = size_t(getCurFragEnd()); + F = reinterpret_cast( + alignToPowerOf2(End, alignof(MCFragment))); + FragSpace -= size_t(F) - End + sizeof(MCFragment); + } else { + F = allocFragSpace(0); + } + new (F) MCFragment(); + F->setParent(Section); + auto &Subsections = Section->Subsections; size_t I = 0, E = Subsections.size(); while (I != E && Subsections[I].first < Subsection) @@ -263,13 +340,16 @@ void MCObjectStreamer::changeSection(MCSection *Section, uint32_t Subsection) { // If the subsection number is not in the sorted Subsections list, create a // new fragment list. if (I == E || Subsections[I].first != Subsection) { - auto *F = getContext().allocFragment(); - F->setParent(Section); Subsections.insert(Subsections.begin() + I, {Subsection, MCSection::FragList{F, F}}); + Section->CurFragList = &Subsections[I].second; + CurFrag = F; + } else { + Section->CurFragList = &Subsections[I].second; + CurFrag = Subsections[I].second.Tail; + // Ensure CurFrag is associated with FragSpace. + addFragment(F); } - Section->CurFragList = &Subsections[I].second; - CurFrag = Section->CurFragList->Tail; // Define the section symbol at subsection 0's initial fragment if required. if (!NewSec) @@ -340,11 +420,11 @@ void MCObjectStreamer::emitInstToData(const MCInst &Inst, MCFragment *F = getCurrentFragment(); // Append the instruction to the data fragment. - size_t CodeOffset = F->getContents().size(); + size_t CodeOffset = getCurFragSize(); + SmallString<16> Content; SmallVector Fixups; - getAssembler().getEmitter().encodeInstruction( - Inst, F->getContentsForAppending(), Fixups, STI); - F->doneAppending(); + getAssembler().getEmitter().encodeInstruction(Inst, Content, Fixups, STI); + appendContents(Content); F->setHasInstructions(STI); if (Fixups.empty()) @@ -352,19 +432,17 @@ void MCObjectStreamer::emitInstToData(const MCInst &Inst, bool MarkedLinkerRelaxable = false; for (auto &Fixup : Fixups) { Fixup.setOffset(Fixup.getOffset() + CodeOffset); - if (!Fixup.isLinkerRelaxable() || MarkedLinkerRelaxable) + if (!Fixup.isLinkerRelaxable()) continue; - MarkedLinkerRelaxable = true; - // Set the fragment's order within the subsection for use by - // MCAssembler::relaxAlign. - auto *Sec = F->getParent(); - if (!Sec->isLinkerRelaxable()) - Sec->setLinkerRelaxable(); + F->setLinkerRelaxable(); // Do not add data after a linker-relaxable instruction. The difference // between a new label and a label at or before the linker-relaxable // instruction cannot be resolved at assemble-time. - F->setLinkerRelaxable(); - newFragment(); + if (!MarkedLinkerRelaxable) { + MarkedLinkerRelaxable = true; + getCurrentSectionOnly()->setLinkerRelaxable(); + newFragment(); + } } F->appendFixups(Fixups); } @@ -538,8 +616,7 @@ void MCObjectStreamer::emitCVFileChecksumOffsetDirective(unsigned FileNo) { void MCObjectStreamer::emitBytes(StringRef Data) { MCDwarfLineEntry::make(this, getCurrentSectionOnly()); - MCFragment *DF = getCurrentFragment(); - DF->appendContents(ArrayRef(Data.data(), Data.size())); + appendContents(ArrayRef(Data.data(), Data.size())); } void MCObjectStreamer::emitValueToAlignment(Align Alignment, int64_t Fill, diff --git a/llvm/lib/MC/MCWin64EH.cpp b/llvm/lib/MC/MCWin64EH.cpp index 72a8dd7031198..e212663d53a79 100644 --- a/llvm/lib/MC/MCWin64EH.cpp +++ b/llvm/lib/MC/MCWin64EH.cpp @@ -318,6 +318,8 @@ static void EmitUnwindInfo(MCStreamer &streamer, WinEH::FrameInfo *info) { // Emit the epilog instructions. if (EnableUnwindV2) { + OS->ensureHeadroom(info->EpilogMap.size() * 2); + bool IsLast = true; for (const auto &Epilog : llvm::reverse(info->EpilogMap)) { if (IsLast) { diff --git a/llvm/lib/MC/MachObjectWriter.cpp b/llvm/lib/MC/MachObjectWriter.cpp index 7b5c3c00f2519..e87696a93cddd 100644 --- a/llvm/lib/MC/MachObjectWriter.cpp +++ b/llvm/lib/MC/MachObjectWriter.cpp @@ -806,7 +806,7 @@ uint64_t MachObjectWriter::writeObject() { } MCSection *Sec = getContext().getMachOSection("__LLVM", "__cg_profile", 0, SectionKind::getMetadata()); - llvm::copy(OS.str(), Sec->curFragList()->Head->getContents().data()); + llvm::copy(OS.str(), Sec->curFragList()->Head->getVarContents().data()); } unsigned NumSections = Asm.end() - Asm.begin(); diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp index d9680c7739a1b..7a8395a2e582b 100644 --- a/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp +++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp @@ -1034,12 +1034,14 @@ MCELFStreamer &MipsTargetELFStreamer::getStreamer() { void MipsTargetELFStreamer::emitGPRel32Value(const MCExpr *Value) { auto &S = getStreamer(); + S.ensureHeadroom(4); S.addFixup(Value, Mips::fixup_Mips_GPREL32); S.appendContents(4, 0); } void MipsTargetELFStreamer::emitGPRel64Value(const MCExpr *Value) { auto &S = getStreamer(); + S.ensureHeadroom(8); // fixup_Mips_GPREL32 desginates R_MIPS_GPREL32+R_MIPS_64 on MIPS64. S.addFixup(Value, Mips::fixup_Mips_GPREL32); S.appendContents(8, 0); @@ -1047,24 +1049,28 @@ void MipsTargetELFStreamer::emitGPRel64Value(const MCExpr *Value) { void MipsTargetELFStreamer::emitDTPRel32Value(const MCExpr *Value) { auto &S = getStreamer(); + S.ensureHeadroom(4); S.addFixup(Value, Mips::fixup_Mips_DTPREL32); S.appendContents(4, 0); } void MipsTargetELFStreamer::emitDTPRel64Value(const MCExpr *Value) { auto &S = getStreamer(); + S.ensureHeadroom(8); S.addFixup(Value, Mips::fixup_Mips_DTPREL64); S.appendContents(8, 0); } void MipsTargetELFStreamer::emitTPRel32Value(const MCExpr *Value) { auto &S = getStreamer(); + S.ensureHeadroom(4); S.addFixup(Value, Mips::fixup_Mips_TPREL32); S.appendContents(4, 0); } void MipsTargetELFStreamer::emitTPRel64Value(const MCExpr *Value) { auto &S = getStreamer(); + S.ensureHeadroom(8); S.addFixup(Value, Mips::fixup_Mips_TPREL64); S.appendContents(8, 0); } From f1d1e9912df370ec0ed455d0239bb6ccfcbd4231 Mon Sep 17 00:00:00 2001 From: Fangrui Song Date: Mon, 28 Jul 2025 22:08:28 -0700 Subject: [PATCH 2/2] . Created using spr 1.3.5-bogner --- llvm/lib/MC/MCWin64EH.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/llvm/lib/MC/MCWin64EH.cpp b/llvm/lib/MC/MCWin64EH.cpp index e212663d53a79..a87648afde7d6 100644 --- a/llvm/lib/MC/MCWin64EH.cpp +++ b/llvm/lib/MC/MCWin64EH.cpp @@ -318,6 +318,7 @@ static void EmitUnwindInfo(MCStreamer &streamer, WinEH::FrameInfo *info) { // Emit the epilog instructions. if (EnableUnwindV2) { + // Ensure the fixups and appended content apply to the same fragment. OS->ensureHeadroom(info->EpilogMap.size() * 2); bool IsLast = true;