diff --git a/llvm/include/llvm/MC/MCObjectStreamer.h b/llvm/include/llvm/MC/MCObjectStreamer.h index 4b43a8fadb925..5ac7aba679ec4 100644 --- a/llvm/include/llvm/MC/MCObjectStreamer.h +++ b/llvm/include/llvm/MC/MCObjectStreamer.h @@ -52,6 +52,10 @@ class MCObjectStreamer : public MCStreamer { DenseMap> pendingAssignments; + SmallVector, 0> FragStorage; + // Available bytes in the current block for trailing data or new fragments. + size_t FragSpace = 0; + void emitInstToData(const MCInst &Inst, const MCSubtargetInfo &); void emitCFIStartProcImpl(MCDwarfFrameInfo &Frame) override; void emitCFIEndProcImpl(MCDwarfFrameInfo &Frame) override; @@ -84,11 +88,18 @@ class MCObjectStreamer : public MCStreamer { // Add a fragment with a variable-size tail and start a new empty fragment. void insert(MCFragment *F); + char *getCurFragEnd() const { + return reinterpret_cast(CurFrag + 1) + CurFrag->getFixedSize(); + } + MCFragment *allocFragSpace(size_t Headroom); // Add a new fragment to the current section without a variable-size tail. void newFragment(); + void ensureHeadroom(size_t Headroom); void appendContents(ArrayRef Contents); void appendContents(size_t Num, char Elt); + // Add a fixup to the current fragment. Call ensureHeadroom beforehand to + // ensure the fixup and appended content apply to the same fragment. void addFixup(const MCExpr *Value, MCFixupKind Kind); void emitLabel(MCSymbol *Symbol, SMLoc Loc = SMLoc()) override; diff --git a/llvm/include/llvm/MC/MCSection.h b/llvm/include/llvm/MC/MCSection.h index 7989310e5a8f2..1579fa6646c3c 100644 --- a/llvm/include/llvm/MC/MCSection.h +++ b/llvm/include/llvm/MC/MCSection.h @@ -93,8 +93,7 @@ class MCFragment { // Track content and fixups for the fixed-size part as fragments are // appended to the section. The content remains immutable, except when // modified by applyFixup. - uint32_t ContentStart = 0; - uint32_t ContentEnd = 0; + uint32_t FixedSize = 0; uint32_t FixupStart = 0; uint32_t FixupEnd = 0; @@ -205,18 +204,6 @@ class MCFragment { //== Content-related functions manage parent's storage using ContentStart and // ContentSize. - // Get a SmallVector reference. The caller should call doneAppending to update - // `ContentEnd`. - SmallVectorImpl &getContentsForAppending(); - void doneAppending(); - void appendContents(ArrayRef Contents) { - getContentsForAppending().append(Contents.begin(), Contents.end()); - doneAppending(); - } - void appendContents(size_t Num, char Elt) { - getContentsForAppending().append(Num, Elt); - doneAppending(); - } MutableArrayRef getContents(); ArrayRef getContents() const; @@ -225,10 +212,10 @@ class MCFragment { MutableArrayRef getVarContents(); ArrayRef getVarContents() const; - size_t getFixedSize() const { return ContentEnd - ContentStart; } + size_t getFixedSize() const { return FixedSize; } size_t getVarSize() const { return VarContentEnd - VarContentStart; } size_t getSize() const { - return ContentEnd - ContentStart + (VarContentEnd - VarContentStart); + return FixedSize + (VarContentEnd - VarContentStart); } //== Fixup-related functions manage parent's storage using FixupStart and @@ -651,28 +638,11 @@ class LLVM_ABI MCSection { bool isBssSection() const { return IsBss; } }; -inline SmallVectorImpl &MCFragment::getContentsForAppending() { - SmallVectorImpl &S = getParent()->ContentStorage; - if (LLVM_UNLIKELY(ContentEnd != S.size())) { - // Move the elements to the end. Reserve space to avoid invalidating - // S.begin()+I for `append`. - auto Size = ContentEnd - ContentStart; - auto I = std::exchange(ContentStart, S.size()); - S.reserve(S.size() + Size); - S.append(S.begin() + I, S.begin() + I + Size); - } - return S; -} -inline void MCFragment::doneAppending() { - ContentEnd = getParent()->ContentStorage.size(); -} inline MutableArrayRef MCFragment::getContents() { - return MutableArrayRef(getParent()->ContentStorage) - .slice(ContentStart, ContentEnd - ContentStart); + return {reinterpret_cast(this + 1), FixedSize}; } inline ArrayRef MCFragment::getContents() const { - return ArrayRef(getParent()->ContentStorage) - .slice(ContentStart, ContentEnd - ContentStart); + return {reinterpret_cast(this + 1), FixedSize}; } inline MutableArrayRef MCFragment::getVarContents() { diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp index e277143ac3899..17d096858d76d 100644 --- a/llvm/lib/MC/MCObjectStreamer.cpp +++ b/llvm/lib/MC/MCObjectStreamer.cpp @@ -46,27 +46,83 @@ MCAssembler *MCObjectStreamer::getAssemblerPtr() { return nullptr; } +constexpr size_t FragBlockSize = 16384; +// Ensure the new fragment can at least store a few bytes. +constexpr size_t NewFragHeadroom = 8; + +static_assert(NewFragHeadroom >= alignof(MCFragment)); +static_assert(FragBlockSize >= sizeof(MCFragment) + NewFragHeadroom); + +MCFragment *MCObjectStreamer::allocFragSpace(size_t Headroom) { + auto Size = std::max(FragBlockSize, sizeof(MCFragment) + Headroom); + FragSpace = Size - sizeof(MCFragment); + auto Chunk = std::unique_ptr(new char[Size]); + auto *F = reinterpret_cast(Chunk.get()); + FragStorage.push_back(std::move(Chunk)); + return F; +} + void MCObjectStreamer::newFragment() { - addFragment(getContext().allocFragment()); + MCFragment *F; + if (LLVM_LIKELY(sizeof(MCFragment) + NewFragHeadroom <= FragSpace)) { + auto End = reinterpret_cast(getCurFragEnd()); + F = reinterpret_cast( + alignToPowerOf2(End, alignof(MCFragment))); + FragSpace -= size_t(F) - End + sizeof(MCFragment); + } else { + F = allocFragSpace(0); + } + new (F) MCFragment(); + addFragment(F); +} + +void MCObjectStreamer::ensureHeadroom(size_t Headroom) { + if (Headroom <= FragSpace) + return; + auto *F = allocFragSpace(Headroom); + new (F) MCFragment(); + addFragment(F); } -void MCObjectStreamer::insert(MCFragment *F) { - assert(F->getKind() != MCFragment::FT_Data && +void MCObjectStreamer::insert(MCFragment *Frag) { + assert(Frag->getKind() != MCFragment::FT_Data && "F should have a variable-size tail"); + // Frag is not connected to FragSpace. Before modifying CurFrag with + // addFragment(Frag), allocate an empty fragment to maintain FragSpace + // connectivity, potentially reusing CurFrag's associated space. + MCFragment *F; + if (LLVM_LIKELY(sizeof(MCFragment) + NewFragHeadroom <= FragSpace)) { + auto End = reinterpret_cast(getCurFragEnd()); + F = reinterpret_cast( + alignToPowerOf2(End, alignof(MCFragment))); + FragSpace -= size_t(F) - End + sizeof(MCFragment); + } else { + F = allocFragSpace(0); + } + new (F) MCFragment(); + + addFragment(Frag); addFragment(F); - newFragment(); } void MCObjectStreamer::appendContents(ArrayRef Contents) { - CurFrag->appendContents(Contents); + ensureHeadroom(Contents.size()); + assert(FragSpace >= Contents.size()); + llvm::copy(Contents, getCurFragEnd()); + CurFrag->FixedSize += Contents.size(); + FragSpace -= Contents.size(); } void MCObjectStreamer::appendContents(size_t Num, char Elt) { - CurFrag->appendContents(Num, Elt); + ensureHeadroom(Num); + MutableArrayRef Data(getCurFragEnd(), Num); + llvm::fill(Data, Elt); + CurFrag->FixedSize += Num; + FragSpace -= Num; } void MCObjectStreamer::addFixup(const MCExpr *Value, MCFixupKind Kind) { - CurFrag->addFixup(MCFixup::create(CurFrag->getFixedSize(), Value, Kind)); + CurFrag->addFixup(MCFixup::create(getCurFragSize(), Value, Kind)); } // As a compile-time optimization, avoid allocating and evaluating an MCExpr @@ -115,6 +171,8 @@ void MCObjectStreamer::reset() { } EmitEHFrame = true; EmitDebugFrame = false; + FragStorage.clear(); + FragSpace = 0; MCStreamer::reset(); } @@ -143,7 +201,6 @@ void MCObjectStreamer::emitCFISections(bool EH, bool Debug, bool SFrame) { void MCObjectStreamer::emitValueImpl(const MCExpr *Value, unsigned Size, SMLoc Loc) { MCStreamer::emitValueImpl(Value, Size, Loc); - MCFragment *DF = getCurrentFragment(); MCDwarfLineEntry::make(this, getCurrentSectionOnly()); @@ -158,9 +215,9 @@ void MCObjectStreamer::emitValueImpl(const MCExpr *Value, unsigned Size, emitIntValue(AbsValue, Size); return; } - DF->addFixup(MCFixup::create(DF->getContents().size(), Value, - MCFixup::getDataKindForSize(Size))); - DF->appendContents(Size, 0); + ensureHeadroom(Size); + addFixup(Value, MCFixup::getDataKindForSize(Size)); + appendContents(Size, 0); } MCSymbol *MCObjectStreamer::emitCFILabel() { @@ -194,7 +251,7 @@ void MCObjectStreamer::emitLabel(MCSymbol *Symbol, SMLoc Loc) { // section. MCFragment *F = CurFrag; Symbol->setFragment(F); - Symbol->setOffset(F->getContents().size()); + Symbol->setOffset(F->getFixedSize()); emitPendingAssignments(Symbol); } @@ -260,6 +317,21 @@ void MCObjectStreamer::changeSection(MCSection *Section, uint32_t Subsection) { F0 = CurFrag; } + // To maintain connectivity between CurFrag and FragSpace when CurFrag is + // modified, allocate an empty fragment and append it to the fragment list. + // (Subsections[I].second.Tail is not connected to FragSpace.) + MCFragment *F; + if (LLVM_LIKELY(sizeof(MCFragment) + NewFragHeadroom <= FragSpace)) { + auto End = reinterpret_cast(getCurFragEnd()); + F = reinterpret_cast( + alignToPowerOf2(End, alignof(MCFragment))); + FragSpace -= size_t(F) - End + sizeof(MCFragment); + } else { + F = allocFragSpace(0); + } + new (F) MCFragment(); + F->setParent(Section); + auto &Subsections = Section->Subsections; size_t I = 0, E = Subsections.size(); while (I != E && Subsections[I].first < Subsection) @@ -267,13 +339,16 @@ void MCObjectStreamer::changeSection(MCSection *Section, uint32_t Subsection) { // If the subsection number is not in the sorted Subsections list, create a // new fragment list. if (I == E || Subsections[I].first != Subsection) { - auto *F = getContext().allocFragment(); - F->setParent(Section); Subsections.insert(Subsections.begin() + I, {Subsection, MCSection::FragList{F, F}}); + Section->CurFragList = &Subsections[I].second; + CurFrag = F; + } else { + Section->CurFragList = &Subsections[I].second; + CurFrag = Subsections[I].second.Tail; + // Ensure CurFrag is associated with FragSpace. + addFragment(F); } - Section->CurFragList = &Subsections[I].second; - CurFrag = Section->CurFragList->Tail; // Define the section symbol at subsection 0's initial fragment if required. if (!NewSec) @@ -344,11 +419,15 @@ void MCObjectStreamer::emitInstToData(const MCInst &Inst, MCFragment *F = getCurrentFragment(); // Append the instruction to the data fragment. - size_t CodeOffset = F->getContents().size(); + size_t CodeOffset = getCurFragSize(); + SmallString<16> Content; SmallVector Fixups; - getAssembler().getEmitter().encodeInstruction( - Inst, F->getContentsForAppending(), Fixups, STI); - F->doneAppending(); + getAssembler().getEmitter().encodeInstruction(Inst, Content, Fixups, STI); + appendContents(Content); + if (CurFrag != F) { + F = CurFrag; + CodeOffset = 0; + } F->setHasInstructions(STI); if (Fixups.empty()) @@ -356,19 +435,17 @@ void MCObjectStreamer::emitInstToData(const MCInst &Inst, bool MarkedLinkerRelaxable = false; for (auto &Fixup : Fixups) { Fixup.setOffset(Fixup.getOffset() + CodeOffset); - if (!Fixup.isLinkerRelaxable() || MarkedLinkerRelaxable) + if (!Fixup.isLinkerRelaxable()) continue; - MarkedLinkerRelaxable = true; - // Set the fragment's order within the subsection for use by - // MCAssembler::relaxAlign. - auto *Sec = F->getParent(); - if (!Sec->isLinkerRelaxable()) - Sec->setLinkerRelaxable(); + F->setLinkerRelaxable(); // Do not add data after a linker-relaxable instruction. The difference // between a new label and a label at or before the linker-relaxable // instruction cannot be resolved at assemble-time. - F->setLinkerRelaxable(); - newFragment(); + if (!MarkedLinkerRelaxable) { + MarkedLinkerRelaxable = true; + getCurrentSectionOnly()->setLinkerRelaxable(); + newFragment(); + } } F->appendFixups(Fixups); } diff --git a/llvm/lib/MC/MCWin64EH.cpp b/llvm/lib/MC/MCWin64EH.cpp index 72a8dd7031198..a87648afde7d6 100644 --- a/llvm/lib/MC/MCWin64EH.cpp +++ b/llvm/lib/MC/MCWin64EH.cpp @@ -318,6 +318,9 @@ static void EmitUnwindInfo(MCStreamer &streamer, WinEH::FrameInfo *info) { // Emit the epilog instructions. if (EnableUnwindV2) { + // Ensure the fixups and appended content apply to the same fragment. + OS->ensureHeadroom(info->EpilogMap.size() * 2); + bool IsLast = true; for (const auto &Epilog : llvm::reverse(info->EpilogMap)) { if (IsLast) { diff --git a/llvm/lib/MC/MCWinCOFFStreamer.cpp b/llvm/lib/MC/MCWinCOFFStreamer.cpp index 1ffe25ccbc473..8be5054ad25e2 100644 --- a/llvm/lib/MC/MCWinCOFFStreamer.cpp +++ b/llvm/lib/MC/MCWinCOFFStreamer.cpp @@ -280,6 +280,7 @@ void MCWinCOFFStreamer::emitCOFFSymbolIndex(MCSymbol const *Symbol) { void MCWinCOFFStreamer::emitCOFFSectionIndex(const MCSymbol *Symbol) { visitUsedSymbol(*Symbol); const MCSymbolRefExpr *SRE = MCSymbolRefExpr::create(Symbol, getContext()); + ensureHeadroom(2); addFixup(SRE, FK_SecRel_2); appendContents(2, 0); } @@ -293,6 +294,7 @@ void MCWinCOFFStreamer::emitCOFFSecRel32(const MCSymbol *Symbol, if (Offset) MCE = MCBinaryExpr::createAdd( MCE, MCConstantExpr::create(Offset, getContext()), getContext()); + ensureHeadroom(4); addFixup(MCE, FK_SecRel_4); // Emit 4 bytes (zeros) to the object file. appendContents(4, 0); @@ -308,6 +310,7 @@ void MCWinCOFFStreamer::emitCOFFImgRel32(const MCSymbol *Symbol, if (Offset) MCE = MCBinaryExpr::createAdd( MCE, MCConstantExpr::create(Offset, getContext()), getContext()); + ensureHeadroom(4); addFixup(MCE, FK_Data_4); // Emit 4 bytes (zeros) to the object file. appendContents(4, 0); @@ -318,6 +321,7 @@ void MCWinCOFFStreamer::emitCOFFSecNumber(MCSymbol const *Symbol) { // Create Symbol for section number. const MCExpr *MCE = MCCOFFSectionNumberTargetExpr::create( *Symbol, this->getWriter(), getContext()); + ensureHeadroom(4); addFixup(MCE, FK_Data_4); // Emit 4 bytes (zeros) to the object file. appendContents(4, 0); @@ -328,6 +332,7 @@ void MCWinCOFFStreamer::emitCOFFSecOffset(MCSymbol const *Symbol) { // Create Symbol for section offset. const MCExpr *MCE = MCCOFFSectionOffsetTargetExpr::create(*Symbol, getContext()); + ensureHeadroom(4); addFixup(MCE, FK_Data_4); // Emit 4 bytes (zeros) to the object file. appendContents(4, 0); diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp index d9680c7739a1b..7a8395a2e582b 100644 --- a/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp +++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsTargetStreamer.cpp @@ -1034,12 +1034,14 @@ MCELFStreamer &MipsTargetELFStreamer::getStreamer() { void MipsTargetELFStreamer::emitGPRel32Value(const MCExpr *Value) { auto &S = getStreamer(); + S.ensureHeadroom(4); S.addFixup(Value, Mips::fixup_Mips_GPREL32); S.appendContents(4, 0); } void MipsTargetELFStreamer::emitGPRel64Value(const MCExpr *Value) { auto &S = getStreamer(); + S.ensureHeadroom(8); // fixup_Mips_GPREL32 desginates R_MIPS_GPREL32+R_MIPS_64 on MIPS64. S.addFixup(Value, Mips::fixup_Mips_GPREL32); S.appendContents(8, 0); @@ -1047,24 +1049,28 @@ void MipsTargetELFStreamer::emitGPRel64Value(const MCExpr *Value) { void MipsTargetELFStreamer::emitDTPRel32Value(const MCExpr *Value) { auto &S = getStreamer(); + S.ensureHeadroom(4); S.addFixup(Value, Mips::fixup_Mips_DTPREL32); S.appendContents(4, 0); } void MipsTargetELFStreamer::emitDTPRel64Value(const MCExpr *Value) { auto &S = getStreamer(); + S.ensureHeadroom(8); S.addFixup(Value, Mips::fixup_Mips_DTPREL64); S.appendContents(8, 0); } void MipsTargetELFStreamer::emitTPRel32Value(const MCExpr *Value) { auto &S = getStreamer(); + S.ensureHeadroom(4); S.addFixup(Value, Mips::fixup_Mips_TPREL32); S.appendContents(4, 0); } void MipsTargetELFStreamer::emitTPRel64Value(const MCExpr *Value) { auto &S = getStreamer(); + S.ensureHeadroom(8); S.addFixup(Value, Mips::fixup_Mips_TPREL64); S.appendContents(8, 0); } diff --git a/llvm/test/MC/ELF/many-instructions.s b/llvm/test/MC/ELF/many-instructions.s new file mode 100644 index 0000000000000..cbdb2a71680d4 --- /dev/null +++ b/llvm/test/MC/ELF/many-instructions.s @@ -0,0 +1,10 @@ +# REQUIRES: asserts +# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o /dev/null -debug-only=mc-dump + +## Test that encodeInstruction may cause a new fragment to be created. +# CHECK: 0 Data Size:16200 +# CHECK: 16200 Data Size:180 + +.rept 16384/10 +movabsq $foo, %rax +.endr