[clang] [clang][bytecode] Add support for (toplevel) array fillers. (PR #164370)
Timm Baeder via cfe-commits
cfe-commits at lists.llvm.org
Tue Oct 21 00:38:47 PDT 2025
https://github.com/tbaederr created https://github.com/llvm/llvm-project/pull/164370
This adds support for array fillers, for global, toplevel descriptors.
Toplevel meaning that this gets an array filler:
```c++
constexpr int foo[200] = {1};
```
But this does not:
```c++
struct S { int i[10] = {3}; };
constexpr S s{};
```
For multidimensional arrays, only the topmost dimension gets an array filler:
```c++
constexpr float arr_f[3][5] = {
{1, 2, 3, 4, 5},
};
```
This means that this patch fixes e.g.
`test/SemaCXX/large-array-init`, but does not fix
`test/CodeGenCXX/cxx11-initializer-aggregate.cpp`, as that contains arrays such as:
```c++
struct B { int n; int arr[1024 * 1024 * 1024 * 2u]; } b = {1, {2}};
// ...
unsigned char data_3[1024][1024][1024] = {{{0}}};
// ...
unsigned char data_12[1024][1024][1024] = {{{1}}};
```
additionally, adding array fillers regresses performance in the common case of no array fillers:
https://llvm-compile-time-tracker.com/compare.php?from=02052caa09b27b422c452a2e1be2e3bfed710156&to=5b654212c1442d814aa8ed1c8de960432e479cdd&stat=instructions:u
>From 0db670a49b72eab7fbcae056c17345f2bc76f283 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Timm=20B=C3=A4der?= <tbaeder at redhat.com>
Date: Mon, 29 Sep 2025 16:12:15 +0200
Subject: [PATCH] [clang][bytecode] Add support for (toplevel) array fillers.
This adds support for array fillers, for global, toplevel descriptors.
Toplevel meaning that this gets an array filler:
```c++
constexpr int foo[200] = {1};
```
But this does not:
```c++
struct S { int i[10] = {3}; };
constexpr S s{};
```
For multidimensional arrays, only the topmost dimension gets an array
filler:
```c++
constexpr float arr_f[3][5] = {
{1, 2, 3, 4, 5},
};
```
This means that this patch fixes e.g.
`test/SemaCXX/large-array-init`, but does not fix
`test/CodeGenCXX/cxx11-initializer-aggregate.cpp`, as that contains
arrays such as:
```c++
struct B { int n; int arr[1024 * 1024 * 1024 * 2u]; } b = {1, {2}};
// ...
unsigned char data_3[1024][1024][1024] = {{{0}}};
// ...
unsigned char data_12[1024][1024][1024] = {{{1}}};
```
additionally, adding array fillers regresses performance in the common
case of no array fillers:
https://llvm-compile-time-tracker.com/compare.php?from=02052caa09b27b422c452a2e1be2e3bfed710156&to=5b654212c1442d814aa8ed1c8de960432e479cdd&stat=instructions:u
---
clang/lib/AST/ByteCode/Compiler.cpp | 65 +++-
clang/lib/AST/ByteCode/Descriptor.cpp | 309 +++++++++++++++++-
clang/lib/AST/ByteCode/Descriptor.h | 49 ++-
clang/lib/AST/ByteCode/Disasm.cpp | 38 ++-
clang/lib/AST/ByteCode/EvaluationResult.cpp | 16 +-
clang/lib/AST/ByteCode/Interp.cpp | 2 +
clang/lib/AST/ByteCode/Interp.h | 49 ++-
clang/lib/AST/ByteCode/InterpBlock.h | 3 +
.../lib/AST/ByteCode/InterpBuiltinBitCast.cpp | 5 +
clang/lib/AST/ByteCode/InterpFrame.cpp | 29 +-
clang/lib/AST/ByteCode/InterpFrame.h | 14 +-
clang/lib/AST/ByteCode/InterpHelpers.cpp | 58 ++++
clang/lib/AST/ByteCode/InterpHelpers.h | 14 +
clang/lib/AST/ByteCode/Opcodes.td | 2 +
clang/lib/AST/ByteCode/Pointer.cpp | 17 +-
clang/lib/AST/ByteCode/Pointer.h | 42 ++-
clang/lib/AST/ByteCode/Program.cpp | 67 +++-
clang/lib/AST/ByteCode/Program.h | 15 +-
clang/lib/AST/CMakeLists.txt | 1 +
clang/test/AST/ByteCode/array-fillers.cpp | 50 +++
20 files changed, 778 insertions(+), 67 deletions(-)
create mode 100644 clang/lib/AST/ByteCode/InterpHelpers.cpp
create mode 100644 clang/lib/AST/ByteCode/InterpHelpers.h
create mode 100644 clang/test/AST/ByteCode/array-fillers.cpp
diff --git a/clang/lib/AST/ByteCode/Compiler.cpp b/clang/lib/AST/ByteCode/Compiler.cpp
index 74cae030bb9bb..5b5bfd5981ea3 100644
--- a/clang/lib/AST/ByteCode/Compiler.cpp
+++ b/clang/lib/AST/ByteCode/Compiler.cpp
@@ -16,6 +16,11 @@
#include "PrimType.h"
#include "Program.h"
#include "clang/AST/Attr.h"
+#include "clang/Basic/UnsignedOrNone.h"
+#include "llvm/ADT/ScopeExit.h"
+#include "llvm/Support/Casting.h"
+
+#define DEBUG_TYPE "exprconstant"
using namespace clang;
using namespace clang::interp;
@@ -1942,9 +1947,9 @@ bool Compiler<Emitter>::visitInitList(ArrayRef<const Expr *> Inits,
const ConstantArrayType *CAT =
Ctx.getASTContext().getAsConstantArrayType(QT);
- uint64_t NumElems = CAT->getZExtSize();
+ uint64_t Capacity = CAT->getZExtSize();
- if (!this->emitCheckArraySize(NumElems, E))
+ if (!this->emitCheckArraySize(Capacity, E))
return false;
OptPrimType InitT = classify(CAT->getElementType());
@@ -1977,15 +1982,59 @@ bool Compiler<Emitter>::visitInitList(ArrayRef<const Expr *> Inits,
}
}
- // Expand the filler expression.
- // FIXME: This should go away.
- if (ArrayFiller) {
- for (; ElementIndex != NumElems; ++ElementIndex) {
- if (!this->visitArrayElemInit(ElementIndex, ArrayFiller, InitT))
+ auto isInit = [&](const Expr *E) -> bool {
+ if (const auto *VD = dyn_cast_if_present<VarDecl>(this->InitializingDecl))
+ return E == VD->getInit();
+ return false;
+ };
+
+ LLVM_DEBUG(llvm::dbgs() << "The number of elements to initialize: "
+ << ElementIndex << ".\n");
+
+ if (isInit(E) && ArraySize::shouldUseFiller(ElementIndex, Capacity) &&
+ Context::shouldBeGloballyIndexed(InitializingDecl)) {
+ assert(ArrayFiller);
+
+ unsigned TempOffset = 0;
+ if (InitT) {
+ TempOffset = this->allocateLocalPrimitive(ArrayFiller, *InitT,
+ /*IsConst=*/false);
+ if (!this->visit(ArrayFiller))
return false;
+ if (!this->emitSetLocal(*InitT, TempOffset, ArrayFiller))
+ return false;
+ } else {
+ if (UnsignedOrNone LocalOffset = this->allocateLocal(ArrayFiller)) {
+ TempOffset = *LocalOffset;
+ if (!this->emitGetPtrLocal(*LocalOffset, E))
+ return false;
+
+ if (!this->visitInitializer(ArrayFiller))
+ return false;
+
+ if (!this->emitFinishInitPop(ArrayFiller))
+ return false;
+ } else {
+ return false;
+ }
+ }
+
+ // Now copy the array filler from the local to the array.
+ if (!this->emitGetPtrLocal(TempOffset, E))
+ return false;
+ if (!this->emitSetArrayFillerPtr(TempOffset, E))
+ return false;
+ } else {
+ // Expand the filler expression.
+ if (ArrayFiller) {
+ for (; ElementIndex != Capacity; ++ElementIndex) {
+ if (!this->visitArrayElemInit(ElementIndex, ArrayFiller, InitT))
+ return false;
+ }
}
}
+ // assert(false);
return this->emitFinishInit(E);
}
@@ -4884,6 +4933,8 @@ Compiler<Emitter>::visitVarDecl(const VarDecl *VD, const Expr *Init,
}
// Local variables.
InitLinkScope<Emitter> ILS(this, InitLink::Decl(VD));
+ this->InitializingDecl = VD;
+ // DeclScope<Emitter> LocalScope(this, VD);
if (VarT) {
unsigned Offset = this->allocateLocalPrimitive(
diff --git a/clang/lib/AST/ByteCode/Descriptor.cpp b/clang/lib/AST/ByteCode/Descriptor.cpp
index 0a819599287ee..2398d93cfbf08 100644
--- a/clang/lib/AST/ByteCode/Descriptor.cpp
+++ b/clang/lib/AST/ByteCode/Descriptor.cpp
@@ -17,6 +17,7 @@
#include "Record.h"
#include "Source.h"
#include "clang/AST/ExprCXX.h"
+#include "llvm/Support/raw_ostream.h"
using namespace clang;
using namespace clang::interp;
@@ -288,10 +289,10 @@ Descriptor::Descriptor(const DeclTy &D, const Type *SourceTy, PrimType Type,
MetadataSize MD, bool IsConst, bool IsTemporary,
bool IsMutable, bool IsVolatile)
: Source(D), SourceType(SourceTy), ElemSize(primSize(Type)), Size(ElemSize),
- MDSize(MD.value_or(0)), AllocSize(align(Size + MDSize)), PrimT(Type),
- IsConst(IsConst), IsMutable(IsMutable), IsTemporary(IsTemporary),
- IsVolatile(IsVolatile), CtorFn(getCtorPrim(Type)),
- DtorFn(getDtorPrim(Type)) {
+ Capacity(Size), MDSize(MD.value_or(0)), AllocSize(align(Size + MDSize)),
+ PrimT(Type), IsConst(IsConst), IsMutable(IsMutable),
+ IsTemporary(IsTemporary), IsVolatile(IsVolatile),
+ CtorFn(getCtorPrim(Type)), DtorFn(getDtorPrim(Type)) {
assert(AllocSize >= Size);
assert(Source && "Missing source");
}
@@ -301,7 +302,7 @@ Descriptor::Descriptor(const DeclTy &D, PrimType Type, MetadataSize MD,
size_t NumElems, bool IsConst, bool IsTemporary,
bool IsMutable)
: Source(D), ElemSize(primSize(Type)), Size(ElemSize * NumElems),
- MDSize(MD.value_or(0)),
+ Capacity(NumElems), MDSize(MD.value_or(0)),
AllocSize(align(MDSize) + align(Size) + sizeof(InitMapPtr)), PrimT(Type),
IsConst(IsConst), IsMutable(IsMutable), IsTemporary(IsTemporary),
IsArray(true), CtorFn(getCtorArrayPrim(Type)),
@@ -310,11 +311,26 @@ Descriptor::Descriptor(const DeclTy &D, PrimType Type, MetadataSize MD,
assert(NumElems <= (MaxArrayElemBytes / ElemSize));
}
+Descriptor::Descriptor(const DeclTy &D, PrimType Type, MetadataSize MD,
+ ArraySize ArrSize, bool IsConst, bool IsTemporary,
+ bool IsMutable)
+ : Source(D), ElemSize(primSize(Type)),
+ Size(ElemSize * (ArrSize.Size + ArrSize.hasFiller())),
+ // Size(ElemSize * (ArrSize.Size)),
+ Capacity(ArrSize.Capacity), MDSize(MD.value_or(0)),
+ AllocSize(align(MDSize) + align(Size) + sizeof(InitMapPtr)), PrimT(Type),
+ IsConst(IsConst), IsMutable(IsMutable), IsTemporary(IsTemporary),
+ IsArray(true), CtorFn(getCtorArrayPrim(Type)),
+ DtorFn(getDtorArrayPrim(Type)) {
+ assert(Source && "Missing source");
+ assert(ArrSize.Size <= (MaxArrayElemBytes / ElemSize));
+}
+
/// Primitive unknown-size arrays.
Descriptor::Descriptor(const DeclTy &D, PrimType Type, MetadataSize MD,
bool IsTemporary, bool IsConst, UnknownSize)
: Source(D), ElemSize(primSize(Type)), Size(UnknownSizeMark),
- MDSize(MD.value_or(0)),
+ Capacity(Size), MDSize(MD.value_or(0)),
AllocSize(MDSize + sizeof(InitMapPtr) + alignof(void *)), PrimT(Type),
IsConst(IsConst), IsMutable(false), IsTemporary(IsTemporary),
IsArray(true), CtorFn(getCtorArrayPrim(Type)),
@@ -329,7 +345,24 @@ Descriptor::Descriptor(const DeclTy &D, const Type *SourceTy,
bool IsMutable)
: Source(D), SourceType(SourceTy),
ElemSize(Elem->getAllocSize() + sizeof(InlineDescriptor)),
- Size(ElemSize * NumElems), MDSize(MD.value_or(0)),
+ Size(ElemSize * NumElems), Capacity(NumElems), MDSize(MD.value_or(0)),
+ AllocSize(std::max<size_t>(alignof(void *), Size) + MDSize),
+ ElemDesc(Elem), IsConst(IsConst), IsMutable(IsMutable),
+ IsTemporary(IsTemporary), IsArray(true), CtorFn(ctorArrayDesc),
+ DtorFn(Elem->DtorFn ? dtorArrayDesc : nullptr) {
+ assert(Source && "Missing source");
+}
+
+/// Arrays of composite elements.
+Descriptor::Descriptor(const DeclTy &D, const Type *SourceTy,
+ const Descriptor *Elem, MetadataSize MD,
+ ArraySize ArrSize, bool IsConst, bool IsTemporary,
+ bool IsMutable)
+ : Source(D), SourceType(SourceTy),
+ ElemSize(Elem->getAllocSize() + sizeof(InlineDescriptor)),
+ Size(ElemSize * (ArrSize.Size + ArrSize.hasFiller())),
+ // Size(ElemSize * (ArrSize.Size)),
+ Capacity(ArrSize.Capacity), MDSize(MD.value_or(0)),
AllocSize(std::max<size_t>(alignof(void *), Size) + MDSize),
ElemDesc(Elem), IsConst(IsConst), IsMutable(IsMutable),
IsTemporary(IsTemporary), IsArray(true), CtorFn(ctorArrayDesc),
@@ -341,7 +374,7 @@ Descriptor::Descriptor(const DeclTy &D, const Type *SourceTy,
Descriptor::Descriptor(const DeclTy &D, const Descriptor *Elem, MetadataSize MD,
bool IsTemporary, UnknownSize)
: Source(D), ElemSize(Elem->getAllocSize() + sizeof(InlineDescriptor)),
- Size(UnknownSizeMark), MDSize(MD.value_or(0)),
+ Size(UnknownSizeMark), Capacity(Size), MDSize(MD.value_or(0)),
AllocSize(MDSize + alignof(void *)), ElemDesc(Elem), IsConst(true),
IsMutable(false), IsTemporary(IsTemporary), IsArray(true),
CtorFn(ctorArrayDesc), DtorFn(Elem->DtorFn ? dtorArrayDesc : nullptr) {
@@ -353,16 +386,16 @@ Descriptor::Descriptor(const DeclTy &D, const Record *R, MetadataSize MD,
bool IsConst, bool IsTemporary, bool IsMutable,
bool IsVolatile)
: Source(D), ElemSize(std::max<size_t>(alignof(void *), R->getFullSize())),
- Size(ElemSize), MDSize(MD.value_or(0)), AllocSize(Size + MDSize),
- ElemRecord(R), IsConst(IsConst), IsMutable(IsMutable),
- IsTemporary(IsTemporary), IsVolatile(IsVolatile), CtorFn(ctorRecord),
- DtorFn(needsRecordDtor(R) ? dtorRecord : nullptr) {
+ Size(ElemSize), Capacity(Size), MDSize(MD.value_or(0)),
+ AllocSize(Size + MDSize), ElemRecord(R), IsConst(IsConst),
+ IsMutable(IsMutable), IsTemporary(IsTemporary), IsVolatile(IsVolatile),
+ CtorFn(ctorRecord), DtorFn(needsRecordDtor(R) ? dtorRecord : nullptr) {
assert(Source && "Missing source");
}
/// Dummy.
Descriptor::Descriptor(const DeclTy &D, MetadataSize MD)
- : Source(D), ElemSize(1), Size(1), MDSize(MD.value_or(0)),
+ : Source(D), ElemSize(1), Size(1), Capacity(Size), MDSize(MD.value_or(0)),
AllocSize(MDSize), ElemRecord(nullptr), IsConst(true), IsMutable(false),
IsTemporary(false) {
assert(Source && "Missing source");
@@ -468,10 +501,255 @@ bool Descriptor::hasTrivialDtor() const {
bool Descriptor::isUnion() const { return isRecord() && ElemRecord->isUnion(); }
+using BlockMoveFn = void (*)(Block *Storage, std::byte *SrcFieldPtr,
+ std::byte *DstFieldPtr,
+ const Descriptor *FieldDesc,
+ const Descriptor *NewDesc);
+
+template <typename T>
+static void moveTy(Block *, std::byte *Src, std::byte *Dst, const Descriptor *,
+ const Descriptor *) {
+
+ auto *SrcPtr = reinterpret_cast<T *>(Src);
+ if constexpr (!std::is_same_v<T, Floating> &&
+ !std::is_same_v<T, MemberPointer> &&
+ !std::is_same_v<T, Pointer>) {
+ }
+ auto *DstPtr = reinterpret_cast<T *>(Dst);
+ new (DstPtr) T(std::move(*SrcPtr));
+}
+static BlockMoveFn getMovePrim(PrimType Type) {
+ TYPE_SWITCH(Type, return moveTy<T>);
+ llvm_unreachable("unknown PrimType");
+}
+
+/// Moving a primitive array.
+template <typename T>
+static void moveArrayTy(Block *, std::byte *Src, std::byte *Dst,
+ const Descriptor *D, const Descriptor *NewDesc) {
+ InitMapPtr &SrcIMP = *reinterpret_cast<InitMapPtr *>(Src);
+ SrcIMP = std::nullopt;
+
+ auto *NewInitMap = new (Dst) InitMapPtr(
+ std::make_pair(false, std::make_shared<InitMap>(NewDesc->getNumElems())));
+
+ Src += sizeof(InitMapPtr);
+ Dst += sizeof(InitMapPtr);
+
+ for (unsigned I = 0, NE = D->getNumElemsWithoutFiller(); I < NE; ++I) {
+ auto *SrcPtr = &reinterpret_cast<T *>(Src)[I];
+ auto *DstPtr = &reinterpret_cast<T *>(Dst)[I];
+ new (DstPtr) T(std::move(*SrcPtr));
+ NewInitMap->value().second->initializeElement(I);
+ }
+}
+
+static BlockMoveFn getMoveArrayPrim(PrimType Type) {
+ TYPE_SWITCH(Type, return moveArrayTy<T>);
+ llvm_unreachable("unknown Expr");
+}
+
+static void moveRecord(Block *B, std::byte *Src, std::byte *Dst,
+ const Descriptor *D, const Descriptor *NewDesc);
+static void moveArrayDesc(Block *B, std::byte *Src, std::byte *Dst,
+ const Descriptor *D, const Descriptor *NewDesc);
+static BlockMoveFn getMoveFn(const Descriptor *D) {
+ if (D->isPrimitive())
+ return getMovePrim(D->getPrimType());
+ if (D->isPrimitiveArray())
+ return getMoveArrayPrim(D->getPrimType());
+ if (D->isCompositeArray())
+ return moveArrayDesc;
+ if (D->isRecord())
+ return moveRecord;
+
+ return nullptr;
+}
+
+static BlockMoveFn getElemMoveFn(const Descriptor *D) {
+ assert(D->isArray());
+ if (D->isPrimitiveArray())
+ return getMovePrim(D->getPrimType());
+ assert(D->ElemDesc);
+ return getMoveFn(D->ElemDesc);
+}
+
+static void moveRecord(Block *B, std::byte *Src, std::byte *Dst,
+ const Descriptor *D, const Descriptor *) {
+ assert(D);
+ assert(D->ElemRecord);
+
+ for (const auto &F : D->ElemRecord->fields()) {
+ auto FieldOffset = F.Offset;
+ const auto *SrcDesc =
+ reinterpret_cast<const InlineDescriptor *>(Src + FieldOffset) - 1;
+ auto *DestDesc =
+ reinterpret_cast<InlineDescriptor *>(Dst + FieldOffset) - 1;
+ std::memcpy(DestDesc, SrcDesc, sizeof(InlineDescriptor));
+
+ if (auto Fn = getMoveFn(F.Desc))
+ Fn(B, Src + FieldOffset, Dst + FieldOffset, F.Desc, nullptr);
+ }
+
+ for (const auto &Base : D->ElemRecord->bases()) {
+ auto BaseOffset = Base.Offset;
+ const auto *SrcDesc =
+ reinterpret_cast<const InlineDescriptor *>(Src + BaseOffset) - 1;
+ auto *DestDesc = reinterpret_cast<InlineDescriptor *>(Dst + BaseOffset) - 1;
+ std::memcpy(DestDesc, SrcDesc, sizeof(InlineDescriptor));
+
+ if (auto Fn = getMoveFn(Base.Desc))
+ Fn(B, Src + BaseOffset, Dst + BaseOffset, Base.Desc, nullptr);
+ }
+
+ for (const auto &VBase : D->ElemRecord->virtual_bases()) {
+ auto VBaseOffset = VBase.Offset;
+ const auto *SrcDesc =
+ reinterpret_cast<const InlineDescriptor *>(Src + VBaseOffset) - 1;
+ auto *DestDesc =
+ reinterpret_cast<InlineDescriptor *>(Dst + VBaseOffset) - 1;
+ std::memcpy(DestDesc, SrcDesc, sizeof(InlineDescriptor));
+ }
+}
+
+static void moveArrayDesc(Block *B, std::byte *Src, std::byte *Dst,
+ const Descriptor *D, const Descriptor *NewDesc) {
+ const unsigned NumElems = D->getNumElemsWithoutFiller();
+ const unsigned ElemSize =
+ D->ElemDesc->getAllocSize() + sizeof(InlineDescriptor);
+
+ unsigned ElemOffset = 0;
+ for (unsigned I = 0; I != NumElems; ++I, ElemOffset += ElemSize) {
+ auto *SrcPtr = Src + ElemOffset;
+ auto *DstPtr = Dst + ElemOffset;
+
+ auto *SrcDesc = reinterpret_cast<InlineDescriptor *>(SrcPtr);
+ auto *SrcElemLoc = reinterpret_cast<std::byte *>(SrcDesc + 1);
+ auto *DstDesc = reinterpret_cast<InlineDescriptor *>(DstPtr);
+ auto *DstElemLoc = reinterpret_cast<std::byte *>(DstDesc + 1);
+
+ *DstDesc = *SrcDesc;
+
+ if (auto Fn = getMoveFn(D->ElemDesc)) {
+ Fn(B, SrcElemLoc, DstElemLoc, D->ElemDesc, NewDesc);
+ }
+ }
+}
+
+void Descriptor::moveArrayData(const Block *From, Block *To) const {
+ assert(From->getDescriptor() == this);
+
+ // First, copy block-level metadata.
+ assert(From->getDescriptor()->getMetadataSize() ==
+ To->getDescriptor()->getMetadataSize());
+ std::memcpy(To->rawData(), From->rawData(), MDSize);
+ unsigned OldNumElems = this->getNumElemsWithoutFiller();
+ unsigned NewNumElems = To->getDescriptor()->getNumElemsWithoutFiller();
+ assert(NewNumElems > OldNumElems);
+ assert(this->hasArrayFiller());
+
+ unsigned ElemSize = getElemSize();
+ // Copy all the old data over.
+
+ if (this->isPrimitiveArray()) {
+ auto MoveFn = getMoveArrayPrim(this->getPrimType());
+ MoveFn(To, const_cast<std::byte *>(From->data()), To->data(), this,
+ To->getDescriptor());
+ } else {
+ assert(isCompositeArray());
+ moveArrayDesc(To, const_cast<std::byte *>(From->data()), To->data(), this,
+ To->getDescriptor());
+ }
+
+ // Now fill the rest by copying over the array filler.
+ unsigned ElemIndex = this->getNumElemsWithoutFiller();
+ assert(ElemSize == To->getDescriptor()->getElemSize());
+ unsigned ArrayFillerIndex = this->getNumElemsWithoutFiller();
+ unsigned ArrayFillerOffset =
+ From->getDescriptor()->getMetadataSize() + (ElemSize * ArrayFillerIndex);
+ unsigned DstOffset = To->getDescriptor()->getMetadataSize();
+ DstOffset += ElemIndex * ElemSize;
+ if (this->isPrimitiveArray()) {
+ DstOffset += sizeof(InitMapPtr);
+ ArrayFillerOffset += sizeof(InitMapPtr);
+ }
+
+ DstOffset = ArrayFillerOffset;
+
+ bool ElemsHaveInlineDesc = this->isCompositeArray();
+
+ for (; ElemIndex != NewNumElems; ++ElemIndex) {
+ if (ElemsHaveInlineDesc) {
+ auto *SrcDesc =
+ reinterpret_cast<InlineDescriptor *>(To->data() + ArrayFillerOffset);
+ auto *DstDesc =
+ reinterpret_cast<InlineDescriptor *>(To->data() + DstOffset);
+ *DstDesc = *SrcDesc;
+ DstOffset += sizeof(InlineDescriptor);
+ } else {
+ InitMapPtr &DstIMP = *reinterpret_cast<InitMapPtr *>(To->data());
+ DstIMP->second->initializeElement(ElemIndex);
+ }
+
+ auto MoveFn = getElemMoveFn(this);
+ if (ElemsHaveInlineDesc)
+ MoveFn(
+ To,
+ const_cast<std::byte *>(
+ From->rawData() + (ArrayFillerOffset + sizeof(InlineDescriptor))),
+ To->rawData() + DstOffset, ElemDesc, To->getDescriptor());
+ else
+ MoveFn(To, const_cast<std::byte *>(From->rawData() + (ArrayFillerOffset)),
+ To->rawData() + DstOffset, ElemDesc, To->getDescriptor());
+
+ if (ElemsHaveInlineDesc)
+ DstOffset += (ElemSize - sizeof(InlineDescriptor));
+ else
+ DstOffset += (ElemSize);
+ }
+
+ // At last, if we didn't expand to the full capacity, we need to carry the
+ // evaluated array filler around with us.
+ if (NewNumElems < To->getDescriptor()->Capacity) {
+ if (ElemsHaveInlineDesc) {
+ auto *SrcDesc =
+ reinterpret_cast<InlineDescriptor *>(To->data() + ArrayFillerOffset);
+ auto *DstDesc =
+ reinterpret_cast<InlineDescriptor *>(To->data() + DstOffset);
+
+ *DstDesc = *SrcDesc;
+ DstOffset += sizeof(InlineDescriptor);
+ } else {
+ InitMapPtr &DstIMP = *reinterpret_cast<InitMapPtr *>(To->data());
+ DstIMP->second->initializeElement(ElemIndex);
+ }
+
+ // Note that DstOffset gets advanced in the loop above and now points to the
+ // element after the filled-in elements.
+ auto MoveFn = getElemMoveFn(this);
+ if (ElemsHaveInlineDesc)
+ MoveFn(
+ To,
+ const_cast<std::byte *>(
+ From->rawData() + (ArrayFillerOffset + sizeof(InlineDescriptor))),
+ To->rawData() + DstOffset, ElemDesc, To->getDescriptor());
+ else
+ MoveFn(To, const_cast<std::byte *>(From->rawData() + (ArrayFillerOffset)),
+ To->rawData() + DstOffset, ElemDesc, To->getDescriptor());
+ }
+}
+
InitMap::InitMap(unsigned N)
- : UninitFields(N), Data(std::make_unique<T[]>(numFields(N))) {}
+ : UninitFields(N), Data(std::make_unique<T[]>(numFields(N))) {
+#ifndef NDEBUG
+ NumFields = N;
+#endif
+}
bool InitMap::initializeElement(unsigned I) {
+#ifndef NDEBUG
+ assert(I < NumFields);
+#endif
unsigned Bucket = I / PER_FIELD;
T Mask = T(1) << (I % PER_FIELD);
if (!(data()[Bucket] & Mask)) {
@@ -482,6 +760,9 @@ bool InitMap::initializeElement(unsigned I) {
}
bool InitMap::isElementInitialized(unsigned I) const {
+#ifndef NDEBUG
+ assert(I < NumFields);
+#endif
unsigned Bucket = I / PER_FIELD;
return data()[Bucket] & (T(1) << (I % PER_FIELD));
}
diff --git a/clang/lib/AST/ByteCode/Descriptor.h b/clang/lib/AST/ByteCode/Descriptor.h
index 90dc2b4aa3111..989247a763fbd 100644
--- a/clang/lib/AST/ByteCode/Descriptor.h
+++ b/clang/lib/AST/ByteCode/Descriptor.h
@@ -29,6 +29,29 @@ enum PrimType : uint8_t;
using DeclTy = llvm::PointerUnion<const Decl *, const Expr *>;
using InitMapPtr = std::optional<std::pair<bool, std::shared_ptr<InitMap>>>;
+struct ArraySize {
+ unsigned Size;
+ unsigned Capacity;
+ ArraySize(unsigned S, unsigned C) : Size(S), Capacity(C) {
+ assert(Size <= Capacity);
+ }
+ bool hasFiller() const { return Capacity != Size; }
+
+ static bool shouldUseFiller(unsigned S, unsigned C) {
+ return (C - S) >= 2;
+ // return C >= 32 && (C - S) >= 16;
+ // return (C - S) >= 5;
+ }
+
+ static ArraySize getNextSize(unsigned S, unsigned C) {
+ // llvm::errs() << S << " / " << C << '\n';
+
+ unsigned S2 = std::min(S * 2, C);
+
+ return ArraySize(S2, C);
+ }
+};
+
/// Invoked whenever a block is created. The constructor method fills in the
/// inline descriptors of all fields and array elements. It also initializes
/// all the fields which contain non-trivial types.
@@ -120,7 +143,7 @@ static_assert(sizeof(GlobalInlineDescriptor) != sizeof(InlineDescriptor), "");
/// Describes a memory block created by an allocation site.
struct Descriptor final {
-private:
+public:
/// Original declaration, used to emit the error message.
const DeclTy Source;
const Type *SourceType = nullptr;
@@ -128,6 +151,7 @@ struct Descriptor final {
const unsigned ElemSize;
/// Size of the storage, in host bytes.
const unsigned Size;
+ const unsigned Capacity;
/// Size of the metadata.
const unsigned MDSize;
/// Size of the allocation (storage + metadata), in host bytes.
@@ -181,6 +205,9 @@ struct Descriptor final {
Descriptor(const DeclTy &D, PrimType Type, MetadataSize MD, size_t NumElems,
bool IsConst, bool IsTemporary, bool IsMutable);
+ Descriptor(const DeclTy &D, PrimType Type, MetadataSize MD, ArraySize ArrSize,
+ bool IsConst, bool IsTemporary, bool IsMutable);
+
/// Allocates a descriptor for an array of primitives of unknown size.
Descriptor(const DeclTy &D, PrimType Type, MetadataSize MDSize, bool IsConst,
bool IsTemporary, UnknownSize);
@@ -190,6 +217,10 @@ struct Descriptor final {
MetadataSize MD, unsigned NumElems, bool IsConst, bool IsTemporary,
bool IsMutable);
+ Descriptor(const DeclTy &D, const Type *SourceTy, const Descriptor *Elem,
+ MetadataSize MD, ArraySize ArrSize, bool IsConst, bool IsTemporary,
+ bool IsMutable);
+
/// Allocates a descriptor for an array of composites of unknown size.
Descriptor(const DeclTy &D, const Descriptor *Elem, MetadataSize MD,
bool IsTemporary, UnknownSize);
@@ -240,7 +271,7 @@ struct Descriptor final {
/// Returns the allocated size, including metadata.
unsigned getAllocSize() const { return AllocSize; }
- /// returns the size of an element when the structure is viewed as an array.
+ /// Returns the size of an element when the structure is viewed as an array.
unsigned getElemSize() const { return ElemSize; }
/// Returns the size of the metadata.
unsigned getMetadataSize() const { return MDSize; }
@@ -250,6 +281,15 @@ struct Descriptor final {
return Size == UnknownSizeMark ? 0 : (getSize() / getElemSize());
}
+ unsigned getNumElemsWithoutFiller() const {
+ assert(!isUnknownSizeArray());
+ unsigned N = getNumElems();
+ return N - (N != Capacity);
+ }
+ bool hasArrayFiller() const { return getNumElems() != Capacity; }
+
+ void moveArrayData(const Block *From, Block *To) const;
+
/// Checks if the descriptor is of an array of primitives.
bool isPrimitiveArray() const { return IsArray && !ElemDesc; }
/// Checks if the descriptor is of an array of composites.
@@ -284,12 +324,15 @@ struct InitMap final {
using T = uint64_t;
/// Bits stored in a single field.
static constexpr uint64_t PER_FIELD = sizeof(T) * CHAR_BIT;
+#ifndef NDEBUG
+ unsigned NumFields;
+#endif
public:
/// Initializes the map with no fields set.
explicit InitMap(unsigned N);
-private:
+public:
friend class Pointer;
/// Returns a pointer to storage.
diff --git a/clang/lib/AST/ByteCode/Disasm.cpp b/clang/lib/AST/ByteCode/Disasm.cpp
index fd0903f2e652c..f14e056526dab 100644
--- a/clang/lib/AST/ByteCode/Disasm.cpp
+++ b/clang/lib/AST/ByteCode/Disasm.cpp
@@ -393,11 +393,17 @@ LLVM_DUMP_METHOD void Descriptor::dump(llvm::raw_ostream &OS) const {
}
// Print a few interesting bits about the descriptor.
- if (isPrimitiveArray())
- OS << " primitive-array";
- else if (isCompositeArray())
- OS << " composite-array";
- else if (isUnion())
+ if (isPrimitiveArray()) {
+ OS << " primitive-array(" << getNumElemsWithoutFiller() << "/" << Capacity
+ << ")";
+ if (hasArrayFiller())
+ OS << " has-filler";
+ } else if (isCompositeArray()) {
+ OS << " composite-array(" << getNumElemsWithoutFiller() << "/" << Capacity
+ << ")";
+ if (hasArrayFiller())
+ OS << " has-filler";
+ } else if (isUnion())
OS << " union";
else if (isRecord())
OS << " record";
@@ -484,8 +490,6 @@ LLVM_DUMP_METHOD void InterpFrame::dump(llvm::raw_ostream &OS,
OS << " (" << F->getName() << ")";
}
OS << "\n";
- OS.indent(Spaces) << "This: " << getThis() << "\n";
- OS.indent(Spaces) << "RVO: " << getRVOPtr() << "\n";
OS.indent(Spaces) << "Depth: " << Depth << "\n";
OS.indent(Spaces) << "ArgSize: " << ArgSize << "\n";
OS.indent(Spaces) << "Args: " << (void *)Args << "\n";
@@ -563,6 +567,26 @@ LLVM_DUMP_METHOD void Block::dump(llvm::raw_ostream &OS) const {
OS << " Dynamic: " << isDynamic() << "\n";
}
+LLVM_DUMP_METHOD void Block::dumpContents() const {
+ llvm::raw_ostream &OS = llvm::errs();
+ const Descriptor *Desc = getDescriptor();
+ assert(Desc);
+
+ Desc->dump(OS);
+ OS << ' ';
+ if (Desc->isPrimitiveArray()) {
+ PrimType ElemT = Desc->getPrimType();
+ Pointer BasePtr = Pointer(const_cast<Block *>(this));
+ for (unsigned I = 0; I != Desc->getNumElems(); ++I) {
+ TYPE_SWITCH(ElemT, { OS << BasePtr.elem<T>(I); });
+ OS << ' ';
+ }
+ OS << '\n';
+ } else {
+ assert(false && "Unimplemented content type in Block::dumpContents()");
+ }
+}
+
LLVM_DUMP_METHOD void EvaluationResult::dump() const {
auto &OS = llvm::errs();
diff --git a/clang/lib/AST/ByteCode/EvaluationResult.cpp b/clang/lib/AST/ByteCode/EvaluationResult.cpp
index 7c3c21cf28251..00360a6ea7ffc 100644
--- a/clang/lib/AST/ByteCode/EvaluationResult.cpp
+++ b/clang/lib/AST/ByteCode/EvaluationResult.cpp
@@ -42,27 +42,27 @@ static bool CheckArrayInitialized(InterpState &S, SourceLocation Loc,
if (ElemType->isRecordType()) {
const Record *R = BasePtr.getElemRecord();
- for (size_t I = 0; I != NumElems; ++I) {
+ for (size_t I = 0; I != BasePtr.getNumAllocatedElems(); ++I) {
Pointer ElemPtr = BasePtr.atIndex(I).narrow();
Result &= CheckFieldsInitialized(S, Loc, ElemPtr, R);
}
} else if (const auto *ElemCAT = dyn_cast<ConstantArrayType>(ElemType)) {
- for (size_t I = 0; I != NumElems; ++I) {
+
+ for (size_t I = 0; I != BasePtr.getNumAllocatedElems(); ++I) {
Pointer ElemPtr = BasePtr.atIndex(I).narrow();
Result &= CheckArrayInitialized(S, Loc, ElemPtr, ElemCAT);
}
} else {
// Primitive arrays.
if (S.getContext().canClassify(ElemType)) {
- if (BasePtr.allElementsInitialized()) {
+ if (BasePtr.allElementsInitialized())
return true;
- } else {
- DiagnoseUninitializedSubobject(S, Loc, BasePtr.getField());
- return false;
- }
+
+ DiagnoseUninitializedSubobject(S, Loc, BasePtr.getField());
+ return false;
}
- for (size_t I = 0; I != NumElems; ++I) {
+ for (size_t I = 0; I != BasePtr.getNumAllocatedElems(); ++I) {
if (!BasePtr.isElementInitialized(I)) {
DiagnoseUninitializedSubobject(S, Loc, BasePtr.getField());
Result = false;
diff --git a/clang/lib/AST/ByteCode/Interp.cpp b/clang/lib/AST/ByteCode/Interp.cpp
index a72282caf5e73..83b784f5b5152 100644
--- a/clang/lib/AST/ByteCode/Interp.cpp
+++ b/clang/lib/AST/ByteCode/Interp.cpp
@@ -1597,6 +1597,8 @@ bool Call(InterpState &S, CodePtr OpPC, const Function *Func,
if (!Func->isFullyCompiled())
compileFunction(S, Func);
+ // Func->dump();
+
if (!CheckCallable(S, OpPC, Func))
return cleanup();
diff --git a/clang/lib/AST/ByteCode/Interp.h b/clang/lib/AST/ByteCode/Interp.h
index 2f7e2d98f3576..fbed3806ea735 100644
--- a/clang/lib/AST/ByteCode/Interp.h
+++ b/clang/lib/AST/ByteCode/Interp.h
@@ -22,6 +22,7 @@
#include "Function.h"
#include "InterpBuiltinBitCast.h"
#include "InterpFrame.h"
+#include "InterpHelpers.h"
#include "InterpStack.h"
#include "InterpState.h"
#include "MemberPointer.h"
@@ -1960,6 +1961,7 @@ template <PrimType Name, class T = typename PrimConv<Name>::T>
bool Store(InterpState &S, CodePtr OpPC) {
const T &Value = S.Stk.pop<T>();
const Pointer &Ptr = S.Stk.peek<Pointer>();
+
if (!CheckStore(S, OpPC, Ptr))
return false;
if (Ptr.canBeInitialized())
@@ -1972,6 +1974,7 @@ template <PrimType Name, class T = typename PrimConv<Name>::T>
bool StorePop(InterpState &S, CodePtr OpPC) {
const T &Value = S.Stk.pop<T>();
const Pointer &Ptr = S.Stk.pop<Pointer>();
+
if (!CheckStore(S, OpPC, Ptr))
return false;
if (Ptr.canBeInitialized())
@@ -2139,7 +2142,10 @@ bool InitElem(InterpState &S, CodePtr OpPC, uint32_t Idx) {
if (!CheckLive(S, OpPC, Ptr, AK_Assign))
return false;
- if (Idx >= Desc->getNumElems()) {
+
+ ensureArraySize(S.P, Ptr, Idx);
+
+ if (Idx >= (Desc->getNumElems())) {
// CheckRange.
if (S.getLangOpts().CPlusPlus) {
const SourceInfo &Loc = S.Current->getSource(OpPC);
@@ -2171,9 +2177,11 @@ bool InitElemPop(InterpState &S, CodePtr OpPC, uint32_t Idx) {
return true;
}
+ ensureArraySize(S.P, Ptr, Idx);
+
if (!CheckLive(S, OpPC, Ptr, AK_Assign))
return false;
- if (Idx >= Desc->getNumElems()) {
+ if (Idx >= Desc->getNumElemsWithoutFiller()) {
// CheckRange.
if (S.getLangOpts().CPlusPlus) {
const SourceInfo &Loc = S.Current->getSource(OpPC);
@@ -2264,7 +2272,7 @@ std::optional<Pointer> OffsetHelper(InterpState &S, CodePtr OpPC,
assert(Ptr.isBlockPointer());
- uint64_t MaxIndex = static_cast<uint64_t>(Ptr.getNumElems());
+ uint64_t MaxIndex = static_cast<uint64_t>(Ptr.getCapacity());
uint64_t Index;
if (Ptr.isOnePastEnd())
Index = MaxIndex;
@@ -2308,8 +2316,9 @@ std::optional<Pointer> OffsetHelper(InterpState &S, CodePtr OpPC,
}
}
- if (Invalid && S.getLangOpts().CPlusPlus)
+ if (Invalid && S.getLangOpts().CPlusPlus) {
return std::nullopt;
+ }
// Offset is valid - compute it on unsigned.
int64_t WideIndex = static_cast<int64_t>(Index);
@@ -2320,6 +2329,8 @@ std::optional<Pointer> OffsetHelper(InterpState &S, CodePtr OpPC,
else
Result = WideIndex - WideOffset;
+ ensureArraySize(S.P, Ptr, Result);
+
// When the pointer is one-past-end, going back to index 0 is the only
// useful thing we can do. Any other index has been diagnosed before and
// we don't get here.
@@ -3098,6 +3109,7 @@ inline bool ArrayElemPtr(InterpState &S, CodePtr OpPC) {
}
if (Offset.isZero()) {
+ ensureArraySize(S.P, Ptr, 0);
if (const Descriptor *Desc = Ptr.getFieldDesc();
Desc && Desc->isArray() && Ptr.getIndex() == 0) {
S.Stk.push<Pointer>(Ptr.atIndex(0).narrow());
@@ -3129,6 +3141,7 @@ inline bool ArrayElemPtrPop(InterpState &S, CodePtr OpPC) {
}
if (Offset.isZero()) {
+ ensureArraySize(S.P, Ptr, 0);
if (const Descriptor *Desc = Ptr.getFieldDesc();
Desc && Desc->isArray() && Ptr.getIndex() == 0) {
S.Stk.push<Pointer>(Ptr.atIndex(0).narrow());
@@ -3164,6 +3177,13 @@ template <PrimType Name, class T = typename PrimConv<Name>::T>
inline bool ArrayElemPop(InterpState &S, CodePtr OpPC, uint32_t Index) {
const Pointer &Ptr = S.Stk.pop<Pointer>();
+ // if (Index == Ptr.getFieldDesc()->getNumElemsWithoutFiller() &&
+ // Index < Ptr.getFieldDesc()->Capacity &&
+ // Ptr.getFieldDesc()->hasArrayFiller()) {
+ // S.Stk.push<T>(Ptr.elem<T>(Ptr.getNumElems()));
+ // return true;
+ // }
+
if (!CheckLoad(S, OpPC, Ptr))
return false;
@@ -3193,6 +3213,27 @@ inline bool CopyArray(InterpState &S, CodePtr OpPC, uint32_t SrcIndex,
return true;
}
+inline bool SetArrayFillerPtr(InterpState &S, CodePtr OpPC,
+ uint32_t LocalIndex) {
+ const auto &FillerValue = S.Stk.pop<Pointer>();
+ const auto &Arr = S.Stk.peek<Pointer>();
+
+ assert(Arr.getFieldDesc()->hasArrayFiller());
+ assert(Arr.isArrayRoot());
+ assert(FillerValue.isRoot());
+ assert(Arr.getNumAllocatedElems() < Arr.getNumAllocatedElemsWithFiller());
+
+ Pointer ArrayFillerDest = Arr.atIndex(Arr.getNumAllocatedElems()).narrow();
+ if (Arr.getFieldDesc()->isPrimitiveArray()) {
+ TYPE_SWITCH(ArrayFillerDest.getFieldDesc()->getPrimType(), {
+ new (&ArrayFillerDest.deref<T>()) T(FillerValue.deref<T>());
+ });
+ return true;
+ }
+
+ return DoMemcpy(S, OpPC, FillerValue, ArrayFillerDest);
+}
+
/// Just takes a pointer and checks if it's an incomplete
/// array type.
inline bool ArrayDecay(InterpState &S, CodePtr OpPC) {
diff --git a/clang/lib/AST/ByteCode/InterpBlock.h b/clang/lib/AST/ByteCode/InterpBlock.h
index 73fdc8d85da11..a549ee9c81c47 100644
--- a/clang/lib/AST/ByteCode/InterpBlock.h
+++ b/clang/lib/AST/ByteCode/InterpBlock.h
@@ -142,8 +142,11 @@ class Block final {
IsInitialized = false;
}
+ void moveArrayData(Block *To) const { Desc->moveArrayData(this, To); }
+
void dump() const { dump(llvm::errs()); }
void dump(llvm::raw_ostream &OS) const;
+ void dumpContents() const;
bool isAccessible() const { return AccessFlags == 0; }
diff --git a/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp b/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp
index 4bd9c66fc9974..f5d51a8147123 100644
--- a/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp
+++ b/clang/lib/AST/ByteCode/InterpBuiltinBitCast.cpp
@@ -11,6 +11,7 @@
#include "Context.h"
#include "Floating.h"
#include "Integral.h"
+#include "InterpHelpers.h"
#include "InterpState.h"
#include "MemberPointer.h"
#include "Pointer.h"
@@ -265,6 +266,8 @@ bool clang::interp::readPointerToBuffer(const Context &Ctx,
Endian TargetEndianness =
ASTCtx.getTargetInfo().isLittleEndian() ? Endian::Little : Endian::Big;
+ ensureArraySize(Ctx.getProgram(), FromPtr);
+
return enumeratePointerFields(
FromPtr, Ctx, Buffer.size(),
[&](const Pointer &P, PrimType T, Bits BitOffset, Bits FullBitWidth,
@@ -384,6 +387,8 @@ bool clang::interp::DoBitCastPtr(InterpState &S, CodePtr OpPC,
readPointerToBuffer(S.getContext(), FromPtr, Buffer,
/*ReturnOnUninit=*/false);
+ ensureArraySize(S.P, ToPtr);
+
// Now read the values out of the buffer again and into ToPtr.
Endian TargetEndianness =
ASTCtx.getTargetInfo().isLittleEndian() ? Endian::Little : Endian::Big;
diff --git a/clang/lib/AST/ByteCode/InterpFrame.cpp b/clang/lib/AST/ByteCode/InterpFrame.cpp
index 039acb5d72b2c..cbde63eedea67 100644
--- a/clang/lib/AST/ByteCode/InterpFrame.cpp
+++ b/clang/lib/AST/ByteCode/InterpFrame.cpp
@@ -81,7 +81,7 @@ void InterpFrame::destroyScopes() {
return;
for (auto &Scope : Func->scopes()) {
for (auto &Local : Scope.locals()) {
- S.deallocate(localBlock(Local.Offset));
+ S.deallocate(localBlock(Local.Offset, false));
}
}
}
@@ -192,6 +192,33 @@ void InterpFrame::describe(llvm::raw_ostream &OS) const {
OS << ")";
}
+void InterpFrame::reallocLocal(Block *Prev, const Descriptor *NewDesc) {
+ // llvm::errs() << __PRETTY_FUNCTION__ << '\n';
+ // llvm::errs() << "Prev: " << Prev << '\n';
+ // this->dump();
+
+ unsigned Offset = ((char *)Prev) - Locals.get() + sizeof(Block);
+
+ // First, we need a new block for the new descriptor.
+ Block *B = new (S.allocate(sizeof(Block) + NewDesc->getAllocSize()))
+ Block(Prev->getEvalID(), Prev->getDeclID(), NewDesc, false, false, false);
+ B->invokeCtor();
+
+ // llvm::errs() << "Reallocated LOCAL in InterpFrame: "<< Prev << " -> " << B
+ // << '\n';
+
+ // Pointer(B).initializeAllElements();
+
+ Prev->moveArrayData(B);
+ Prev->movePointersTo(B);
+
+ assert(!Prev->hasPointers());
+
+ ReallocatedLocals[Offset] = B;
+ // llvm::errs() << " !!!!!!!!!!!! " << Offset << " should now point to " << B
+ // << '\n';
+}
+
SourceRange InterpFrame::getCallRange() const {
if (!Caller->Func) {
if (SourceRange NullRange = S.getRange(nullptr, {}); NullRange.isValid())
diff --git a/clang/lib/AST/ByteCode/InterpFrame.h b/clang/lib/AST/ByteCode/InterpFrame.h
index fa9de2e1e7c6d..c79a80587b080 100644
--- a/clang/lib/AST/ByteCode/InterpFrame.h
+++ b/clang/lib/AST/ByteCode/InterpFrame.h
@@ -142,6 +142,7 @@ class InterpFrame final : public Frame {
void dump() const { dump(llvm::errs(), 0); }
void dump(llvm::raw_ostream &OS, unsigned Indent = 0) const;
+ void reallocLocal(Block *Prev, const Descriptor *NewDesc);
private:
/// Returns an original argument from the stack.
@@ -156,7 +157,16 @@ class InterpFrame final : public Frame {
}
/// Returns a pointer to a local's block.
- Block *localBlock(unsigned Offset) const {
+ Block *localBlock(unsigned Offset, bool CheckReallocs = true) const {
+ if (CheckReallocs) {
+ // llvm::errs() << __PRETTY_FUNCTION__ << ": " << Offset << ". Reallocated
+ // locals: " << ReallocatedLocals.size() << '\n';
+ if (auto It = ReallocatedLocals.find(Offset);
+ It != ReallocatedLocals.end()) {
+ // llvm::errs() << "AAAAAAAha! localblock() on a reallocated local!\n";
+ return It->second;
+ }
+ }
return reinterpret_cast<Block *>(Locals.get() + Offset - sizeof(Block));
}
@@ -186,6 +196,8 @@ class InterpFrame final : public Frame {
const size_t FrameOffset;
/// Mapping from arg offsets to their argument blocks.
llvm::DenseMap<unsigned, std::unique_ptr<char[]>> Params;
+
+ llvm::DenseMap<unsigned, Block *> ReallocatedLocals;
};
} // namespace interp
diff --git a/clang/lib/AST/ByteCode/InterpHelpers.cpp b/clang/lib/AST/ByteCode/InterpHelpers.cpp
new file mode 100644
index 0000000000000..fd458b224e0a2
--- /dev/null
+++ b/clang/lib/AST/ByteCode/InterpHelpers.cpp
@@ -0,0 +1,58 @@
+
+
+#include "InterpHelpers.h"
+#include "Descriptor.h"
+#include "InterpBlock.h"
+#include "Program.h"
+
+namespace clang {
+namespace interp {
+
+void ensureArraySize(Program &P, const Pointer &Ptr, unsigned RequestedIndex) {
+ if (!Ptr.isBlockPointer())
+ return;
+
+ assert(Ptr.getFieldDesc());
+ assert(Ptr.getDeclDesc());
+ if (!Ptr.getDeclDesc()->isArray())
+ return;
+
+ // No fillers for these.
+ if (!Ptr.isStatic() || Ptr.isUnknownSizeArray() || Ptr.block()->isDynamic())
+ return;
+
+ assert(Ptr.getFieldDesc()->isArray());
+
+ bool NeedsRealloc = RequestedIndex >= Ptr.getNumAllocatedElems() &&
+ RequestedIndex < Ptr.getCapacity();
+
+ // llvm::errs() << "NeedsRealloc: " << NeedsRealloc << '\n';
+ if (!NeedsRealloc)
+ return;
+
+ assert(Ptr.getFieldDesc()->hasArrayFiller());
+ unsigned RequestedSize = RequestedIndex + 1;
+ assert(RequestedSize <= Ptr.getNumElems());
+
+ const Descriptor *D = Ptr.getFieldDesc();
+ ArraySize NewArraySize = ArraySize::getNextSize(RequestedSize, D->Capacity);
+
+ const Descriptor *NewDesc = nullptr;
+ if (D->isPrimitiveArray()) {
+ NewDesc = P.allocateDescriptor(D->Source, D->getPrimType(),
+ Descriptor::GlobalMD, NewArraySize,
+ D->IsConst, D->IsTemporary, D->IsMutable);
+ } else if (D->isCompositeArray()) {
+ NewDesc = P.allocateDescriptor(D->Source, D->SourceType, D->ElemDesc,
+ Descriptor::GlobalMD, NewArraySize,
+ D->IsConst, D->IsTemporary, D->IsMutable);
+ } else {
+ llvm_unreachable("Should be either a primitive or composite array");
+ }
+
+ assert(NewDesc);
+ P.reallocGlobal(const_cast<Block *>(Ptr.block()), NewDesc);
+}
+
+} // namespace interp
+} // namespace clang
diff --git a/clang/lib/AST/ByteCode/InterpHelpers.h b/clang/lib/AST/ByteCode/InterpHelpers.h
new file mode 100644
index 0000000000000..1eaa54900788f
--- /dev/null
+++ b/clang/lib/AST/ByteCode/InterpHelpers.h
@@ -0,0 +1,14 @@
+
+#include "Pointer.h"
+
+namespace clang {
+namespace interp {
+void ensureArraySize(Program &P, const Pointer &Ptr, unsigned RequestedIndex);
+
+inline void ensureArraySize(Program &P, const Pointer &Ptr) {
+ if (!Ptr.getFieldDesc()->isArray())
+ return;
+ ensureArraySize(P, Ptr, Ptr.getNumElems() - 1);
+}
+} // namespace interp
+} // namespace clang
diff --git a/clang/lib/AST/ByteCode/Opcodes.td b/clang/lib/AST/ByteCode/Opcodes.td
index 532c4448e6f40..63a065cfe2ce8 100644
--- a/clang/lib/AST/ByteCode/Opcodes.td
+++ b/clang/lib/AST/ByteCode/Opcodes.td
@@ -381,6 +381,8 @@ def CopyArray : Opcode {
let HasGroup = 1;
}
+def SetArrayFillerPtr : Opcode { let Args = [ArgUint32]; }
+
//===----------------------------------------------------------------------===//
// Direct field accessors
//===----------------------------------------------------------------------===//
diff --git a/clang/lib/AST/ByteCode/Pointer.cpp b/clang/lib/AST/ByteCode/Pointer.cpp
index e417bdfb81b8f..a785e342fa457 100644
--- a/clang/lib/AST/ByteCode/Pointer.cpp
+++ b/clang/lib/AST/ByteCode/Pointer.cpp
@@ -812,9 +812,12 @@ std::optional<APValue> Pointer::toRValue(const Context &Ctx,
}
if (const auto *AT = Ty->getAsArrayTypeUnsafe()) {
- const size_t NumElems = Ptr.getNumElems();
+ const Descriptor *Desc = Ptr.getFieldDesc();
+
+ size_t Capacity = Desc->Capacity;
+ const size_t NumElems = Ptr.getNumAllocatedElems();
QualType ElemTy = AT->getElementType();
- R = APValue(APValue::UninitArray{}, NumElems, NumElems);
+ R = APValue(APValue::UninitArray{}, NumElems, Desc->Capacity);
bool Ok = true;
OptPrimType ElemT = Ctx.classify(ElemTy);
@@ -826,6 +829,16 @@ std::optional<APValue> Pointer::toRValue(const Context &Ctx,
Ok &= Composite(ElemTy, Ptr.atIndex(I).narrow(), Slot);
}
}
+
+ if (NumElems != Capacity) {
+ APValue &Slot = R.getArrayFiller();
+ if (ElemT) {
+ TYPE_SWITCH(*ElemT, Slot = Ptr.elem<T>(NumElems).toAPValue(ASTCtx));
+ } else {
+ Ok &= Composite(ElemTy, Ptr.atIndex(NumElems).narrow(), Slot);
+ }
+ }
+
return Ok;
}
diff --git a/clang/lib/AST/ByteCode/Pointer.h b/clang/lib/AST/ByteCode/Pointer.h
index cd738ce8b2a3e..958d3467ef24c 100644
--- a/clang/lib/AST/ByteCode/Pointer.h
+++ b/clang/lib/AST/ByteCode/Pointer.h
@@ -596,6 +596,33 @@ class Pointer {
unsigned getNumElems() const {
if (!isBlockPointer())
return ~0u;
+ // return getFieldDesc()->getNumElemsWithoutFiller();
+ // return getSize() / elemSize();
+ return getCapacity();
+ }
+
+ unsigned getNumAllocatedElems() const {
+ if (!isBlockPointer())
+ return ~0u;
+
+ assert(getFieldDesc()->isArray());
+ // return getSize() / elemSize();
+ return getFieldDesc()->getNumElemsWithoutFiller();
+ }
+
+ unsigned getNumAllocatedElemsWithFiller() const {
+ if (!isBlockPointer())
+ return ~0u;
+
+ assert(getFieldDesc()->isArray());
+ return getFieldDesc()->getNumElems(); // XXX WITH FILLER
+ }
+
+ unsigned getCapacity() const {
+ if (!isBlockPointer())
+ return ~0u;
+ if (getFieldDesc()->IsArray)
+ return getFieldDesc()->Capacity;
return getSize() / elemSize();
}
@@ -636,6 +663,15 @@ class Pointer {
if (isUnknownSizeArray())
return false;
+ if (isPastEnd())
+ return true;
+
+ return getIndex() == getNumElems();
+
+ // if (getFieldDesc()->IsArray && getIndex() >=
+ // getFieldDesc()->getNumElems() && getIndex() < getFieldDesc()->Capacity)
+ // return false;
+
return isPastEnd() || (getSize() == getOffset());
}
@@ -644,6 +680,10 @@ class Pointer {
if (isIntegralPointer())
return false;
+ // if (getFieldDesc()->IsArray && getIndex() >=
+ // getFieldDesc()->getNumElems() && getIndex() < getFieldDesc()->Capacity)
+ // return false;
+
return !isZero() && Offset > BS.Pointee->getSize();
}
@@ -682,7 +722,7 @@ class Pointer {
assert(BS.Pointee);
assert(isDereferencable());
assert(getFieldDesc()->isPrimitiveArray());
- assert(I < getFieldDesc()->getNumElems());
+ // assert(I < getFieldDesc()->getNumElems());
unsigned ElemByteOffset = I * getFieldDesc()->getElemSize();
unsigned ReadOffset = BS.Base + sizeof(InitMapPtr) + ElemByteOffset;
diff --git a/clang/lib/AST/ByteCode/Program.cpp b/clang/lib/AST/ByteCode/Program.cpp
index e0b2852f0e906..ac6b7145634f0 100644
--- a/clang/lib/AST/ByteCode/Program.cpp
+++ b/clang/lib/AST/ByteCode/Program.cpp
@@ -14,6 +14,7 @@
#include "clang/AST/Decl.h"
#include "clang/AST/DeclCXX.h"
#include "clang/AST/DeclTemplate.h"
+#include "clang/AST/Expr.h"
using namespace clang;
using namespace clang::interp;
@@ -262,7 +263,7 @@ UnsignedOrNone Program::createGlobal(const DeclTy &D, QualType Ty,
IsTemporary, /*IsMutable=*/false, IsVolatile);
else
Desc = createDescriptor(D, Ty.getTypePtr(), Descriptor::GlobalMD, IsConst,
- IsTemporary, /*IsMutable=*/false, IsVolatile);
+ IsTemporary, /*IsMutable=*/false, IsVolatile, Init);
if (!Desc)
return std::nullopt;
@@ -283,6 +284,23 @@ UnsignedOrNone Program::createGlobal(const DeclTy &D, QualType Ty,
return I;
}
+void Program::reallocGlobal(Block *Prev, const Descriptor *NewDesc) {
+ assert(Prev->getDescriptor()->IsArray);
+ assert(NewDesc->IsArray);
+
+ auto *G = new (Allocator, NewDesc->getAllocSize())
+ Global(Ctx.getEvalID(), Prev->getDeclID(), NewDesc, true, false, false);
+ G->block()->invokeCtor();
+
+ auto [_, PrevIndex] =
+ *GlobalIndices.find(Prev->getDescriptor()->getSource().getOpaqueValue());
+
+ Prev->moveArrayData(G->block());
+
+ Globals[PrevIndex] = G;
+ Prev->movePointersTo(G->block());
+}
+
Function *Program::getFunction(const FunctionDecl *F) {
F = F->getCanonicalDecl();
assert(F);
@@ -394,6 +412,17 @@ Record *Program::getOrCreateRecord(const RecordDecl *RD) {
return R;
}
+static unsigned getNumInits(const Expr *E, unsigned Capacity) {
+ unsigned N = Capacity;
+ if (const auto *ILE = dyn_cast_if_present<InitListExpr>(E))
+ N = ILE->getNumInitsWithEmbedExpanded();
+ if (const auto *PE = dyn_cast_if_present<ParenListExpr>(E))
+ N = PE->getNumExprs();
+ if (ArraySize::shouldUseFiller(N, Capacity))
+ return N;
+ return Capacity;
+}
+
Descriptor *Program::createDescriptor(const DeclTy &D, const Type *Ty,
Descriptor::MetadataSize MDSize,
bool IsConst, bool IsTemporary,
@@ -413,27 +442,37 @@ Descriptor *Program::createDescriptor(const DeclTy &D, const Type *Ty,
QualType ElemTy = ArrayType->getElementType();
// Array of well-known bounds.
if (const auto *CAT = dyn_cast<ConstantArrayType>(ArrayType)) {
+ size_t Capacity = CAT->getZExtSize();
+ size_t Size = getNumInits(Init, Capacity);
size_t NumElems = CAT->getZExtSize();
+
+ if (const auto *VD =
+ dyn_cast_if_present<ValueDecl>(D.dyn_cast<const Decl *>())) {
+ if (!Context::shouldBeGloballyIndexed(VD))
+ Size = Capacity;
+ }
+
if (OptPrimType T = Ctx.classify(ElemTy)) {
// Arrays of primitives.
unsigned ElemSize = primSize(*T);
if (std::numeric_limits<unsigned>::max() / ElemSize <= NumElems) {
return {};
}
- return allocateDescriptor(D, *T, MDSize, NumElems, IsConst, IsTemporary,
- IsMutable);
+ return allocateDescriptor(D, *T, MDSize, ArraySize(Size, Capacity),
+ IsConst, IsTemporary, IsMutable);
}
- // Arrays of composites. In this case, the array is a list of pointers,
- // followed by the actual elements.
- const Descriptor *ElemDesc = createDescriptor(
- D, ElemTy.getTypePtr(), std::nullopt, IsConst, IsTemporary);
- if (!ElemDesc)
- return nullptr;
- unsigned ElemSize = ElemDesc->getAllocSize() + sizeof(InlineDescriptor);
- if (std::numeric_limits<unsigned>::max() / ElemSize <= NumElems)
- return {};
- return allocateDescriptor(D, Ty, ElemDesc, MDSize, NumElems, IsConst,
- IsTemporary, IsMutable);
+
+ // Arrays of composites.
+ const Descriptor *ElemDesc = createDescriptor(
+ D, ElemTy.getTypePtr(), std::nullopt, IsConst, IsTemporary);
+ if (!ElemDesc)
+ return nullptr;
+ unsigned ElemSize = ElemDesc->getAllocSize() + sizeof(InlineDescriptor);
+ if (std::numeric_limits<unsigned>::max() / ElemSize <= NumElems)
+ return {};
+ return allocateDescriptor(D, Ty, ElemDesc, MDSize,
+ ArraySize(Size, Capacity), IsConst, IsTemporary,
+ IsMutable);
}
// Array of unknown bounds - cannot be accessed and pointer arithmetic
diff --git a/clang/lib/AST/ByteCode/Program.h b/clang/lib/AST/ByteCode/Program.h
index 28fcc97f5339d..19154541e0b9e 100644
--- a/clang/lib/AST/ByteCode/Program.h
+++ b/clang/lib/AST/ByteCode/Program.h
@@ -161,6 +161,12 @@ class Program final {
return std::nullopt;
return CurrentDeclaration;
}
+ /// Creates a new descriptor.
+ template <typename... Ts> Descriptor *allocateDescriptor(Ts &&...Args) {
+ return new (Allocator) Descriptor(std::forward<Ts>(Args)...);
+ }
+
+ void reallocGlobal(Block *Prev, const Descriptor *NewDesc);
private:
friend class DeclScope;
@@ -204,6 +210,10 @@ class Program final {
Block *block() { return &B; }
const Block *block() const { return &B; }
+ GlobalInlineDescriptor getInlineDesc() const {
+ return *reinterpret_cast<const GlobalInlineDescriptor *>(B.rawData());
+ }
+
private:
/// Required metadata - does not actually track pointers.
Block B;
@@ -223,11 +233,6 @@ class Program final {
/// Dummy parameter to generate pointers from.
llvm::DenseMap<const void *, unsigned> DummyVariables;
- /// Creates a new descriptor.
- template <typename... Ts> Descriptor *allocateDescriptor(Ts &&...Args) {
- return new (Allocator) Descriptor(std::forward<Ts>(Args)...);
- }
-
/// No declaration ID.
static constexpr unsigned NoDeclaration = ~0u;
/// Last declaration ID.
diff --git a/clang/lib/AST/CMakeLists.txt b/clang/lib/AST/CMakeLists.txt
index d4fd7a7f16d53..b1fe7211f5ce1 100644
--- a/clang/lib/AST/CMakeLists.txt
+++ b/clang/lib/AST/CMakeLists.txt
@@ -82,6 +82,7 @@ add_clang_library(clangAST
ByteCode/EvaluationResult.cpp
ByteCode/DynamicAllocator.cpp
ByteCode/Interp.cpp
+ ByteCode/InterpHelpers.cpp
ByteCode/InterpBlock.cpp
ByteCode/InterpFrame.cpp
ByteCode/InterpStack.cpp
diff --git a/clang/test/AST/ByteCode/array-fillers.cpp b/clang/test/AST/ByteCode/array-fillers.cpp
new file mode 100644
index 0000000000000..395c93ff61d29
--- /dev/null
+++ b/clang/test/AST/ByteCode/array-fillers.cpp
@@ -0,0 +1,50 @@
+// RUN: %clang_cc1 %s -std=c++20 -verify=both,expected -fexperimental-new-constant-interpreter
+
+// both-no-diagnostics
+
+
+constexpr int F[100] = {1,2};
+static_assert(F[98] == 0);
+static_assert(F[99] == 0);
+static_assert(F[0] == 1);
+static_assert(F[1] == 2);
+static_assert(F[2] == 0);
+
+constexpr _Complex double Doubles[4] = {{1.0, 2.0}};
+static_assert(__real(Doubles[0]) == 1.0, "");
+static_assert(__imag(Doubles[0]) == 2.0, "");
+
+static_assert(__real(Doubles[1]) == 0.0, "");
+static_assert(__imag(Doubles[1]) == 0.0, "");
+
+static_assert(__real(Doubles[2]) == 0.0, "");
+static_assert(__imag(Doubles[2]) == 0.0, "");
+static_assert(__real(Doubles[3]) == 0.0, "");
+static_assert(__imag(Doubles[3]) == 0.0, "");
+
+static_assert(__real(Doubles[0]) == 1.0, "");
+static_assert(__imag(Doubles[0]) == 2.0, "");
+
+struct S {
+ int x = 20;
+};
+constexpr S s[20] = {};
+static_assert(s[0].x == 20);
+static_assert(s[1].x == 20);
+static_assert(s[2].x == 20);
+static_assert(s[3].x == 20);
+static_assert(s[4].x == 20);
+
+constexpr int test() {
+ int a[4] = {};
+ int r = a[2];
+ return r;
+}
+static_assert(test() == 0);
+
+constexpr int test2() {
+ char buff[2] = {};
+ buff[0] = 'B';
+ return buff[1] == '\0' && buff[0] == 'B';
+}
+static_assert(test2());
More information about the cfe-commits
mailing list