[clang] [HLSL][RootSignature] Implement Lexing of DescriptorTables (PR #122981)
Finn Plummer via cfe-commits
cfe-commits at lists.llvm.org
Wed Jan 22 11:30:23 PST 2025
https://github.com/inbelic updated https://github.com/llvm/llvm-project/pull/122981
>From c1fc823abf07dfb8326b6190672d9881890bbb15 Mon Sep 17 00:00:00 2001
From: Finn Plummer <canadienfinn at gmail.com>
Date: Tue, 14 Jan 2025 22:22:45 +0000
Subject: [PATCH 1/7] [HLSL][RootSignature] Implement Lexing of
DescriptorTables
- Define required tokens to parse a Descriptor Table in TokenKinds.def
- Implements a Lexer to handle all of the defined tokens in
ParseHLSLRootSignature
---
.../Parse/HLSLRootSignatureTokenKinds.def | 121 ++++++++++++++
.../clang/Parse/ParseHLSLRootSignature.h | 96 +++++++++++
clang/lib/Parse/CMakeLists.txt | 1 +
clang/lib/Parse/ParseHLSLRootSignature.cpp | 151 ++++++++++++++++++
clang/unittests/CMakeLists.txt | 1 +
clang/unittests/Parse/CMakeLists.txt | 26 +++
.../Parse/ParseHLSLRootSignatureTest.cpp | 130 +++++++++++++++
7 files changed, 526 insertions(+)
create mode 100644 clang/include/clang/Parse/HLSLRootSignatureTokenKinds.def
create mode 100644 clang/include/clang/Parse/ParseHLSLRootSignature.h
create mode 100644 clang/lib/Parse/ParseHLSLRootSignature.cpp
create mode 100644 clang/unittests/Parse/CMakeLists.txt
create mode 100644 clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp
diff --git a/clang/include/clang/Parse/HLSLRootSignatureTokenKinds.def b/clang/include/clang/Parse/HLSLRootSignatureTokenKinds.def
new file mode 100644
index 00000000000000..5e47af8d4b1364
--- /dev/null
+++ b/clang/include/clang/Parse/HLSLRootSignatureTokenKinds.def
@@ -0,0 +1,121 @@
+//===--- HLSLRootSignature.def - Tokens and Enum Database -------*- C++ -*-===//
+
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the TokenKinds used in the Root Signature DSL. This
+// includes keywords, enums and a small subset of punctuators. Users of this
+// file must optionally #define the TOK, KEYWORD, ENUM or specific ENUM macros
+// to make use of this file.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef TOK
+#define TOK(X)
+#endif
+#ifndef PUNCTUATOR
+#define PUNCTUATOR(X,Y) TOK(pu_ ## X)
+#endif
+#ifndef KEYWORD
+#define KEYWORD(X) TOK(kw_ ## X)
+#endif
+#ifndef ENUM
+#define ENUM(NAME, LIT) TOK(en_ ## NAME)
+#endif
+
+// Defines the various types of enum
+#ifndef DESCRIPTOR_RANGE_OFFSET_ENUM
+#define DESCRIPTOR_RANGE_OFFSET_ENUM(NAME, LIT) ENUM(NAME, LIT)
+#endif
+#ifndef ROOT_DESCRIPTOR_FLAG_ENUM
+#define ROOT_DESCRIPTOR_FLAG_ENUM(NAME, LIT) ENUM(NAME, LIT)
+#endif
+// Note: ON denotes that the flag is unique from the above Root Descriptor
+// Flags. This is required to avoid token kind enum conflicts.
+#ifndef DESCRIPTOR_RANGE_FLAG_ENUM_OFF
+#define DESCRIPTOR_RANGE_FLAG_ENUM_OFF(NAME, LIT)
+#endif
+#ifndef DESCRIPTOR_RANGE_FLAG_ENUM_ON
+#define DESCRIPTOR_RANGE_FLAG_ENUM_ON(NAME, LIT) ENUM(NAME, LIT)
+#endif
+#ifndef DESCRIPTOR_RANGE_FLAG_ENUM
+#define DESCRIPTOR_RANGE_FLAG_ENUM(NAME, LIT, ON) DESCRIPTOR_RANGE_FLAG_ENUM_##ON(NAME, LIT)
+#endif
+#ifndef SHADER_VISIBILITY_ENUM
+#define SHADER_VISIBILITY_ENUM(NAME, LIT) ENUM(NAME, LIT)
+#endif
+
+// General Tokens:
+TOK(invalid)
+TOK(int_literal)
+
+// Register Tokens:
+TOK(bReg)
+TOK(tReg)
+TOK(uReg)
+TOK(sReg)
+
+// Punctuators:
+PUNCTUATOR(l_paren, '(')
+PUNCTUATOR(r_paren, ')')
+PUNCTUATOR(comma, ',')
+PUNCTUATOR(or, '|')
+PUNCTUATOR(equal, '=')
+
+// RootElement Keywords:
+KEYWORD(DescriptorTable)
+
+// DescriptorTable Keywords:
+KEYWORD(CBV)
+KEYWORD(SRV)
+KEYWORD(UAV)
+KEYWORD(Sampler)
+
+// General Parameter Keywords:
+KEYWORD(space)
+KEYWORD(visibility)
+KEYWORD(flags)
+
+// View Parameter Keywords:
+KEYWORD(numDescriptors)
+KEYWORD(offset)
+
+// Descriptor Range Offset Enum:
+DESCRIPTOR_RANGE_OFFSET_ENUM(DescriptorRangeOffsetAppend, "DESCRIPTOR_RANGE_OFFSET_APPEND")
+
+// Root Descriptor Flag Enums:
+ROOT_DESCRIPTOR_FLAG_ENUM(DataVolatile, "DATA_VOLATILE")
+ROOT_DESCRIPTOR_FLAG_ENUM(DataStaticWhileSetAtExecute, "DATA_STATIC_WHILE_SET_AT_EXECUTE")
+ROOT_DESCRIPTOR_FLAG_ENUM(DataStatic, "DATA_STATIC")
+
+// Descriptor Range Flag Enums:
+DESCRIPTOR_RANGE_FLAG_ENUM(DescriptorsVolatile, "DESCRIPTORS_VOLATILE", ON)
+DESCRIPTOR_RANGE_FLAG_ENUM(DataVolatile, "DATA_VOLATILE", OFF)
+DESCRIPTOR_RANGE_FLAG_ENUM(DataStaticWhileSetAtExecute, "DATA_STATIC_WHILE_SET_AT_EXECUTE", OFF)
+DESCRIPTOR_RANGE_FLAG_ENUM(DataStatic, "DATA_STATIC", OFF)
+DESCRIPTOR_RANGE_FLAG_ENUM(DescriptorsStaticKeepingBufferBoundsChecks, "DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS", ON)
+
+// Shader Visibiliy Enums:
+SHADER_VISIBILITY_ENUM(All, "SHADER_VISIBILITY_ALL")
+SHADER_VISIBILITY_ENUM(Vertex, "SHADER_VISIBILITY_VERTEX")
+SHADER_VISIBILITY_ENUM(Hull, "SHADER_VISIBILITY_HULL")
+SHADER_VISIBILITY_ENUM(Domain, "SHADER_VISIBILITY_DOMAIN")
+SHADER_VISIBILITY_ENUM(Geometry, "SHADER_VISIBILITY_GEOMETRY")
+SHADER_VISIBILITY_ENUM(Pixel, "SHADER_VISIBILITY_PIXEL")
+SHADER_VISIBILITY_ENUM(Amplification, "SHADER_VISIBILITY_AMPLIFICATION")
+SHADER_VISIBILITY_ENUM(Mesh, "SHADER_VISIBILITY_MESH")
+
+#undef SHADER_VISIBILITY_ENUM
+#undef DESCRIPTOR_RANGE_FLAG_ENUM
+#undef DESCRIPTOR_RANGE_FLAG_ENUM_OFF
+#undef DESCRIPTOR_RANGE_FLAG_ENUM_ON
+#undef ROOT_DESCRIPTOR_FLAG_ENUM
+#undef DESCRIPTOR_RANGE_OFFSET_ENUM
+#undef ENUM
+#undef KEYWORD
+#undef PUNCTUATOR
+#undef TOK
diff --git a/clang/include/clang/Parse/ParseHLSLRootSignature.h b/clang/include/clang/Parse/ParseHLSLRootSignature.h
new file mode 100644
index 00000000000000..6c534411e754a0
--- /dev/null
+++ b/clang/include/clang/Parse/ParseHLSLRootSignature.h
@@ -0,0 +1,96 @@
+//===--- ParseHLSLRootSignature.h -------------------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file defines the ParseHLSLRootSignature interface.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_PARSE_PARSEHLSLROOTSIGNATURE_H
+#define LLVM_CLANG_PARSE_PARSEHLSLROOTSIGNATURE_H
+
+#include "clang/Lex/LiteralSupport.h"
+#include "clang/Lex/Preprocessor.h"
+
+#include "llvm/ADT/APInt.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/ADT/StringSwitch.h"
+
+namespace llvm {
+namespace hlsl {
+namespace root_signature {
+
+struct RootSignatureToken {
+ enum Kind {
+#define TOK(X) X,
+#include "clang/Parse/HLSLRootSignatureTokenKinds.def"
+ };
+
+ Kind Kind = Kind::invalid;
+
+ // Retain the SouceLocation of the token for diagnostics
+ clang::SourceLocation TokLoc;
+
+ // Retain if the uint32_t bits represent a signed integer
+ bool Signed = false;
+ union {
+ uint32_t IntLiteral = 0;
+ float FloatLiteral;
+ };
+
+ // Constructors
+ RootSignatureToken() {}
+ RootSignatureToken(clang::SourceLocation TokLoc) : TokLoc(TokLoc) {}
+};
+using TokenKind = enum RootSignatureToken::Kind;
+
+class RootSignatureLexer {
+public:
+ RootSignatureLexer(StringRef Signature, clang::SourceLocation SourceLoc,
+ clang::Preprocessor &PP)
+ : Buffer(Signature), SourceLoc(SourceLoc), PP(PP) {}
+
+ // Consumes the internal buffer as a list of tokens and will emplace them
+ // onto the given tokens.
+ //
+ // It will consume until it successfully reaches the end of the buffer,
+ // or, until the first error is encountered. The return value denotes if
+ // there was a failure.
+ bool Lex(SmallVector<RootSignatureToken> &Tokens);
+
+ // Get the current source location of the lexer
+ clang::SourceLocation GetLocation() { return SourceLoc; };
+
+private:
+ // Internal buffer to iterate over
+ StringRef Buffer;
+
+ // Passed down parameters from Sema
+ clang::SourceLocation SourceLoc;
+ clang::Preprocessor &PP;
+
+ bool LexNumber(RootSignatureToken &Result);
+
+ // Consumes the internal buffer for a single token.
+ //
+ // The return value denotes if there was a failure.
+ bool LexToken(RootSignatureToken &Token);
+
+ // Advance the buffer by the specified number of characters. Updates the
+ // SourceLocation appropriately.
+ void AdvanceBuffer(unsigned NumCharacters = 1) {
+ Buffer = Buffer.drop_front(NumCharacters);
+ SourceLoc = SourceLoc.getLocWithOffset(NumCharacters);
+ }
+};
+
+} // namespace root_signature
+} // namespace hlsl
+} // namespace llvm
+
+#endif // LLVM_CLANG_PARSE_PARSEHLSLROOTSIGNATURE_H
diff --git a/clang/lib/Parse/CMakeLists.txt b/clang/lib/Parse/CMakeLists.txt
index 22e902f7e1bc50..00fde537bb9c60 100644
--- a/clang/lib/Parse/CMakeLists.txt
+++ b/clang/lib/Parse/CMakeLists.txt
@@ -14,6 +14,7 @@ add_clang_library(clangParse
ParseExpr.cpp
ParseExprCXX.cpp
ParseHLSL.cpp
+ ParseHLSLRootSignature.cpp
ParseInit.cpp
ParseObjc.cpp
ParseOpenMP.cpp
diff --git a/clang/lib/Parse/ParseHLSLRootSignature.cpp b/clang/lib/Parse/ParseHLSLRootSignature.cpp
new file mode 100644
index 00000000000000..491e4cafa04b3c
--- /dev/null
+++ b/clang/lib/Parse/ParseHLSLRootSignature.cpp
@@ -0,0 +1,151 @@
+#include "clang/Parse/ParseHLSLRootSignature.h"
+
+namespace llvm {
+namespace hlsl {
+namespace root_signature {
+
+// Lexer Definitions
+
+static bool IsPreprocessorNumberChar(char C) {
+ // TODO: extend for float support with or without hexadecimal/exponent
+ return isdigit(C); // integer support
+}
+
+bool RootSignatureLexer::LexNumber(RootSignatureToken &Result) {
+ // NumericLiteralParser does not handle the sign so we will manually apply it
+ Result.Signed = Buffer.front() == '-';
+ if (Result.Signed)
+ AdvanceBuffer();
+
+ // Retrieve the possible number
+ StringRef NumSpelling = Buffer.take_while(IsPreprocessorNumberChar);
+
+ // Parse the numeric value and so semantic checks on its specification
+ clang::NumericLiteralParser Literal(NumSpelling, SourceLoc,
+ PP.getSourceManager(), PP.getLangOpts(),
+ PP.getTargetInfo(), PP.getDiagnostics());
+ if (Literal.hadError)
+ return true; // Error has already been reported so just return
+
+ // Retrieve the number value to store into the token
+ if (Literal.isIntegerLiteral()) {
+ Result.Kind = TokenKind::int_literal;
+
+ APSInt X = APSInt(32, Result.Signed);
+ if (Literal.GetIntegerValue(X))
+ return true; // TODO: Report overflow error
+
+ X = Result.Signed ? -X : X;
+ Result.IntLiteral = (uint32_t)X.getZExtValue();
+ } else {
+ return true; // TODO: report unsupported number literal specification
+ }
+
+ AdvanceBuffer(NumSpelling.size());
+ return false;
+}
+
+bool RootSignatureLexer::Lex(SmallVector<RootSignatureToken> &Tokens) {
+ // Discard any leading whitespace
+ AdvanceBuffer(Buffer.take_while(isspace).size());
+
+ while (!Buffer.empty()) {
+ RootSignatureToken Result;
+ if (LexToken(Result))
+ return true;
+
+ // Successfully Lexed the token so we can store it
+ Tokens.push_back(Result);
+
+ // Discard any trailing whitespace
+ AdvanceBuffer(Buffer.take_while(isspace).size());
+ }
+
+ return false;
+}
+
+bool RootSignatureLexer::LexToken(RootSignatureToken &Result) {
+ // Record where this token is in the text for usage in parser diagnostics
+ Result.TokLoc = SourceLoc;
+
+ char C = Buffer.front();
+
+ // Punctuators
+ switch (C) {
+#define PUNCTUATOR(X, Y) \
+ case Y: { \
+ Result.Kind = TokenKind::pu_##X; \
+ AdvanceBuffer(); \
+ return false; \
+ }
+#include "clang/Parse/HLSLRootSignatureTokenKinds.def"
+ default:
+ break;
+ }
+
+ // Numeric constant
+ if (isdigit(C) || C == '-')
+ return LexNumber(Result);
+
+ // All following tokens require at least one additional character
+ if (Buffer.size() <= 1)
+ return true; // TODO: Report invalid token error
+
+ // Peek at the next character to deteremine token type
+ char NextC = Buffer[1];
+
+ // Registers: [tsub][0-9+]
+ if ((C == 't' || C == 's' || C == 'u' || C == 'b') && isdigit(NextC)) {
+ AdvanceBuffer();
+
+ if (LexNumber(Result))
+ return true;
+
+ // Lex number could also parse a float so ensure it was an unsigned int
+ if (Result.Kind != TokenKind::int_literal || Result.Signed)
+ return true; // Return invalid number literal for register error
+
+ // Convert character to the register type.
+ // This is done after LexNumber to override the TokenKind
+ switch (C) {
+ case 'b':
+ Result.Kind = TokenKind::bReg;
+ break;
+ case 't':
+ Result.Kind = TokenKind::tReg;
+ break;
+ case 'u':
+ Result.Kind = TokenKind::uReg;
+ break;
+ case 's':
+ Result.Kind = TokenKind::sReg;
+ break;
+ default:
+ llvm_unreachable("Switch for an expected token was not provided");
+ return true;
+ }
+ return false;
+ }
+
+ // Keywords and Enums:
+ StringRef TokSpelling =
+ Buffer.take_while([](char C) { return isalnum(C) || C == '_'; });
+
+ // Define a large string switch statement for all the keywords and enums
+ auto Switch = llvm::StringSwitch<TokenKind>(TokSpelling);
+#define KEYWORD(NAME) Switch.Case(#NAME, TokenKind::kw_##NAME);
+#define ENUM(NAME, LIT) Switch.CaseLower(LIT, TokenKind::en_##NAME);
+#include "clang/Parse/HLSLRootSignatureTokenKinds.def"
+
+ // Then attempt to retreive a string from it
+ auto Kind = Switch.Default(TokenKind::invalid);
+ if (Kind == TokenKind::invalid)
+ return true; // TODO: Report invalid identifier
+
+ Result.Kind = Kind;
+ AdvanceBuffer(TokSpelling.size());
+ return false;
+}
+
+} // namespace root_signature
+} // namespace hlsl
diff --git a/clang/unittests/CMakeLists.txt b/clang/unittests/CMakeLists.txt
index 85d265426ec80b..9b3ce8aa7de739 100644
--- a/clang/unittests/CMakeLists.txt
+++ b/clang/unittests/CMakeLists.txt
@@ -25,6 +25,7 @@ endfunction()
add_subdirectory(Basic)
add_subdirectory(Lex)
+add_subdirectory(Parse)
add_subdirectory(Driver)
if(CLANG_ENABLE_STATIC_ANALYZER)
add_subdirectory(Analysis)
diff --git a/clang/unittests/Parse/CMakeLists.txt b/clang/unittests/Parse/CMakeLists.txt
new file mode 100644
index 00000000000000..1b7eb4934a46c8
--- /dev/null
+++ b/clang/unittests/Parse/CMakeLists.txt
@@ -0,0 +1,26 @@
+set(LLVM_LINK_COMPONENTS
+ Support
+ )
+
+add_clang_unittest(ParseTests
+ ParseHLSLRootSignatureTest.cpp
+ )
+
+clang_target_link_libraries(ParseTests
+ PRIVATE
+ clangAST
+ clangASTMatchers
+ clangBasic
+ clangFrontend
+ clangParse
+ clangSema
+ clangSerialization
+ clangTooling
+ )
+
+target_link_libraries(ParseTests
+ PRIVATE
+ LLVMTestingAnnotations
+ LLVMTestingSupport
+ clangTesting
+ )
diff --git a/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp b/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp
new file mode 100644
index 00000000000000..c430b0657dacf5
--- /dev/null
+++ b/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp
@@ -0,0 +1,130 @@
+//=== ParseHLSLRootSignatureTest.cpp - Parse Root Signature tests ---------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Basic/Diagnostic.h"
+#include "clang/Basic/DiagnosticOptions.h"
+#include "clang/Basic/FileManager.h"
+#include "clang/Basic/LangOptions.h"
+#include "clang/Basic/SourceLocation.h"
+#include "clang/Basic/SourceManager.h"
+#include "clang/Basic/TargetInfo.h"
+#include "clang/Lex/HeaderSearch.h"
+#include "clang/Lex/HeaderSearchOptions.h"
+#include "clang/Lex/Lexer.h"
+#include "clang/Lex/ModuleLoader.h"
+#include "clang/Lex/Preprocessor.h"
+#include "clang/Lex/PreprocessorOptions.h"
+
+#include "clang/Parse/ParseHLSLRootSignature.h"
+#include "gtest/gtest.h"
+
+using namespace llvm::hlsl::root_signature;
+using namespace clang;
+
+namespace {
+
+// The test fixture.
+class ParseHLSLRootSignatureTest : public ::testing::Test {
+protected:
+ ParseHLSLRootSignatureTest()
+ : FileMgr(FileMgrOpts), DiagID(new DiagnosticIDs()),
+ Diags(DiagID, new DiagnosticOptions, new IgnoringDiagConsumer()),
+ SourceMgr(Diags, FileMgr), TargetOpts(new TargetOptions) {
+ TargetOpts->Triple = "x86_64-apple-darwin11.1.0";
+ Target = TargetInfo::CreateTargetInfo(Diags, TargetOpts);
+ }
+
+ Preprocessor *CreatePP(StringRef Source, TrivialModuleLoader &ModLoader) {
+ std::unique_ptr<llvm::MemoryBuffer> Buf =
+ llvm::MemoryBuffer::getMemBuffer(Source);
+ SourceMgr.setMainFileID(SourceMgr.createFileID(std::move(Buf)));
+
+ HeaderSearch HeaderInfo(std::make_shared<HeaderSearchOptions>(), SourceMgr,
+ Diags, LangOpts, Target.get());
+ Preprocessor *PP =
+ new Preprocessor(std::make_shared<PreprocessorOptions>(), Diags,
+ LangOpts, SourceMgr, HeaderInfo, ModLoader,
+ /*IILookup =*/nullptr,
+ /*OwnsHeaderSearch =*/false);
+ PP->Initialize(*Target);
+ PP->EnterMainSourceFile();
+ return PP;
+ }
+
+ void CheckTokens(SmallVector<RootSignatureToken> &Computed,
+ SmallVector<TokenKind> &Expected) {
+ ASSERT_EQ(Computed.size(), Expected.size());
+ for (unsigned I = 0, E = Expected.size(); I != E; ++I) {
+ ASSERT_EQ(Computed[I].Kind, Expected[I]);
+ }
+ }
+
+ FileSystemOptions FileMgrOpts;
+ FileManager FileMgr;
+ IntrusiveRefCntPtr<DiagnosticIDs> DiagID;
+ DiagnosticsEngine Diags;
+ SourceManager SourceMgr;
+ LangOptions LangOpts;
+ std::shared_ptr<TargetOptions> TargetOpts;
+ IntrusiveRefCntPtr<TargetInfo> Target;
+};
+
+TEST_F(ParseHLSLRootSignatureTest, LexValidTokensTest) {
+ const llvm::StringLiteral Source = R"cc(
+ -42
+
+ b0 t43 u987 s234
+
+ (),|=
+
+ DescriptorTable
+
+ CBV SRV UAV Sampler
+ space visibility flags
+ numDescriptors offset
+
+ DESCRIPTOR_RANGE_OFFSET_APPEND
+
+ DATA_VOLATILE
+ DATA_STATIC_WHILE_SET_AT_EXECUTE
+ DATA_STATIC
+ DESCRIPTORS_VOLATILE
+ DESCRIPTORS_STATIC_KEEPING_BUFFER_BOUNDS_CHECKS
+
+ shader_visibility_all
+ shader_visibility_vertex
+ shader_visibility_hull
+ shader_visibility_domain
+ shader_visibility_geometry
+ shader_visibility_pixel
+ shader_visibility_amplification
+ shader_visibility_mesh
+ )cc";
+
+ TrivialModuleLoader ModLoader;
+ Preprocessor *PP = CreatePP(Source, ModLoader);
+ auto TokLoc = SourceLocation();
+
+ RootSignatureLexer Lexer(Source, TokLoc, *PP);
+
+ SmallVector<RootSignatureToken> Tokens = {
+ RootSignatureToken() // invalid token for completeness
+ };
+ ASSERT_FALSE(Lexer.Lex(Tokens));
+
+ SmallVector<TokenKind> Expected = {
+#define TOK(NAME) TokenKind::NAME,
+#include "clang/Parse/HLSLRootSignatureTokenKinds.def"
+ };
+
+ CheckTokens(Tokens, Expected);
+
+ delete PP;
+}
+
+} // anonymous namespace
>From 949b50af4c60eae588595260d9384d01324245e5 Mon Sep 17 00:00:00 2001
From: Finn Plummer <canadienfinn at gmail.com>
Date: Tue, 14 Jan 2025 23:07:46 +0000
Subject: [PATCH 2/7] fix typo when breaking up commits
---
clang/lib/Parse/ParseHLSLRootSignature.cpp | 1 +
1 file changed, 1 insertion(+)
diff --git a/clang/lib/Parse/ParseHLSLRootSignature.cpp b/clang/lib/Parse/ParseHLSLRootSignature.cpp
index 491e4cafa04b3c..baa9950163a5c6 100644
--- a/clang/lib/Parse/ParseHLSLRootSignature.cpp
+++ b/clang/lib/Parse/ParseHLSLRootSignature.cpp
@@ -149,3 +149,4 @@ bool RootSignatureLexer::LexToken(RootSignatureToken &Result) {
} // namespace root_signature
} // namespace hlsl
+} // namespace llvm
>From a3286192694fd8599b74690b79f333482289f548 Mon Sep 17 00:00:00 2001
From: Finn Plummer <canadienfinn at gmail.com>
Date: Thu, 16 Jan 2025 17:10:05 +0000
Subject: [PATCH 3/7] review comments:
- fix typo
- use std::unique_ptr for testing mem management
---
clang/lib/Parse/ParseHLSLRootSignature.cpp | 2 +-
.../Parse/ParseHLSLRootSignatureTest.cpp | 17 ++++++++---------
2 files changed, 9 insertions(+), 10 deletions(-)
diff --git a/clang/lib/Parse/ParseHLSLRootSignature.cpp b/clang/lib/Parse/ParseHLSLRootSignature.cpp
index baa9950163a5c6..9f2154471ec3f1 100644
--- a/clang/lib/Parse/ParseHLSLRootSignature.cpp
+++ b/clang/lib/Parse/ParseHLSLRootSignature.cpp
@@ -20,7 +20,7 @@ bool RootSignatureLexer::LexNumber(RootSignatureToken &Result) {
// Retrieve the possible number
StringRef NumSpelling = Buffer.take_while(IsPreprocessorNumberChar);
- // Parse the numeric value and so semantic checks on its specification
+ // Parse the numeric value and do semantic checks on its specification
clang::NumericLiteralParser Literal(NumSpelling, SourceLoc,
PP.getSourceManager(), PP.getLangOpts(),
PP.getTargetInfo(), PP.getDiagnostics());
diff --git a/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp b/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp
index c430b0657dacf5..4fd60488c6ff20 100644
--- a/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp
+++ b/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp
@@ -39,18 +39,19 @@ class ParseHLSLRootSignatureTest : public ::testing::Test {
Target = TargetInfo::CreateTargetInfo(Diags, TargetOpts);
}
- Preprocessor *CreatePP(StringRef Source, TrivialModuleLoader &ModLoader) {
+ std::unique_ptr<Preprocessor> CreatePP(StringRef Source,
+ TrivialModuleLoader &ModLoader) {
std::unique_ptr<llvm::MemoryBuffer> Buf =
llvm::MemoryBuffer::getMemBuffer(Source);
SourceMgr.setMainFileID(SourceMgr.createFileID(std::move(Buf)));
HeaderSearch HeaderInfo(std::make_shared<HeaderSearchOptions>(), SourceMgr,
Diags, LangOpts, Target.get());
- Preprocessor *PP =
- new Preprocessor(std::make_shared<PreprocessorOptions>(), Diags,
- LangOpts, SourceMgr, HeaderInfo, ModLoader,
- /*IILookup =*/nullptr,
- /*OwnsHeaderSearch =*/false);
+ std::unique_ptr<Preprocessor> PP = std::make_unique<Preprocessor>(
+ std::make_shared<PreprocessorOptions>(), Diags, LangOpts, SourceMgr,
+ HeaderInfo, ModLoader,
+ /*IILookup =*/nullptr,
+ /*OwnsHeaderSearch =*/false);
PP->Initialize(*Target);
PP->EnterMainSourceFile();
return PP;
@@ -107,7 +108,7 @@ TEST_F(ParseHLSLRootSignatureTest, LexValidTokensTest) {
)cc";
TrivialModuleLoader ModLoader;
- Preprocessor *PP = CreatePP(Source, ModLoader);
+ auto PP = CreatePP(Source, ModLoader);
auto TokLoc = SourceLocation();
RootSignatureLexer Lexer(Source, TokLoc, *PP);
@@ -123,8 +124,6 @@ TEST_F(ParseHLSLRootSignatureTest, LexValidTokensTest) {
};
CheckTokens(Tokens, Expected);
-
- delete PP;
}
} // anonymous namespace
>From 8e88fa70156ae5a5d3d0103d65ea99028fbfdf0b Mon Sep 17 00:00:00 2001
From: Finn Plummer <canadienfinn at gmail.com>
Date: Thu, 16 Jan 2025 17:23:47 +0000
Subject: [PATCH 4/7] review comments: fix handling of signed ints
- update Lexer to allow '+' prefix to denote a signed positive integer
- add missing errors when we expect an unsigned int for parameters
- add tests to show lexing of positive signed integer and unsigned
integer
---
clang/lib/Parse/ParseHLSLRootSignature.cpp | 14 ++++++++------
.../unittests/Parse/ParseHLSLRootSignatureTest.cpp | 4 ++++
2 files changed, 12 insertions(+), 6 deletions(-)
diff --git a/clang/lib/Parse/ParseHLSLRootSignature.cpp b/clang/lib/Parse/ParseHLSLRootSignature.cpp
index 9f2154471ec3f1..0cdd2273489406 100644
--- a/clang/lib/Parse/ParseHLSLRootSignature.cpp
+++ b/clang/lib/Parse/ParseHLSLRootSignature.cpp
@@ -6,19 +6,20 @@ namespace root_signature {
// Lexer Definitions
-static bool IsPreprocessorNumberChar(char C) {
+static bool IsNumberChar(char C) {
// TODO: extend for float support with or without hexadecimal/exponent
return isdigit(C); // integer support
}
bool RootSignatureLexer::LexNumber(RootSignatureToken &Result) {
// NumericLiteralParser does not handle the sign so we will manually apply it
- Result.Signed = Buffer.front() == '-';
+ bool Negative = Buffer.front() == '-';
+ Result.Signed = Negative || Buffer.front() == '+';
if (Result.Signed)
AdvanceBuffer();
// Retrieve the possible number
- StringRef NumSpelling = Buffer.take_while(IsPreprocessorNumberChar);
+ StringRef NumSpelling = Buffer.take_while(IsNumberChar);
// Parse the numeric value and do semantic checks on its specification
clang::NumericLiteralParser Literal(NumSpelling, SourceLoc,
@@ -35,7 +36,7 @@ bool RootSignatureLexer::LexNumber(RootSignatureToken &Result) {
if (Literal.GetIntegerValue(X))
return true; // TODO: Report overflow error
- X = Result.Signed ? -X : X;
+ X = Negative ? -X : X;
Result.IntLiteral = (uint32_t)X.getZExtValue();
} else {
return true; // TODO: report unsupported number literal specification
@@ -84,7 +85,7 @@ bool RootSignatureLexer::LexToken(RootSignatureToken &Result) {
}
// Numeric constant
- if (isdigit(C) || C == '-')
+ if (isdigit(C) || C == '-' || C == '+')
return LexNumber(Result);
// All following tokens require at least one additional character
@@ -101,7 +102,8 @@ bool RootSignatureLexer::LexToken(RootSignatureToken &Result) {
if (LexNumber(Result))
return true;
- // Lex number could also parse a float so ensure it was an unsigned int
+ // Lex number could also parse a signed int/float so ensure it was an
+ // unsigned int
if (Result.Kind != TokenKind::int_literal || Result.Signed)
return true; // Return invalid number literal for register error
diff --git a/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp b/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp
index 4fd60488c6ff20..42c57731ddb804 100644
--- a/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp
+++ b/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp
@@ -105,6 +105,8 @@ TEST_F(ParseHLSLRootSignatureTest, LexValidTokensTest) {
shader_visibility_pixel
shader_visibility_amplification
shader_visibility_mesh
+
+ 42 +42
)cc";
TrivialModuleLoader ModLoader;
@@ -121,6 +123,8 @@ TEST_F(ParseHLSLRootSignatureTest, LexValidTokensTest) {
SmallVector<TokenKind> Expected = {
#define TOK(NAME) TokenKind::NAME,
#include "clang/Parse/HLSLRootSignatureTokenKinds.def"
+ TokenKind::int_literal,
+ TokenKind::int_literal,
};
CheckTokens(Tokens, Expected);
>From a76b907053462405287a2a5b09454765ff1bf73c Mon Sep 17 00:00:00 2001
From: Finn Plummer <canadienfinn at gmail.com>
Date: Thu, 16 Jan 2025 17:25:20 +0000
Subject: [PATCH 5/7] review comments: use early exit code convention
---
clang/lib/Parse/ParseHLSLRootSignature.cpp | 19 +++++++++----------
1 file changed, 9 insertions(+), 10 deletions(-)
diff --git a/clang/lib/Parse/ParseHLSLRootSignature.cpp b/clang/lib/Parse/ParseHLSLRootSignature.cpp
index 0cdd2273489406..fac4a92f1920be 100644
--- a/clang/lib/Parse/ParseHLSLRootSignature.cpp
+++ b/clang/lib/Parse/ParseHLSLRootSignature.cpp
@@ -28,19 +28,18 @@ bool RootSignatureLexer::LexNumber(RootSignatureToken &Result) {
if (Literal.hadError)
return true; // Error has already been reported so just return
+ if (!Literal.isIntegerLiteral())
+ return true; // TODO: report unsupported number literal specification
+
// Retrieve the number value to store into the token
- if (Literal.isIntegerLiteral()) {
- Result.Kind = TokenKind::int_literal;
+ Result.Kind = TokenKind::int_literal;
- APSInt X = APSInt(32, Result.Signed);
- if (Literal.GetIntegerValue(X))
- return true; // TODO: Report overflow error
+ APSInt X = APSInt(32, Result.Signed);
+ if (Literal.GetIntegerValue(X))
+ return true; // TODO: Report overflow error
- X = Negative ? -X : X;
- Result.IntLiteral = (uint32_t)X.getZExtValue();
- } else {
- return true; // TODO: report unsupported number literal specification
- }
+ X = Negative ? -X : X;
+ Result.IntLiteral = (uint32_t)X.getZExtValue();
AdvanceBuffer(NumSpelling.size());
return false;
>From 7a228d3028cff5d1082f1d68d3932a8a8dcd0bed Mon Sep 17 00:00:00 2001
From: Finn Plummer <canadienfinn at gmail.com>
Date: Wed, 22 Jan 2025 19:28:23 +0000
Subject: [PATCH 6/7] self-review: update namespacing
- parser moved to clang dir so namespace is changed to clang
- objects are prefixed with RootSignature[Parser] etc so remove the
root_signature namespace
---
.../clang/Parse/ParseHLSLRootSignature.h | 6 ++----
clang/lib/Parse/ParseHLSLRootSignature.cpp | 8 +++-----
.../Parse/ParseHLSLRootSignatureTest.cpp | 19 +++++++++----------
3 files changed, 14 insertions(+), 19 deletions(-)
diff --git a/clang/include/clang/Parse/ParseHLSLRootSignature.h b/clang/include/clang/Parse/ParseHLSLRootSignature.h
index 6c534411e754a0..40e0c79ca5768f 100644
--- a/clang/include/clang/Parse/ParseHLSLRootSignature.h
+++ b/clang/include/clang/Parse/ParseHLSLRootSignature.h
@@ -21,9 +21,8 @@
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/StringSwitch.h"
-namespace llvm {
+namespace clang {
namespace hlsl {
-namespace root_signature {
struct RootSignatureToken {
enum Kind {
@@ -89,8 +88,7 @@ class RootSignatureLexer {
}
};
-} // namespace root_signature
} // namespace hlsl
-} // namespace llvm
+} // namespace clang
#endif // LLVM_CLANG_PARSE_PARSEHLSLROOTSIGNATURE_H
diff --git a/clang/lib/Parse/ParseHLSLRootSignature.cpp b/clang/lib/Parse/ParseHLSLRootSignature.cpp
index fac4a92f1920be..1ea7f94b3ec1f8 100644
--- a/clang/lib/Parse/ParseHLSLRootSignature.cpp
+++ b/clang/lib/Parse/ParseHLSLRootSignature.cpp
@@ -1,8 +1,7 @@
#include "clang/Parse/ParseHLSLRootSignature.h"
-namespace llvm {
+namespace clang {
namespace hlsl {
-namespace root_signature {
// Lexer Definitions
@@ -34,7 +33,7 @@ bool RootSignatureLexer::LexNumber(RootSignatureToken &Result) {
// Retrieve the number value to store into the token
Result.Kind = TokenKind::int_literal;
- APSInt X = APSInt(32, Result.Signed);
+ llvm::APSInt X = llvm::APSInt(32, Result.Signed);
if (Literal.GetIntegerValue(X))
return true; // TODO: Report overflow error
@@ -148,6 +147,5 @@ bool RootSignatureLexer::LexToken(RootSignatureToken &Result) {
return false;
}
-} // namespace root_signature
} // namespace hlsl
-} // namespace llvm
+} // namespace clang
diff --git a/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp b/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp
index 42c57731ddb804..84c582b41d0d04 100644
--- a/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp
+++ b/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp
@@ -23,7 +23,6 @@
#include "clang/Parse/ParseHLSLRootSignature.h"
#include "gtest/gtest.h"
-using namespace llvm::hlsl::root_signature;
using namespace clang;
namespace {
@@ -57,8 +56,8 @@ class ParseHLSLRootSignatureTest : public ::testing::Test {
return PP;
}
- void CheckTokens(SmallVector<RootSignatureToken> &Computed,
- SmallVector<TokenKind> &Expected) {
+ void CheckTokens(SmallVector<hlsl::RootSignatureToken> &Computed,
+ SmallVector<hlsl::TokenKind> &Expected) {
ASSERT_EQ(Computed.size(), Expected.size());
for (unsigned I = 0, E = Expected.size(); I != E; ++I) {
ASSERT_EQ(Computed[I].Kind, Expected[I]);
@@ -113,18 +112,18 @@ TEST_F(ParseHLSLRootSignatureTest, LexValidTokensTest) {
auto PP = CreatePP(Source, ModLoader);
auto TokLoc = SourceLocation();
- RootSignatureLexer Lexer(Source, TokLoc, *PP);
+ hlsl::RootSignatureLexer Lexer(Source, TokLoc, *PP);
- SmallVector<RootSignatureToken> Tokens = {
- RootSignatureToken() // invalid token for completeness
+ SmallVector<hlsl::RootSignatureToken> Tokens = {
+ hlsl::RootSignatureToken() // invalid token for completeness
};
ASSERT_FALSE(Lexer.Lex(Tokens));
- SmallVector<TokenKind> Expected = {
-#define TOK(NAME) TokenKind::NAME,
+ SmallVector<hlsl::TokenKind> Expected = {
+#define TOK(NAME) hlsl::TokenKind::NAME,
#include "clang/Parse/HLSLRootSignatureTokenKinds.def"
- TokenKind::int_literal,
- TokenKind::int_literal,
+ hlsl::TokenKind::int_literal,
+ hlsl::TokenKind::int_literal,
};
CheckTokens(Tokens, Expected);
>From 0a1df31e105557e99f4cc8d4858f2f01286b2bb6 Mon Sep 17 00:00:00 2001
From: Finn Plummer <canadienfinn at gmail.com>
Date: Wed, 22 Jan 2025 19:29:45 +0000
Subject: [PATCH 7/7] review: add testcase description
---
clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp | 2 ++
1 file changed, 2 insertions(+)
diff --git a/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp b/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp
index 84c582b41d0d04..62b0a213cc0d22 100644
--- a/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp
+++ b/clang/unittests/Parse/ParseHLSLRootSignatureTest.cpp
@@ -75,6 +75,8 @@ class ParseHLSLRootSignatureTest : public ::testing::Test {
};
TEST_F(ParseHLSLRootSignatureTest, LexValidTokensTest) {
+ // This test will check that we can lex all defined tokens as defined in
+ // HLSLRootSignatureTokenKinds.def, plus some additional integer variations
const llvm::StringLiteral Source = R"cc(
-42
More information about the cfe-commits
mailing list