[llvm] r318581 - [asan] Add a full redzone after every stack variable

[Walter Lee via llvm-commits]

Author: waltl
Date: Fri Nov 17 17:13:18 2017
New Revision: 318581

URL: http://llvm.org/viewvc/llvm-project?rev=318581&view=rev
Log:
[asan] Add a full redzone after every stack variable

We were not doing that for large shadow granularity.  Also add more
stack frame layout tests for large shadow granularity.

Differential Revision: https://reviews.llvm.org/D39475

Modified:
    llvm/trunk/lib/Transforms/Utils/ASanStackFrameLayout.cpp
    llvm/trunk/unittests/Transforms/Utils/ASanStackFrameLayoutTest.cpp

Modified: llvm/trunk/lib/Transforms/Utils/ASanStackFrameLayout.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Utils/ASanStackFrameLayout.cpp?rev=318581&r1=318580&r2=318581&view=diff
==============================================================================
--- llvm/trunk/lib/Transforms/Utils/ASanStackFrameLayout.cpp (original)
+++ llvm/trunk/lib/Transforms/Utils/ASanStackFrameLayout.cpp Fri Nov 17 17:13:18 2017
@@ -36,9 +36,11 @@ static inline bool CompareVars(const ASa
 // with e.g. alignment 1 and alignment 16 do not get reordered by CompareVars.
 static const size_t kMinAlignment = 16;
 
+// We want to add a full redzone after every variable.
 // The larger the variable Size the larger is the redzone.
 // The resulting frame size is a multiple of Alignment.
-static size_t VarAndRedzoneSize(size_t Size, size_t Alignment) {
+static size_t VarAndRedzoneSize(size_t Size, size_t Granularity,
+                                size_t Alignment) {
   size_t Res = 0;
   if (Size <= 4)  Res = 16;
   else if (Size <= 16) Res = 32;
@@ -46,7 +48,7 @@ static size_t VarAndRedzoneSize(size_t S
   else if (Size <= 512) Res = Size + 64;
   else if (Size <= 4096) Res = Size + 128;
   else                   Res = Size + 256;
-  return alignTo(Res, Alignment);
+  return alignTo(std::max(Res, 2 * Granularity), Alignment);
 }
 
 ASanStackFrameLayout
@@ -80,7 +82,8 @@ ComputeASanStackFrameLayout(SmallVectorI
     assert(Size > 0);
     size_t NextAlignment = IsLast ? Granularity
                    : std::max(Granularity, Vars[i + 1].Alignment);
-    size_t SizeWithRedzone = VarAndRedzoneSize(Size, NextAlignment);
+    size_t SizeWithRedzone = VarAndRedzoneSize(Size, Granularity,
+                                               NextAlignment);
     Vars[i].Offset = Offset;
     Offset += SizeWithRedzone;
   }

Modified: llvm/trunk/unittests/Transforms/Utils/ASanStackFrameLayoutTest.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/unittests/Transforms/Utils/ASanStackFrameLayoutTest.cpp?rev=318581&r1=318580&r2=318581&view=diff
==============================================================================
--- llvm/trunk/unittests/Transforms/Utils/ASanStackFrameLayoutTest.cpp (original)
+++ llvm/trunk/unittests/Transforms/Utils/ASanStackFrameLayoutTest.cpp Fri Nov 17 17:13:18 2017
@@ -68,9 +68,12 @@ TEST(ASanStackFrameLayout, Test) {
   VAR(a, 16, 16, 1, 0);
   VAR(a, 41, 9, 1, 7);
   VAR(a, 105, 103, 1, 0);
+  VAR(a, 200, 97, 1, 0);
 
   TEST_LAYOUT({a1_1}, 8, 16, "1 16 1 4 a1_1", "LL1R", "LL1R");
-  TEST_LAYOUT({a1_1}, 64, 64, "1 64 1 4 a1_1", "L1", "L1");
+  TEST_LAYOUT({a1_1}, 16, 16, "1 16 1 4 a1_1", "L1R", "L1R");
+  TEST_LAYOUT({a1_1}, 32, 32, "1 32 1 4 a1_1", "L1R", "L1R");
+  TEST_LAYOUT({a1_1}, 64, 64, "1 64 1 4 a1_1", "L1R", "L1R");
   TEST_LAYOUT({p1_32}, 8, 32, "1 32 1 8 p1_32:15", "LLLL1RRR", "LLLL1RRR");
   TEST_LAYOUT({p1_32}, 8, 64, "1 64 1 8 p1_32:15", "LLLLLLLL1RRRRRRR",
               "LLLLLLLL1RRRRRRR");
@@ -103,6 +106,30 @@ TEST(ASanStackFrameLayout, Test) {
     TEST_LAYOUT(t, 8, 32, "3 32 1 4 a1_1 48 16 5 a16_1 80 41 7 a41_1:7",
                 "LLLL1M00MM000001RRRR", "LLLL1MSSMMSS0001RRRR");
   }
+
+  TEST_LAYOUT({a2_1}, 32, 32, "1 32 2 4 a2_1", "L2R", "L2R");
+  TEST_LAYOUT({a9_1}, 32, 32, "1 32 9 4 a9_1", "L9R", "L9R");
+  TEST_LAYOUT({a16_1}, 32, 32, "1 32 16 5 a16_1", "L16R", "LSR");
+  TEST_LAYOUT({p1_256}, 32, 32, "1 256 1 11 p1_256:2700",
+              "LLLLLLLL1R", "LLLLLLLL1R");
+  TEST_LAYOUT({a41_1}, 32, 32, "1 32 41 7 a41_1:7", "L09R",
+              "LS9R");
+  TEST_LAYOUT({a105_1}, 32, 32, "1 32 105 6 a105_1", "L0009R",
+              "LSSSSR");
+  TEST_LAYOUT({a200_1}, 32, 32, "1 32 200 6 a200_1", "L0000008RR",
+              "LSSSS008RR");
+
+  {
+    SmallVector<ASanStackVariableDescription, 10> t = {a1_1, p1_256};
+    TEST_LAYOUT(t, 32, 32, "2 256 1 11 p1_256:2700 320 1 4 a1_1",
+                "LLLLLLLL1M1R", "LLLLLLLL1M1R");
+  }
+
+  {
+    SmallVector<ASanStackVariableDescription, 10> t = {a1_1, a16_1, a41_1};
+    TEST_LAYOUT(t, 32, 32, "3 32 1 4 a1_1 96 16 5 a16_1 160 41 7 a41_1:7",
+                "L1M16M09R", "L1MSMS9R");
+  }
 #undef VAR
 #undef TEST_LAYOUT
 }