<div dir="ltr">Pleas use a test with one of the existing tools instead of a unit test.<div><br></div><div>In fact, you should be able to just add this to test/Other/ResponseFile.ll.</div></div><div class="gmail_extra"><br><div class="gmail_quote">On 23 January 2015 at 18:44, Yunzhong Gao <span dir="ltr"><<a href="mailto:Yunzhong_Gao@playstation.sony.com" target="_blank">Yunzhong_Gao@playstation.sony.com</a>></span> wrote:<br><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex">Hi all,<br>
This is spun off from D7133, based on Rafael's suggestion that the UTF-8 BOM changes should be in its separate patch.<br>
In the process of writing a regression test case, I looked at what was done for UTF-16 BOM, and tried to add a similar unit test for UTF-8 BOM in the same file, but this also means that hasUTF8ByteOrderMark() needs be exposed as an external function instead of a static helper function. I hope that is okay.<br>
- Gao<br>
<br>
<a href="http://reviews.llvm.org/D7156" target="_blank">http://reviews.llvm.org/D7156</a><br>
<br>
Files:<br>
llvm/include/llvm/Support/ConvertUTF.h<br>
llvm/lib/Support/CommandLine.cpp<br>
llvm/lib/Support/ConvertUTFWrapper.cpp<br>
llvm/test/Other/ResponseFile.ll<br>
llvm/unittests/Support/ConvertUTFTest.cpp<br>
<br>
Index: llvm/include/llvm/Support/ConvertUTF.h<br>
===================================================================<br>
--- llvm/include/llvm/Support/ConvertUTF.h<br>
+++ llvm/include/llvm/Support/ConvertUTF.h<br>
@@ -243,6 +243,13 @@<br>
bool hasUTF16ByteOrderMark(ArrayRef<char> SrcBytes);<br>
<br>
/**<br>
+ * Returns true if a blob of text starts with a UTF-8 byte order mark.<br>
+ * UTF-8 BOM is a sequence of bytes on Windows and is not affected by the host<br>
+ * system's endianness.<br>
+ */<br>
+bool hasUTF8ByteOrderMark(ArrayRef<char> SrcBytes);<br>
+<br>
+/**<br>
* Converts a stream of raw bytes assumed to be UTF16 into a UTF8 std::string.<br>
*<br>
* \param [in] SrcBytes A buffer of what is assumed to be UTF-16 encoded text.<br>
Index: llvm/lib/Support/CommandLine.cpp<br>
===================================================================<br>
--- llvm/lib/Support/CommandLine.cpp<br>
+++ llvm/lib/Support/CommandLine.cpp<br>
@@ -674,6 +674,11 @@<br>
return false;<br>
Str = StringRef(UTF8Buf);<br>
}<br>
+ // If we see UTF-8 BOM sequence at the beginning of a file, we shall remove<br>
+ // these bytes before parsing.<br>
+ // Reference: <a href="http://en.wikipedia.org/wiki/UTF-8#Byte_order_mark" target="_blank">http://en.wikipedia.org/wiki/UTF-8#Byte_order_mark</a><br>
+ else if (hasUTF8ByteOrderMark(BufRef))<br>
+ Str = StringRef(BufRef.data() + 3, BufRef.size() - 3);<br>
<br>
// Tokenize the contents into NewArgv.<br>
Tokenizer(Str, Saver, NewArgv, MarkEOLs);<br>
Index: llvm/lib/Support/ConvertUTFWrapper.cpp<br>
===================================================================<br>
--- llvm/lib/Support/ConvertUTFWrapper.cpp<br>
+++ llvm/lib/Support/ConvertUTFWrapper.cpp<br>
@@ -81,6 +81,13 @@<br>
(S[0] == '\xfe' && S[1] == '\xff')));<br>
}<br>
<br>
+// It is called byte order marker but the UTF-8 BOM is actually not affected<br>
+// by the host system's endianness.<br>
+bool hasUTF8ByteOrderMark(ArrayRef<char> S) {<br>
+ return (S.size() >= 3 &&<br>
+ S[0] == '\xef' && S[1] == '\xbb' && S[2] == '\xbf');<br>
+}<br>
+<br>
bool convertUTF16ToUTF8String(ArrayRef<char> SrcBytes, std::string &Out) {<br>
assert(Out.empty());<br>
<br>
Index: llvm/test/Other/ResponseFile.ll<br>
===================================================================<br>
--- llvm/test/Other/ResponseFile.ll<br>
+++ llvm/test/Other/ResponseFile.ll<br>
@@ -6,6 +6,13 @@<br>
; RUN: llvm-as @%t.list2 -o %t.bc<br>
; RUN: llvm-nm %t.bc 2>&1 | FileCheck %s<br>
<br>
+; When the response file begins with UTF8 BOM sequence, we shall remove them.<br>
+; RUN: echo -e "\xef\xbb\xbf" > %t.list3<br>
+; RUN: echo %s >> %t.list3<br>
+; RUN: echo -e "\xef\xbb\xbf-time-passes @%t.list3" > %t.list4<br>
+; RUN: llvm-as @%t.list4 -o %t.bc<br>
+; RUN: llvm-nm %t.bc 2>&1 | FileCheck %s<br>
+<br>
; CHECK: T foobar<br>
<br>
define void @foobar() {<br>
Index: llvm/unittests/Support/ConvertUTFTest.cpp<br>
===================================================================<br>
--- llvm/unittests/Support/ConvertUTFTest.cpp<br>
+++ llvm/unittests/Support/ConvertUTFTest.cpp<br>
@@ -66,6 +66,20 @@<br>
EXPECT_FALSE(HasBOM);<br>
}<br>
<br>
+TEST(ConvertUTFTest, HasUTF8BOM) {<br>
+ bool HasBOM = hasUTF8ByteOrderMark(makeArrayRef("\xef\xbb\xbf", 3));<br>
+ EXPECT_TRUE(HasBOM);<br>
+ HasBOM = hasUTF8ByteOrderMark(makeArrayRef("\xef\xbb\xbf ", 4));<br>
+ EXPECT_TRUE(HasBOM);<br>
+ HasBOM = hasUTF8ByteOrderMark(makeArrayRef("\xef\xbb\xbf\x00asdf", 7));<br>
+ EXPECT_TRUE(HasBOM);<br>
+<br>
+ HasBOM = hasUTF8ByteOrderMark(None);<br>
+ EXPECT_FALSE(HasBOM);<br>
+ HasBOM = hasUTF8ByteOrderMark(makeArrayRef("\xef", 1));<br>
+ EXPECT_FALSE(HasBOM);<br>
+}<br>
+<br>
struct ConvertUTFResultContainer {<br>
ConversionResult ErrorCode;<br>
std::vector<unsigned> UnicodeScalars;<br>
<br>
EMAIL PREFERENCES<br>
<a href="http://reviews.llvm.org/settings/panel/emailpreferences/" target="_blank">http://reviews.llvm.org/settings/panel/emailpreferences/</a><br>
</blockquote></div><br></div>