[llvm] e0b259f - [llvm] [Support] Add CURL HTTP Client.
Noah Shutty via llvm-commits
llvm-commits at lists.llvm.org
Thu Dec 2 12:35:36 PST 2021
Author: Noah Shutty
Date: 2021-12-02T20:30:59Z
New Revision: e0b259f22c003ffe909693c6ab0d508d1814434d
URL: https://github.com/llvm/llvm-project/commit/e0b259f22c003ffe909693c6ab0d508d1814434d
DIFF: https://github.com/llvm/llvm-project/commit/e0b259f22c003ffe909693c6ab0d508d1814434d.diff
LOG: [llvm] [Support] Add CURL HTTP Client.
Provides an implementation of `HTTPClient` that wraps libcurl.
Reviewed By: dblaikie
Differential Revision: https://reviews.llvm.org/D112753
Added:
Modified:
llvm/CMakeLists.txt
llvm/include/llvm/Config/llvm-config.h.cmake
llvm/include/llvm/Support/HTTPClient.h
llvm/lib/Support/CMakeLists.txt
llvm/lib/Support/HTTPClient.cpp
llvm/lib/Support/InitLLVM.cpp
llvm/unittests/Support/HTTPClient.cpp
Removed:
################################################################################
diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt
index f9d270592e207..c06143e679bf0 100644
--- a/llvm/CMakeLists.txt
+++ b/llvm/CMakeLists.txt
@@ -392,6 +392,8 @@ endif()
set(LLVM_ENABLE_ZLIB "ON" CACHE STRING "Use zlib for compression/decompression if available. Can be ON, OFF, or FORCE_ON")
+set(LLVM_ENABLE_CURL "ON" CACHE STRING "Use libcurl for the HTTP client if available. Can be ON, OFF, or FORCE_ON")
+
set(LLVM_Z3_INSTALL_DIR "" CACHE STRING "Install directory of the Z3 solver.")
option(LLVM_ENABLE_Z3_SOLVER
diff --git a/llvm/include/llvm/Config/llvm-config.h.cmake b/llvm/include/llvm/Config/llvm-config.h.cmake
index 169f59695e5ed..4493bb65d4442 100644
--- a/llvm/include/llvm/Config/llvm-config.h.cmake
+++ b/llvm/include/llvm/Config/llvm-config.h.cmake
@@ -85,6 +85,9 @@
/* Define if we have z3 and want to build it */
#cmakedefine LLVM_WITH_Z3 ${LLVM_WITH_Z3}
+/* Define if we have curl and want to use it */
+#cmakedefine LLVM_ENABLE_CURL ${LLVM_ENABLE_CURL}
+
/* Define if LLVM was built with a dependency to the libtensorflow dynamic library */
#cmakedefine LLVM_HAVE_TF_API
diff --git a/llvm/include/llvm/Support/HTTPClient.h b/llvm/include/llvm/Support/HTTPClient.h
index 3172610c2d8b1..687b1b62addae 100644
--- a/llvm/include/llvm/Support/HTTPClient.h
+++ b/llvm/include/llvm/Support/HTTPClient.h
@@ -77,6 +77,11 @@ class BufferedHTTPResponseHandler final : public HTTPResponseHandler {
/// A reusable client that can perform HTTPRequests through a network socket.
class HTTPClient {
+#ifdef LLVM_ENABLE_CURL
+ void *Curl = nullptr;
+ static bool IsInitialized;
+#endif
+
public:
HTTPClient();
~HTTPClient();
diff --git a/llvm/lib/Support/CMakeLists.txt b/llvm/lib/Support/CMakeLists.txt
index 516dc4959fa52..6ce682d911771 100644
--- a/llvm/lib/Support/CMakeLists.txt
+++ b/llvm/lib/Support/CMakeLists.txt
@@ -74,6 +74,11 @@ if(LLVM_WITH_Z3)
set(system_libs ${system_libs} ${Z3_LIBRARIES})
endif()
+# Link LibCURL if the user wants it
+if (LLVM_ENABLE_CURL)
+ set(system_libs ${system_libs} ${CURL_LIBRARIES})
+endif()
+
# Override the C runtime allocator on Windows and embed it into LLVM tools & libraries
if(LLVM_INTEGRATED_CRT_ALLOC)
if (CMAKE_BUILD_TYPE AND NOT ${LLVM_USE_CRT_${uppercase_CMAKE_BUILD_TYPE}} MATCHES "^(MT|MTd)$")
diff --git a/llvm/lib/Support/HTTPClient.cpp b/llvm/lib/Support/HTTPClient.cpp
index 68ba56d1fe502..52178e054f686 100644
--- a/llvm/lib/Support/HTTPClient.cpp
+++ b/llvm/lib/Support/HTTPClient.cpp
@@ -19,6 +19,9 @@
#include "llvm/Support/Errc.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/MemoryBuffer.h"
+#ifdef LLVM_ENABLE_CURL
+#include <curl/curl.h>
+#endif
using namespace llvm;
@@ -81,12 +84,120 @@ Expected<HTTPResponseBuffer> HTTPClient::get(StringRef Url) {
return perform(Request);
}
+#ifdef LLVM_ENABLE_CURL
+
+bool HTTPClient::isAvailable() { return true; }
+
+bool HTTPClient::IsInitialized = false;
+
+void HTTPClient::initialize() {
+ if (!IsInitialized) {
+ curl_global_init(CURL_GLOBAL_ALL);
+ IsInitialized = true;
+ }
+}
+
+void HTTPClient::cleanup() {
+ if (IsInitialized) {
+ curl_global_cleanup();
+ IsInitialized = false;
+ }
+}
+
+void HTTPClient::setTimeout(std::chrono::milliseconds Timeout) {
+ if (Timeout < std::chrono::milliseconds(0))
+ Timeout = std::chrono::milliseconds(0);
+ curl_easy_setopt(Curl, CURLOPT_TIMEOUT_MS, Timeout.count());
+}
+
+/// CurlHTTPRequest and the curl{Header,Write}Function are implementation
+/// details used to work with Curl. Curl makes callbacks with a single
+/// customizable pointer parameter.
+struct CurlHTTPRequest {
+ CurlHTTPRequest(HTTPResponseHandler &Handler) : Handler(Handler) {}
+ void storeError(Error Err) {
+ ErrorState = joinErrors(std::move(Err), std::move(ErrorState));
+ }
+ HTTPResponseHandler &Handler;
+ llvm::Error ErrorState = Error::success();
+};
+
+static size_t curlHeaderFunction(char *Contents, size_t Size, size_t NMemb,
+ CurlHTTPRequest *CurlRequest) {
+ assert(Size == 1 && "The Size passed by libCURL to CURLOPT_HEADERFUNCTION "
+ "should always be 1.");
+ if (Error Err =
+ CurlRequest->Handler.handleHeaderLine(StringRef(Contents, NMemb))) {
+ CurlRequest->storeError(std::move(Err));
+ return 0;
+ }
+ return NMemb;
+}
+
+static size_t curlWriteFunction(char *Contents, size_t Size, size_t NMemb,
+ CurlHTTPRequest *CurlRequest) {
+ Size *= NMemb;
+ if (Error Err =
+ CurlRequest->Handler.handleBodyChunk(StringRef(Contents, Size))) {
+ CurlRequest->storeError(std::move(Err));
+ return 0;
+ }
+ return Size;
+}
+
+HTTPClient::HTTPClient() {
+ assert(IsInitialized &&
+ "Must call HTTPClient::initialize() at the beginning of main().");
+ if (Curl)
+ return;
+ assert((Curl = curl_easy_init()) && "Curl could not be initialized.");
+ // Set the callback hooks.
+ curl_easy_setopt(Curl, CURLOPT_WRITEFUNCTION, curlWriteFunction);
+ curl_easy_setopt(Curl, CURLOPT_HEADERFUNCTION, curlHeaderFunction);
+}
+
+HTTPClient::~HTTPClient() { curl_easy_cleanup(Curl); }
+
+Error HTTPClient::perform(const HTTPRequest &Request,
+ HTTPResponseHandler &Handler) {
+ if (Request.Method != HTTPMethod::GET)
+ return createStringError(errc::invalid_argument,
+ "Unsupported CURL request method.");
+
+ SmallString<128> Url = Request.Url;
+ curl_easy_setopt(Curl, CURLOPT_URL, Url.c_str());
+ curl_easy_setopt(Curl, CURLOPT_FOLLOWLOCATION, Request.FollowRedirects);
+
+ CurlHTTPRequest CurlRequest(Handler);
+ curl_easy_setopt(Curl, CURLOPT_WRITEDATA, &CurlRequest);
+ curl_easy_setopt(Curl, CURLOPT_HEADERDATA, &CurlRequest);
+ CURLcode CurlRes = curl_easy_perform(Curl);
+ if (CurlRes != CURLE_OK)
+ return joinErrors(std::move(CurlRequest.ErrorState),
+ createStringError(errc::io_error,
+ "curl_easy_perform() failed: %s\n",
+ curl_easy_strerror(CurlRes)));
+ if (CurlRequest.ErrorState)
+ return std::move(CurlRequest.ErrorState);
+
+ unsigned Code;
+ curl_easy_getinfo(Curl, CURLINFO_RESPONSE_CODE, &Code);
+ if (Error Err = Handler.handleStatusCode(Code))
+ return joinErrors(std::move(CurlRequest.ErrorState), std::move(Err));
+
+ return std::move(CurlRequest.ErrorState);
+}
+
+#else
+
HTTPClient::HTTPClient() = default;
HTTPClient::~HTTPClient() = default;
bool HTTPClient::isAvailable() { return false; }
+void HTTPClient::initialize() {}
+
void HTTPClient::cleanup() {}
void HTTPClient::setTimeout(std::chrono::milliseconds Timeout) {}
@@ -95,3 +206,5 @@ Error HTTPClient::perform(const HTTPRequest &Request,
HTTPResponseHandler &Handler) {
llvm_unreachable("No HTTP Client implementation available.");
}
+
+#endif
diff --git a/llvm/lib/Support/InitLLVM.cpp b/llvm/lib/Support/InitLLVM.cpp
index 152de6ebae0ac..8a0068feeadac 100644
--- a/llvm/lib/Support/InitLLVM.cpp
+++ b/llvm/lib/Support/InitLLVM.cpp
@@ -8,6 +8,7 @@
#include "llvm/Support/InitLLVM.h"
#include "llvm/Support/Error.h"
+#include "llvm/Support/HTTPClient.h"
#include "llvm/Support/ManagedStatic.h"
#include "llvm/Support/PrettyStackTrace.h"
#include "llvm/Support/Process.h"
@@ -58,6 +59,11 @@ InitLLVM::InitLLVM(int &Argc, const char **&Argv,
Argc = Args.size() - 1;
Argv = Args.data();
#endif
+
+ HTTPClient::initialize();
}
-InitLLVM::~InitLLVM() { llvm_shutdown(); }
+InitLLVM::~InitLLVM() {
+ HTTPClient::cleanup();
+ llvm_shutdown();
+}
diff --git a/llvm/unittests/Support/HTTPClient.cpp b/llvm/unittests/Support/HTTPClient.cpp
index 30ae67e43ad83..e4f965bb463c3 100644
--- a/llvm/unittests/Support/HTTPClient.cpp
+++ b/llvm/unittests/Support/HTTPClient.cpp
@@ -86,3 +86,9 @@ TEST(BufferedHTTPResponseHandler, MalformedContentLength) {
EXPECT_THAT_ERROR(Handler.handleBodyChunk("non-empty body content"),
Failed<llvm::StringError>());
}
+
+#ifdef LLVM_ENABLE_CURL
+
+TEST(HTTPClient, isAvailable) { EXPECT_TRUE(HTTPClient::isAvailable()); }
+
+#endif
More information about the llvm-commits
mailing list