Support reading from S3 directly
Change-Id: Ice18f6739a7e315bea223a2a6f634b6c4c725c11
Signed-off-by: Austin Schuh <austin.schuh@bluerivertech.com>
diff --git a/WORKSPACE b/WORKSPACE
index 13a8726..6ef1eb9 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -1231,6 +1231,41 @@
path = "third_party/com_github_nlohmann_json",
)
+# https://curl.haxx.se/download/curl-7.69.1.tar.gz
+http_archive(
+ name = "com_github_curl_curl",
+ build_file = "//debian:curl.BUILD",
+ sha256 = "01ae0c123dee45b01bbaef94c0bc00ed2aec89cb2ee0fd598e0d302a6b5e0a98",
+ strip_prefix = "curl-7.69.1",
+ url = "https://www.frc971.org/Build-Dependencies/curl-7.69.1.tar.gz",
+)
+
+http_archive(
+ # No official name exists. Names used in our external dependencies include
+ # zlib, madler_zlib, com_github_madler_zlib.
+ name = "zlib",
+ build_file = "//debian:BUILD.zlib.bazel",
+ sha256 = "629380c90a77b964d896ed37163f5c3a34f6e6d897311f1df2a7016355c45eff",
+ strip_prefix = "zlib-1.2.11",
+ urls = [
+ "https://github.com/madler/zlib/archive/v1.2.11.tar.gz",
+ ],
+)
+
+# This one is tricky to get an archive because it has recursive submodules. These semi-automated steps do work though:
+# git clone -b version1.9 --recurse-submodules --depth=1 https://github.com/aws/aws-sdk-cpp
+# cd aws-sdk-cpp
+# echo bsdtar -a -cf aws_sdk-version.tar.gz --ignore-zeros @\<\(git archive HEAD\) $(git submodule foreach --recursive --quiet 'echo @\<\(cd $displaypath \&\& git archive HEAD --prefix=$displaypath/\)')
+# Now run the command that printed, and the output will be at aws_sdk-version.tar.gz.
+http_archive(
+ name = "aws_sdk",
+ build_file = "//debian:aws_sdk.BUILD",
+ patch_args = ["-p1"],
+ patches = ["//debian:aws_sdk.patch"],
+ sha256 = "1a2668722e5b5b2608a6ab21c876c1d98b5fd8c7d613129ced9561f5520817dc",
+ url = "https://www.frc971.org/Build-Dependencies/aws_sdk-19.0.0-RC1.tar.gz",
+)
+
http_file(
name = "com_github_foxglove_mcap_mcap",
executable = True,
diff --git a/aos/events/logging/BUILD b/aos/events/logging/BUILD
index 82e7061..7201e70 100644
--- a/aos/events/logging/BUILD
+++ b/aos/events/logging/BUILD
@@ -48,6 +48,10 @@
"logfile_sorting.h",
"logfile_utils.h",
],
+ copts = select({
+ "//tools:cpu_k8": ["-DENABLE_S3=1"],
+ "//conditions:default": ["-DENABLE_S3=0"],
+ }),
target_compatible_with = ["@platforms//os:linux"],
visibility = ["//visibility:public"],
deps = [
@@ -69,7 +73,10 @@
"@com_google_absl//absl/types:span",
"@boringssl//:crypto",
] + select({
- "//tools:cpu_k8": [":lzma_encoder"],
+ "//tools:cpu_k8": [
+ ":s3_fetcher",
+ ":lzma_encoder",
+ ],
"//tools:cpu_arm64": [":lzma_encoder"],
"//conditions:default": [],
}),
@@ -200,6 +207,27 @@
],
)
+cc_library(
+ name = "s3_fetcher",
+ srcs = [
+ "s3_fetcher.cc",
+ ],
+ hdrs = [
+ "s3_fetcher.h",
+ ],
+ target_compatible_with = ["@platforms//os:linux"],
+ visibility = ["//visibility:public"],
+ deps = [
+ ":buffer_encoder",
+ "//aos/containers:resizeable_buffer",
+ "@aws_sdk//:core",
+ "@aws_sdk//:s3",
+ "@com_github_google_glog//:glog",
+ "@com_google_absl//absl/strings",
+ "@com_google_absl//absl/types:span",
+ ],
+)
+
cc_test(
name = "lzma_encoder_test",
srcs = [
diff --git a/aos/events/logging/logfile_sorting.cc b/aos/events/logging/logfile_sorting.cc
index 1a16456..89801b7 100644
--- a/aos/events/logging/logfile_sorting.cc
+++ b/aos/events/logging/logfile_sorting.cc
@@ -16,14 +16,17 @@
#include "openssl/sha.h"
#include "sys/stat.h"
+#if ENABLE_S3
+#include "aos/events/logging/s3_fetcher.h"
+#endif
+
DEFINE_bool(quiet_sorting, false,
"If true, sort with minimal messages about truncated files.");
namespace aos {
namespace logger {
-namespace chrono = std::chrono;
-
namespace {
+namespace chrono = std::chrono;
// Check if string ends with ending
bool EndsWith(std::string_view str, std::string_view ending) {
@@ -62,7 +65,36 @@
EndsWith(filename, ".bfbs.sz");
}
+#if ENABLE_S3
+class S3FileOperations final : public FileOperations {
+ public:
+ S3FileOperations(std::string_view url) : object_urls_(ListS3Objects(url)) {}
+
+ bool Exists() override { return !object_urls_.empty(); }
+ void FindLogs(std::vector<std::string> *files) override {
+ // We already have a recursive listing, so just grab all the objects from
+ // there.
+ for (const std::string &object_url : object_urls_) {
+ if (IsValidFilename(object_url)) {
+ files->push_back(object_url);
+ }
+ }
+ }
+
+ private:
+ const std::vector<std::string> object_urls_;
+};
+#endif
+
std::unique_ptr<FileOperations> MakeFileOperations(std::string_view filename) {
+ static constexpr std::string_view kS3 = "s3:";
+ if (filename.substr(0, kS3.size()) == kS3) {
+#if ENABLE_S3
+ return std::make_unique<S3FileOperations>(filename);
+#else
+ LOG(FATAL) << "Reading files from S3 not supported on this platform";
+#endif
+ }
if (filename.find("://") != filename.npos) {
LOG(FATAL) << "This looks like a URL of an unknown type: " << filename;
}
@@ -391,7 +423,7 @@
void PartsSorter::PopulateFromFiles(const std::vector<std::string> &parts) {
// Now extract everything into our datastructures above for sorting.
for (const std::string &part : parts) {
- if (part_readers.size() > 200) {
+ if (part_readers.size() > 50) {
// Don't leave arbitrary numbers of readers open, because they each take
// resources, so close a big batch at once periodically.
part_readers.clear();
diff --git a/aos/events/logging/logfile_utils.cc b/aos/events/logging/logfile_utils.cc
index 4c06f0a..a76c1a8 100644
--- a/aos/events/logging/logfile_utils.cc
+++ b/aos/events/logging/logfile_utils.cc
@@ -28,6 +28,9 @@
#if ENABLE_LZMA
#include "aos/events/logging/lzma_encoder.h"
#endif
+#if ENABLE_S3
+#include "aos/events/logging/s3_fetcher.h"
+#endif
DEFINE_int32(flush_size, 128 * 1024,
"Number of outstanding bytes to allow before flushing to disk.");
@@ -862,7 +865,16 @@
SpanReader::SpanReader(std::string_view filename, bool quiet)
: filename_(filename) {
- decoder_ = std::make_unique<DummyDecoder>(filename);
+ static constexpr std::string_view kS3 = "s3:";
+ if (filename.substr(0, kS3.size()) == kS3) {
+#if ENABLE_S3
+ decoder_ = std::make_unique<S3Fetcher>(filename);
+#else
+ LOG(FATAL) << "Reading files from S3 not supported on this platform";
+#endif
+ } else {
+ decoder_ = std::make_unique<DummyDecoder>(filename);
+ }
static constexpr std::string_view kXz = ".xz";
static constexpr std::string_view kSnappy = SnappyDecoder::kExtension;
diff --git a/aos/events/logging/s3_fetcher.cc b/aos/events/logging/s3_fetcher.cc
new file mode 100644
index 0000000..f44a20b
--- /dev/null
+++ b/aos/events/logging/s3_fetcher.cc
@@ -0,0 +1,208 @@
+#include "aos/events/logging/s3_fetcher.h"
+
+#include <aws/core/Aws.h>
+#include <aws/s3/model/ListObjectsV2Request.h>
+
+#include "absl/strings/str_cat.h"
+#include "glog/logging.h"
+
+// When we first start reading a log folder, we end up reading the first part of
+// each file twice. We could speed this up by restructuring the API so all the
+// downloads could be started in parallel, and the initial chunk of data cached.
+// However, even though this initial part is slower than necessary,
+// decompressing and sorting the main part of the log file is still the slowest
+// part, and this implementation does parallelize downloads with that, so it's
+// good enough for now.
+
+namespace aos::logger {
+namespace {
+
+struct AwsAPIOwner {
+ Aws::SDKOptions options;
+
+ AwsAPIOwner() {
+ options.httpOptions.installSigPipeHandler = true;
+ Aws::InitAPI(options);
+ }
+ ~AwsAPIOwner() { Aws::ShutdownAPI(options); }
+};
+
+// If this doesn't fit the largest message, the per-request overhead very
+// quickly dominates log reading time, because there's no processing in between
+// reading all the pieces of a single message. Bigger takes more memory, but our
+// logs aren't split across all that many files usually, so this can be fairly
+// large without increasing memory requirements for log reading too much.
+constexpr int kChunkSize = 10 * 1024 * 1024;
+
+void InitAwsAPI() { static AwsAPIOwner api_owner; }
+
+Aws::Client::ClientConfiguration MakeClientConfiguration() {
+ InitAwsAPI();
+ Aws::Client::ClientConfiguration config;
+ config.region = Aws::Region::AWS_GLOBAL;
+ return config;
+}
+
+struct ParsedRange {
+ uint64_t start, end, total_size;
+};
+
+ParsedRange ParseRange(std::string_view string) {
+ static constexpr std::string_view kBytes = "bytes ";
+ CHECK(string.substr(0, kBytes.size()) == kBytes)
+ << ": Invalid range: " << string;
+ string = string.substr(kBytes.size());
+
+ const size_t dash = string.find('-');
+ CHECK(dash != string.npos) << ": Invalid range: " << string;
+ const size_t slash = string.find('/');
+ CHECK(slash != string.npos) << ": Invalid range: " << string;
+
+ ParsedRange result;
+ const std::string_view start_string = string.substr(0, dash);
+ CHECK(absl::SimpleAtoi(start_string, &result.start))
+ << ": failed to parse " << start_string << " from " << string;
+ const std::string_view end_string = string.substr(dash + 1, slash - dash - 1);
+ CHECK(absl::SimpleAtoi(end_string, &result.end))
+ << ": failed to parse " << end_string << " from " << string;
+ const std::string_view total_string = string.substr(slash + 1);
+ CHECK(absl::SimpleAtoi(total_string, &result.total_size))
+ << ": failed to parse " << total_string << " from " << string;
+ return result;
+}
+
+} // namespace
+
+ObjectName ParseUrl(std::string_view url) {
+ static constexpr std::string_view kS3 = "s3://";
+ if (url.substr(0, kS3.size()) != kS3) {
+ LOG(FATAL) << "Not an S3 URL: " << url;
+ }
+ url = url.substr(kS3.size());
+ const size_t slash = url.find('/');
+ CHECK(slash != url.npos) << ": Invalid S3 URL: " << url;
+ ObjectName result;
+ result.bucket = url.substr(0, slash);
+ result.key = url.substr(slash + 1);
+ return result;
+}
+
+// This client is thread-safe, so it should be used globally. Destroying it can
+// take a while to shut down all the threads.
+Aws::S3::S3Client &GetS3Client() {
+ static Aws::S3::S3Client result(MakeClientConfiguration());
+ return result;
+}
+
+S3Fetcher::S3Fetcher(std::string_view url) : url_(url) {
+ VLOG(1) << "opening " << url;
+ // Start the initial request now.
+ StartRequest();
+}
+
+size_t S3Fetcher::Read(uint8_t *begin, uint8_t *end) {
+ VLOG(1) << "looking to read " << (end - begin);
+ size_t total_read = 0;
+
+ while (true) {
+ // First copy any data we already have.
+ const size_t current_size =
+ std::min<size_t>(current_chunk_.size(), end - begin - total_read);
+ memcpy(begin + total_read, current_chunk_.data(), current_size);
+ total_read += current_size;
+ current_chunk_.erase_front(current_size);
+ if (static_cast<ssize_t>(total_read) == end - begin) {
+ VLOG(1) << "Got all " << total_read;
+ // Got all of what the caller wants, done now.
+ return total_read;
+ }
+ CHECK_EQ(current_chunk_.size(), 0u)
+ << ": Should have already copied this data out";
+ if (end_of_object_) {
+ VLOG(1) << "At end after " << total_read;
+ // Nothing more to read.
+ return total_read;
+ }
+
+ // Read data from the last request.
+ CHECK(get_next_chunk_.valid()) << ": Should have a request started already";
+ Aws::S3::Model::GetObjectOutcome get_outcome = get_next_chunk_.get();
+ if (!get_outcome.IsSuccess()) {
+ if (next_byte_to_request_ == 0 &&
+ get_outcome.GetError().GetResponseCode() ==
+ Aws::Http::HttpResponseCode::REQUESTED_RANGE_NOT_SATISFIABLE) {
+ VLOG(1) << "At beginning of empty file";
+ // This is what happens with an empty file.
+ // TODO(Brian): Do a List operation to verify it's actually empty?
+ CHECK_EQ(0u, total_read);
+ end_of_object_ = true;
+ return 0;
+ }
+ LOG(FATAL) << ": GET for " << url_
+ << " failed: " << get_outcome.GetError();
+ }
+ const ParsedRange content_range =
+ ParseRange(get_outcome.GetResult().GetContentRange());
+ const uint64_t content_bytes = content_range.end - content_range.start + 1;
+ CHECK_EQ(content_range.start, next_byte_to_request_);
+ next_byte_to_request_ += kChunkSize;
+
+ auto &stream = get_outcome.GetResult().GetBody();
+ current_chunk_.resize(content_bytes);
+ stream.read(reinterpret_cast<char *>(current_chunk_.data()), content_bytes);
+ const size_t stream_read = stream.gcount();
+ VLOG(1) << "got " << stream_read << " from "
+ << get_outcome.GetResult().GetContentRange();
+ CHECK_EQ(stream_read, content_bytes);
+ if (content_range.end + 1 == content_range.total_size) {
+ end_of_object_ = true;
+ continue;
+ }
+
+ // Kick off the next request.
+ StartRequest();
+ }
+
+ return total_read;
+}
+
+void S3Fetcher::StartRequest() {
+ Aws::S3::Model::GetObjectRequest get_request;
+ const ObjectName object_name = ParseUrl(url_);
+ get_request.SetBucket(object_name.bucket);
+ get_request.SetKey(object_name.key);
+ const uint64_t last_byte_to_request = next_byte_to_request_ + kChunkSize;
+ get_request.SetRange(absl::StrCat("bytes=", next_byte_to_request_, "-",
+ last_byte_to_request - 1));
+ VLOG(1) << "request for " << next_byte_to_request_ << "-"
+ << last_byte_to_request << ": " << get_request.GetRange();
+ get_next_chunk_ = GetS3Client().GetObjectCallable(get_request);
+}
+
+std::vector<std::string> ListS3Objects(std::string_view url) {
+ Aws::S3::Model::ListObjectsV2Request list_request;
+ const ObjectName object_name = ParseUrl(url);
+ list_request.SetBucket(object_name.bucket);
+ list_request.SetPrefix(object_name.key);
+ Aws::S3::Model::ListObjectsV2Outcome list_outcome =
+ GetS3Client().ListObjectsV2(list_request);
+ std::vector<std::string> result;
+ while (true) {
+ CHECK(list_outcome.IsSuccess()) << ": Listing objects for " << url
+ << " failed: " << list_outcome.GetError();
+ auto &list_result = list_outcome.GetResult();
+ for (const Aws::S3::Model::Object &object : list_result.GetContents()) {
+ result.push_back(absl::StrCat("s3://", list_outcome.GetResult().GetName(),
+ "/", object.GetKey()));
+ VLOG(2) << "got " << result.back();
+ }
+ if (!list_result.GetIsTruncated()) {
+ break;
+ }
+ list_request.SetContinuationToken(list_result.GetNextContinuationToken());
+ list_outcome = GetS3Client().ListObjectsV2(list_request);
+ }
+ return result;
+}
+
+} // namespace aos::logger
diff --git a/aos/events/logging/s3_fetcher.h b/aos/events/logging/s3_fetcher.h
new file mode 100644
index 0000000..cf00226
--- /dev/null
+++ b/aos/events/logging/s3_fetcher.h
@@ -0,0 +1,62 @@
+
+#ifndef AOS_EVENTS_LOGGING_S3_FETCHER_H_
+#define AOS_EVENTS_LOGGING_S3_FETCHER_H_
+
+#include <aws/s3/S3Client.h>
+#include <aws/s3/model/GetObjectRequest.h>
+
+#include <future>
+#include <string_view>
+
+#include "aos/containers/resizeable_buffer.h"
+#include "aos/events/logging/buffer_encoder.h"
+
+namespace aos::logger {
+
+// Fetches data from an S3 URL.
+class S3Fetcher final : public DataDecoder {
+ public:
+ explicit S3Fetcher(std::string_view url);
+ S3Fetcher(const S3Fetcher &) = delete;
+ S3Fetcher &operator=(const S3Fetcher &) = delete;
+
+ size_t Read(uint8_t *begin, uint8_t *end) final;
+ std::string_view filename() const final { return url_; }
+
+ private:
+ const std::string url_;
+
+ // The current chunk we're reading from. Empty if there is no current chunk or
+ // we've read all of it.
+ ResizeableBuffer current_chunk_;
+ // If valid, the next chunk which we've triggered to be retrieved in the
+ // background.
+ std::future<Aws::S3::Model::GetObjectOutcome> get_next_chunk_;
+
+ // The next byte index we're going to request. This means we've already made
+ // requests for all prior bytes, but not necessarily received them.
+ uint64_t next_byte_to_request_ = 0;
+
+ // Set once we've received data for the end of the object. Some of it may
+ // still be in current_chunk_ though.
+ bool end_of_object_ = false;
+
+ // Kicks off a request for the next chunk.
+ void StartRequest();
+};
+
+Aws::S3::S3Client& GetS3Client();
+
+struct ObjectName {
+ std::string bucket, key;
+};
+
+ObjectName ParseUrl(std::string_view url);
+
+// Does an S3 object listing with the given URL prefix. Returns the URLs for all
+// the objects under it.
+std::vector<std::string> ListS3Objects(std::string_view url);
+
+} // namespace aos::logger
+
+#endif // AOS_EVENTS_LOGGING_S3_FETCHER_H_
diff --git a/debian/BUILD b/debian/BUILD
index c920457..de5144d 100644
--- a/debian/BUILD
+++ b/debian/BUILD
@@ -504,4 +504,6 @@
exports_files([
"ssh_wrapper.sh",
+ "curl.BUILD",
+ "BUILD.zlib.bazel",
])
diff --git a/debian/BUILD.zlib.bazel b/debian/BUILD.zlib.bazel
new file mode 100644
index 0000000..c0f81fa
--- /dev/null
+++ b/debian/BUILD.zlib.bazel
@@ -0,0 +1,22 @@
+licenses(["notice"]) # BSD/MIT-like license
+
+cc_library(
+ name = "zlib",
+ srcs = glob(["*.c"]),
+ hdrs = glob(["*.h"]),
+ # Use -Dverbose=-1 to turn off zlib's trace logging.
+ copts = [
+ "-w",
+ "-Dverbose=-1",
+ ],
+ includes = [
+ ".",
+ ],
+ visibility = ["//visibility:public"],
+)
+
+alias(
+ name = "z",
+ actual = ":zlib",
+ visibility = ["//visibility:public"],
+)
diff --git a/debian/aws_sdk.BUILD b/debian/aws_sdk.BUILD
new file mode 100644
index 0000000..a77f5bb
--- /dev/null
+++ b/debian/aws_sdk.BUILD
@@ -0,0 +1,354 @@
+load("@org_frc971//tools/build_rules:select.bzl", "compiler_select")
+
+cc_library(
+ name = "s3",
+ srcs = glob(["aws-cpp-sdk-s3/source/**/*.cpp"]),
+ hdrs = glob(["aws-cpp-sdk-s3/include/**/*.h"]),
+ includes = ["aws-cpp-sdk-s3/include"],
+ target_compatible_with = ["@platforms//os:linux"],
+ visibility = ["//visibility:public"],
+ deps = [
+ ":aws-c-auth",
+ ":core",
+ ],
+)
+
+genrule(
+ name = "gen_SDKConfig",
+ outs = ["aws-cpp-sdk-core/include/aws/core/SDKConfig.h"],
+ cmd = "echo '#undef USE_AWS_MEMORY_MANAGEMENT' > $@",
+ target_compatible_with = ["@platforms//os:linux"],
+)
+
+cc_library(
+ name = "core",
+ srcs = glob(
+ include = ["aws-cpp-sdk-core/source/**/*.cpp"],
+ exclude = [
+ "aws-cpp-sdk-core/source/utils/crypto/*/*.cpp",
+ "aws-cpp-sdk-core/source/platform/**/*.cpp",
+ "aws-cpp-sdk-core/source/platform/windows/**/*.cpp",
+ # net/*.cpp is for not-(linux or windows), so exclude everything in there.
+ "aws-cpp-sdk-core/source/net/**/*.cpp",
+ "aws-cpp-sdk-core/source/http/windows/**/*.cpp",
+ ],
+ ) + glob([
+ "aws-cpp-sdk-core/source/utils/crypto/openssl/*.cpp",
+ "aws-cpp-sdk-core/source/utils/crypto/factory/*.cpp",
+ "aws-cpp-sdk-core/source/platform/linux-shared/**/*.cpp",
+ "aws-cpp-sdk-core/source/net/linux-shared/*.cpp",
+ ]) + [
+ ":gen_SDKConfig",
+ ],
+ hdrs = glob(
+ include = ["aws-cpp-sdk-core/include/**/*.h"],
+ exclude = [
+ "aws-cpp-sdk-core/include/aws/core/utils/crypto/*/*.h",
+ "aws-cpp-sdk-core/include/aws/core/http/windows/**/*.h",
+ ],
+ ) + glob([
+ "aws-cpp-sdk-core/include/aws/core/utils/crypto/openssl/*.h",
+ ]),
+ copts = [
+ "-DAWS_SDK_VERSION_MAJOR=19",
+ "-DAWS_SDK_VERSION_MINOR=0",
+ "-DAWS_SDK_VERSION_PATCH=\"\\\"0-RC1\"\\\"",
+ "-DENABLE_OPENSSL_ENCRYPTION",
+ "-DENABLE_CURL_CLIENT",
+ "-Wno-cast-align",
+ "-Wno-cast-qual",
+ "-Wno-format-nonliteral",
+ ],
+ includes = ["aws-cpp-sdk-core/include"],
+ target_compatible_with = ["@platforms//os:linux"],
+ visibility = ["//visibility:public"],
+ deps = [
+ ":aws-c-auth",
+ ":aws-c-common",
+ ":aws-c-http",
+ ":crt",
+ "@boringssl//:crypto",
+ "@com_github_curl_curl//:curl",
+ ],
+)
+
+genrule(
+ name = "gen_Config",
+ outs = ["crt/aws-crt-cpp/include/aws/crt/Config.h"],
+ cmd = "echo '#define AWS_CRT_CPP_VERSION \"19.0.0-RC1\"' > $@",
+ target_compatible_with = ["@platforms//os:linux"],
+)
+
+cc_library(
+ name = "crt",
+ srcs = glob(["crt/aws-crt-cpp/source/**/*.cpp"]),
+ hdrs = glob(["crt/aws-crt-cpp/include/**/*.h"]) + [
+ ":gen_Config",
+ ],
+ copts = [
+ "-Wno-sign-compare",
+ "-Wno-cast-qual",
+ ],
+ includes = ["crt/aws-crt-cpp/include"],
+ target_compatible_with = ["@platforms//os:linux"],
+ deps = [
+ ":aws-c-auth",
+ ":aws-c-common",
+ ":aws-c-event-stream",
+ ":aws-c-mqtt",
+ ":aws-c-s3",
+ ],
+)
+
+genrule(
+ name = "gen_config",
+ outs = ["crt/aws-crt-cpp/crt/aws-c-common/include/aws/common/config.h"],
+ cmd = "\n".join([
+ "cat >$@ <<END",
+ "#define AWS_HAVE_GCC_OVERFLOW_MATH_EXTENSIONS 1",
+ "#define AWS_HAVE_GCC_INLINE_ASM 1",
+ "#undef AWS_HAVE_MSVC_MULX",
+ "#define AWS_HAVE_EXECINFO 1",
+ "END",
+ ]),
+ target_compatible_with = ["@platforms//os:linux"],
+)
+
+cc_library(
+ name = "aws-c-common",
+ srcs = glob([
+ "crt/aws-crt-cpp/crt/aws-c-common/source/*.c",
+ "crt/aws-crt-cpp/crt/aws-c-common/source/posix/*.c",
+ ]) + [
+ ":gen_config",
+ ] + select({
+ # See the paths in crt/aws-crt-cpp/crt/aws-c-common/CMakeLists.txt for the appropriate globs for each architecture.
+ "@//tools:cpu_k8": glob(
+ include = [
+ "crt/aws-crt-cpp/crt/aws-c-common/source/arch/intel/*.c",
+ "crt/aws-crt-cpp/crt/aws-c-common/source/arch/intel/asm/*.c",
+ ],
+ exclude = [
+ # We don't build with AVX, see crt/aws-crt-cpp/crt/aws-c-common/CMakeLists.txt for details of the macros that need to be set if this is enabled.
+ "crt/aws-crt-cpp/crt/aws-c-common/source/arch/intel/encoding_avx2.c",
+ ],
+ ),
+ "@//tools:cpu_arm64": glob([
+ "crt/aws-crt-cpp/crt/aws-c-common/source/arch/arm/asm/*.c",
+ ]),
+ "@//tools:cpu_armv7": glob([
+ "crt/aws-crt-cpp/crt/aws-c-common/source/arch/arm/asm/*.c",
+ ]),
+ "//conditions:default": [],
+ }),
+ hdrs = glob(["crt/aws-crt-cpp/crt/aws-c-common/include/**/*.h"]),
+ copts = [
+ "-Wno-cast-align",
+ "-Wno-cast-qual",
+ "-Wno-sign-compare",
+ "-Wno-format-nonliteral",
+ ] + compiler_select({
+ "clang": [],
+ "gcc": [
+ "-Wno-old-style-declaration",
+ ],
+ }),
+ includes = ["crt/aws-crt-cpp/crt/aws-c-common/include"],
+ target_compatible_with = ["@platforms//os:linux"],
+ textual_hdrs = glob(["crt/aws-crt-cpp/crt/aws-c-common/include/**/*.inl"]),
+)
+
+# -march=armv8-a+crc
+cc_library(
+ name = "aws-c-event-stream",
+ srcs = glob(["crt/aws-crt-cpp/crt/aws-c-event-stream/source/*.c"]) + select({
+ "@//tools:cpu_k8": glob(["crt/aws-crt-cpp/crt/aws-c-event-stream/source/intel/asm/*.c"]),
+ "@//tools:cpu_arm64": glob(["crt/aws-crt-cpp/crt/aws-c-event-stream/source/arm/*.c"]),
+ "@//tools:cpu_armv7": glob(["crt/aws-crt-cpp/crt/aws-c-event-stream/source/arm/*.c"]),
+ "//conditions:default": [],
+ }),
+ hdrs = glob(["crt/aws-crt-cpp/crt/aws-c-event-stream/include/**/*.h"]),
+ copts = [
+ "-Wno-cast-align",
+ "-Wno-cast-qual",
+ ],
+ includes = ["crt/aws-crt-cpp/crt/aws-c-event-stream/include"],
+ deps = [
+ ":aws-c-common",
+ ":aws-c-io",
+ ":aws-checksums",
+ ],
+)
+
+cc_library(
+ name = "aws-checksums",
+ srcs = glob(["crt/aws-crt-cpp/crt/aws-checksums/source/*.c"]) + select({
+ "@//tools:cpu_k8": glob(["crt/aws-crt-cpp/crt/aws-checksums/source/intel/asm/*.c"]),
+ "@//tools:cpu_arm64": glob(["crt/aws-crt-cpp/crt/aws-checksums/source/arm/*.c"]),
+ "@//tools:cpu_armv7": glob(["crt/aws-crt-cpp/crt/aws-checksums/source/arm/*.c"]),
+ "//conditions:default": [],
+ }),
+ hdrs = glob(["crt/aws-crt-cpp/crt/aws-checksums/include/**/*.h"]),
+ copts = [
+ "-Wno-cast-qual",
+ "-Wno-cast-align",
+ "-Wno-implicit-function-declaration",
+ ],
+ includes = ["crt/aws-crt-cpp/crt/aws-checksums/include"],
+ target_compatible_with = ["@platforms//os:linux"],
+ deps = [
+ ":aws-c-common",
+ ],
+)
+
+cc_library(
+ name = "aws-c-cal",
+ srcs = glob([
+ "crt/aws-crt-cpp/crt/aws-c-cal/source/*.c",
+ "crt/aws-crt-cpp/crt/aws-c-cal/source/unix/*.c",
+ ]),
+ hdrs = glob(["crt/aws-crt-cpp/crt/aws-c-cal/include/**/*.h"]),
+ copts = [
+ "-Wno-incompatible-pointer-types",
+ "-Wno-unused-function",
+ "-Wno-cast-align",
+ "-Wno-cast-qual",
+ ],
+ includes = ["crt/aws-crt-cpp/crt/aws-c-cal/include"],
+ target_compatible_with = ["@platforms//os:linux"],
+ deps = [
+ ":aws-c-common",
+ "@boringssl//:crypto",
+ ],
+)
+
+cc_library(
+ name = "aws-c-s3",
+ srcs = glob(["crt/aws-crt-cpp/crt/aws-c-s3/source/**/*.c"]),
+ hdrs = glob(["crt/aws-crt-cpp/crt/aws-c-s3/include/**/*.h"]),
+ copts = [
+ "-Wno-cast-align",
+ "-Wno-cast-qual",
+ ],
+ includes = ["crt/aws-crt-cpp/crt/aws-c-s3/include"],
+ target_compatible_with = ["@platforms//os:linux"],
+ deps = [
+ ":aws-c-auth",
+ ":aws-c-common",
+ ],
+)
+
+cc_library(
+ name = "aws-c-compression",
+ srcs = glob(["crt/aws-crt-cpp/crt/aws-c-compression/source/*.c"]),
+ hdrs = glob(["crt/aws-crt-cpp/crt/aws-c-compression/include/**/*.h"]),
+ includes = ["crt/aws-crt-cpp/crt/aws-c-compression/include"],
+ target_compatible_with = ["@platforms//os:linux"],
+ deps = [
+ ":aws-c-common",
+ ],
+)
+
+cc_library(
+ name = "aws-c-http",
+ srcs = glob(["crt/aws-crt-cpp/crt/aws-c-http/source/**/*.c"]),
+ hdrs = glob(["crt/aws-crt-cpp/crt/aws-c-http/include/**/*.h"]),
+ copts = [
+ "-Wno-unused-but-set-variable",
+ "-Wno-cast-align",
+ "-Wno-cast-qual",
+ ],
+ includes = ["crt/aws-crt-cpp/crt/aws-c-http/include"],
+ target_compatible_with = ["@platforms//os:linux"],
+ textual_hdrs = glob(["crt/aws-crt-cpp/crt/aws-c-http/include/**/*.def"]),
+ deps = [
+ ":aws-c-common",
+ ":aws-c-compression",
+ ":aws-c-io",
+ ],
+)
+
+cc_library(
+ name = "aws-c-auth",
+ srcs = glob(["crt/aws-crt-cpp/crt/aws-c-auth/source/**/*.c"]),
+ hdrs = glob(["crt/aws-crt-cpp/crt/aws-c-auth/include/**/*.h"]),
+ copts = [
+ "-Wno-cast-align",
+ "-Wno-cast-qual",
+ ],
+ includes = ["crt/aws-crt-cpp/crt/aws-c-auth/include"],
+ target_compatible_with = ["@platforms//os:linux"],
+ deps = [
+ ":aws-c-common",
+ ":aws-c-http",
+ ":aws-c-io",
+ ],
+)
+
+cc_library(
+ name = "aws-c-mqtt",
+ srcs = glob(["crt/aws-crt-cpp/crt/aws-c-mqtt/source/**/*.c"]),
+ hdrs = glob(["crt/aws-crt-cpp/crt/aws-c-mqtt/include/**/*.h"]),
+ copts = [
+ "-Wno-cast-qual",
+ "-Wno-cast-align",
+ ],
+ includes = ["crt/aws-crt-cpp/crt/aws-c-mqtt/include"],
+ target_compatible_with = ["@platforms//os:linux"],
+ deps = [
+ ":aws-c-common",
+ ":aws-c-io",
+ ],
+)
+
+cc_library(
+ name = "aws-c-io",
+ srcs = glob([
+ "crt/aws-crt-cpp/crt/aws-c-io/source/*.c",
+ "crt/aws-crt-cpp/crt/aws-c-io/source/linux/*.c",
+ "crt/aws-crt-cpp/crt/aws-c-io/source/s2n/*.c",
+ "crt/aws-crt-cpp/crt/aws-c-io/source/posix/*.c",
+ ]),
+ hdrs = glob(["crt/aws-crt-cpp/crt/aws-c-io/include/**/*.h"]),
+ copts = [
+ "-DAWS_USE_EPOLL",
+ "-Wno-cast-align",
+ "-Wno-cast-qual",
+ "-Wno-sign-compare",
+ "-Wno-unused-parameter",
+ ],
+ includes = ["crt/aws-crt-cpp/crt/aws-c-io/include"],
+ target_compatible_with = ["@platforms//os:linux"],
+ deps = [
+ ":aws-c-cal",
+ ":aws-c-common",
+ ":s2n",
+ ],
+)
+
+cc_library(
+ name = "s2n",
+ srcs = glob([
+ "crt/aws-crt-cpp/crt/s2n/**/*.h",
+ "crt/aws-crt-cpp/crt/s2n/tls/**/*.c",
+ "crt/aws-crt-cpp/crt/s2n/error/**/*.c",
+ "crt/aws-crt-cpp/crt/s2n/utils/**/*.c",
+ "crt/aws-crt-cpp/crt/s2n/stuffer/**/*.c",
+ "crt/aws-crt-cpp/crt/s2n/crypto/**/*.c",
+ ]),
+ hdrs = ["crt/aws-crt-cpp/crt/s2n/api/s2n.h"],
+ copts = [
+ "-Iexternal/aws_sdk/crt/aws-crt-cpp/crt/s2n",
+ "-DS2N_NO_PQ",
+ "-Wno-unknown-pragmas",
+ "-Wno-cast-align",
+ "-Wno-cast-qual",
+ "-Wno-unused-parameter",
+ "-Wno-sign-compare",
+ ],
+ includes = ["crt/aws-crt-cpp/crt/s2n/api"],
+ target_compatible_with = ["@platforms//os:linux"],
+ deps = [
+ "@boringssl//:crypto",
+ ],
+)
diff --git a/debian/aws_sdk.patch b/debian/aws_sdk.patch
new file mode 100644
index 0000000..df613f5
--- /dev/null
+++ b/debian/aws_sdk.patch
@@ -0,0 +1,92 @@
+Submodule crt/aws-crt-cpp contains modified content
+Submodule crt/aws-c-cal contains modified content
+diff --git a/crt/aws-crt-cpp/crt/aws-c-cal/source/unix/openssl_platform_init.c b/crt/aws-crt-cpp/crt/aws-c-cal/source/unix/openssl_platform_init.c
+index 761455b..fc434ba 100644
+--- a/crt/aws-crt-cpp/crt/aws-c-cal/source/unix/openssl_platform_init.c
++++ b/crt/aws-crt-cpp/crt/aws-c-cal/source/unix/openssl_platform_init.c
+@@ -34,7 +34,7 @@ struct openssl_evp_md_ctx_table *g_aws_openssl_evp_md_ctx_table = NULL;
+ /* 1.1 */
+ extern HMAC_CTX *HMAC_CTX_new(void) __attribute__((weak)) __attribute__((used));
+ extern void HMAC_CTX_free(HMAC_CTX *) __attribute__((weak)) __attribute__((used));
+-extern int HMAC_CTX_reset(HMAC_CTX *) __attribute__((weak)) __attribute__((used));
++//extern int HMAC_CTX_reset(HMAC_CTX *) __attribute__((weak)) __attribute__((used));
+
+ /* 1.0.2 */
+ extern void HMAC_CTX_init(HMAC_CTX *) __attribute__((weak)) __attribute__((used));
+@@ -43,8 +43,8 @@ extern void HMAC_CTX_cleanup(HMAC_CTX *) __attribute__((weak)) __attribute__((us
+ /* common */
+ extern int HMAC_Update(HMAC_CTX *, const unsigned char *, size_t) __attribute__((weak)) __attribute__((used));
+ extern int HMAC_Final(HMAC_CTX *, unsigned char *, unsigned int *) __attribute__((weak)) __attribute__((used));
+-extern int HMAC_Init_ex(HMAC_CTX *, const void *, int, const EVP_MD *, ENGINE *) __attribute__((weak))
+-__attribute__((used));
++//extern int HMAC_Init_ex(HMAC_CTX *, const void *, int, const EVP_MD *, ENGINE *) __attribute__((weak))
++//__attribute__((used));
+
+ /* libcrypto 1.1 stub for init */
+ static void s_hmac_ctx_init_noop(HMAC_CTX *ctx) {
+@@ -393,9 +393,9 @@ void aws_cal_platform_init(struct aws_allocator *allocator) {
+ }
+ }
+
+- if (!CRYPTO_get_id_callback()) {
+- CRYPTO_set_id_callback(s_id_fn);
+- }
++ //if (!CRYPTO_get_id_callback()) {
++ //CRYPTO_set_id_callback(s_id_fn);
++ //}
+ }
+
+ void aws_cal_platform_clean_up(void) {
+@@ -408,9 +408,9 @@ void aws_cal_platform_clean_up(void) {
+ aws_mem_release(s_libcrypto_allocator, s_libcrypto_locks);
+ }
+
+- if (CRYPTO_get_id_callback() == s_id_fn) {
+- CRYPTO_set_id_callback(NULL);
+- }
++ //if (CRYPTO_get_id_callback() == s_id_fn) {
++ //CRYPTO_set_id_callback(NULL);
++ //}
+ }
+ #if !defined(__GNUC__) || (__GNUC__ >= 4 && __GNUC_MINOR__ > 1)
+ # pragma GCC diagnostic pop
+Submodule crt/s2n contains modified content
+diff --git a/crt/aws-crt-cpp/crt/s2n/utils/s2n_asn1_time.c b/crt/aws-crt-cpp/crt/s2n/utils/s2n_asn1_time.c
+index 84dbc6df..d3566b81 100755
+--- a/crt/aws-crt-cpp/crt/s2n/utils/s2n_asn1_time.c
++++ b/crt/aws-crt-cpp/crt/s2n/utils/s2n_asn1_time.c
+@@ -46,7 +46,7 @@ typedef enum parser_state {
+ } parser_state;
+
+ static inline long get_gmt_offset(struct tm *t) {
+-#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__ANDROID__) || defined(ANDROID) || defined(__APPLE__) && defined(__MACH__)
++#if defined(__FreeBSD__) || defined(__NetBSD__) || defined(__ANDROID__) || defined(ANDROID) || defined(__APPLE__) && defined(__MACH__) || defined(__USE_MISC)
+ return t->tm_gmtoff;
+ #else
+ return t->__tm_gmtoff;
+diff --git a/crt/aws-crt-cpp/crt/s2n/utils/s2n_init.c b/crt/aws-crt-cpp/crt/s2n/utils/s2n_init.c
+index 0f79f959..ae8122fb 100644
+--- a/crt/aws-crt-cpp/crt/s2n/utils/s2n_init.c
++++ b/crt/aws-crt-cpp/crt/s2n/utils/s2n_init.c
+@@ -45,7 +45,7 @@ int s2n_init(void)
+ GUARD_POSIX(s2n_security_policies_init());
+ GUARD_POSIX(s2n_config_defaults_init());
+ GUARD_POSIX(s2n_extension_type_init());
+- GUARD_AS_POSIX(s2n_pq_init());
++ //GUARD_AS_POSIX(s2n_pq_init());
+
+ S2N_ERROR_IF(atexit(s2n_cleanup_atexit) != 0, S2N_ERR_ATEXIT);
+
+diff --git a/crt/aws-crt-cpp/crt/aws-c-common/include/aws/common/private/lookup3.inl b/crt/aws-crt-cpp/crt/aws-c-common/include/aws/common/private/lookup3.inl
+index 0f79f959..ae8122fb 100644
+--- a/crt/aws-crt-cpp/crt/aws-c-common/include/aws/common/private/lookup3.inl
++++ b/crt/aws-crt-cpp/crt/aws-c-common/include/aws/common/private/lookup3.inl
+@@ -533,7 +533,7 @@
+ * "CPROVER check pop". The masking trick does make the hash noticably
+ * faster for short strings (like English words).
+ */
+-#ifndef VALGRIND
++#if !defined(VALGRIND) && !__has_feature(address_sanitizer) && !__has_feature(memory_sanitizer)
+ #ifdef CBMC
+ # pragma CPROVER check push
+ # pragma CPROVER check disable "pointer"
diff --git a/debian/curl.BUILD b/debian/curl.BUILD
new file mode 100644
index 0000000..95edb78
--- /dev/null
+++ b/debian/curl.BUILD
@@ -0,0 +1,766 @@
+# This is based on:
+# https://github.com/googleapis/google-cloud-cpp/blob/v1.26.1/bazel/curl.BUILD
+# which is based on:
+# https://github.com/tensorflow/tensorflow/blob/51d480498b07346b8b6e2ee3fbd3dc486f60ed96/third_party/curl.BUILD
+# Description:
+# curl is a tool for talking to web servers.
+
+licenses(["notice"]) # MIT/X derivative license
+
+exports_files(["COPYING"])
+
+config_setting(
+ name = "windows",
+ values = {"cpu": "x64_windows"},
+ visibility = ["//visibility:public"],
+)
+
+config_setting(
+ name = "linux_x86_64",
+ constraint_values = [
+ "@platforms//cpu:x86_64",
+ "@platforms//os:linux",
+ ],
+)
+
+config_setting(
+ name = "linux_arm64",
+ constraint_values = [
+ "@platforms//cpu:arm64",
+ "@platforms//os:linux",
+ ],
+)
+
+config_setting(
+ name = "linux_armv7",
+ constraint_values = [
+ "@platforms//cpu:armv7",
+ "@platforms//os:linux",
+ ],
+)
+
+config_setting(
+ name = "macos",
+ values = {"cpu": "darwin"},
+ visibility = ["//visibility:public"],
+)
+
+# On Linux, libcurl needs to know, at compile time, the location for the
+# Certificate Authority (CA) bundle file. By default we dynamically guess the
+# location for most common Linux distribution, but this guessing makes the build
+# not cacheable.
+#
+# In our CI builds we define the CA bundle location using a --define option.
+# This makes the CI results more amenable to caching, at the cost of "some"
+# complexity in this BUILD file.
+#
+# Note that builds issued by developers in the command-line should continue to
+# work, it is just that the build results are less cacheable than they make
+# wish.
+
+# First convert the --define ca_bundle_style=... option into a config_setting
+# rule:
+config_setting(
+ name = "ca_bundle_style_is_redhat",
+ define_values = {
+ "ca_bundle_style": "redhat",
+ },
+)
+
+config_setting(
+ name = "ca_bundle_style_is_debian",
+ define_values = {
+ "ca_bundle_style": "debian",
+ },
+)
+
+# This just generates a header file with the right value for the CURL_CA_BUNDLE
+# macro. The file is included by curl_config.h if the macro is *not* provided
+# in the build line using -DCURL_CA_BUNDLE=<some-path>:
+genrule(
+ name = "gen-ca-bundle-linux",
+ outs = ["include/curl_ca_bundle_location.h"],
+ cmd = """
+ if [ -f /etc/fedora-release ]; then
+ echo '#define CURL_CA_BUNDLE "/etc/pki/tls/certs/ca-bundle.crt"'
+ elif [ -f /etc/redhat-release ]; then
+ echo '#define CURL_CA_BUNDLE "/etc/pki/tls/certs/ca-bundle.crt"'
+ elif [ -f /etc/debian_version ]; then
+ echo '#define CURL_CA_BUNDLE "/etc/ssl/certs/ca-certificates.crt"'
+ elif [ -f /etc/arch-release ]; then
+ echo '#define CURL_CA_BUNDLE "/etc/ssl/certs/ca-certificates.crt"'
+ else
+ >&2 echo "Unknown platform, cannot guess location of CA bundle"
+ exit 1
+ fi >$@
+ """,
+)
+
+# This is the default depending to define CURL_CA_BUNDLE on Linux, it is tagged
+# as `no-cache` because the output on one host cannot be used in another host.
+cc_library(
+ name = "define-ca-bundle-location-guess",
+ hdrs = ["include/curl_ca_bundle_location.h"],
+ tags = ["no-cache"],
+)
+
+# This library is used on redhat-like hosts (RHEL, CentOS, and Fedora for
+# example), because all such hosts use the same value, we can cache this
+# library.
+cc_library(
+ name = "define-ca-bundle-location-redhat",
+ hdrs = [],
+ defines = ["""CURL_CA_BUNDLE='"/etc/pki/tls/certs/ca-bundle.crt"'"""],
+ tags = [],
+)
+
+# This library is used on debian-like hosts (Debian, and Ubuntu for example),
+# because all such hosts use the same value, we can cache this library.
+cc_library(
+ name = "define-ca-bundle-location-debian",
+ hdrs = [],
+ defines = ["""CURL_CA_BUNDLE='"/etc/ssl/certs/ca-certificates.crt"'"""],
+ tags = [],
+)
+
+# This library uses the `ca_bundle_style*` config setting to branch the
+# dependency
+cc_library(
+ name = "define-ca-bundle-location-linux",
+ deps = select({
+ ":ca_bundle_style_is_redhat": [":define-ca-bundle-location-redhat"],
+ ":ca_bundle_style_is_debian": [":define-ca-bundle-location-debian"],
+ "//conditions:default": [":define-ca-bundle-location-guess"],
+ }),
+)
+
+# Finally, on Windows and macOS we do not need to define CURL_CA_BUNDLE at all,
+# so on those platforms we skip the branch of the dependencies altogether.
+cc_library(
+ name = "define-ca-bundle-location",
+ deps = select({
+ ":windows": [],
+ ":macos": [],
+ "//conditions:default": [":define-ca-bundle-location-linux"],
+ }),
+)
+
+CURL_WIN_COPTS = [
+ "/Iexternal/com_github_curl_curl/lib",
+ "/DBUILDING_LIBCURL",
+ "/DHAVE_CONFIG_H",
+ "/DCURL_DISABLE_FTP",
+ "/DCURL_DISABLE_NTLM",
+ "/DCURL_DISABLE_PROXY",
+ "/DHAVE_LIBZ",
+ "/DHAVE_ZLIB_H",
+ # Defining _USING_V110_SDK71_ is hackery to defeat curl's incorrect
+ # detection of what OS releases we can build on with VC 2012. This
+ # may not be needed (or may have to change) if the WINVER setting
+ # changes in //third_party/msvc/vc_12_0/CROSSTOOL.
+ "/D_USING_V110_SDK71_",
+]
+
+CURL_WIN_SRCS = [
+ "lib/asyn-thread.c",
+ "lib/inet_ntop.c",
+ "lib/system_win32.c",
+ "lib/x509asn1.c",
+ "lib/vtls/schannel.c",
+ "lib/vtls/schannel_verify.c",
+ "lib/idn_win32.c",
+]
+
+cc_library(
+ name = "curl",
+ srcs = [
+ "include/curl_config.h",
+ "lib/amigaos.h",
+ "lib/arpa_telnet.h",
+ "lib/asyn.h",
+ "lib/asyn-ares.c",
+ "lib/base64.c",
+ "lib/config-win32.h",
+ "lib/conncache.c",
+ "lib/conncache.h",
+ "lib/connect.c",
+ "lib/connect.h",
+ "lib/content_encoding.c",
+ "lib/content_encoding.h",
+ "lib/cookie.c",
+ "lib/cookie.h",
+ "lib/curl_addrinfo.c",
+ "lib/curl_addrinfo.h",
+ "lib/curl_base64.h",
+ "lib/curl_ctype.c",
+ "lib/curl_ctype.h",
+ "lib/curl_des.h",
+ "lib/curl_endian.h",
+ "lib/curl_fnmatch.c",
+ "lib/curl_fnmatch.h",
+ "lib/curl_gethostname.c",
+ "lib/curl_gethostname.h",
+ "lib/curl_gssapi.h",
+ "lib/curl_hmac.h",
+ "lib/curl_ldap.h",
+ "lib/curl_md4.h",
+ "lib/curl_md5.h",
+ "lib/curl_memory.h",
+ "lib/curl_memrchr.c",
+ "lib/curl_memrchr.h",
+ "lib/curl_multibyte.c",
+ "lib/curl_multibyte.h",
+ "lib/curl_ntlm_core.h",
+ "lib/curl_ntlm_wb.h",
+ "lib/curl_printf.h",
+ "lib/curl_rtmp.c",
+ "lib/curl_rtmp.h",
+ "lib/curl_sasl.c",
+ "lib/curl_sasl.h",
+ "lib/curl_sec.h",
+ "lib/curl_setup.h",
+ "lib/curl_setup_once.h",
+ "lib/curl_sha256.h",
+ "lib/curl_sspi.c",
+ "lib/curl_sspi.h",
+ "lib/curl_threads.c",
+ "lib/curl_threads.h",
+ "lib/curlx.h",
+ "lib/dict.h",
+ "lib/dotdot.c",
+ "lib/dotdot.h",
+ "lib/easy.c",
+ "lib/easyif.h",
+ "lib/escape.c",
+ "lib/escape.h",
+ "lib/file.h",
+ "lib/fileinfo.c",
+ "lib/fileinfo.h",
+ "lib/formdata.c",
+ "lib/formdata.h",
+ "lib/ftp.h",
+ "lib/ftplistparser.h",
+ "lib/getenv.c",
+ "lib/getinfo.c",
+ "lib/getinfo.h",
+ "lib/gopher.h",
+ "lib/hash.c",
+ "lib/hash.h",
+ "lib/hmac.c",
+ "lib/hostasyn.c",
+ "lib/hostcheck.c",
+ "lib/hostcheck.h",
+ "lib/hostip.c",
+ "lib/hostip.h",
+ "lib/hostip4.c",
+ "lib/hostip6.c",
+ "lib/hostsyn.c",
+ "lib/http.c",
+ "lib/http.h",
+ "lib/http2.c",
+ "lib/http2.h",
+ "lib/http_chunks.c",
+ "lib/http_chunks.h",
+ "lib/http_digest.c",
+ "lib/http_digest.h",
+ "lib/http_negotiate.h",
+ "lib/http_ntlm.h",
+ "lib/http_proxy.c",
+ "lib/http_proxy.h",
+ "lib/if2ip.c",
+ "lib/if2ip.h",
+ "lib/imap.h",
+ "lib/inet_ntop.h",
+ "lib/inet_pton.c",
+ "lib/inet_pton.h",
+ "lib/krb5.c",
+ "lib/llist.c",
+ "lib/llist.h",
+ "lib/md4.c",
+ "lib/md5.c",
+ "lib/memdebug.c",
+ "lib/memdebug.h",
+ "lib/mime.c",
+ "lib/mime.h",
+ "lib/mprintf.c",
+ "lib/multi.c",
+ "lib/multihandle.h",
+ "lib/multiif.h",
+ "lib/netrc.c",
+ "lib/netrc.h",
+ "lib/non-ascii.h",
+ "lib/nonblock.c",
+ "lib/nonblock.h",
+ "lib/nwlib.c",
+ "lib/nwos.c",
+ "lib/parsedate.c",
+ "lib/parsedate.h",
+ "lib/pingpong.h",
+ "lib/pingpong.c",
+ "lib/pop3.h",
+ "lib/progress.c",
+ "lib/progress.h",
+ "lib/quic.h",
+ "lib/rand.c",
+ "lib/rand.h",
+ "lib/rename.h",
+ "lib/rename.c",
+ "lib/rtsp.c",
+ "lib/rtsp.h",
+ "lib/security.c",
+ "lib/select.c",
+ "lib/select.h",
+ "lib/sendf.c",
+ "lib/sendf.h",
+ "lib/setopt.c",
+ "lib/setopt.h",
+ "lib/setup-os400.h",
+ "lib/setup-vms.h",
+ "lib/sha256.c",
+ "lib/share.c",
+ "lib/share.h",
+ "lib/sigpipe.h",
+ "lib/slist.c",
+ "lib/slist.h",
+ "lib/smb.h",
+ "lib/smtp.h",
+ "lib/sockaddr.h",
+ "lib/socketpair.h",
+ "lib/socketpair.c",
+ "lib/socks.c",
+ "lib/socks.h",
+ "lib/speedcheck.c",
+ "lib/speedcheck.h",
+ "lib/splay.c",
+ "lib/splay.h",
+ "lib/strcase.c",
+ "lib/strcase.h",
+ "lib/strdup.c",
+ "lib/strdup.h",
+ "lib/strerror.c",
+ "lib/strerror.h",
+ "lib/strtok.c",
+ "lib/strtok.h",
+ "lib/strtoofft.c",
+ "lib/strtoofft.h",
+ "lib/system_win32.h",
+ "lib/telnet.h",
+ "lib/tftp.h",
+ "lib/timeval.c",
+ "lib/timeval.h",
+ "lib/transfer.c",
+ "lib/transfer.h",
+ "lib/url.c",
+ "lib/url.h",
+ "lib/urldata.h",
+ "lib/vauth/cleartext.c",
+ "lib/vauth/cram.c",
+ "lib/vauth/digest.c",
+ "lib/vauth/digest.h",
+ "lib/vauth/ntlm.h",
+ "lib/vauth/oauth2.c",
+ "lib/vauth/vauth.c",
+ "lib/vauth/vauth.h",
+ "lib/version.c",
+ "lib/vssh/ssh.h",
+ "lib/vtls/bearssl.h",
+ "lib/vtls/gskit.h",
+ "lib/vtls/gtls.h",
+ "lib/vtls/mbedtls.h",
+ "lib/vtls/nssg.h",
+ "lib/vtls/openssl.h",
+ "lib/vtls/schannel.h",
+ "lib/vtls/vtls.c",
+ "lib/vtls/vtls.h",
+ "lib/vtls/wolfssl.h",
+ "lib/warnless.c",
+ "lib/warnless.h",
+ "lib/wildcard.c",
+ "lib/wildcard.h",
+ "lib/x509asn1.h",
+ "lib/psl.h",
+ "lib/psl.c",
+ "lib/vtls/sectransp.h",
+ "lib/vtls/mesalink.h",
+ "lib/vtls/mesalink.c",
+ "lib/curl_get_line.h",
+ "lib/curl_get_line.c",
+ "lib/urlapi-int.h",
+ "lib/urlapi.c",
+ "lib/altsvc.h",
+ "lib/altsvc.c",
+ "lib/doh.h",
+ "lib/doh.c",
+ ] + select({
+ ":macos": [
+ "lib/vtls/sectransp.c",
+ ],
+ ":windows": CURL_WIN_SRCS,
+ "//conditions:default": [
+ "lib/vtls/openssl.c",
+ ],
+ }),
+ hdrs = [
+ "include/curl/curl.h",
+ "include/curl/curlver.h",
+ "include/curl/easy.h",
+ "include/curl/mprintf.h",
+ "include/curl/multi.h",
+ "include/curl/stdcheaders.h",
+ "include/curl/system.h",
+ "include/curl/typecheck-gcc.h",
+ "include/curl/urlapi.h",
+ ],
+ copts = select({
+ ":windows": CURL_WIN_COPTS,
+ "//conditions:default": [
+ "-Iexternal/com_github_curl_curl/lib",
+ "-D_GNU_SOURCE",
+ "-DBUILDING_LIBCURL",
+ "-DHAVE_CONFIG_H",
+ "-DCURL_DISABLE_FTP",
+ "-DCURL_DISABLE_NTLM", # turning it off in configure is not enough
+ "-DHAVE_LIBZ",
+ "-DHAVE_ZLIB_H",
+ "-Wno-string-plus-int",
+ "-Wno-cast-qual",
+ "-Wno-format-nonliteral",
+ "-Wno-tautological-type-limit-compare",
+ ],
+ }) + select({
+ ":macos": [
+ "-fno-constant-cfstrings",
+ ],
+ "//conditions:default": [],
+ }),
+ defines = ["CURL_STATICLIB"] + select({
+ ":windows": [
+ # See curl.h for discussion of write size and Windows
+ "CURL_MAX_WRITE_SIZE=16384",
+ ],
+ "//conditions:default": [
+ "CURL_MAX_WRITE_SIZE=65536",
+ ],
+ }),
+ includes = ["include"],
+ linkopts = select({
+ ":macos": [
+ "-Wl,-framework",
+ "-Wl,CoreFoundation",
+ "-Wl,-framework",
+ "-Wl,Security",
+ ],
+ ":windows": [
+ "-DEFAULTLIB:ws2_32.lib",
+ "-DEFAULTLIB:advapi32.lib",
+ "-DEFAULTLIB:crypt32.lib",
+ "-DEFAULTLIB:Normaliz.lib",
+ ],
+ "//conditions:default": [
+ "-lrt",
+ ],
+ }),
+ visibility = ["//visibility:public"],
+ deps = [
+ # Use the same version of zlib that gRPC does.
+ "@zlib//:zlib",
+ ":define-ca-bundle-location",
+ ] + select({
+ ":windows": [],
+ "//conditions:default": [
+ "@boringssl//:ssl",
+ ],
+ }),
+)
+
+genrule(
+ name = "configure",
+ outs = ["include/curl_config.h"],
+ cmd = "\n".join([
+ "cat <<'EOF' >$@",
+ "#ifndef EXTERNAL_CURL_INCLUDE_CURL_CONFIG_H_",
+ "#define EXTERNAL_CURL_INCLUDE_CURL_CONFIG_H_",
+ "",
+ "#if !defined(_WIN32) && !defined(__APPLE__)",
+ "# include <openssl/opensslv.h>",
+ "# if defined(OPENSSL_IS_BORINGSSL)",
+ "# define HAVE_BORINGSSL 1",
+ "# endif",
+ "#endif",
+ "",
+ "#if defined(_WIN32)",
+ "# include \"lib/config-win32.h\"",
+ "# define BUILDING_LIBCURL 1",
+ "# define CURL_DISABLE_CRYPTO_AUTH 1",
+ "# define CURL_DISABLE_DICT 1",
+ "# define CURL_DISABLE_FILE 1",
+ "# define CURL_DISABLE_GOPHER 1",
+ "# define CURL_DISABLE_IMAP 1",
+ "# define CURL_DISABLE_LDAP 1",
+ "# define CURL_DISABLE_LDAPS 1",
+ "# define CURL_DISABLE_POP3 1",
+ "# define CURL_PULL_WS2TCPIP_H 1",
+ "# define CURL_DISABLE_SMTP 1",
+ "# define CURL_DISABLE_TELNET 1",
+ "# define CURL_DISABLE_TFTP 1",
+ "# define CURL_PULL_WS2TCPIP_H 1",
+ "# define USE_WINDOWS_SSPI 1",
+ "# define USE_WIN32_IDN 1",
+ "# define USE_SCHANNEL 1",
+ "# define WANT_IDN_PROTOTYPES 1",
+ "#elif defined(__APPLE__)",
+ "# define HAVE_FSETXATTR_6 1",
+ "# define HAVE_SETMODE 1",
+ "# define HAVE_SYS_FILIO_H 1",
+ "# define HAVE_SYS_SOCKIO_H 1",
+ "# define OS \"x86_64-apple-darwin15.5.0\"",
+ "# define USE_SECTRANSP 1",
+ "#else",
+ "# if !defined(CURL_CA_BUNDLE)",
+ "# include \"include/curl_ca_bundle_location.h\"",
+ "# endif // CURL_CA_BUNDLE",
+ "# define GETSERVBYPORT_R_ARGS 6",
+ "# define GETSERVBYPORT_R_BUFSIZE 4096",
+ "# define HAVE_BORINGSSL 1",
+ "# define HAVE_CLOCK_GETTIME_MONOTONIC 1",
+ "# define HAVE_CONNECT 1",
+ "# define HAVE_CRYPTO_CLEANUP_ALL_EX_DATA 1",
+ "# define HAVE_FSETXATTR_5 1",
+ "# define HAVE_GETHOSTBYADDR_R 1",
+ "# define HAVE_GETHOSTBYADDR_R_8 1",
+ "# define HAVE_GETHOSTBYNAME_R 1",
+ "# define HAVE_GETHOSTBYNAME_R_6 1",
+ "# define HAVE_GETSERVBYPORT_R 1",
+ "# define HAVE_LIBSSL 1",
+ "# define HAVE_MALLOC_H 1",
+ "# define HAVE_MSG_NOSIGNAL 1",
+ "# define HAVE_OPENSSL_CRYPTO_H 1",
+ "# define HAVE_OPENSSL_ERR_H 1",
+ "# define HAVE_OPENSSL_PEM_H 1",
+ "# define HAVE_OPENSSL_PKCS12_H 1",
+ "# define HAVE_OPENSSL_RSA_H 1",
+ "# define HAVE_OPENSSL_SSL_H 1",
+ "# define HAVE_OPENSSL_X509_H 1",
+ "# define HAVE_RAND_EGD 1",
+ "# define HAVE_RAND_STATUS 1",
+ "# define HAVE_SSL_GET_SHUTDOWN 1",
+ "# define HAVE_TERMIOS_H 1",
+ "# define OS \"x86_64-pc-linux-gnu\"",
+ "# define RANDOM_FILE \"/dev/urandom\"",
+ "# define USE_OPENSSL 1",
+ "#endif",
+ "",
+ "#if !defined(_WIN32)",
+ "# define CURL_DISABLE_DICT 1",
+ "# define CURL_DISABLE_FILE 1",
+ "# define CURL_DISABLE_GOPHER 1",
+ "# define CURL_DISABLE_IMAP 1",
+ "# define CURL_DISABLE_LDAP 1",
+ "# define CURL_DISABLE_LDAPS 1",
+ "# define CURL_DISABLE_POP3 1",
+ "# define CURL_DISABLE_SMTP 1",
+ "# define CURL_DISABLE_TELNET 1",
+ "# define CURL_DISABLE_TFTP 1",
+ "# define CURL_EXTERN_SYMBOL __attribute__ ((__visibility__ (\"default\")))",
+ "# define ENABLE_IPV6 1",
+ "# define GETHOSTNAME_TYPE_ARG2 size_t",
+ "# define GETNAMEINFO_QUAL_ARG1 const",
+ "# define GETNAMEINFO_TYPE_ARG1 struct sockaddr *",
+ "# define GETNAMEINFO_TYPE_ARG2 socklen_t",
+ "# define GETNAMEINFO_TYPE_ARG46 socklen_t",
+ "# define GETNAMEINFO_TYPE_ARG7 int",
+ "# define HAVE_ALARM 1",
+ "# define HAVE_ALLOCA_H 1",
+ "# define HAVE_ARPA_INET_H 1",
+ "# define HAVE_ARPA_TFTP_H 1",
+ "# define HAVE_ASSERT_H 1",
+ "# define HAVE_BASENAME 1",
+ "# define HAVE_BOOL_T 1",
+ "# define HAVE_CONNECT 1",
+ "# define HAVE_DLFCN_H 1",
+ "# define HAVE_ERRNO_H 1",
+ "# define HAVE_FCNTL 1",
+ "# define HAVE_FCNTL_H 1",
+ "# define HAVE_FCNTL_O_NONBLOCK 1",
+ "# define HAVE_FDOPEN 1",
+ "# define HAVE_FORK 1",
+ "# define HAVE_FREEADDRINFO 1",
+ "# define HAVE_FREEIFADDRS 1",
+ "# if !defined(__ANDROID__)",
+ "# define HAVE_FSETXATTR 1",
+ "# endif",
+ "# define HAVE_FTRUNCATE 1",
+ "# define HAVE_GAI_STRERROR 1",
+ "# define HAVE_GETADDRINFO 1",
+ "# define HAVE_GETADDRINFO_THREADSAFE 1",
+ "# define HAVE_GETEUID 1",
+ "# define HAVE_GETHOSTBYADDR 1",
+ "# define HAVE_GETHOSTBYNAME 1",
+ "# define HAVE_GETHOSTNAME 1",
+ "# if !defined(__ANDROID__)",
+ "# define HAVE_GETIFADDRS 1",
+ "# endif",
+ "# define HAVE_GETNAMEINFO 1",
+ "# define HAVE_GETPEERNAME 1",
+ "# define HAVE_GETPPID 1",
+ "# define HAVE_GETPROTOBYNAME 1",
+ "# define HAVE_GETPWUID 1",
+ "# if !defined(__ANDROID__)",
+ "# define HAVE_GETPWUID_R 1",
+ "# endif",
+ "# define HAVE_GETRLIMIT 1",
+ "# define HAVE_GETSOCKNAME 1",
+ "# define HAVE_GETTIMEOFDAY 1",
+ "# define HAVE_GMTIME_R 1",
+ "# if !defined(__ANDROID__)",
+ "# define HAVE_IFADDRS_H 1",
+ "# endif",
+ "# define HAVE_IF_NAMETOINDEX 1",
+ "# define HAVE_INET_ADDR 1",
+ "# define HAVE_INET_NTOP 1",
+ "# define HAVE_INET_PTON 1",
+ "# define HAVE_INTTYPES_H 1",
+ "# define HAVE_IOCTL 1",
+ "# define HAVE_IOCTL_FIONBIO 1",
+ "# define HAVE_IOCTL_SIOCGIFADDR 1",
+ "# define HAVE_LIBGEN_H 1",
+ "# define HAVE_LIBZ 1",
+ "# define HAVE_LIMITS_H 1",
+ "# define HAVE_LL 1",
+ "# define HAVE_LOCALE_H 1",
+ "# define HAVE_LOCALTIME_R 1",
+ "# define HAVE_LONGLONG 1",
+ "# define HAVE_MEMORY_H 1",
+ "# define HAVE_NETDB_H 1",
+ "# define HAVE_NETINET_IN_H 1",
+ "# define HAVE_NETINET_TCP_H 1",
+ "# define HAVE_NET_IF_H 1",
+ "# define HAVE_PERROR 1",
+ "# define HAVE_PIPE 1",
+ "# define HAVE_POLL 1",
+ "# define HAVE_POLL_FINE 1",
+ "# define HAVE_POLL_H 1",
+ "# define HAVE_POSIX_STRERROR_R 1",
+ "# define HAVE_PWD_H 1",
+ "# define HAVE_RECV 1",
+ "# define HAVE_SELECT 1",
+ "# define HAVE_SEND 1",
+ "# define HAVE_SETJMP_H 1",
+ "# define HAVE_SETLOCALE 1",
+ "# define HAVE_SETRLIMIT 1",
+ "# define HAVE_SETSOCKOPT 1",
+ "# define HAVE_SGTTY_H 1",
+ "# define HAVE_SIGACTION 1",
+ "# define HAVE_SIGINTERRUPT 1",
+ "# define HAVE_SIGNAL 1",
+ "# define HAVE_SIGNAL_H 1",
+ "# define HAVE_SIGSETJMP 1",
+ "# define HAVE_SIG_ATOMIC_T 1",
+ "# define HAVE_SOCKADDR_IN6_SIN6_SCOPE_ID 1",
+ "# define HAVE_SOCKET 1",
+ "# define HAVE_SOCKETPAIR 1",
+ "# define HAVE_STDBOOL_H 1",
+ "# define HAVE_STDINT_H 1",
+ "# define HAVE_STDIO_H 1",
+ "# define HAVE_STDLIB_H 1",
+ "# define HAVE_STRCASECMP 1",
+ "# define HAVE_STRDUP 1",
+ "# define HAVE_STRERROR_R 1",
+ "# define HAVE_STRINGS_H 1",
+ "# define HAVE_STRING_H 1",
+ "# define HAVE_STRNCASECMP 1",
+ "# define HAVE_STRSTR 1",
+ "# define HAVE_STRTOK_R 1",
+ "# define HAVE_STRTOLL 1",
+ "# define HAVE_STRUCT_SOCKADDR_STORAGE 1",
+ "# define HAVE_STRUCT_TIMEVAL 1",
+ "# define HAVE_SYS_IOCTL_H 1",
+ "# define HAVE_SYS_PARAM_H 1",
+ "# define HAVE_SYS_POLL_H 1",
+ "# define HAVE_SYS_RESOURCE_H 1",
+ "# define HAVE_SYS_SELECT_H 1",
+ "# define HAVE_SYS_SOCKET_H 1",
+ "# define HAVE_SYS_STAT_H 1",
+ "# define HAVE_SYS_TIME_H 1",
+ "# define HAVE_SYS_TYPES_H 1",
+ "# define HAVE_SYS_UIO_H 1",
+ "# define HAVE_SYS_UN_H 1",
+ "# define HAVE_SYS_WAIT_H 1",
+ "# define HAVE_SYS_XATTR_H 1",
+ "# define HAVE_TIME_H 1",
+ "# define HAVE_UNAME 1",
+ "# define HAVE_UNISTD_H 1",
+ "# define HAVE_UTIME 1",
+ "# define HAVE_UTIME_H 1",
+ "# define HAVE_VARIADIC_MACROS_C99 1",
+ "# define HAVE_VARIADIC_MACROS_GCC 1",
+ "# define HAVE_WRITABLE_ARGV 1",
+ "# define HAVE_WRITEV 1",
+ "# define HAVE_ZLIB_H 1",
+ "# define LT_OBJDIR \".libs/\"",
+ "# define PACKAGE \"curl\"",
+ "# define PACKAGE_BUGREPORT \"a suitable curl mailing list: https://curl.haxx.se/mail/\"",
+ "# define PACKAGE_NAME \"curl\"",
+ "# define PACKAGE_STRING \"curl -\"",
+ "# define PACKAGE_TARNAME \"curl\"",
+ "# define PACKAGE_URL \"\"",
+ "# define PACKAGE_VERSION \"-\"",
+ "# define RECV_TYPE_ARG1 int",
+ "# define RECV_TYPE_ARG2 void *",
+ "# define RECV_TYPE_ARG3 size_t",
+ "# define RECV_TYPE_ARG4 int",
+ "# define RECV_TYPE_RETV ssize_t",
+ "# define RETSIGTYPE void",
+ "# define SELECT_QUAL_ARG5",
+ "# define SELECT_TYPE_ARG1 int",
+ "# define SELECT_TYPE_ARG234 fd_set *",
+ "# define SELECT_TYPE_ARG5 struct timeval *",
+ "# define SELECT_TYPE_RETV int",
+ "# define SEND_QUAL_ARG2 const",
+ "# define SEND_TYPE_ARG1 int",
+ "# define SEND_TYPE_ARG2 void *",
+ "# define SEND_TYPE_ARG3 size_t",
+ "# define SEND_TYPE_ARG4 int",
+ "# define SEND_TYPE_RETV ssize_t",
+ "# define SIZEOF_INT 4",
+ "# define SIZEOF_SHORT 2",
+ "# define STDC_HEADERS 1",
+ "# define STRERROR_R_TYPE_ARG3 size_t",
+ "# define TIME_WITH_SYS_TIME 1",
+ "# define VERSION \"-\"",
+ "# ifndef _DARWIN_USE_64_BIT_INODE",
+ "# define _DARWIN_USE_64_BIT_INODE 1",
+ "# endif",
+ "#endif",
+ "",
+ "#endif // EXTERNAL_CURL_INCLUDE_CURL_CONFIG_H_",
+ ]) + "\n" + select({
+ ":linux_x86_64": "\n".join([
+ "# define SIZEOF_LONG 8",
+ "# define SIZEOF_OFF_T 8",
+ "# define SIZEOF_CURL_OFF_T 8",
+ "# define SIZEOF_SIZE_T 8",
+ "# define SIZEOF_TIME_T 8",
+ "# define SIZEOF_VOIDP 8",
+ ]),
+ ":linux_arm64": "\n".join([
+ "# define SIZEOF_LONG 8",
+ "# define SIZEOF_OFF_T 8",
+ "# define SIZEOF_CURL_OFF_T 8",
+ "# define SIZEOF_SIZE_T 8",
+ "# define SIZEOF_TIME_T 8",
+ "# define SIZEOF_VOIDP 8",
+ ]),
+ ":linux_armv7": "\n".join(
+ [
+ "# define SIZEOF_LONG 4",
+ "# define SIZEOF_OFF_T 4",
+ "# define SIZEOF_CURL_OFF_T 4",
+ "# define SIZEOF_SIZE_T 4",
+ "# define SIZEOF_TIME_T 4",
+ "# define SIZEOF_VOIDP 4",
+ ],
+ ),
+ "//conditions:default": "",
+ }) + "\nEOF",
+)
diff --git a/tools/BUILD b/tools/BUILD
index d64d602..8ea984a 100644
--- a/tools/BUILD
+++ b/tools/BUILD
@@ -56,6 +56,13 @@
)
config_setting(
+ name = "cpu_armv7",
+ constraint_values = [
+ "@platforms//cpu:armv7",
+ ],
+)
+
+config_setting(
name = "cpu_arm64",
constraint_values = ["@platforms//cpu:arm64"],
)