MCPcopy Create free account
hub / github.com/ml-explore/mlx-data / AWSFileFetcher

Method AWSFileFetcher

mlx/data/core/AWSFileFetcher.cpp:44–83  ·  view source on GitHub ↗

Source from the content-addressed store, hash-verified

42}
43
44AWSFileFetcher::AWSFileFetcher(
45 const std::string& bucket,
46 const AWSFileFetcherOptions& opt)
47 : FileFetcher(
48 opt.num_prefetch_max,
49 opt.num_prefetch_threads,
50 opt.num_kept_files,
51 opt.verbose),
52 bucket_(bucket),
53 prefix_(opt.prefix),
54 local_prefix_(opt.local_prefix),
55 buffer_size_(opt.buffer_size),
56 num_threads_(opt.num_threads),
57 dtor_called_(false) {
58 config_ = std::make_unique<Aws::Client::ClientConfiguration>();
59 config_->endpointOverride = opt.endpoint;
60 config_->verifySSL = opt.verify_ssl;
61 config_->connectTimeoutMs = opt.connect_timeout_ms;
62 config_->retryStrategy =
63 std::make_shared<Aws::Client::DefaultRetryStrategy>(opt.num_retry_max);
64 config_->maxConnections = opt.num_connection_max;
65 config_->region = opt.region;
66
67 // C++ SDK does not check AWS_CA_BUNDLE, so we do it ourselves
68 if (opt.ca_bundle.empty()) {
69 auto aws_ca_bundle_env = std::getenv("AWS_CA_BUNDLE");
70 if (aws_ca_bundle_env) {
71 config_->caFile = std::string(aws_ca_bundle_env);
72 }
73 } else {
74 config_->caFile = opt.ca_bundle;
75 }
76 config_virtual_host_ = opt.virtual_host;
77
78 update_credentials(
79 opt.access_key_id,
80 opt.secret_access_key,
81 opt.session_token,
82 opt.expiration);
83}
84
85void AWSFileFetcher::update_credentials(
86 const std::string& access_key_id,

Callers

nothing calls this directly

Calls

no outgoing calls

Tested by

no test coverage detected