Class AWSFileFetcher

mlx/data/core/AWSFileFetcher.h:51–101

};

class AWSFileFetcher : public FileFetcher {
 public:
  AWSFileFetcher(
      const std::string& bucket,
      const AWSFileFetcherOptions& opt = AWSFileFetcherOptions());

  int64_t get_size(const std::string& filename) const;

  virtual void backend_fetch(const std::string& filename) const override;
  virtual void backend_erase(const std::string& filename) const override;

  void update_credentials(
      const std::string& access_key_id = "",
      const std::string& secret_access_key = "",
      const std::string& session_token = "",
      const std::string& expiration = "") const;

  void update_credentials_with_callback(
      std::function<
          std::tuple<std::string, std::string, std::string, std::string>()>
          callback,
      int64_t period = 0);

  bool are_credentials_expired() const;

  virtual ~AWSFileFetcher();

 protected:
  void check_credentials() const;

  std::string bucket_;
  std::filesystem::path prefix_;
  std::filesystem::path local_prefix_;
  std::unique_ptr<Aws::Client::ClientConfiguration> config_;
  bool config_virtual_host_;
  int64_t buffer_size_;
  int num_threads_;
  mutable std::unique_ptr<Aws::Auth::AWSCredentials> credentials_;
  mutable std::unique_ptr<Aws::S3::S3Client> client_;
  mutable std::atomic<bool> dtor_called_;
  mutable std::shared_mutex client_mutex_;

  // credentials periodic update
  std::function<
      std::tuple<std::string, std::string, std::string, std::string>()>
      credentials_callback_;
  int64_t credentials_period_;
  mutable std::chrono::time_point<std::chrono::system_clock>
      credentials_timestamp_;
  mutable std::shared_mutex credentials_mutex_;
};

} // namespace core
} // namespace data