Upload file to S3 storage
(
self,
path: str,
buf: BytesIO,
info: MediaPipeline.SpiderInfo,
meta: dict[str, Any] | None = None,
headers: dict[str, str] | None = None,
)
| 210 | ) |
| 211 | |
| 212 | def persist_file( |
| 213 | self, |
| 214 | path: str, |
| 215 | buf: BytesIO, |
| 216 | info: MediaPipeline.SpiderInfo, |
| 217 | meta: dict[str, Any] | None = None, |
| 218 | headers: dict[str, str] | None = None, |
| 219 | ) -> Deferred[Any]: |
| 220 | """Upload file to S3 storage""" |
| 221 | key_name = f"{self.prefix}{path}" |
| 222 | buf.seek(0) |
| 223 | extra = self._headers_to_botocore_kwargs(self.HEADERS) |
| 224 | if headers: |
| 225 | extra.update(self._headers_to_botocore_kwargs(headers)) |
| 226 | return deferred_from_coro( |
| 227 | run_in_thread( |
| 228 | self.s3_client.put_object, # type: ignore[attr-defined] |
| 229 | Bucket=self.bucket, |
| 230 | Key=key_name, |
| 231 | Body=buf, |
| 232 | Metadata={k: str(v) for k, v in meta.items()} if meta else {}, |
| 233 | ACL=self.POLICY, |
| 234 | **extra, |
| 235 | ) |
| 236 | ) |
| 237 | |
| 238 | def _headers_to_botocore_kwargs(self, headers: dict[str, Any]) -> dict[str, Any]: |
| 239 | """Convert headers to botocore keyword arguments.""" |