OptimalPartInfo - calculate the optimal part info for a given object size. NOTE: Assumption here is that for any object to be uploaded to any S3 compatible object storage it will have the following parameters as constants. maxPartsCount - 10000 minPartSize - 16MiB maxObjectSize - ~48.83TiB (max
(objectSize int64, configuredPartSize uint64)
| 69 | // minPartSize - 16MiB |
| 70 | // maxObjectSize - ~48.83TiB (maxPartSize * maxPartsCount) |
| 71 | func OptimalPartInfo(objectSize int64, configuredPartSize uint64) (totalPartsCount int, partSize, lastPartSize int64, err error) { |
| 72 | // When object size is unknown (-1), default to 5TiB to limit memory usage. |
| 73 | // This results in ~537MiB part sizes. For larger objects (up to ~48.83TiB), |
| 74 | // callers should set configuredPartSize explicitly to control memory usage. |
| 75 | var unknownSize bool |
| 76 | if objectSize == -1 { |
| 77 | unknownSize = true |
| 78 | objectSize = maxMultipartPutObjectSize |
| 79 | } |
| 80 | |
| 81 | // object size is larger than supported maximum. |
| 82 | if objectSize > maxObjectSize { |
| 83 | err = errEntityTooLarge(objectSize, maxObjectSize, "", "") |
| 84 | return totalPartsCount, partSize, lastPartSize, err |
| 85 | } |
| 86 | |
| 87 | var partSizeFlt float64 |
| 88 | if configuredPartSize > 0 { |
| 89 | if int64(configuredPartSize) > objectSize { |
| 90 | err = errEntityTooLarge(int64(configuredPartSize), objectSize, "", "") |
| 91 | return totalPartsCount, partSize, lastPartSize, err |
| 92 | } |
| 93 | |
| 94 | if !unknownSize { |
| 95 | if objectSize > (int64(configuredPartSize) * maxPartsCount) { |
| 96 | err = errInvalidArgument("Part size * max_parts(10000) is lesser than input objectSize.") |
| 97 | return totalPartsCount, partSize, lastPartSize, err |
| 98 | } |
| 99 | } |
| 100 | |
| 101 | if configuredPartSize < absMinPartSize { |
| 102 | err = errInvalidArgument("Input part size is smaller than allowed minimum of 5MiB.") |
| 103 | return totalPartsCount, partSize, lastPartSize, err |
| 104 | } |
| 105 | |
| 106 | if configuredPartSize > maxPartSize { |
| 107 | err = errInvalidArgument("Input part size is bigger than allowed maximum of 5GiB.") |
| 108 | return totalPartsCount, partSize, lastPartSize, err |
| 109 | } |
| 110 | |
| 111 | partSizeFlt = float64(configuredPartSize) |
| 112 | if unknownSize { |
| 113 | // If input has unknown size and part size is configured |
| 114 | // keep it to maximum allowed as per 10000 parts. |
| 115 | objectSize = int64(configuredPartSize) * maxPartsCount |
| 116 | } |
| 117 | } else { |
| 118 | configuredPartSize = minPartSize |
| 119 | // Use floats for part size for all calculations to avoid |
| 120 | // overflows during float64 to int64 conversions. |
| 121 | partSizeFlt = float64(objectSize / maxPartsCount) |
| 122 | partSizeFlt = math.Ceil(partSizeFlt/float64(configuredPartSize)) * float64(configuredPartSize) |
| 123 | } |
| 124 | |
| 125 | // Total parts count. |
| 126 | totalPartsCount = int(math.Ceil(float64(objectSize) / partSizeFlt)) |
| 127 | // Part size. |
| 128 | partSize = int64(partSizeFlt) |