| 197 | } |
| 198 | |
| 199 | func (b *streamingBlock) Complete() (int, error) { |
| 200 | // Flush final row group |
| 201 | b.index.Flush() |
| 202 | b.meta.TotalRecords++ |
| 203 | err := b.pw.Flush() |
| 204 | if err != nil { |
| 205 | return 0, err |
| 206 | } |
| 207 | |
| 208 | // Close parquet file. This writes the footer and metadata. |
| 209 | err = b.pw.Close() |
| 210 | if err != nil { |
| 211 | return 0, err |
| 212 | } |
| 213 | |
| 214 | // Now Flush and close out in-memory buffer |
| 215 | n := b.bw.Len() |
| 216 | b.meta.Size_ += uint64(n) |
| 217 | err = b.bw.Flush() |
| 218 | if err != nil { |
| 219 | return 0, err |
| 220 | } |
| 221 | |
| 222 | err = b.bw.Close() |
| 223 | if err != nil { |
| 224 | return 0, err |
| 225 | } |
| 226 | |
| 227 | err = b.w.Close() |
| 228 | if err != nil { |
| 229 | return 0, err |
| 230 | } |
| 231 | |
| 232 | // Read the footer size out of the parquet footer |
| 233 | buf := make([]byte, 8) |
| 234 | err = b.r.ReadRange(b.ctx, DataFileName, (uuid.UUID)(b.meta.BlockID), b.meta.TenantID, b.meta.Size_-8, buf, nil) |
| 235 | if err != nil { |
| 236 | return 0, fmt.Errorf("error reading parquet file footer: %w", err) |
| 237 | } |
| 238 | if string(buf[4:8]) != "PAR1" { |
| 239 | return 0, errors.New("failed to confirm magic footer while writing a new parquet block") |
| 240 | } |
| 241 | b.meta.FooterSize = binary.LittleEndian.Uint32(buf[0:4]) |
| 242 | |
| 243 | b.meta.BloomShardCount = uint32(b.bloom.GetShardCount()) |
| 244 | |
| 245 | if b.withNoCompactFlag { |
| 246 | // write nocompact flag first to prevent compaction before completion |
| 247 | err := b.to.WriteNoCompactFlag(b.ctx, (uuid.UUID)(b.meta.BlockID), b.meta.TenantID) |
| 248 | if err != nil { |
| 249 | return 0, fmt.Errorf("unexpected error writing nocompact flag: %w", err) |
| 250 | } |
| 251 | } |
| 252 | |
| 253 | return n, writeBlockMeta(b.ctx, b.to, b.meta, b.bloom, b.index) |
| 254 | } |
| 255 | |
| 256 | // estimateMarshalledSizeFromTrace attempts to estimate the size of trace in bytes. This is used to make choose |