s3api_copy_size_calculation.go 6.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239
  1. package s3api
  2. import (
  3. "net/http"
  4. "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
  5. "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
  6. )
  7. // CopySizeCalculator handles size calculations for different copy scenarios
  8. type CopySizeCalculator struct {
  9. srcSize int64
  10. srcEncrypted bool
  11. dstEncrypted bool
  12. srcType EncryptionType
  13. dstType EncryptionType
  14. isCompressed bool
  15. }
  16. // EncryptionType represents different encryption types
  17. type EncryptionType int
  18. const (
  19. EncryptionTypeNone EncryptionType = iota
  20. EncryptionTypeSSEC
  21. EncryptionTypeSSEKMS
  22. EncryptionTypeSSES3
  23. )
  24. // NewCopySizeCalculator creates a new size calculator for copy operations
  25. func NewCopySizeCalculator(entry *filer_pb.Entry, r *http.Request) *CopySizeCalculator {
  26. calc := &CopySizeCalculator{
  27. srcSize: int64(entry.Attributes.FileSize),
  28. isCompressed: isCompressedEntry(entry),
  29. }
  30. // Determine source encryption type
  31. calc.srcType, calc.srcEncrypted = getSourceEncryptionType(entry.Extended)
  32. // Determine destination encryption type
  33. calc.dstType, calc.dstEncrypted = getDestinationEncryptionType(r)
  34. return calc
  35. }
  36. // CalculateTargetSize calculates the expected size of the target object
  37. func (calc *CopySizeCalculator) CalculateTargetSize() int64 {
  38. // For compressed objects, size calculation is complex
  39. if calc.isCompressed {
  40. return -1 // Indicates unknown size
  41. }
  42. switch {
  43. case !calc.srcEncrypted && !calc.dstEncrypted:
  44. // Plain → Plain: no size change
  45. return calc.srcSize
  46. case !calc.srcEncrypted && calc.dstEncrypted:
  47. // Plain → Encrypted: no overhead since IV is in metadata
  48. return calc.srcSize
  49. case calc.srcEncrypted && !calc.dstEncrypted:
  50. // Encrypted → Plain: no overhead since IV is in metadata
  51. return calc.srcSize
  52. case calc.srcEncrypted && calc.dstEncrypted:
  53. // Encrypted → Encrypted: no overhead since IV is in metadata
  54. return calc.srcSize
  55. default:
  56. return calc.srcSize
  57. }
  58. }
  59. // CalculateActualSize calculates the actual unencrypted size of the content
  60. func (calc *CopySizeCalculator) CalculateActualSize() int64 {
  61. // With IV in metadata, encrypted and unencrypted sizes are the same
  62. return calc.srcSize
  63. }
  64. // CalculateEncryptedSize calculates the encrypted size for the given encryption type
  65. func (calc *CopySizeCalculator) CalculateEncryptedSize(encType EncryptionType) int64 {
  66. // With IV in metadata, encrypted size equals actual size
  67. return calc.CalculateActualSize()
  68. }
  69. // getSourceEncryptionType determines the encryption type of the source object
  70. func getSourceEncryptionType(metadata map[string][]byte) (EncryptionType, bool) {
  71. if IsSSECEncrypted(metadata) {
  72. return EncryptionTypeSSEC, true
  73. }
  74. if IsSSEKMSEncrypted(metadata) {
  75. return EncryptionTypeSSEKMS, true
  76. }
  77. if IsSSES3EncryptedInternal(metadata) {
  78. return EncryptionTypeSSES3, true
  79. }
  80. return EncryptionTypeNone, false
  81. }
  82. // getDestinationEncryptionType determines the encryption type for the destination
  83. func getDestinationEncryptionType(r *http.Request) (EncryptionType, bool) {
  84. if IsSSECRequest(r) {
  85. return EncryptionTypeSSEC, true
  86. }
  87. if IsSSEKMSRequest(r) {
  88. return EncryptionTypeSSEKMS, true
  89. }
  90. if IsSSES3RequestInternal(r) {
  91. return EncryptionTypeSSES3, true
  92. }
  93. return EncryptionTypeNone, false
  94. }
  95. // isCompressedEntry checks if the entry represents a compressed object
  96. func isCompressedEntry(entry *filer_pb.Entry) bool {
  97. // Check for compression indicators in metadata
  98. if compressionType, exists := entry.Extended["compression"]; exists {
  99. return string(compressionType) != ""
  100. }
  101. // Check MIME type for compressed formats
  102. mimeType := entry.Attributes.Mime
  103. compressedMimeTypes := []string{
  104. "application/gzip",
  105. "application/x-gzip",
  106. "application/zip",
  107. "application/x-compress",
  108. "application/x-compressed",
  109. }
  110. for _, compressedType := range compressedMimeTypes {
  111. if mimeType == compressedType {
  112. return true
  113. }
  114. }
  115. return false
  116. }
  117. // SizeTransitionInfo provides detailed information about size changes during copy
  118. type SizeTransitionInfo struct {
  119. SourceSize int64
  120. TargetSize int64
  121. ActualSize int64
  122. SizeChange int64
  123. SourceType EncryptionType
  124. TargetType EncryptionType
  125. IsCompressed bool
  126. RequiresResize bool
  127. }
  128. // GetSizeTransitionInfo returns detailed size transition information
  129. func (calc *CopySizeCalculator) GetSizeTransitionInfo() *SizeTransitionInfo {
  130. targetSize := calc.CalculateTargetSize()
  131. actualSize := calc.CalculateActualSize()
  132. info := &SizeTransitionInfo{
  133. SourceSize: calc.srcSize,
  134. TargetSize: targetSize,
  135. ActualSize: actualSize,
  136. SizeChange: targetSize - calc.srcSize,
  137. SourceType: calc.srcType,
  138. TargetType: calc.dstType,
  139. IsCompressed: calc.isCompressed,
  140. RequiresResize: targetSize != calc.srcSize,
  141. }
  142. return info
  143. }
  144. // String returns a string representation of the encryption type
  145. func (e EncryptionType) String() string {
  146. switch e {
  147. case EncryptionTypeNone:
  148. return "None"
  149. case EncryptionTypeSSEC:
  150. return s3_constants.SSETypeC
  151. case EncryptionTypeSSEKMS:
  152. return s3_constants.SSETypeKMS
  153. case EncryptionTypeSSES3:
  154. return s3_constants.SSETypeS3
  155. default:
  156. return "Unknown"
  157. }
  158. }
  159. // OptimizedSizeCalculation provides size calculations optimized for different scenarios
  160. type OptimizedSizeCalculation struct {
  161. Strategy UnifiedCopyStrategy
  162. SourceSize int64
  163. TargetSize int64
  164. ActualContentSize int64
  165. EncryptionOverhead int64
  166. CanPreallocate bool
  167. RequiresStreaming bool
  168. }
  169. // CalculateOptimizedSizes calculates sizes optimized for the copy strategy
  170. func CalculateOptimizedSizes(entry *filer_pb.Entry, r *http.Request, strategy UnifiedCopyStrategy) *OptimizedSizeCalculation {
  171. calc := NewCopySizeCalculator(entry, r)
  172. info := calc.GetSizeTransitionInfo()
  173. result := &OptimizedSizeCalculation{
  174. Strategy: strategy,
  175. SourceSize: info.SourceSize,
  176. TargetSize: info.TargetSize,
  177. ActualContentSize: info.ActualSize,
  178. CanPreallocate: !info.IsCompressed && info.TargetSize > 0,
  179. RequiresStreaming: info.IsCompressed || info.TargetSize < 0,
  180. }
  181. // Calculate encryption overhead for the target
  182. // With IV in metadata, all encryption overhead is 0
  183. result.EncryptionOverhead = 0
  184. // Adjust based on strategy
  185. switch strategy {
  186. case CopyStrategyDirect:
  187. // Direct copy: no size change
  188. result.TargetSize = result.SourceSize
  189. result.CanPreallocate = true
  190. case CopyStrategyKeyRotation:
  191. // Key rotation: size might change slightly due to different IVs
  192. if info.SourceType == EncryptionTypeSSEC && info.TargetType == EncryptionTypeSSEC {
  193. // SSE-C key rotation: same overhead
  194. result.TargetSize = result.SourceSize
  195. }
  196. result.CanPreallocate = true
  197. case CopyStrategyEncrypt, CopyStrategyDecrypt, CopyStrategyReencrypt:
  198. // Size changes based on encryption transition
  199. result.TargetSize = info.TargetSize
  200. result.CanPreallocate = !info.IsCompressed
  201. }
  202. return result
  203. }