| 12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267 |
- package sse_test
- import (
- "bytes"
- "context"
- "crypto/md5"
- "crypto/rand"
- "encoding/base64"
- "fmt"
- "io"
- "strings"
- "testing"
- "time"
- "github.com/aws/aws-sdk-go-v2/aws"
- "github.com/aws/aws-sdk-go-v2/config"
- "github.com/aws/aws-sdk-go-v2/credentials"
- "github.com/aws/aws-sdk-go-v2/service/s3"
- "github.com/aws/aws-sdk-go-v2/service/s3/types"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/require"
- )
- // assertDataEqual compares two byte slices using MD5 hashes and provides a concise error message
- func assertDataEqual(t *testing.T, expected, actual []byte, msgAndArgs ...interface{}) {
- if len(expected) == len(actual) && bytes.Equal(expected, actual) {
- return // Data matches, no need to fail
- }
- expectedMD5 := md5.Sum(expected)
- actualMD5 := md5.Sum(actual)
- // Create preview of first 1K bytes for debugging
- previewSize := 1024
- if len(expected) < previewSize {
- previewSize = len(expected)
- }
- expectedPreview := expected[:previewSize]
- actualPreviewSize := previewSize
- if len(actual) < actualPreviewSize {
- actualPreviewSize = len(actual)
- }
- actualPreview := actual[:actualPreviewSize]
- // Format the assertion failure message
- msg := fmt.Sprintf("Data mismatch:\nExpected length: %d, MD5: %x\nActual length: %d, MD5: %x\nExpected preview (first %d bytes): %x\nActual preview (first %d bytes): %x",
- len(expected), expectedMD5, len(actual), actualMD5,
- len(expectedPreview), expectedPreview, len(actualPreview), actualPreview)
- if len(msgAndArgs) > 0 {
- if format, ok := msgAndArgs[0].(string); ok {
- msg = fmt.Sprintf(format, msgAndArgs[1:]...) + "\n" + msg
- }
- }
- t.Error(msg)
- }
- // min returns the minimum of two integers
- func min(a, b int) int {
- if a < b {
- return a
- }
- return b
- }
- // S3SSETestConfig holds configuration for S3 SSE integration tests
- type S3SSETestConfig struct {
- Endpoint string
- AccessKey string
- SecretKey string
- Region string
- BucketPrefix string
- UseSSL bool
- SkipVerifySSL bool
- }
- // Default test configuration
- var defaultConfig = &S3SSETestConfig{
- Endpoint: "http://127.0.0.1:8333",
- AccessKey: "some_access_key1",
- SecretKey: "some_secret_key1",
- Region: "us-east-1",
- BucketPrefix: "test-sse-",
- UseSSL: false,
- SkipVerifySSL: true,
- }
- // Test data sizes for comprehensive coverage
- var testDataSizes = []int{
- 0, // Empty file
- 1, // Single byte
- 16, // One AES block
- 31, // Just under two blocks
- 32, // Exactly two blocks
- 100, // Small file
- 1024, // 1KB
- 8192, // 8KB
- 64 * 1024, // 64KB
- 1024 * 1024, // 1MB
- }
- // SSECKey represents an SSE-C encryption key for testing
- type SSECKey struct {
- Key []byte
- KeyB64 string
- KeyMD5 string
- }
- // generateSSECKey generates a random SSE-C key for testing
- func generateSSECKey() *SSECKey {
- key := make([]byte, 32) // 256-bit key
- rand.Read(key)
- keyB64 := base64.StdEncoding.EncodeToString(key)
- keyMD5Hash := md5.Sum(key)
- keyMD5 := base64.StdEncoding.EncodeToString(keyMD5Hash[:])
- return &SSECKey{
- Key: key,
- KeyB64: keyB64,
- KeyMD5: keyMD5,
- }
- }
- // createS3Client creates an S3 client for testing
- func createS3Client(ctx context.Context, cfg *S3SSETestConfig) (*s3.Client, error) {
- customResolver := aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
- return aws.Endpoint{
- URL: cfg.Endpoint,
- HostnameImmutable: true,
- }, nil
- })
- awsCfg, err := config.LoadDefaultConfig(ctx,
- config.WithRegion(cfg.Region),
- config.WithEndpointResolverWithOptions(customResolver),
- config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(
- cfg.AccessKey,
- cfg.SecretKey,
- "",
- )),
- )
- if err != nil {
- return nil, err
- }
- return s3.NewFromConfig(awsCfg, func(o *s3.Options) {
- o.UsePathStyle = true
- }), nil
- }
- // generateTestData generates random test data of specified size
- func generateTestData(size int) []byte {
- data := make([]byte, size)
- rand.Read(data)
- return data
- }
- // createTestBucket creates a test bucket with a unique name
- func createTestBucket(ctx context.Context, client *s3.Client, prefix string) (string, error) {
- bucketName := fmt.Sprintf("%s%d", prefix, time.Now().UnixNano())
- _, err := client.CreateBucket(ctx, &s3.CreateBucketInput{
- Bucket: aws.String(bucketName),
- })
- return bucketName, err
- }
- // cleanupTestBucket removes a test bucket and all its objects
- func cleanupTestBucket(ctx context.Context, client *s3.Client, bucketName string) error {
- // List and delete all objects first
- listResp, err := client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{
- Bucket: aws.String(bucketName),
- })
- if err != nil {
- return err
- }
- if len(listResp.Contents) > 0 {
- var objectIds []types.ObjectIdentifier
- for _, obj := range listResp.Contents {
- objectIds = append(objectIds, types.ObjectIdentifier{
- Key: obj.Key,
- })
- }
- _, err = client.DeleteObjects(ctx, &s3.DeleteObjectsInput{
- Bucket: aws.String(bucketName),
- Delete: &types.Delete{
- Objects: objectIds,
- },
- })
- if err != nil {
- return err
- }
- }
- // Delete the bucket
- _, err = client.DeleteBucket(ctx, &s3.DeleteBucketInput{
- Bucket: aws.String(bucketName),
- })
- return err
- }
- // TestSSECIntegrationBasic tests basic SSE-C functionality end-to-end
- func TestSSECIntegrationBasic(t *testing.T) {
- ctx := context.Background()
- client, err := createS3Client(ctx, defaultConfig)
- require.NoError(t, err, "Failed to create S3 client")
- bucketName, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"ssec-basic-")
- require.NoError(t, err, "Failed to create test bucket")
- defer cleanupTestBucket(ctx, client, bucketName)
- // Generate test key
- sseKey := generateSSECKey()
- testData := []byte("Hello, SSE-C integration test!")
- objectKey := "test-object-ssec"
- t.Run("PUT with SSE-C", func(t *testing.T) {
- // Upload object with SSE-C
- _, err := client.PutObject(ctx, &s3.PutObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- Body: bytes.NewReader(testData),
- SSECustomerAlgorithm: aws.String("AES256"),
- SSECustomerKey: aws.String(sseKey.KeyB64),
- SSECustomerKeyMD5: aws.String(sseKey.KeyMD5),
- })
- require.NoError(t, err, "Failed to upload SSE-C object")
- })
- t.Run("GET with correct SSE-C key", func(t *testing.T) {
- // Retrieve object with correct key
- resp, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- SSECustomerAlgorithm: aws.String("AES256"),
- SSECustomerKey: aws.String(sseKey.KeyB64),
- SSECustomerKeyMD5: aws.String(sseKey.KeyMD5),
- })
- require.NoError(t, err, "Failed to retrieve SSE-C object")
- defer resp.Body.Close()
- // Verify decrypted content matches original
- retrievedData, err := io.ReadAll(resp.Body)
- require.NoError(t, err, "Failed to read retrieved data")
- assertDataEqual(t, testData, retrievedData, "Decrypted data does not match original")
- // Verify SSE headers are present
- assert.Equal(t, "AES256", aws.ToString(resp.SSECustomerAlgorithm))
- assert.Equal(t, sseKey.KeyMD5, aws.ToString(resp.SSECustomerKeyMD5))
- })
- t.Run("GET without SSE-C key should fail", func(t *testing.T) {
- // Try to retrieve object without encryption key - should fail
- _, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- })
- assert.Error(t, err, "Should fail to retrieve SSE-C object without key")
- })
- t.Run("GET with wrong SSE-C key should fail", func(t *testing.T) {
- wrongKey := generateSSECKey()
- // Try to retrieve object with wrong key - should fail
- _, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- SSECustomerAlgorithm: aws.String("AES256"),
- SSECustomerKey: aws.String(wrongKey.KeyB64),
- SSECustomerKeyMD5: aws.String(wrongKey.KeyMD5),
- })
- assert.Error(t, err, "Should fail to retrieve SSE-C object with wrong key")
- })
- }
- // TestSSECIntegrationVariousDataSizes tests SSE-C with various data sizes
- func TestSSECIntegrationVariousDataSizes(t *testing.T) {
- ctx := context.Background()
- client, err := createS3Client(ctx, defaultConfig)
- require.NoError(t, err, "Failed to create S3 client")
- bucketName, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"ssec-sizes-")
- require.NoError(t, err, "Failed to create test bucket")
- defer cleanupTestBucket(ctx, client, bucketName)
- sseKey := generateSSECKey()
- for _, size := range testDataSizes {
- t.Run(fmt.Sprintf("Size_%d_bytes", size), func(t *testing.T) {
- testData := generateTestData(size)
- objectKey := fmt.Sprintf("test-object-size-%d", size)
- // Upload with SSE-C
- _, err := client.PutObject(ctx, &s3.PutObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- Body: bytes.NewReader(testData),
- SSECustomerAlgorithm: aws.String("AES256"),
- SSECustomerKey: aws.String(sseKey.KeyB64),
- SSECustomerKeyMD5: aws.String(sseKey.KeyMD5),
- })
- require.NoError(t, err, "Failed to upload object of size %d", size)
- // Retrieve with SSE-C
- resp, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- SSECustomerAlgorithm: aws.String("AES256"),
- SSECustomerKey: aws.String(sseKey.KeyB64),
- SSECustomerKeyMD5: aws.String(sseKey.KeyMD5),
- })
- require.NoError(t, err, "Failed to retrieve object of size %d", size)
- defer resp.Body.Close()
- // Verify content matches
- retrievedData, err := io.ReadAll(resp.Body)
- require.NoError(t, err, "Failed to read retrieved data of size %d", size)
- assertDataEqual(t, testData, retrievedData, "Data mismatch for size %d", size)
- // Verify content length is correct (this would have caught the IV-in-stream bug!)
- assert.Equal(t, int64(size), aws.ToInt64(resp.ContentLength),
- "Content length mismatch for size %d", size)
- })
- }
- }
- // TestSSEKMSIntegrationBasic tests basic SSE-KMS functionality end-to-end
- func TestSSEKMSIntegrationBasic(t *testing.T) {
- ctx := context.Background()
- client, err := createS3Client(ctx, defaultConfig)
- require.NoError(t, err, "Failed to create S3 client")
- bucketName, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"ssekms-basic-")
- require.NoError(t, err, "Failed to create test bucket")
- defer cleanupTestBucket(ctx, client, bucketName)
- testData := []byte("Hello, SSE-KMS integration test!")
- objectKey := "test-object-ssekms"
- kmsKeyID := "test-key-123" // Test key ID
- t.Run("PUT with SSE-KMS", func(t *testing.T) {
- // Upload object with SSE-KMS
- _, err := client.PutObject(ctx, &s3.PutObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- Body: bytes.NewReader(testData),
- ServerSideEncryption: types.ServerSideEncryptionAwsKms,
- SSEKMSKeyId: aws.String(kmsKeyID),
- })
- require.NoError(t, err, "Failed to upload SSE-KMS object")
- })
- t.Run("GET SSE-KMS object", func(t *testing.T) {
- // Retrieve object - no additional headers needed for GET
- resp, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- })
- require.NoError(t, err, "Failed to retrieve SSE-KMS object")
- defer resp.Body.Close()
- // Verify decrypted content matches original
- retrievedData, err := io.ReadAll(resp.Body)
- require.NoError(t, err, "Failed to read retrieved data")
- assertDataEqual(t, testData, retrievedData, "Decrypted data does not match original")
- // Verify SSE-KMS headers are present
- assert.Equal(t, types.ServerSideEncryptionAwsKms, resp.ServerSideEncryption)
- assert.Equal(t, kmsKeyID, aws.ToString(resp.SSEKMSKeyId))
- })
- t.Run("HEAD SSE-KMS object", func(t *testing.T) {
- // Test HEAD operation to verify metadata
- resp, err := client.HeadObject(ctx, &s3.HeadObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- })
- require.NoError(t, err, "Failed to HEAD SSE-KMS object")
- // Verify SSE-KMS metadata
- assert.Equal(t, types.ServerSideEncryptionAwsKms, resp.ServerSideEncryption)
- assert.Equal(t, kmsKeyID, aws.ToString(resp.SSEKMSKeyId))
- assert.Equal(t, int64(len(testData)), aws.ToInt64(resp.ContentLength))
- })
- }
- // TestSSEKMSIntegrationVariousDataSizes tests SSE-KMS with various data sizes
- func TestSSEKMSIntegrationVariousDataSizes(t *testing.T) {
- ctx := context.Background()
- client, err := createS3Client(ctx, defaultConfig)
- require.NoError(t, err, "Failed to create S3 client")
- bucketName, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"ssekms-sizes-")
- require.NoError(t, err, "Failed to create test bucket")
- defer cleanupTestBucket(ctx, client, bucketName)
- kmsKeyID := "test-key-size-tests"
- for _, size := range testDataSizes {
- t.Run(fmt.Sprintf("Size_%d_bytes", size), func(t *testing.T) {
- testData := generateTestData(size)
- objectKey := fmt.Sprintf("test-object-kms-size-%d", size)
- // Upload with SSE-KMS
- _, err := client.PutObject(ctx, &s3.PutObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- Body: bytes.NewReader(testData),
- ServerSideEncryption: types.ServerSideEncryptionAwsKms,
- SSEKMSKeyId: aws.String(kmsKeyID),
- })
- require.NoError(t, err, "Failed to upload KMS object of size %d", size)
- // Retrieve with SSE-KMS
- resp, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- })
- require.NoError(t, err, "Failed to retrieve KMS object of size %d", size)
- defer resp.Body.Close()
- // Verify content matches
- retrievedData, err := io.ReadAll(resp.Body)
- require.NoError(t, err, "Failed to read retrieved KMS data of size %d", size)
- assertDataEqual(t, testData, retrievedData, "Data mismatch for KMS size %d", size)
- // Verify content length is correct
- assert.Equal(t, int64(size), aws.ToInt64(resp.ContentLength),
- "Content length mismatch for KMS size %d", size)
- })
- }
- }
- // TestSSECObjectCopyIntegration tests SSE-C object copying end-to-end
- func TestSSECObjectCopyIntegration(t *testing.T) {
- ctx := context.Background()
- client, err := createS3Client(ctx, defaultConfig)
- require.NoError(t, err, "Failed to create S3 client")
- bucketName, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"ssec-copy-")
- require.NoError(t, err, "Failed to create test bucket")
- defer cleanupTestBucket(ctx, client, bucketName)
- // Generate test keys
- sourceKey := generateSSECKey()
- destKey := generateSSECKey()
- testData := []byte("Hello, SSE-C copy integration test!")
- // Upload source object
- sourceObjectKey := "source-object"
- _, err = client.PutObject(ctx, &s3.PutObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(sourceObjectKey),
- Body: bytes.NewReader(testData),
- SSECustomerAlgorithm: aws.String("AES256"),
- SSECustomerKey: aws.String(sourceKey.KeyB64),
- SSECustomerKeyMD5: aws.String(sourceKey.KeyMD5),
- })
- require.NoError(t, err, "Failed to upload source SSE-C object")
- t.Run("Copy SSE-C to SSE-C with different key", func(t *testing.T) {
- destObjectKey := "dest-object-ssec"
- copySource := fmt.Sprintf("%s/%s", bucketName, sourceObjectKey)
- // Copy object with different SSE-C key
- _, err := client.CopyObject(ctx, &s3.CopyObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(destObjectKey),
- CopySource: aws.String(copySource),
- CopySourceSSECustomerAlgorithm: aws.String("AES256"),
- CopySourceSSECustomerKey: aws.String(sourceKey.KeyB64),
- CopySourceSSECustomerKeyMD5: aws.String(sourceKey.KeyMD5),
- SSECustomerAlgorithm: aws.String("AES256"),
- SSECustomerKey: aws.String(destKey.KeyB64),
- SSECustomerKeyMD5: aws.String(destKey.KeyMD5),
- })
- require.NoError(t, err, "Failed to copy SSE-C object")
- // Retrieve copied object with destination key
- resp, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(destObjectKey),
- SSECustomerAlgorithm: aws.String("AES256"),
- SSECustomerKey: aws.String(destKey.KeyB64),
- SSECustomerKeyMD5: aws.String(destKey.KeyMD5),
- })
- require.NoError(t, err, "Failed to retrieve copied SSE-C object")
- defer resp.Body.Close()
- // Verify content matches original
- retrievedData, err := io.ReadAll(resp.Body)
- require.NoError(t, err, "Failed to read copied data")
- assertDataEqual(t, testData, retrievedData, "Copied data does not match original")
- })
- t.Run("Copy SSE-C to plain", func(t *testing.T) {
- destObjectKey := "dest-object-plain"
- copySource := fmt.Sprintf("%s/%s", bucketName, sourceObjectKey)
- // Copy SSE-C object to plain object
- _, err := client.CopyObject(ctx, &s3.CopyObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(destObjectKey),
- CopySource: aws.String(copySource),
- CopySourceSSECustomerAlgorithm: aws.String("AES256"),
- CopySourceSSECustomerKey: aws.String(sourceKey.KeyB64),
- CopySourceSSECustomerKeyMD5: aws.String(sourceKey.KeyMD5),
- // No destination encryption headers = plain object
- })
- require.NoError(t, err, "Failed to copy SSE-C to plain object")
- // Retrieve plain object (no encryption headers needed)
- resp, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(destObjectKey),
- })
- require.NoError(t, err, "Failed to retrieve plain copied object")
- defer resp.Body.Close()
- // Verify content matches original
- retrievedData, err := io.ReadAll(resp.Body)
- require.NoError(t, err, "Failed to read plain copied data")
- assertDataEqual(t, testData, retrievedData, "Plain copied data does not match original")
- })
- }
- // TestSSEKMSObjectCopyIntegration tests SSE-KMS object copying end-to-end
- func TestSSEKMSObjectCopyIntegration(t *testing.T) {
- ctx := context.Background()
- client, err := createS3Client(ctx, defaultConfig)
- require.NoError(t, err, "Failed to create S3 client")
- bucketName, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"ssekms-copy-")
- require.NoError(t, err, "Failed to create test bucket")
- defer cleanupTestBucket(ctx, client, bucketName)
- testData := []byte("Hello, SSE-KMS copy integration test!")
- sourceKeyID := "source-test-key-123"
- destKeyID := "dest-test-key-456"
- // Upload source object with SSE-KMS
- sourceObjectKey := "source-object-kms"
- _, err = client.PutObject(ctx, &s3.PutObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(sourceObjectKey),
- Body: bytes.NewReader(testData),
- ServerSideEncryption: types.ServerSideEncryptionAwsKms,
- SSEKMSKeyId: aws.String(sourceKeyID),
- })
- require.NoError(t, err, "Failed to upload source SSE-KMS object")
- t.Run("Copy SSE-KMS with different key", func(t *testing.T) {
- destObjectKey := "dest-object-kms"
- copySource := fmt.Sprintf("%s/%s", bucketName, sourceObjectKey)
- // Copy object with different SSE-KMS key
- _, err := client.CopyObject(ctx, &s3.CopyObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(destObjectKey),
- CopySource: aws.String(copySource),
- ServerSideEncryption: types.ServerSideEncryptionAwsKms,
- SSEKMSKeyId: aws.String(destKeyID),
- })
- require.NoError(t, err, "Failed to copy SSE-KMS object")
- // Retrieve copied object
- resp, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(destObjectKey),
- })
- require.NoError(t, err, "Failed to retrieve copied SSE-KMS object")
- defer resp.Body.Close()
- // Verify content matches original
- retrievedData, err := io.ReadAll(resp.Body)
- require.NoError(t, err, "Failed to read copied KMS data")
- assertDataEqual(t, testData, retrievedData, "Copied KMS data does not match original")
- // Verify new key ID is used
- assert.Equal(t, destKeyID, aws.ToString(resp.SSEKMSKeyId))
- })
- }
- // TestSSEMultipartUploadIntegration tests SSE multipart uploads end-to-end
- func TestSSEMultipartUploadIntegration(t *testing.T) {
- ctx := context.Background()
- client, err := createS3Client(ctx, defaultConfig)
- require.NoError(t, err, "Failed to create S3 client")
- bucketName, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"sse-multipart-")
- require.NoError(t, err, "Failed to create test bucket")
- defer cleanupTestBucket(ctx, client, bucketName)
- t.Run("SSE-C Multipart Upload", func(t *testing.T) {
- sseKey := generateSSECKey()
- objectKey := "multipart-ssec-object"
- // Create multipart upload
- createResp, err := client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- SSECustomerAlgorithm: aws.String("AES256"),
- SSECustomerKey: aws.String(sseKey.KeyB64),
- SSECustomerKeyMD5: aws.String(sseKey.KeyMD5),
- })
- require.NoError(t, err, "Failed to create SSE-C multipart upload")
- uploadID := aws.ToString(createResp.UploadId)
- // Upload parts
- partSize := 5 * 1024 * 1024 // 5MB
- part1Data := generateTestData(partSize)
- part2Data := generateTestData(partSize)
- // Upload part 1
- part1Resp, err := client.UploadPart(ctx, &s3.UploadPartInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- PartNumber: aws.Int32(1),
- UploadId: aws.String(uploadID),
- Body: bytes.NewReader(part1Data),
- SSECustomerAlgorithm: aws.String("AES256"),
- SSECustomerKey: aws.String(sseKey.KeyB64),
- SSECustomerKeyMD5: aws.String(sseKey.KeyMD5),
- })
- require.NoError(t, err, "Failed to upload part 1")
- // Upload part 2
- part2Resp, err := client.UploadPart(ctx, &s3.UploadPartInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- PartNumber: aws.Int32(2),
- UploadId: aws.String(uploadID),
- Body: bytes.NewReader(part2Data),
- SSECustomerAlgorithm: aws.String("AES256"),
- SSECustomerKey: aws.String(sseKey.KeyB64),
- SSECustomerKeyMD5: aws.String(sseKey.KeyMD5),
- })
- require.NoError(t, err, "Failed to upload part 2")
- // Complete multipart upload
- _, err = client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- UploadId: aws.String(uploadID),
- MultipartUpload: &types.CompletedMultipartUpload{
- Parts: []types.CompletedPart{
- {
- ETag: part1Resp.ETag,
- PartNumber: aws.Int32(1),
- },
- {
- ETag: part2Resp.ETag,
- PartNumber: aws.Int32(2),
- },
- },
- },
- })
- require.NoError(t, err, "Failed to complete SSE-C multipart upload")
- // Retrieve and verify the complete object
- resp, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- SSECustomerAlgorithm: aws.String("AES256"),
- SSECustomerKey: aws.String(sseKey.KeyB64),
- SSECustomerKeyMD5: aws.String(sseKey.KeyMD5),
- })
- require.NoError(t, err, "Failed to retrieve multipart SSE-C object")
- defer resp.Body.Close()
- retrievedData, err := io.ReadAll(resp.Body)
- require.NoError(t, err, "Failed to read multipart data")
- // Verify data matches concatenated parts
- expectedData := append(part1Data, part2Data...)
- assertDataEqual(t, expectedData, retrievedData, "Multipart data does not match original")
- assert.Equal(t, int64(len(expectedData)), aws.ToInt64(resp.ContentLength),
- "Multipart content length mismatch")
- })
- t.Run("SSE-KMS Multipart Upload", func(t *testing.T) {
- kmsKeyID := "test-multipart-key"
- objectKey := "multipart-kms-object"
- // Create multipart upload
- createResp, err := client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- ServerSideEncryption: types.ServerSideEncryptionAwsKms,
- SSEKMSKeyId: aws.String(kmsKeyID),
- })
- require.NoError(t, err, "Failed to create SSE-KMS multipart upload")
- uploadID := aws.ToString(createResp.UploadId)
- // Upload parts
- partSize := 5 * 1024 * 1024 // 5MB
- part1Data := generateTestData(partSize)
- part2Data := generateTestData(partSize / 2) // Different size
- // Upload part 1
- part1Resp, err := client.UploadPart(ctx, &s3.UploadPartInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- PartNumber: aws.Int32(1),
- UploadId: aws.String(uploadID),
- Body: bytes.NewReader(part1Data),
- })
- require.NoError(t, err, "Failed to upload KMS part 1")
- // Upload part 2
- part2Resp, err := client.UploadPart(ctx, &s3.UploadPartInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- PartNumber: aws.Int32(2),
- UploadId: aws.String(uploadID),
- Body: bytes.NewReader(part2Data),
- })
- require.NoError(t, err, "Failed to upload KMS part 2")
- // Complete multipart upload
- _, err = client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- UploadId: aws.String(uploadID),
- MultipartUpload: &types.CompletedMultipartUpload{
- Parts: []types.CompletedPart{
- {
- ETag: part1Resp.ETag,
- PartNumber: aws.Int32(1),
- },
- {
- ETag: part2Resp.ETag,
- PartNumber: aws.Int32(2),
- },
- },
- },
- })
- require.NoError(t, err, "Failed to complete SSE-KMS multipart upload")
- // Retrieve and verify the complete object
- resp, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- })
- require.NoError(t, err, "Failed to retrieve multipart SSE-KMS object")
- defer resp.Body.Close()
- retrievedData, err := io.ReadAll(resp.Body)
- require.NoError(t, err, "Failed to read multipart KMS data")
- // Verify data matches concatenated parts
- expectedData := append(part1Data, part2Data...)
- // Debug: Print some information about the sizes and first few bytes
- t.Logf("Expected data size: %d, Retrieved data size: %d", len(expectedData), len(retrievedData))
- if len(expectedData) > 0 && len(retrievedData) > 0 {
- t.Logf("Expected first 32 bytes: %x", expectedData[:min(32, len(expectedData))])
- t.Logf("Retrieved first 32 bytes: %x", retrievedData[:min(32, len(retrievedData))])
- }
- assertDataEqual(t, expectedData, retrievedData, "Multipart KMS data does not match original")
- // Verify KMS metadata
- assert.Equal(t, types.ServerSideEncryptionAwsKms, resp.ServerSideEncryption)
- assert.Equal(t, kmsKeyID, aws.ToString(resp.SSEKMSKeyId))
- })
- }
- // TestDebugSSEMultipart helps debug the multipart SSE-KMS data mismatch
- func TestDebugSSEMultipart(t *testing.T) {
- ctx := context.Background()
- client, err := createS3Client(ctx, defaultConfig)
- require.NoError(t, err, "Failed to create S3 client")
- bucketName, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"debug-multipart-")
- require.NoError(t, err, "Failed to create test bucket")
- defer cleanupTestBucket(ctx, client, bucketName)
- objectKey := "debug-multipart-object"
- kmsKeyID := "test-multipart-key"
- // Create multipart upload
- createResp, err := client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- ServerSideEncryption: types.ServerSideEncryptionAwsKms,
- SSEKMSKeyId: aws.String(kmsKeyID),
- })
- require.NoError(t, err, "Failed to create SSE-KMS multipart upload")
- uploadID := aws.ToString(createResp.UploadId)
- // Upload two parts - exactly like the failing test
- partSize := 5 * 1024 * 1024 // 5MB
- part1Data := generateTestData(partSize) // 5MB
- part2Data := generateTestData(partSize / 2) // 2.5MB
- // Upload part 1
- part1Resp, err := client.UploadPart(ctx, &s3.UploadPartInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- PartNumber: aws.Int32(1),
- UploadId: aws.String(uploadID),
- Body: bytes.NewReader(part1Data),
- })
- require.NoError(t, err, "Failed to upload part 1")
- // Upload part 2
- part2Resp, err := client.UploadPart(ctx, &s3.UploadPartInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- PartNumber: aws.Int32(2),
- UploadId: aws.String(uploadID),
- Body: bytes.NewReader(part2Data),
- })
- require.NoError(t, err, "Failed to upload part 2")
- // Complete multipart upload
- _, err = client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- UploadId: aws.String(uploadID),
- MultipartUpload: &types.CompletedMultipartUpload{
- Parts: []types.CompletedPart{
- {ETag: part1Resp.ETag, PartNumber: aws.Int32(1)},
- {ETag: part2Resp.ETag, PartNumber: aws.Int32(2)},
- },
- },
- })
- require.NoError(t, err, "Failed to complete multipart upload")
- // Retrieve the object
- resp, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- })
- require.NoError(t, err, "Failed to retrieve object")
- defer resp.Body.Close()
- retrievedData, err := io.ReadAll(resp.Body)
- require.NoError(t, err, "Failed to read retrieved data")
- // Expected data
- expectedData := append(part1Data, part2Data...)
- t.Logf("=== DATA COMPARISON DEBUG ===")
- t.Logf("Expected size: %d, Retrieved size: %d", len(expectedData), len(retrievedData))
- // Find exact point of divergence
- divergePoint := -1
- minLen := len(expectedData)
- if len(retrievedData) < minLen {
- minLen = len(retrievedData)
- }
- for i := 0; i < minLen; i++ {
- if expectedData[i] != retrievedData[i] {
- divergePoint = i
- break
- }
- }
- if divergePoint >= 0 {
- t.Logf("Data diverges at byte %d (0x%x)", divergePoint, divergePoint)
- t.Logf("Expected: 0x%02x, Retrieved: 0x%02x", expectedData[divergePoint], retrievedData[divergePoint])
- // Show context around divergence point
- start := divergePoint - 10
- if start < 0 {
- start = 0
- }
- end := divergePoint + 10
- if end > minLen {
- end = minLen
- }
- t.Logf("Context [%d:%d]:", start, end)
- t.Logf("Expected: %x", expectedData[start:end])
- t.Logf("Retrieved: %x", retrievedData[start:end])
- // Identify chunk boundaries
- if divergePoint >= 4194304 {
- t.Logf("Divergence is in chunk 2 or 3 (after 4MB boundary)")
- }
- if divergePoint >= 5242880 {
- t.Logf("Divergence is in chunk 3 (part 2, after 5MB boundary)")
- }
- } else if len(expectedData) != len(retrievedData) {
- t.Logf("Data lengths differ but common part matches")
- } else {
- t.Logf("Data matches completely!")
- }
- // Test completed successfully
- t.Logf("SSE comparison test completed - data matches completely!")
- }
- // TestSSEErrorConditions tests various error conditions in SSE
- func TestSSEErrorConditions(t *testing.T) {
- ctx := context.Background()
- client, err := createS3Client(ctx, defaultConfig)
- require.NoError(t, err, "Failed to create S3 client")
- bucketName, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"sse-errors-")
- require.NoError(t, err, "Failed to create test bucket")
- defer cleanupTestBucket(ctx, client, bucketName)
- t.Run("SSE-C Invalid Key Length", func(t *testing.T) {
- invalidKey := base64.StdEncoding.EncodeToString([]byte("too-short"))
- _, err := client.PutObject(ctx, &s3.PutObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String("invalid-key-test"),
- Body: strings.NewReader("test"),
- SSECustomerAlgorithm: aws.String("AES256"),
- SSECustomerKey: aws.String(invalidKey),
- SSECustomerKeyMD5: aws.String("invalid-md5"),
- })
- assert.Error(t, err, "Should fail with invalid SSE-C key")
- })
- t.Run("SSE-KMS Invalid Key ID", func(t *testing.T) {
- // Empty key ID should be rejected
- _, err := client.PutObject(ctx, &s3.PutObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String("invalid-kms-key-test"),
- Body: strings.NewReader("test"),
- ServerSideEncryption: types.ServerSideEncryptionAwsKms,
- SSEKMSKeyId: aws.String(""), // Invalid empty key
- })
- assert.Error(t, err, "Should fail with empty KMS key ID")
- })
- }
- // BenchmarkSSECThroughput benchmarks SSE-C throughput
- func BenchmarkSSECThroughput(b *testing.B) {
- ctx := context.Background()
- client, err := createS3Client(ctx, defaultConfig)
- require.NoError(b, err, "Failed to create S3 client")
- bucketName, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"ssec-bench-")
- require.NoError(b, err, "Failed to create test bucket")
- defer cleanupTestBucket(ctx, client, bucketName)
- sseKey := generateSSECKey()
- testData := generateTestData(1024 * 1024) // 1MB
- b.ResetTimer()
- b.SetBytes(int64(len(testData)))
- for i := 0; i < b.N; i++ {
- objectKey := fmt.Sprintf("bench-object-%d", i)
- // Upload
- _, err := client.PutObject(ctx, &s3.PutObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- Body: bytes.NewReader(testData),
- SSECustomerAlgorithm: aws.String("AES256"),
- SSECustomerKey: aws.String(sseKey.KeyB64),
- SSECustomerKeyMD5: aws.String(sseKey.KeyMD5),
- })
- require.NoError(b, err, "Failed to upload in benchmark")
- // Download
- resp, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- SSECustomerAlgorithm: aws.String("AES256"),
- SSECustomerKey: aws.String(sseKey.KeyB64),
- SSECustomerKeyMD5: aws.String(sseKey.KeyMD5),
- })
- require.NoError(b, err, "Failed to download in benchmark")
- _, err = io.ReadAll(resp.Body)
- require.NoError(b, err, "Failed to read data in benchmark")
- resp.Body.Close()
- }
- }
- // TestSSECRangeRequests tests SSE-C with HTTP Range requests
- func TestSSECRangeRequests(t *testing.T) {
- ctx := context.Background()
- client, err := createS3Client(ctx, defaultConfig)
- require.NoError(t, err, "Failed to create S3 client")
- bucketName, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"ssec-range-")
- require.NoError(t, err, "Failed to create test bucket")
- defer cleanupTestBucket(ctx, client, bucketName)
- sseKey := generateSSECKey()
- // Create test data that's large enough for meaningful range tests
- testData := generateTestData(2048) // 2KB
- objectKey := "test-range-object"
- // Upload with SSE-C
- _, err = client.PutObject(ctx, &s3.PutObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- Body: bytes.NewReader(testData),
- SSECustomerAlgorithm: aws.String("AES256"),
- SSECustomerKey: aws.String(sseKey.KeyB64),
- SSECustomerKeyMD5: aws.String(sseKey.KeyMD5),
- })
- require.NoError(t, err, "Failed to upload SSE-C object")
- // Test various range requests
- testCases := []struct {
- name string
- start int64
- end int64
- }{
- {"First 100 bytes", 0, 99},
- {"Middle 100 bytes", 500, 599},
- {"Last 100 bytes", int64(len(testData) - 100), int64(len(testData) - 1)},
- {"Single byte", 42, 42},
- {"Cross boundary", 15, 17}, // Test AES block boundary crossing
- }
- for _, tc := range testCases {
- t.Run(tc.name, func(t *testing.T) {
- // Get range with SSE-C
- resp, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- Range: aws.String(fmt.Sprintf("bytes=%d-%d", tc.start, tc.end)),
- SSECustomerAlgorithm: aws.String("AES256"),
- SSECustomerKey: aws.String(sseKey.KeyB64),
- SSECustomerKeyMD5: aws.String(sseKey.KeyMD5),
- })
- require.NoError(t, err, "Failed to get range %d-%d from SSE-C object", tc.start, tc.end)
- defer resp.Body.Close()
- // Range requests should return partial content status
- // Note: AWS SDK Go v2 doesn't expose HTTP status code directly in GetObject response
- // The fact that we get a successful response with correct range data indicates 206 status
- // Read the range data
- rangeData, err := io.ReadAll(resp.Body)
- require.NoError(t, err, "Failed to read range data")
- // Verify content matches expected range
- expectedLength := tc.end - tc.start + 1
- expectedData := testData[tc.start : tc.start+expectedLength]
- assertDataEqual(t, expectedData, rangeData, "Range data mismatch for %s", tc.name)
- // Verify content length header
- assert.Equal(t, expectedLength, aws.ToInt64(resp.ContentLength), "Content length mismatch for %s", tc.name)
- // Verify SSE headers are present
- assert.Equal(t, "AES256", aws.ToString(resp.SSECustomerAlgorithm))
- assert.Equal(t, sseKey.KeyMD5, aws.ToString(resp.SSECustomerKeyMD5))
- })
- }
- }
- // TestSSEKMSRangeRequests tests SSE-KMS with HTTP Range requests
- func TestSSEKMSRangeRequests(t *testing.T) {
- ctx := context.Background()
- client, err := createS3Client(ctx, defaultConfig)
- require.NoError(t, err, "Failed to create S3 client")
- bucketName, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"ssekms-range-")
- require.NoError(t, err, "Failed to create test bucket")
- defer cleanupTestBucket(ctx, client, bucketName)
- kmsKeyID := "test-range-key"
- // Create test data that's large enough for meaningful range tests
- testData := generateTestData(2048) // 2KB
- objectKey := "test-kms-range-object"
- // Upload with SSE-KMS
- _, err = client.PutObject(ctx, &s3.PutObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- Body: bytes.NewReader(testData),
- ServerSideEncryption: types.ServerSideEncryptionAwsKms,
- SSEKMSKeyId: aws.String(kmsKeyID),
- })
- require.NoError(t, err, "Failed to upload SSE-KMS object")
- // Test various range requests
- testCases := []struct {
- name string
- start int64
- end int64
- }{
- {"First 100 bytes", 0, 99},
- {"Middle 100 bytes", 500, 599},
- {"Last 100 bytes", int64(len(testData) - 100), int64(len(testData) - 1)},
- {"Single byte", 42, 42},
- {"Cross boundary", 15, 17}, // Test AES block boundary crossing
- }
- for _, tc := range testCases {
- t.Run(tc.name, func(t *testing.T) {
- // Get range with SSE-KMS (no additional headers needed for GET)
- resp, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- Range: aws.String(fmt.Sprintf("bytes=%d-%d", tc.start, tc.end)),
- })
- require.NoError(t, err, "Failed to get range %d-%d from SSE-KMS object", tc.start, tc.end)
- defer resp.Body.Close()
- // Range requests should return partial content status
- // Note: AWS SDK Go v2 doesn't expose HTTP status code directly in GetObject response
- // The fact that we get a successful response with correct range data indicates 206 status
- // Read the range data
- rangeData, err := io.ReadAll(resp.Body)
- require.NoError(t, err, "Failed to read range data")
- // Verify content matches expected range
- expectedLength := tc.end - tc.start + 1
- expectedData := testData[tc.start : tc.start+expectedLength]
- assertDataEqual(t, expectedData, rangeData, "Range data mismatch for %s", tc.name)
- // Verify content length header
- assert.Equal(t, expectedLength, aws.ToInt64(resp.ContentLength), "Content length mismatch for %s", tc.name)
- // Verify SSE headers are present
- assert.Equal(t, types.ServerSideEncryptionAwsKms, resp.ServerSideEncryption)
- assert.Equal(t, kmsKeyID, aws.ToString(resp.SSEKMSKeyId))
- })
- }
- }
- // BenchmarkSSEKMSThroughput benchmarks SSE-KMS throughput
- func BenchmarkSSEKMSThroughput(b *testing.B) {
- ctx := context.Background()
- client, err := createS3Client(ctx, defaultConfig)
- require.NoError(b, err, "Failed to create S3 client")
- bucketName, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"ssekms-bench-")
- require.NoError(b, err, "Failed to create test bucket")
- defer cleanupTestBucket(ctx, client, bucketName)
- kmsKeyID := "bench-test-key"
- testData := generateTestData(1024 * 1024) // 1MB
- b.ResetTimer()
- b.SetBytes(int64(len(testData)))
- for i := 0; i < b.N; i++ {
- objectKey := fmt.Sprintf("bench-kms-object-%d", i)
- // Upload
- _, err := client.PutObject(ctx, &s3.PutObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- Body: bytes.NewReader(testData),
- ServerSideEncryption: types.ServerSideEncryptionAwsKms,
- SSEKMSKeyId: aws.String(kmsKeyID),
- })
- require.NoError(b, err, "Failed to upload in KMS benchmark")
- // Download
- resp, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- })
- require.NoError(b, err, "Failed to download in KMS benchmark")
- _, err = io.ReadAll(resp.Body)
- require.NoError(b, err, "Failed to read KMS data in benchmark")
- resp.Body.Close()
- }
- }
- // TestSSES3IntegrationBasic tests basic SSE-S3 upload and download functionality
- func TestSSES3IntegrationBasic(t *testing.T) {
- ctx := context.Background()
- client, err := createS3Client(ctx, defaultConfig)
- require.NoError(t, err, "Failed to create S3 client")
- bucketName, err := createTestBucket(ctx, client, "sse-s3-basic")
- require.NoError(t, err, "Failed to create test bucket")
- defer cleanupTestBucket(ctx, client, bucketName)
- testData := []byte("Hello, SSE-S3! This is a test of server-side encryption with S3-managed keys.")
- objectKey := "test-sse-s3-object.txt"
- t.Run("SSE-S3 Upload", func(t *testing.T) {
- // Upload object with SSE-S3
- _, err := client.PutObject(ctx, &s3.PutObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- Body: bytes.NewReader(testData),
- ServerSideEncryption: types.ServerSideEncryptionAes256,
- })
- require.NoError(t, err, "Failed to upload object with SSE-S3")
- })
- t.Run("SSE-S3 Download", func(t *testing.T) {
- // Download and verify object
- resp, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- })
- require.NoError(t, err, "Failed to download SSE-S3 object")
- // Verify SSE-S3 headers in response
- assert.Equal(t, types.ServerSideEncryptionAes256, resp.ServerSideEncryption, "Server-side encryption header mismatch")
- // Read and verify content
- downloadedData, err := io.ReadAll(resp.Body)
- require.NoError(t, err, "Failed to read downloaded data")
- resp.Body.Close()
- assertDataEqual(t, testData, downloadedData, "Downloaded data doesn't match original")
- })
- t.Run("SSE-S3 HEAD Request", func(t *testing.T) {
- // HEAD request should also return SSE headers
- resp, err := client.HeadObject(ctx, &s3.HeadObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- })
- require.NoError(t, err, "Failed to HEAD SSE-S3 object")
- // Verify SSE-S3 headers
- assert.Equal(t, types.ServerSideEncryptionAes256, resp.ServerSideEncryption, "SSE-S3 header missing in HEAD response")
- })
- }
- // TestSSES3IntegrationVariousDataSizes tests SSE-S3 with various data sizes
- func TestSSES3IntegrationVariousDataSizes(t *testing.T) {
- ctx := context.Background()
- client, err := createS3Client(ctx, defaultConfig)
- require.NoError(t, err, "Failed to create S3 client")
- bucketName, err := createTestBucket(ctx, client, "sse-s3-sizes")
- require.NoError(t, err, "Failed to create test bucket")
- defer cleanupTestBucket(ctx, client, bucketName)
- // Test various data sizes including edge cases
- testSizes := []int{
- 0, // Empty file
- 1, // Single byte
- 16, // One AES block
- 31, // Just under two blocks
- 32, // Exactly two blocks
- 100, // Small file
- 1024, // 1KB
- 8192, // 8KB
- 65536, // 64KB
- 1024 * 1024, // 1MB
- }
- for _, size := range testSizes {
- t.Run(fmt.Sprintf("Size_%d_bytes", size), func(t *testing.T) {
- testData := generateTestData(size)
- objectKey := fmt.Sprintf("test-sse-s3-%d.dat", size)
- // Upload with SSE-S3
- _, err := client.PutObject(ctx, &s3.PutObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- Body: bytes.NewReader(testData),
- ServerSideEncryption: types.ServerSideEncryptionAes256,
- })
- require.NoError(t, err, "Failed to upload SSE-S3 object of size %d", size)
- // Download and verify
- resp, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- })
- require.NoError(t, err, "Failed to download SSE-S3 object of size %d", size)
- // Verify encryption headers
- assert.Equal(t, types.ServerSideEncryptionAes256, resp.ServerSideEncryption, "Missing SSE-S3 header for size %d", size)
- // Verify content
- downloadedData, err := io.ReadAll(resp.Body)
- require.NoError(t, err, "Failed to read downloaded data for size %d", size)
- resp.Body.Close()
- assertDataEqual(t, testData, downloadedData, "Data mismatch for size %d", size)
- })
- }
- }
- // TestSSES3WithUserMetadata tests SSE-S3 with user-defined metadata
- func TestSSES3WithUserMetadata(t *testing.T) {
- ctx := context.Background()
- client, err := createS3Client(ctx, defaultConfig)
- require.NoError(t, err, "Failed to create S3 client")
- bucketName, err := createTestBucket(ctx, client, "sse-s3-metadata")
- require.NoError(t, err, "Failed to create test bucket")
- defer cleanupTestBucket(ctx, client, bucketName)
- testData := []byte("SSE-S3 with custom metadata")
- objectKey := "test-object-with-metadata.txt"
- userMetadata := map[string]string{
- "author": "test-user",
- "version": "1.0",
- "environment": "test",
- }
- t.Run("Upload with Metadata", func(t *testing.T) {
- // Upload object with SSE-S3 and user metadata
- _, err := client.PutObject(ctx, &s3.PutObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- Body: bytes.NewReader(testData),
- ServerSideEncryption: types.ServerSideEncryptionAes256,
- Metadata: userMetadata,
- })
- require.NoError(t, err, "Failed to upload object with SSE-S3 and metadata")
- })
- t.Run("Verify Metadata and Encryption", func(t *testing.T) {
- // HEAD request to check metadata and encryption
- resp, err := client.HeadObject(ctx, &s3.HeadObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- })
- require.NoError(t, err, "Failed to HEAD SSE-S3 object with metadata")
- // Verify SSE-S3 headers
- assert.Equal(t, types.ServerSideEncryptionAes256, resp.ServerSideEncryption, "SSE-S3 header missing with metadata")
- // Verify user metadata
- for key, expectedValue := range userMetadata {
- actualValue, exists := resp.Metadata[key]
- assert.True(t, exists, "Metadata key %s not found", key)
- assert.Equal(t, expectedValue, actualValue, "Metadata value mismatch for key %s", key)
- }
- })
- t.Run("Download and Verify Content", func(t *testing.T) {
- // Download and verify content
- resp, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- })
- require.NoError(t, err, "Failed to download SSE-S3 object with metadata")
- // Verify SSE-S3 headers
- assert.Equal(t, types.ServerSideEncryptionAes256, resp.ServerSideEncryption, "SSE-S3 header missing in GET response")
- // Verify content
- downloadedData, err := io.ReadAll(resp.Body)
- require.NoError(t, err, "Failed to read downloaded data")
- resp.Body.Close()
- assertDataEqual(t, testData, downloadedData, "Downloaded data doesn't match original")
- })
- }
- // TestSSES3RangeRequests tests SSE-S3 with HTTP range requests
- func TestSSES3RangeRequests(t *testing.T) {
- ctx := context.Background()
- client, err := createS3Client(ctx, defaultConfig)
- require.NoError(t, err, "Failed to create S3 client")
- bucketName, err := createTestBucket(ctx, client, "sse-s3-range")
- require.NoError(t, err, "Failed to create test bucket")
- defer cleanupTestBucket(ctx, client, bucketName)
- // Create test data large enough to ensure multipart storage
- testData := generateTestData(1024 * 1024) // 1MB to ensure multipart chunking
- objectKey := "test-sse-s3-range.dat"
- // Upload object with SSE-S3
- _, err = client.PutObject(ctx, &s3.PutObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- Body: bytes.NewReader(testData),
- ServerSideEncryption: types.ServerSideEncryptionAes256,
- })
- require.NoError(t, err, "Failed to upload SSE-S3 object for range testing")
- testCases := []struct {
- name string
- rangeHeader string
- expectedStart int
- expectedEnd int
- }{
- {"First 100 bytes", "bytes=0-99", 0, 99},
- {"Middle range", "bytes=100000-199999", 100000, 199999},
- {"Last 100 bytes", "bytes=1048476-1048575", 1048476, 1048575},
- {"From offset to end", "bytes=500000-", 500000, len(testData) - 1},
- }
- for _, tc := range testCases {
- t.Run(tc.name, func(t *testing.T) {
- // Request range
- resp, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- Range: aws.String(tc.rangeHeader),
- })
- require.NoError(t, err, "Failed to get range %s", tc.rangeHeader)
- // Verify SSE-S3 headers are present in range response
- assert.Equal(t, types.ServerSideEncryptionAes256, resp.ServerSideEncryption, "SSE-S3 header missing in range response")
- // Read range data
- rangeData, err := io.ReadAll(resp.Body)
- require.NoError(t, err, "Failed to read range data")
- resp.Body.Close()
- // Calculate expected data
- endIndex := tc.expectedEnd
- if tc.expectedEnd >= len(testData) {
- endIndex = len(testData) - 1
- }
- expectedData := testData[tc.expectedStart : endIndex+1]
- // Verify range data
- assertDataEqual(t, expectedData, rangeData, "Range data mismatch for %s", tc.rangeHeader)
- })
- }
- }
- // TestSSES3BucketDefaultEncryption tests bucket-level default encryption with SSE-S3
- func TestSSES3BucketDefaultEncryption(t *testing.T) {
- ctx := context.Background()
- client, err := createS3Client(ctx, defaultConfig)
- require.NoError(t, err, "Failed to create S3 client")
- bucketName, err := createTestBucket(ctx, client, "sse-s3-default")
- require.NoError(t, err, "Failed to create test bucket")
- defer cleanupTestBucket(ctx, client, bucketName)
- t.Run("Set Bucket Default Encryption", func(t *testing.T) {
- // Set bucket encryption configuration
- _, err := client.PutBucketEncryption(ctx, &s3.PutBucketEncryptionInput{
- Bucket: aws.String(bucketName),
- ServerSideEncryptionConfiguration: &types.ServerSideEncryptionConfiguration{
- Rules: []types.ServerSideEncryptionRule{
- {
- ApplyServerSideEncryptionByDefault: &types.ServerSideEncryptionByDefault{
- SSEAlgorithm: types.ServerSideEncryptionAes256,
- },
- },
- },
- },
- })
- require.NoError(t, err, "Failed to set bucket default encryption")
- })
- t.Run("Upload Object Without Encryption Headers", func(t *testing.T) {
- testData := []byte("This object should be automatically encrypted with SSE-S3 due to bucket default policy.")
- objectKey := "test-default-encrypted-object.txt"
- // Upload object WITHOUT any encryption headers
- _, err := client.PutObject(ctx, &s3.PutObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- Body: bytes.NewReader(testData),
- // No ServerSideEncryption specified - should use bucket default
- })
- require.NoError(t, err, "Failed to upload object without encryption headers")
- // Download and verify it was automatically encrypted
- resp, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- })
- require.NoError(t, err, "Failed to download object")
- // Verify SSE-S3 headers are present (indicating automatic encryption)
- assert.Equal(t, types.ServerSideEncryptionAes256, resp.ServerSideEncryption, "Object should have been automatically encrypted with SSE-S3")
- // Verify content is correct (decryption works)
- downloadedData, err := io.ReadAll(resp.Body)
- require.NoError(t, err, "Failed to read downloaded data")
- resp.Body.Close()
- assertDataEqual(t, testData, downloadedData, "Downloaded data doesn't match original")
- })
- t.Run("Get Bucket Encryption Configuration", func(t *testing.T) {
- // Verify we can retrieve the bucket encryption configuration
- resp, err := client.GetBucketEncryption(ctx, &s3.GetBucketEncryptionInput{
- Bucket: aws.String(bucketName),
- })
- require.NoError(t, err, "Failed to get bucket encryption configuration")
- require.Len(t, resp.ServerSideEncryptionConfiguration.Rules, 1, "Should have one encryption rule")
- rule := resp.ServerSideEncryptionConfiguration.Rules[0]
- assert.Equal(t, types.ServerSideEncryptionAes256, rule.ApplyServerSideEncryptionByDefault.SSEAlgorithm, "Encryption algorithm should be AES256")
- })
- t.Run("Delete Bucket Encryption Configuration", func(t *testing.T) {
- // Remove bucket encryption configuration
- _, err := client.DeleteBucketEncryption(ctx, &s3.DeleteBucketEncryptionInput{
- Bucket: aws.String(bucketName),
- })
- require.NoError(t, err, "Failed to delete bucket encryption configuration")
- // Verify it's removed by trying to get it (should fail)
- _, err = client.GetBucketEncryption(ctx, &s3.GetBucketEncryptionInput{
- Bucket: aws.String(bucketName),
- })
- require.Error(t, err, "Getting bucket encryption should fail after deletion")
- })
- t.Run("Upload After Removing Default Encryption", func(t *testing.T) {
- testData := []byte("This object should NOT be encrypted after removing bucket default.")
- objectKey := "test-no-default-encryption.txt"
- // Upload object without encryption headers (should not be encrypted now)
- _, err := client.PutObject(ctx, &s3.PutObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- Body: bytes.NewReader(testData),
- })
- require.NoError(t, err, "Failed to upload object")
- // Verify it's NOT encrypted
- resp, err := client.HeadObject(ctx, &s3.HeadObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- })
- require.NoError(t, err, "Failed to HEAD object")
- // ServerSideEncryption should be empty/nil when no encryption is applied
- assert.Empty(t, resp.ServerSideEncryption, "Object should not be encrypted after removing bucket default")
- })
- }
- // TestSSES3MultipartUploads tests SSE-S3 multipart upload functionality
- func TestSSES3MultipartUploads(t *testing.T) {
- ctx := context.Background()
- client, err := createS3Client(ctx, defaultConfig)
- require.NoError(t, err, "Failed to create S3 client")
- bucketName, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"sse-s3-multipart-")
- require.NoError(t, err, "Failed to create test bucket")
- defer cleanupTestBucket(ctx, client, bucketName)
- t.Run("Large_File_Multipart_Upload", func(t *testing.T) {
- objectKey := "test-sse-s3-multipart-large.dat"
- // Create 10MB test data to ensure multipart upload
- testData := generateTestData(10 * 1024 * 1024)
- // Upload with SSE-S3
- _, err = client.PutObject(ctx, &s3.PutObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- Body: bytes.NewReader(testData),
- ServerSideEncryption: types.ServerSideEncryptionAes256,
- })
- require.NoError(t, err, "SSE-S3 multipart upload failed")
- // Verify encryption headers
- headResp, err := client.HeadObject(ctx, &s3.HeadObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- })
- require.NoError(t, err, "Failed to head object")
- assert.Equal(t, types.ServerSideEncryptionAes256, headResp.ServerSideEncryption, "Expected SSE-S3 encryption")
- // Download and verify content
- getResp, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- })
- require.NoError(t, err, "Failed to download SSE-S3 multipart object")
- defer getResp.Body.Close()
- downloadedData, err := io.ReadAll(getResp.Body)
- require.NoError(t, err, "Failed to read downloaded data")
- assert.Equal(t, testData, downloadedData, "SSE-S3 multipart upload data should match")
- // Test range requests on multipart SSE-S3 object
- t.Run("Range_Request_On_Multipart", func(t *testing.T) {
- start := int64(1024 * 1024) // 1MB offset
- end := int64(2*1024*1024 - 1) // 2MB - 1
- expectedLength := end - start + 1
- rangeResp, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- Range: aws.String(fmt.Sprintf("bytes=%d-%d", start, end)),
- })
- require.NoError(t, err, "Failed to get range from SSE-S3 multipart object")
- defer rangeResp.Body.Close()
- rangeData, err := io.ReadAll(rangeResp.Body)
- require.NoError(t, err, "Failed to read range data")
- assert.Equal(t, expectedLength, int64(len(rangeData)), "Range length should match")
- // Verify range content matches original data
- expectedRange := testData[start : end+1]
- assert.Equal(t, expectedRange, rangeData, "Range content should match for SSE-S3 multipart object")
- })
- })
- t.Run("Explicit_Multipart_Upload_API", func(t *testing.T) {
- objectKey := "test-sse-s3-explicit-multipart.dat"
- testData := generateTestData(15 * 1024 * 1024) // 15MB
- // Create multipart upload with SSE-S3
- createResp, err := client.CreateMultipartUpload(ctx, &s3.CreateMultipartUploadInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- ServerSideEncryption: types.ServerSideEncryptionAes256,
- })
- require.NoError(t, err, "Failed to create SSE-S3 multipart upload")
- uploadID := *createResp.UploadId
- var parts []types.CompletedPart
- // Upload parts (5MB each, except the last part)
- partSize := 5 * 1024 * 1024
- for i := 0; i < len(testData); i += partSize {
- partNumber := int32(len(parts) + 1)
- endIdx := i + partSize
- if endIdx > len(testData) {
- endIdx = len(testData)
- }
- partData := testData[i:endIdx]
- uploadPartResp, err := client.UploadPart(ctx, &s3.UploadPartInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- PartNumber: aws.Int32(partNumber),
- UploadId: aws.String(uploadID),
- Body: bytes.NewReader(partData),
- })
- require.NoError(t, err, "Failed to upload part %d", partNumber)
- parts = append(parts, types.CompletedPart{
- ETag: uploadPartResp.ETag,
- PartNumber: aws.Int32(partNumber),
- })
- }
- // Complete multipart upload
- _, err = client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- UploadId: aws.String(uploadID),
- MultipartUpload: &types.CompletedMultipartUpload{
- Parts: parts,
- },
- })
- require.NoError(t, err, "Failed to complete SSE-S3 multipart upload")
- // Verify the completed object
- headResp, err := client.HeadObject(ctx, &s3.HeadObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- })
- require.NoError(t, err, "Failed to head completed multipart object")
- assert.Equal(t, types.ServerSideEncryptionAes256, headResp.ServerSideEncryption, "Expected SSE-S3 encryption on completed multipart object")
- // Download and verify content
- getResp, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- })
- require.NoError(t, err, "Failed to download completed SSE-S3 multipart object")
- defer getResp.Body.Close()
- downloadedData, err := io.ReadAll(getResp.Body)
- require.NoError(t, err, "Failed to read downloaded data")
- assert.Equal(t, testData, downloadedData, "Explicit SSE-S3 multipart upload data should match")
- })
- }
- // TestCrossSSECopy tests copying objects between different SSE encryption types
- func TestCrossSSECopy(t *testing.T) {
- ctx := context.Background()
- client, err := createS3Client(ctx, defaultConfig)
- require.NoError(t, err, "Failed to create S3 client")
- bucketName, err := createTestBucket(ctx, client, defaultConfig.BucketPrefix+"sse-cross-copy-")
- require.NoError(t, err, "Failed to create test bucket")
- defer cleanupTestBucket(ctx, client, bucketName)
- // Test data
- testData := []byte("Cross-SSE copy test data")
- // Generate proper SSE-C key
- sseKey := generateSSECKey()
- t.Run("SSE-S3_to_Unencrypted", func(t *testing.T) {
- sourceKey := "source-sse-s3-obj"
- destKey := "dest-unencrypted-obj"
- // Upload with SSE-S3
- _, err = client.PutObject(ctx, &s3.PutObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(sourceKey),
- Body: bytes.NewReader(testData),
- ServerSideEncryption: types.ServerSideEncryptionAes256,
- })
- require.NoError(t, err, "SSE-S3 upload failed")
- // Copy to unencrypted
- _, err = client.CopyObject(ctx, &s3.CopyObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(destKey),
- CopySource: aws.String(fmt.Sprintf("%s/%s", bucketName, sourceKey)),
- })
- require.NoError(t, err, "Copy SSE-S3 to unencrypted failed")
- // Verify destination is unencrypted and content matches
- getResp, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(destKey),
- })
- require.NoError(t, err, "GET failed")
- defer getResp.Body.Close()
- assert.Empty(t, getResp.ServerSideEncryption, "Should be unencrypted")
- downloadedData, err := io.ReadAll(getResp.Body)
- require.NoError(t, err, "Read failed")
- assertDataEqual(t, testData, downloadedData)
- })
- t.Run("Unencrypted_to_SSE-S3", func(t *testing.T) {
- sourceKey := "source-unencrypted-obj"
- destKey := "dest-sse-s3-obj"
- // Upload unencrypted
- _, err = client.PutObject(ctx, &s3.PutObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(sourceKey),
- Body: bytes.NewReader(testData),
- })
- require.NoError(t, err, "Unencrypted upload failed")
- // Copy to SSE-S3
- _, err = client.CopyObject(ctx, &s3.CopyObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(destKey),
- CopySource: aws.String(fmt.Sprintf("%s/%s", bucketName, sourceKey)),
- ServerSideEncryption: types.ServerSideEncryptionAes256,
- })
- require.NoError(t, err, "Copy unencrypted to SSE-S3 failed")
- // Verify destination is SSE-S3 encrypted and content matches
- getResp, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(destKey),
- })
- require.NoError(t, err, "GET failed")
- defer getResp.Body.Close()
- assert.Equal(t, types.ServerSideEncryptionAes256, getResp.ServerSideEncryption, "Expected SSE-S3")
- downloadedData, err := io.ReadAll(getResp.Body)
- require.NoError(t, err, "Read failed")
- assertDataEqual(t, testData, downloadedData)
- })
- t.Run("SSE-C_to_SSE-S3", func(t *testing.T) {
- sourceKey := "source-sse-c-obj"
- destKey := "dest-sse-s3-obj"
- // Upload with SSE-C
- _, err = client.PutObject(ctx, &s3.PutObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(sourceKey),
- Body: bytes.NewReader(testData),
- SSECustomerAlgorithm: aws.String("AES256"),
- SSECustomerKey: aws.String(sseKey.KeyB64),
- SSECustomerKeyMD5: aws.String(sseKey.KeyMD5),
- })
- require.NoError(t, err, "SSE-C upload failed")
- // Copy to SSE-S3
- _, err = client.CopyObject(ctx, &s3.CopyObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(destKey),
- CopySource: aws.String(fmt.Sprintf("%s/%s", bucketName, sourceKey)),
- CopySourceSSECustomerAlgorithm: aws.String("AES256"),
- CopySourceSSECustomerKey: aws.String(sseKey.KeyB64),
- CopySourceSSECustomerKeyMD5: aws.String(sseKey.KeyMD5),
- ServerSideEncryption: types.ServerSideEncryptionAes256,
- })
- require.NoError(t, err, "Copy SSE-C to SSE-S3 failed")
- // Verify destination encryption and content
- headResp, err := client.HeadObject(ctx, &s3.HeadObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(destKey),
- })
- require.NoError(t, err, "HEAD failed")
- assert.Equal(t, types.ServerSideEncryptionAes256, headResp.ServerSideEncryption, "Expected SSE-S3")
- getResp, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(destKey),
- })
- require.NoError(t, err, "GET failed")
- defer getResp.Body.Close()
- downloadedData, err := io.ReadAll(getResp.Body)
- require.NoError(t, err, "Read failed")
- assertDataEqual(t, testData, downloadedData)
- })
- t.Run("SSE-S3_to_SSE-C", func(t *testing.T) {
- sourceKey := "source-sse-s3-obj"
- destKey := "dest-sse-c-obj"
- // Upload with SSE-S3
- _, err = client.PutObject(ctx, &s3.PutObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(sourceKey),
- Body: bytes.NewReader(testData),
- ServerSideEncryption: types.ServerSideEncryptionAes256,
- })
- require.NoError(t, err, "Failed to upload SSE-S3 source object")
- // Copy to SSE-C
- _, err = client.CopyObject(ctx, &s3.CopyObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(destKey),
- CopySource: aws.String(fmt.Sprintf("%s/%s", bucketName, sourceKey)),
- SSECustomerAlgorithm: aws.String("AES256"),
- SSECustomerKey: aws.String(sseKey.KeyB64),
- SSECustomerKeyMD5: aws.String(sseKey.KeyMD5),
- })
- require.NoError(t, err, "Copy SSE-S3 to SSE-C failed")
- // Verify destination encryption and content
- getResp, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(destKey),
- SSECustomerAlgorithm: aws.String("AES256"),
- SSECustomerKey: aws.String(sseKey.KeyB64),
- SSECustomerKeyMD5: aws.String(sseKey.KeyMD5),
- })
- require.NoError(t, err, "GET with SSE-C failed")
- defer getResp.Body.Close()
- assert.Equal(t, "AES256", aws.ToString(getResp.SSECustomerAlgorithm), "Expected SSE-C")
- downloadedData, err := io.ReadAll(getResp.Body)
- require.NoError(t, err, "Read failed")
- assertDataEqual(t, testData, downloadedData)
- })
- }
- // REGRESSION TESTS FOR CRITICAL BUGS FIXED
- // These tests specifically target the IV storage bugs that were fixed
- // TestSSES3IVStorageRegression tests that IVs are properly stored for explicit SSE-S3 uploads
- // This test would have caught the critical bug where IVs were discarded in putToFiler
- func TestSSES3IVStorageRegression(t *testing.T) {
- ctx := context.Background()
- client, err := createS3Client(ctx, defaultConfig)
- require.NoError(t, err, "Failed to create S3 client")
- bucketName, err := createTestBucket(ctx, client, "sse-s3-iv-regression")
- require.NoError(t, err, "Failed to create test bucket")
- defer cleanupTestBucket(ctx, client, bucketName)
- t.Run("Explicit SSE-S3 IV Storage and Retrieval", func(t *testing.T) {
- testData := []byte("This tests the critical IV storage bug that was fixed - the IV must be stored on the key object for decryption to work.")
- objectKey := "explicit-sse-s3-iv-test.txt"
- // Upload with explicit SSE-S3 header (this used to discard the IV)
- putResp, err := client.PutObject(ctx, &s3.PutObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- Body: bytes.NewReader(testData),
- ServerSideEncryption: types.ServerSideEncryptionAes256,
- })
- require.NoError(t, err, "Failed to upload explicit SSE-S3 object")
- // Verify PUT response has SSE-S3 headers
- assert.Equal(t, types.ServerSideEncryptionAes256, putResp.ServerSideEncryption, "PUT response should indicate SSE-S3")
- // Critical test: Download and decrypt the object
- // This would have FAILED with the original bug because IV was discarded
- getResp, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- })
- require.NoError(t, err, "Failed to download explicit SSE-S3 object")
- // Verify GET response has SSE-S3 headers
- assert.Equal(t, types.ServerSideEncryptionAes256, getResp.ServerSideEncryption, "GET response should indicate SSE-S3")
- // This is the critical test - verify data can be decrypted correctly
- downloadedData, err := io.ReadAll(getResp.Body)
- require.NoError(t, err, "Failed to read decrypted data")
- getResp.Body.Close()
- // This assertion would have FAILED with the original bug
- assertDataEqual(t, testData, downloadedData, "CRITICAL: Decryption failed - IV was not stored properly")
- })
- t.Run("Multiple Explicit SSE-S3 Objects", func(t *testing.T) {
- // Test multiple objects to ensure each gets its own unique IV
- numObjects := 5
- testDataSet := make([][]byte, numObjects)
- objectKeys := make([]string, numObjects)
- // Upload multiple objects with explicit SSE-S3
- for i := 0; i < numObjects; i++ {
- testDataSet[i] = []byte(fmt.Sprintf("Test data for object %d - verifying unique IV storage", i))
- objectKeys[i] = fmt.Sprintf("explicit-sse-s3-multi-%d.txt", i)
- _, err := client.PutObject(ctx, &s3.PutObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKeys[i]),
- Body: bytes.NewReader(testDataSet[i]),
- ServerSideEncryption: types.ServerSideEncryptionAes256,
- })
- require.NoError(t, err, "Failed to upload explicit SSE-S3 object %d", i)
- }
- // Download and verify each object decrypts correctly
- for i := 0; i < numObjects; i++ {
- getResp, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKeys[i]),
- })
- require.NoError(t, err, "Failed to download explicit SSE-S3 object %d", i)
- downloadedData, err := io.ReadAll(getResp.Body)
- require.NoError(t, err, "Failed to read decrypted data for object %d", i)
- getResp.Body.Close()
- assertDataEqual(t, testDataSet[i], downloadedData, "Decryption failed for object %d - IV not unique/stored", i)
- }
- })
- }
- // TestSSES3BucketDefaultIVStorageRegression tests bucket default SSE-S3 IV storage
- // This test would have caught the critical bug where IVs were not stored on key objects in bucket defaults
- func TestSSES3BucketDefaultIVStorageRegression(t *testing.T) {
- ctx := context.Background()
- client, err := createS3Client(ctx, defaultConfig)
- require.NoError(t, err, "Failed to create S3 client")
- bucketName, err := createTestBucket(ctx, client, "sse-s3-default-iv-regression")
- require.NoError(t, err, "Failed to create test bucket")
- defer cleanupTestBucket(ctx, client, bucketName)
- // Set bucket default encryption to SSE-S3
- _, err = client.PutBucketEncryption(ctx, &s3.PutBucketEncryptionInput{
- Bucket: aws.String(bucketName),
- ServerSideEncryptionConfiguration: &types.ServerSideEncryptionConfiguration{
- Rules: []types.ServerSideEncryptionRule{
- {
- ApplyServerSideEncryptionByDefault: &types.ServerSideEncryptionByDefault{
- SSEAlgorithm: types.ServerSideEncryptionAes256,
- },
- },
- },
- },
- })
- require.NoError(t, err, "Failed to set bucket default SSE-S3 encryption")
- t.Run("Bucket Default SSE-S3 IV Storage", func(t *testing.T) {
- testData := []byte("This tests the bucket default SSE-S3 IV storage bug - IV must be stored on key object for decryption.")
- objectKey := "bucket-default-sse-s3-iv-test.txt"
- // Upload WITHOUT encryption headers - should use bucket default SSE-S3
- // This used to fail because applySSES3DefaultEncryption didn't store IV on key
- putResp, err := client.PutObject(ctx, &s3.PutObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- Body: bytes.NewReader(testData),
- // No ServerSideEncryption specified - should use bucket default
- })
- require.NoError(t, err, "Failed to upload object for bucket default SSE-S3")
- // Verify bucket default encryption was applied
- assert.Equal(t, types.ServerSideEncryptionAes256, putResp.ServerSideEncryption, "PUT response should show bucket default SSE-S3")
- // Critical test: Download and decrypt the object
- // This would have FAILED with the original bug because IV wasn't stored on key object
- getResp, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- })
- require.NoError(t, err, "Failed to download bucket default SSE-S3 object")
- // Verify GET response shows SSE-S3 was applied
- assert.Equal(t, types.ServerSideEncryptionAes256, getResp.ServerSideEncryption, "GET response should show SSE-S3")
- // This is the critical test - verify decryption works
- downloadedData, err := io.ReadAll(getResp.Body)
- require.NoError(t, err, "Failed to read decrypted data")
- getResp.Body.Close()
- // This assertion would have FAILED with the original bucket default bug
- assertDataEqual(t, testData, downloadedData, "CRITICAL: Bucket default SSE-S3 decryption failed - IV not stored on key object")
- })
- t.Run("Multiple Bucket Default Objects", func(t *testing.T) {
- // Test multiple objects with bucket default encryption
- numObjects := 3
- testDataSet := make([][]byte, numObjects)
- objectKeys := make([]string, numObjects)
- // Upload multiple objects without encryption headers
- for i := 0; i < numObjects; i++ {
- testDataSet[i] = []byte(fmt.Sprintf("Bucket default test data %d - verifying IV storage works", i))
- objectKeys[i] = fmt.Sprintf("bucket-default-multi-%d.txt", i)
- _, err := client.PutObject(ctx, &s3.PutObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKeys[i]),
- Body: bytes.NewReader(testDataSet[i]),
- // No encryption headers - bucket default should apply
- })
- require.NoError(t, err, "Failed to upload bucket default object %d", i)
- }
- // Verify each object was encrypted and can be decrypted
- for i := 0; i < numObjects; i++ {
- getResp, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKeys[i]),
- })
- require.NoError(t, err, "Failed to download bucket default object %d", i)
- // Verify SSE-S3 was applied by bucket default
- assert.Equal(t, types.ServerSideEncryptionAes256, getResp.ServerSideEncryption, "Object %d should be SSE-S3 encrypted", i)
- downloadedData, err := io.ReadAll(getResp.Body)
- require.NoError(t, err, "Failed to read decrypted data for object %d", i)
- getResp.Body.Close()
- assertDataEqual(t, testDataSet[i], downloadedData, "Bucket default SSE-S3 decryption failed for object %d", i)
- }
- })
- }
- // TestSSES3EdgeCaseRegression tests edge cases that could cause IV storage issues
- func TestSSES3EdgeCaseRegression(t *testing.T) {
- ctx := context.Background()
- client, err := createS3Client(ctx, defaultConfig)
- require.NoError(t, err, "Failed to create S3 client")
- bucketName, err := createTestBucket(ctx, client, "sse-s3-edge-regression")
- require.NoError(t, err, "Failed to create test bucket")
- defer cleanupTestBucket(ctx, client, bucketName)
- t.Run("Empty Object SSE-S3", func(t *testing.T) {
- // Test edge case: empty objects with SSE-S3 (IV storage still required)
- objectKey := "empty-sse-s3-object"
- _, err := client.PutObject(ctx, &s3.PutObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- Body: bytes.NewReader([]byte{}),
- ServerSideEncryption: types.ServerSideEncryptionAes256,
- })
- require.NoError(t, err, "Failed to upload empty SSE-S3 object")
- // Verify empty object can be retrieved (IV must be stored even for empty objects)
- getResp, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- })
- require.NoError(t, err, "Failed to download empty SSE-S3 object")
- downloadedData, err := io.ReadAll(getResp.Body)
- require.NoError(t, err, "Failed to read empty decrypted data")
- getResp.Body.Close()
- assert.Equal(t, []byte{}, downloadedData, "Empty object content mismatch")
- assert.Equal(t, types.ServerSideEncryptionAes256, getResp.ServerSideEncryption, "Empty object should be SSE-S3 encrypted")
- })
- t.Run("Large Object SSE-S3", func(t *testing.T) {
- // Test large objects to ensure IV storage works for chunked uploads
- largeData := generateTestData(1024 * 1024) // 1MB
- objectKey := "large-sse-s3-object"
- _, err := client.PutObject(ctx, &s3.PutObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- Body: bytes.NewReader(largeData),
- ServerSideEncryption: types.ServerSideEncryptionAes256,
- })
- require.NoError(t, err, "Failed to upload large SSE-S3 object")
- // Verify large object can be decrypted (IV must be stored properly)
- getResp, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- })
- require.NoError(t, err, "Failed to download large SSE-S3 object")
- downloadedData, err := io.ReadAll(getResp.Body)
- require.NoError(t, err, "Failed to read large decrypted data")
- getResp.Body.Close()
- assertDataEqual(t, largeData, downloadedData, "Large object decryption failed - IV storage issue")
- assert.Equal(t, types.ServerSideEncryptionAes256, getResp.ServerSideEncryption, "Large object should be SSE-S3 encrypted")
- })
- }
- // TestSSES3ErrorHandlingRegression tests error handling improvements that were added
- func TestSSES3ErrorHandlingRegression(t *testing.T) {
- ctx := context.Background()
- client, err := createS3Client(ctx, defaultConfig)
- require.NoError(t, err, "Failed to create S3 client")
- bucketName, err := createTestBucket(ctx, client, "sse-s3-error-regression")
- require.NoError(t, err, "Failed to create test bucket")
- defer cleanupTestBucket(ctx, client, bucketName)
- t.Run("SSE-S3 With Other Valid Operations", func(t *testing.T) {
- // Ensure SSE-S3 works with other S3 operations (metadata, tagging, etc.)
- testData := []byte("Testing SSE-S3 with metadata and other operations")
- objectKey := "sse-s3-with-metadata"
- // Upload with SSE-S3 and metadata
- _, err := client.PutObject(ctx, &s3.PutObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- Body: bytes.NewReader(testData),
- ServerSideEncryption: types.ServerSideEncryptionAes256,
- Metadata: map[string]string{
- "test-key": "test-value",
- "purpose": "regression-test",
- },
- })
- require.NoError(t, err, "Failed to upload SSE-S3 object with metadata")
- // HEAD request to verify metadata and encryption
- headResp, err := client.HeadObject(ctx, &s3.HeadObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- })
- require.NoError(t, err, "Failed to HEAD SSE-S3 object")
- assert.Equal(t, types.ServerSideEncryptionAes256, headResp.ServerSideEncryption, "HEAD should show SSE-S3")
- assert.Equal(t, "test-value", headResp.Metadata["test-key"], "Metadata should be preserved")
- assert.Equal(t, "regression-test", headResp.Metadata["purpose"], "Metadata should be preserved")
- // GET to verify decryption still works with metadata
- getResp, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- })
- require.NoError(t, err, "Failed to GET SSE-S3 object")
- downloadedData, err := io.ReadAll(getResp.Body)
- require.NoError(t, err, "Failed to read decrypted data")
- getResp.Body.Close()
- assertDataEqual(t, testData, downloadedData, "SSE-S3 with metadata decryption failed")
- assert.Equal(t, types.ServerSideEncryptionAes256, getResp.ServerSideEncryption, "GET should show SSE-S3")
- assert.Equal(t, "test-value", getResp.Metadata["test-key"], "GET metadata should be preserved")
- })
- }
- // TestSSES3FunctionalityCompletion tests that SSE-S3 feature is now fully functional
- func TestSSES3FunctionalityCompletion(t *testing.T) {
- ctx := context.Background()
- client, err := createS3Client(ctx, defaultConfig)
- require.NoError(t, err, "Failed to create S3 client")
- bucketName, err := createTestBucket(ctx, client, "sse-s3-completion")
- require.NoError(t, err, "Failed to create test bucket")
- defer cleanupTestBucket(ctx, client, bucketName)
- t.Run("All SSE-S3 Scenarios Work", func(t *testing.T) {
- scenarios := []struct {
- name string
- setupBucket func() error
- encryption *types.ServerSideEncryption
- expectSSES3 bool
- }{
- {
- name: "Explicit SSE-S3 Header",
- setupBucket: func() error { return nil },
- encryption: &[]types.ServerSideEncryption{types.ServerSideEncryptionAes256}[0],
- expectSSES3: true,
- },
- {
- name: "Bucket Default SSE-S3",
- setupBucket: func() error {
- _, err := client.PutBucketEncryption(ctx, &s3.PutBucketEncryptionInput{
- Bucket: aws.String(bucketName),
- ServerSideEncryptionConfiguration: &types.ServerSideEncryptionConfiguration{
- Rules: []types.ServerSideEncryptionRule{
- {
- ApplyServerSideEncryptionByDefault: &types.ServerSideEncryptionByDefault{
- SSEAlgorithm: types.ServerSideEncryptionAes256,
- },
- },
- },
- },
- })
- return err
- },
- encryption: nil,
- expectSSES3: true,
- },
- }
- for i, scenario := range scenarios {
- t.Run(scenario.name, func(t *testing.T) {
- // Setup bucket if needed
- err := scenario.setupBucket()
- require.NoError(t, err, "Failed to setup bucket for scenario %s", scenario.name)
- testData := []byte(fmt.Sprintf("Test data for scenario: %s", scenario.name))
- objectKey := fmt.Sprintf("completion-test-%d", i)
- // Upload object
- putInput := &s3.PutObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- Body: bytes.NewReader(testData),
- }
- if scenario.encryption != nil {
- putInput.ServerSideEncryption = *scenario.encryption
- }
- putResp, err := client.PutObject(ctx, putInput)
- require.NoError(t, err, "Failed to upload object for scenario %s", scenario.name)
- if scenario.expectSSES3 {
- assert.Equal(t, types.ServerSideEncryptionAes256, putResp.ServerSideEncryption, "Should use SSE-S3 for %s", scenario.name)
- }
- // Download and verify
- getResp, err := client.GetObject(ctx, &s3.GetObjectInput{
- Bucket: aws.String(bucketName),
- Key: aws.String(objectKey),
- })
- require.NoError(t, err, "Failed to download object for scenario %s", scenario.name)
- if scenario.expectSSES3 {
- assert.Equal(t, types.ServerSideEncryptionAes256, getResp.ServerSideEncryption, "Should return SSE-S3 for %s", scenario.name)
- }
- downloadedData, err := io.ReadAll(getResp.Body)
- require.NoError(t, err, "Failed to read data for scenario %s", scenario.name)
- getResp.Body.Close()
- // This is the ultimate test - decryption must work
- assertDataEqual(t, testData, downloadedData, "Decryption failed for scenario %s", scenario.name)
- // Clean up bucket encryption for next scenario
- client.DeleteBucketEncryption(ctx, &s3.DeleteBucketEncryptionInput{
- Bucket: aws.String(bucketName),
- })
- })
- }
- })
- }
|