使用go sdk进行ceph s3 分片上传和分段下载

可以使用 AWS SDK for Go 和 go-ceph 库来实现 Ceph S3 分片上传和分段下载。

以下是一个简单的示例代码，演示如何使用 AWS SDK for Go 和 go-ceph 库进行分片上传和分段下载：

package main

import (
	"bytes"
	"fmt"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/beyondstorage/go-service-ceph"
	"github.com/beyondstorage/go-storage/v4/types"
	"io/ioutil"
)

func main() {
	// 创建 Ceph 对象存储服务
	store, err := ceph.NewStorager(
		ceph.WithCredential("access_key", "secret_key"),
		ceph.WithEndpoint("http://ceph-gateway:7480"),
		ceph.WithLocation("us-east-1"),
	)
	if err != nil {
		fmt.Printf("fail to create ceph storager: %v\n", err)
		return
	}

	// 创建 S3 服务
	s3Client := s3.New(session.Must(session.NewSession(&aws.Config{
		Credentials:      credentials.NewStaticCredentials("access_key", "secret_key", ""),
		Endpoint:         aws.String("http://ceph-gateway:7480"),
		Region:           aws.String("us-east-1"),
		S3ForcePathStyle: aws.Bool(true),
	})))

	// 分片上传文件
	putObjectInput := &s3.CreateMultipartUploadInput{
		Bucket: aws.String("my-bucket"),
		Key:    aws.String("my-file"),
	}
	createMultipartUploadOutput, err := s3Client.CreateMultipartUpload(putObjectInput)
	if err != nil {
		fmt.Printf("fail to create multipart upload: %v\n", err)
		return
	}

	// 将文件分成多个部分
	fileBytes, err := ioutil.ReadFile("my-file")
	if err != nil {
		fmt.Printf("fail to read file: %v\n", err)
		return
	}
	partSize := int64(5 * 1024 * 1024) // 5MB
	var parts []*s3.CompletedPart
	for i := int64(0); i < int64(len(fileBytes)); i += partSize {
		partNumber := int64(i/partSize) + 1
		partBytes := fileBytes[i:min(i+partSize, int64(len(fileBytes)))]
		partInput := &s3.UploadPartInput{
			Bucket:        aws.String("my-bucket"),
			Key:           aws.String("my-file"),
			UploadId:      createMultipartUploadOutput.UploadId,
			PartNumber:    aws.Int64(partNumber),
			ContentLength: aws.Int64(int64(len(partBytes))),
			Body:          bytes.NewReader(partBytes),
		}
		partOutput, err := s3Client.UploadPart(partInput)
		if err != nil {
			fmt.Printf("fail to upload part: %v\n", err)
			return
		}
		parts = append(parts, &s3.CompletedPart{
			ETag:       partOutput.ETag,
			PartNumber: aws.Int64(partNumber),
		})
	}

	// 完成上传
	completeInput := &s3.CompleteMultipartUploadInput{
		Bucket:   aws.String("my-bucket"),
		Key:      aws.String("my-file"),
		UploadId: createMultipartUploadOutput.UploadId,
		MultipartUpload: &s3.CompletedMultipartUpload{
			Parts: parts,
		},
	}
	_, err = s3Client.CompleteMultipartUpload(completeInput)
	if err != nil {
		fmt.Printf("fail to complete multipart upload: %v\n", err)
		return
	}

	// 分段下载文件
	getObjectInput := &s3.GetObjectInput{
		Bucket: aws.String("my-bucket"),
		Key:    aws.String("my-file"),
	}
	getObjectOutput, err := s3Client.GetObject(getObjectInput)
	if err != nil {
		fmt.Printf("fail to get object: %v\n", err)
		return
	}
	defer getObjectOutput.Body.Close()
	objectSize := *getObjectOutput.ContentLength
	partSize = int64(5 * 1024 * 1024) // 5MB
	var fileData []byte
	for i := int64(0); i < objectSize; i += partSize {
		partInput := &s3.GetObjectInput{
			Bucket: aws.String("my-bucket"),
			Key:    aws.String("my-file"),
			Range:  aws.String(fmt.Sprintf("bytes=%d-%d", i, min(i+partSize, objectSize)-1)),
		}
		partOutput, err := s3Client.GetObject(partInput)
		if err != nil {
			fmt.Printf("fail to get object part: %v\n", err)
			return
		}
		defer partOutput.Body.Close()
		partData, err := ioutil.ReadAll(partOutput.Body)
		if err != nil {
			fmt.Printf("fail to read object part: %v\n", err)
			return
		}
		fileData = append(fileData, partData...)
	}

	// 将文件数据写入本地文件
	err = ioutil.WriteFile("my-file-downloaded", fileData, 0644)
	if err != nil {
		fmt.Printf("fail to write file: %v\n", err)
		return
	}
	fmt.Println("done")
}

func min(a, b int64) int64 {
	if a < b {
		return a
	}
	return b
}

在这个例子中，我们首先创建了一个 Ceph 对象存储服务和一个 AWS S3 服务。然后，我们使用 S3 服务进行分片上传文件，将文件分成多个部分，并将每个部分上传到 Ceph S3 存储桶。最后，我们使用 S3 服务进行分段下载文件，将文件分成多个部分，并将每个部分从 Ceph S3 存储桶下载到本地文件系统，最后将所有部分组合成一个文件。