Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Discover EFS FileSystemId instead of specifying it in storage class #1492

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

Discover EFS FileSystemId instead of specifying it in storage class #1492

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
65af3c8
feat: support for efs discovery
avanish23 Oct 13, 2024
a693243
used aws-sdk-go-v2
avanish23 Nov 1, 2024
954eb4c
Merge branch 'master' into efs-discovery
avanish23 Dec 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ require (
k8s.io/kubernetes v1.27.16
k8s.io/mount-utils v0.27.16
k8s.io/pod-security-admission v0.27.16
sigs.k8s.io/yaml v1.3.0
)

require (
Expand Down Expand Up @@ -136,7 +137,6 @@ require (
sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.1.2 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
)

replace (
Expand Down
6 changes: 4 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,8 @@ github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4Kfc
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=
github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
github.com/emicklei/go-restful/v3 v3.12.0 h1:y2DdzBAURM29NFF94q6RaY4vjIH1rtwDapwQtU84iWk=
github.com/emicklei/go-restful/v3 v3.12.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
Expand Down Expand Up @@ -385,8 +387,8 @@ github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5
github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
Expand Down
43 changes: 40 additions & 3 deletions pkg/cloud/cloud.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,9 @@ var (
)

type FileSystem struct {
FileSystemId string
FileSystemId string
FileSystemArn string
Tags map[string]string
}

type AccessPoint struct {
Expand Down Expand Up @@ -105,7 +107,8 @@ type Cloud interface {
DescribeAccessPoint(ctx context.Context, accessPointId string) (accessPoint *AccessPoint, err error)
FindAccessPointByClientToken(ctx context.Context, clientToken, fileSystemId string) (accessPoint *AccessPoint, err error)
ListAccessPoints(ctx context.Context, fileSystemId string) (accessPoints []*AccessPoint, err error)
DescribeFileSystem(ctx context.Context, fileSystemId string) (fs *FileSystem, err error)
DescribeFileSystemById(ctx context.Context, fileSystemId string) (fs *FileSystem, err error)
DescribeFileSystemByToken(ctx context.Context, creationToken string) (fs []*FileSystem, err error)
DescribeMountTargets(ctx context.Context, fileSystemId, az string) (fs *MountTarget, err error)
}

Expand Down Expand Up @@ -322,7 +325,41 @@ func (c *cloud) ListAccessPoints(ctx context.Context, fileSystemId string) (acce
return
}

func (c *cloud) DescribeFileSystem(ctx context.Context, fileSystemId string) (fs *FileSystem, err error) {
func (c *cloud) DescribeFileSystemByToken(ctx context.Context, creationToken string) (fs []*FileSystem, err error) {
var describeFsInput *efs.DescribeFileSystemsInput
if creationToken == "" {
describeFsInput = &efs.DescribeFileSystemsInput{}
} else {
describeFsInput = &efs.DescribeFileSystemsInput{CreationToken: &creationToken}
}
klog.V(5).Infof("Calling DescribeFileSystems with input: %+v", *describeFsInput)
res, err := c.efs.DescribeFileSystems(ctx, describeFsInput)
if err != nil {
if isAccessDenied(err) {
return nil, ErrAccessDenied
}
if isFileSystemNotFound(err) {
return nil, ErrNotFound
}
return nil, fmt.Errorf("Describe File System failed: %v", err)
}

var efsList = make([]*FileSystem, 0)
for _, fileSystem := range res.FileSystems {
var tagsList = make(map[string]string, 0)
for _, tag := range fileSystem.Tags {
tagsList[*tag.Key] = *tag.Value
}
efsList = append(efsList, &FileSystem{
FileSystemId: *fileSystem.FileSystemId,
FileSystemArn: *fileSystem.FileSystemArn,
Tags: tagsList,
})
}
return efsList, nil
}

func (c *cloud) DescribeFileSystemById(ctx context.Context, fileSystemId string) (fs *FileSystem, err error) {
describeFsInput := &efs.DescribeFileSystemsInput{FileSystemId: &fileSystemId}
klog.V(5).Infof("Calling DescribeFileSystems with input: %+v", *describeFsInput)
res, err := c.efs.DescribeFileSystems(ctx, describeFsInput)
Expand Down
220 changes: 214 additions & 6 deletions pkg/cloud/cloud_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -785,7 +785,7 @@ func TestDescribeFileSystem(t *testing.T) {

ctx := context.Background()
mockEfs.EXPECT().DescribeFileSystems(gomock.Eq(ctx), gomock.Any()).Return(output, nil)
res, err := c.DescribeFileSystem(ctx, fsId)
res, err := c.DescribeFileSystemById(ctx, fsId)
if err != nil {
t.Fatalf("Describe File System failed: %v", err)
}
Expand Down Expand Up @@ -813,7 +813,7 @@ func TestDescribeFileSystem(t *testing.T) {

ctx := context.Background()
mockEfs.EXPECT().DescribeFileSystems(gomock.Eq(ctx), gomock.Any()).Return(output, nil)
_, err := c.DescribeFileSystem(ctx, fsId)
_, err := c.DescribeFileSystemById(ctx, fsId)
if err == nil {
t.Fatalf("DescribeFileSystem did not fail")
}
Expand Down Expand Up @@ -848,7 +848,7 @@ func TestDescribeFileSystem(t *testing.T) {

ctx := context.Background()
mockEfs.EXPECT().DescribeFileSystems(gomock.Eq(ctx), gomock.Any()).Return(output, nil)
_, err := c.DescribeFileSystem(ctx, fsId)
_, err := c.DescribeFileSystemById(ctx, fsId)
if err == nil {
t.Fatalf("DescribeFileSystem did not fail")
}
Expand All @@ -867,7 +867,7 @@ func TestDescribeFileSystem(t *testing.T) {
&types.FileSystemNotFound{
Message: aws.String("File System not found"),
})
_, err := c.DescribeFileSystem(ctx, fsId)
_, err := c.DescribeFileSystemById(ctx, fsId)
if err == nil {
t.Fatalf("DescribeFileSystem did not fail")
}
Expand All @@ -890,7 +890,7 @@ func TestDescribeFileSystem(t *testing.T) {
Code: AccessDeniedException,
Message: "Access Denied",
})
_, err := c.DescribeFileSystem(ctx, fsId)
_, err := c.DescribeFileSystemById(ctx, fsId)
if err == nil {
t.Fatalf("DescribeFileSystem did not fail")
}
Expand All @@ -909,7 +909,7 @@ func TestDescribeFileSystem(t *testing.T) {

ctx := context.Background()
mockEfs.EXPECT().DescribeFileSystems(gomock.Eq(ctx), gomock.Any()).Return(nil, errors.New("DescribeFileSystem failed"))
_, err := c.DescribeFileSystem(ctx, fsId)
_, err := c.DescribeFileSystemById(ctx, fsId)
if err == nil {
t.Fatalf("DescribeFileSystem did not fail")
}
Expand Down Expand Up @@ -1050,6 +1050,214 @@ func TestDescribeMountTargets(t *testing.T) {
}
}

func TestDescribeFileSystemByToken(t *testing.T) {
var (
fsId = []string{"fs-abcd1234", "fs-efgh5678"}
fsArn = []string{"arn:aws:elasticfilesystem:us-west-2:1111333322228888:file-system/fs-0123456789abcdef8", "arn:aws:elasticfilesystem:us-west-2:1111333322228888:file-system/fs-987654321abcdef0"}
creationToken = "efs-for-discovery"
az = "us-east-1a"
)

testCases := []struct {
name string
testFunc func(t *testing.T)
}{
{
name: "Success: Normal flow",
testFunc: func(t *testing.T) {
mockctl := gomock.NewController(t)
mockEfs := mocks.NewMockEfs(mockctl)
c := &cloud{efs: mockEfs}

fs := &efs.DescribeFileSystemsOutput{
FileSystems: []types.FileSystemDescription{
{
FileSystemId: aws.String(fsId[0]),
FileSystemArn: aws.String(fsArn[0]),
Encrypted: aws.Bool(true),
CreationToken: aws.String("efs-for-discovery"),
AvailabilityZoneName: aws.String(az),
Tags: []types.Tag{
{
Key: aws.String("env"),
Value: aws.String("prod"),
},
{
Key: aws.String("owner"),
Value: aws.String("[email protected]"),
},
},
},
{
FileSystemId: aws.String(fsId[1]),
FileSystemArn: aws.String(fsArn[1]),
Encrypted: aws.Bool(true),
CreationToken: aws.String("efs-not-for-discovery"),
AvailabilityZoneName: aws.String(az),
Tags: []types.Tag{
{
Key: aws.String("env"),
Value: aws.String("prod"),
},
{
Key: aws.String("owner"),
Value: aws.String("[email protected]"),
},
},
},
},
}

ctx := context.Background()
mockEfs.EXPECT().DescribeFileSystems(gomock.Eq(ctx), gomock.Any()).DoAndReturn(func(ctx context.Context, input *efs.DescribeFileSystemsInput, opts ...func(*efs.Options)) (*efs.DescribeFileSystemsOutput, error) {
res := &efs.DescribeFileSystemsOutput{}
for _, fileSystem := range fs.FileSystems {
if input.CreationToken != nil && *fileSystem.CreationToken == *input.CreationToken {
res.FileSystems = append(res.FileSystems, fileSystem)
} else if input.CreationToken == nil {
res.FileSystems = append(res.FileSystems, fileSystem)
}
}
return res, nil
})

efsList, err := c.DescribeFileSystemByToken(ctx, creationToken)
if err != nil {
t.Fatalf("DescribeFileSystem failed")
}
if len(efsList) != 1 {
t.Fatalf("Expected 1 fileSystems got %d", len(efsList))
}
mockctl.Finish()
},
},
{
name: "Success: Normal flow without creation token",
testFunc: func(t *testing.T) {
mockctl := gomock.NewController(t)
mockEfs := mocks.NewMockEfs(mockctl)
c := &cloud{efs: mockEfs}

fs := &efs.DescribeFileSystemsOutput{
FileSystems: []types.FileSystemDescription{
{
FileSystemId: aws.String(fsId[0]),
FileSystemArn: aws.String(fsArn[0]),
Encrypted: aws.Bool(true),
CreationToken: aws.String("efs-for-discovery"),
AvailabilityZoneName: aws.String(az),
Tags: []types.Tag{
{
Key: aws.String("env"),
Value: aws.String("prod"),
},
{
Key: aws.String("owner"),
Value: aws.String("[email protected]"),
},
},
},
{
FileSystemId: aws.String(fsId[1]),
FileSystemArn: aws.String(fsArn[1]),
Encrypted: aws.Bool(true),
CreationToken: aws.String("efs-not-for-discovery"),
AvailabilityZoneName: aws.String(az),
Tags: []types.Tag{
{
Key: aws.String("env"),
Value: aws.String("prod"),
},
{
Key: aws.String("owner"),
Value: aws.String("[email protected]"),
},
},
},
},
}

ctx := context.Background()
mockEfs.EXPECT().DescribeFileSystems(gomock.Eq(ctx), gomock.Any()).DoAndReturn(func(ctx context.Context, input *efs.DescribeFileSystemsInput, opts ...func(*efs.Options)) (*efs.DescribeFileSystemsOutput, error) {
res := &efs.DescribeFileSystemsOutput{}
for _, fileSystem := range fs.FileSystems {
if input.CreationToken != nil && *fileSystem.CreationToken == *input.CreationToken {
res.FileSystems = append(res.FileSystems, fileSystem)
} else if input.CreationToken == nil {
res.FileSystems = append(res.FileSystems, fileSystem)
}
}
return res, nil
})

efsList, err := c.DescribeFileSystemByToken(ctx, "")
if err != nil {
t.Fatalf("DescribeFileSystem failed")
}
if len(efsList) != len(fs.FileSystems) {
t.Fatalf("Expected 1 fileSystems got %d", len(efsList))
}
for i, fileSystem := range fs.FileSystems {
for _, v := range fileSystem.Tags {
if val, exists := efsList[i].Tags[*v.Key]; !exists || val != *v.Value {
t.Fatalf("Tags list is corrupted, expected %s for %s but got %s", *v.Value, *v.Key, val)
}
}
}
mockctl.Finish()
},
},
{
name: "Fail: Access Denied",
testFunc: func(t *testing.T) {
mockctl := gomock.NewController(t)
mockEfs := mocks.NewMockEfs(mockctl)
c := &cloud{efs: mockEfs}

ctx := context.Background()
mockEfs.EXPECT().DescribeFileSystems(gomock.Eq(ctx), gomock.Any()).Return(nil, &smithy.GenericAPIError{
Code: AccessDeniedException,
Message: "Access Denied",
})

_, err := c.DescribeFileSystemByToken(ctx, "efs-discovery")
if err == nil {
t.Fatalf("DescribeFileSystemByToken did not fail")
}
if err != ErrAccessDenied {
t.Fatalf("Failed. Expected: %v, Actual:%v", ErrAccessDenied, err)
}
mockctl.Finish()
},
},
{
name: "Fail: File System not found",
testFunc: func(t *testing.T) {
mockctl := gomock.NewController(t)
mockEfs := mocks.NewMockEfs(mockctl)
c := &cloud{efs: mockEfs}

ctx := context.Background()
mockEfs.EXPECT().DescribeFileSystems(gomock.Eq(ctx), gomock.Any()).Return(nil, &types.FileSystemNotFound{
Message: aws.String("File System not found"),
})

_, err := c.DescribeFileSystemByToken(ctx, "efs-discovery")
if err == nil {
t.Fatalf("DescribeFileSystemByToken did not fail")
}
if err != ErrNotFound {
t.Fatalf("Failed. Expected: %v, Actual:%v", ErrNotFound, err)
}
mockctl.Finish()
},
},
}
for _, tc := range testCases {
t.Run(tc.name, tc.testFunc)
}
}

func testResult(t *testing.T, funcName string, ret interface{}, err error, expectError errtyp) {
if expectError.message == "" {
if err != nil {
Expand Down
Loading