Skip to content

Commit 387f79b

Browse files
mohammedadnan21bpradipt
authored andcommitted
Add LUKS encryption support for cloud volumes via CDH
Integrate with CDH's secure_mount TTRPC API to support LUKS2 encrypted block volumes in Peer Pods. When a StorageClass specifies encrypt-type and kbs-key-id, the interceptor inside the PodVM delegates to CDH which handles LUKS formatting/opening with keys fetched from KBS after remote attestation. Changes: - Extend CloudVolumeAnnotation with EncryptType and KeyID fields - Proxy reads encrypt-type/kbs-key-id from mountInfo.json metadata - Add CDH TTRPC client using generated proto types, with context-aware connection retry logic (10 attempts, respects CreateContainer cancel) - Interceptor branches on EncryptType: CDH path vs plain formatAndMount - Pass predictable mapperName to CDH for reliable cryptsetup cleanup; fallback to /proc/mounts lookup if name is unavailable - Validate encrypt type (only luks/luks2 accepted, always send luks2 to CDH since that is what CDH accepts) - isLuks detection via magic-byte read distinguishes fresh disks from existing LUKS volumes; errors are hard failures to prevent CDH from reformatting an encrypted disk due to a transient read error - Resolve mapper name before unmount and skip cryptsetup close if unmount fails - unmountCloudVolumes handles cryptsetup close for encrypted mounts - Unit tests for annotation flow, proto round-trip, encrypt type validation, CDH request format, encrypted mount tracking Signed-off-by: Mohammed Adnan <muhammedadnan50007@gmail.com>
1 parent 9639c3d commit 387f79b

6 files changed

Lines changed: 508 additions & 26 deletions

File tree

src/cloud-api-adaptor/pkg/adaptor/proxy/cloud_volumes_test.go

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -365,3 +365,94 @@ func TestCloudVolumes_SkipsInvalidMountInfoJSON(t *testing.T) {
365365
_, ok := req.OCI.Annotations["io.confidentialcontainers.org.cloud_volumes"]
366366
assert.False(t, ok, "annotation should not be set with invalid JSON")
367367
}
368+
369+
func TestCloudVolumes_EncryptionAnnotation(t *testing.T) {
370+
dir := t.TempDir()
371+
overrideKataDirectVolumesDir(t, dir)
372+
373+
service, cleanup := setupMockAgentAndService(t)
374+
defer cleanup()
375+
376+
podUID := "pod-uid-enc-333"
377+
volPath := "/var/lib/kubelet/pods/" + podUID + "/volumes/kubernetes.io~csi/pvc-encrypted/mount"
378+
379+
writeTestMountInfo(t, dir, volPath, map[string]interface{}{
380+
"device": "/subscriptions/sub/disks/csi-vol-pvc-encrypted",
381+
"fstype": "ext4",
382+
"metadata": map[string]interface{}{
383+
"cloud-volume-path": "/subscriptions/sub/disks/csi-vol-pvc-encrypted",
384+
"encrypt-type": "LUKS",
385+
"kbs-key-id": "default/key/volume-enc-key",
386+
},
387+
})
388+
389+
req := newCreateContainerRequest("test-encrypted-vol").
390+
withAnnotations(map[string]string{
391+
"io.kubernetes.cri.sandbox-uid": podUID,
392+
}).
393+
withMounts(&pb.Mount{
394+
Destination: "/mnt/secret",
395+
Source: volPath,
396+
Type: "bind",
397+
}).
398+
build()
399+
400+
_, err := service.CreateContainer(context.Background(), req)
401+
require.NoError(t, err)
402+
403+
cvJSON, ok := req.OCI.Annotations[util.CloudVolumesAnnotationKey]
404+
require.True(t, ok, "cloud_volumes annotation should be set")
405+
406+
var cloudVolumes map[string]util.CloudVolumeAnnotation
407+
require.NoError(t, json.Unmarshal([]byte(cvJSON), &cloudVolumes))
408+
409+
require.Contains(t, cloudVolumes, "vol-0")
410+
vol := cloudVolumes["vol-0"]
411+
assert.Equal(t, "/mnt/secret", vol.MountPoint)
412+
assert.Equal(t, "ext4", vol.FSType)
413+
assert.Equal(t, "0", vol.LUN)
414+
assert.Equal(t, "/subscriptions/sub/disks/csi-vol-pvc-encrypted", vol.DiskID)
415+
assert.Equal(t, "LUKS", vol.EncryptType)
416+
assert.Equal(t, "default/key/volume-enc-key", vol.KeyID)
417+
}
418+
419+
func TestCloudVolumes_NoEncryptionParamsWhenAbsent(t *testing.T) {
420+
dir := t.TempDir()
421+
overrideKataDirectVolumesDir(t, dir)
422+
423+
service, cleanup := setupMockAgentAndService(t)
424+
defer cleanup()
425+
426+
podUID := "pod-uid-plain-444"
427+
volPath := "/var/lib/kubelet/pods/" + podUID + "/volumes/kubernetes.io~csi/pvc-plain/mount"
428+
429+
writeTestMountInfo(t, dir, volPath, map[string]interface{}{
430+
"device": "vol-abc123def",
431+
"fstype": "xfs",
432+
})
433+
434+
req := newCreateContainerRequest("test-plain-vol").
435+
withAnnotations(map[string]string{
436+
"io.kubernetes.cri.sandbox-uid": podUID,
437+
}).
438+
withMounts(&pb.Mount{
439+
Destination: "/mnt/data",
440+
Source: volPath,
441+
Type: "bind",
442+
}).
443+
build()
444+
445+
_, err := service.CreateContainer(context.Background(), req)
446+
require.NoError(t, err)
447+
448+
cvJSON, ok := req.OCI.Annotations[util.CloudVolumesAnnotationKey]
449+
require.True(t, ok, "cloud_volumes annotation should be set")
450+
451+
var cloudVolumes map[string]util.CloudVolumeAnnotation
452+
require.NoError(t, json.Unmarshal([]byte(cvJSON), &cloudVolumes))
453+
454+
require.Contains(t, cloudVolumes, "vol-0")
455+
vol := cloudVolumes["vol-0"]
456+
assert.Equal(t, "", vol.EncryptType)
457+
assert.Equal(t, "", vol.KeyID)
458+
}

src/cloud-api-adaptor/pkg/adaptor/proxy/service.go

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -139,10 +139,18 @@ func (s *proxyService) CreateContainer(ctx context.Context, req *pb.CreateContai
139139
}
140140

141141
diskID := ""
142+
encryptType := ""
143+
keyID := ""
142144
if md, ok := mountInfo["metadata"].(map[string]interface{}); ok {
143145
if cp, ok := md["cloud-volume-path"].(string); ok && cp != "" {
144146
diskID = cp
145147
}
148+
if et, ok := md["encrypt-type"].(string); ok {
149+
encryptType = et
150+
}
151+
if kid, ok := md["kbs-key-id"].(string); ok {
152+
keyID = kid
153+
}
146154
}
147155
if diskID == "" {
148156
if d, ok := mountInfo["device"].(string); ok {
@@ -174,12 +182,14 @@ func (s *proxyService) CreateContainer(ctx context.Context, req *pb.CreateContai
174182

175183
volKey := fmt.Sprintf("vol-%d", canonicalIdx)
176184
cloudVolumes[volKey] = util.CloudVolumeAnnotation{
177-
MountPoint: mountDest,
178-
FSType: fsType,
179-
LUN: fmt.Sprintf("%d", canonicalIdx),
180-
DiskID: diskID,
185+
MountPoint: mountDest,
186+
FSType: fsType,
187+
LUN: fmt.Sprintf("%d", canonicalIdx),
188+
DiskID: diskID,
189+
EncryptType: encryptType,
190+
KeyID: keyID,
181191
}
182-
logger.Printf("Detected cloud volume %s -> %s (lun=%d, disk=%s, fs=%s)", volKey, mountDest, canonicalIdx, diskID, fsType)
192+
logger.Printf("Detected cloud volume %s -> %s (lun=%d, disk=%s, fs=%s, encrypt=%s)", volKey, mountDest, canonicalIdx, diskID, fsType, encryptType)
183193
canonicalIdx++
184194
}
185195
}
Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
// (C) Copyright Confidential Containers Contributors
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package interceptor
5+
6+
import (
7+
"context"
8+
"fmt"
9+
"net"
10+
"time"
11+
12+
"github.com/containerd/ttrpc"
13+
14+
"github.com/confidential-containers/cloud-api-adaptor/src/cloud-api-adaptor/pkg/forwarder/interceptor/cdhpb"
15+
)
16+
17+
const cdhServiceName = "api.SecureMountService"
18+
19+
type cdhClient struct {
20+
conn net.Conn
21+
client *ttrpc.Client
22+
}
23+
24+
func newCDHClient(ctx context.Context, socketPath string) (*cdhClient, error) {
25+
const maxAttempts = 10
26+
const retryDelay = 2 * time.Second
27+
28+
dialer := &net.Dialer{Timeout: 5 * time.Second}
29+
30+
var conn net.Conn
31+
var err error
32+
for attempt := 1; attempt <= maxAttempts; attempt++ {
33+
conn, err = dialer.DialContext(ctx, "unix", socketPath)
34+
if err == nil {
35+
break
36+
}
37+
if attempt == maxAttempts {
38+
break
39+
}
40+
logger.Printf("CDH socket %s not ready (attempt %d/%d): %v", socketPath, attempt, maxAttempts, err)
41+
select {
42+
case <-ctx.Done():
43+
return nil, fmt.Errorf("dialing CDH socket %s: %w", socketPath, ctx.Err())
44+
case <-time.After(retryDelay):
45+
}
46+
}
47+
if err != nil {
48+
return nil, fmt.Errorf("dialing CDH socket %s after %d attempts: %w", socketPath, maxAttempts, err)
49+
}
50+
51+
client := ttrpc.NewClient(conn)
52+
return &cdhClient{conn: conn, client: client}, nil
53+
}
54+
55+
func (c *cdhClient) close() {
56+
c.conn.Close()
57+
if err := c.client.Close(); err != nil {
58+
logger.Printf("WARNING: closing CDH ttrpc client: %v", err)
59+
}
60+
}
61+
62+
func (c *cdhClient) secureMount(ctx context.Context, volumeType string, options map[string]string, flags []string, mountPoint string) error {
63+
req := &cdhpb.SecureMountRequest{
64+
VolumeType: volumeType,
65+
Options: options,
66+
Flags: flags,
67+
MountPoint: mountPoint,
68+
}
69+
resp := &cdhpb.SecureMountResponse{}
70+
71+
if err := c.client.Call(ctx, cdhServiceName, "SecureMount", req, resp); err != nil {
72+
return fmt.Errorf("CDH SecureMount RPC failed: %w", err)
73+
}
74+
75+
return nil
76+
}

0 commit comments

Comments
 (0)