feat(glance): support nfs as backend storage (#25240)

wanyaoqi · web-flow · commit 0b6c26c45bd6 · 2026-08-03T16:00:21.000+08:00
diff --git a/pkg/apis/image/consts.go b/pkg/apis/image/consts.go
@@ -51,9 +51,13 @@ const (
 
 	LocalFilePrefix = "file://"
 	S3Prefix        = "s3://"
+	NfsPrefix       = "nfs://"
+
+	NfsSubDirName = "images"
 
 	IMAGE_STORAGE_DRIVER_LOCAL = "local"
 	IMAGE_STORAGE_DRIVER_S3    = "s3"
+	IMAGE_STORAGE_DRIVER_NFS   = "nfs"
 
 	// image properties
 	IMAGE_OS_ARCH              = "os_arch"
diff --git a/pkg/compute/hostdrivers/esxi.go b/pkg/compute/hostdrivers/esxi.go
@@ -79,7 +79,7 @@ func (self *SESXiHostDriver) CheckAndSetCacheImage(ctx context.Context, userCred
 	hostCacheImage := models.StoragecachedimageManager.GetStoragecachedimage(storageCache.GetId(), cacheImage.GetId())
 	if hostCacheImage == nil {
 		zone, _ := host.GetZone()
-		srcHostCacheImage, err = cacheImage.ChooseSourceStoragecacheInRange(api.HOST_TYPE_ESXI, []string{host.Id},
+		srcHostCacheImage, err = cacheImage.ChooseSourceStoragecacheInRange([]string{api.HOST_TYPE_ESXI}, []string{host.Id},
 			[]interface{}{zone, host.GetCloudprovider()})
 		if err != nil {
 			return err
diff --git a/pkg/compute/hostdrivers/kvm.go b/pkg/compute/hostdrivers/kvm.go
@@ -220,7 +220,7 @@ func (self *SKVMHostDriver) CheckAndSetCacheImage(ctx context.Context, userCred
 	if srcHost != nil {
 		rangeObjs = append(rangeObjs, srcHost)
 	}
-	srcHostCacheImage, err := cacheImage.ChooseSourceStoragecacheInRange(api.HOST_TYPE_HYPERVISOR, []string{host.Id}, rangeObjs)
+	srcHostCacheImage, err := cacheImage.ChooseSourceStoragecacheInRange([]string{api.HOST_TYPE_HYPERVISOR, api.HOST_TYPE_CONTAINER}, []string{host.Id}, rangeObjs)
 	if err != nil {
 		return errors.Wrapf(err, "Choose source storagecache")
 	}
diff --git a/pkg/compute/models/cachedimages.go b/pkg/compute/models/cachedimages.go
@@ -611,7 +611,7 @@ func (cachedImage *SCachedimage) addRefCount() {
 	}
 }
 
-func (cachedImage *SCachedimage) ChooseSourceStoragecacheInRange(hostType string, excludes []string, rangeObjs []interface{}) (*SStoragecachedimage, error) {
+func (cachedImage *SCachedimage) ChooseSourceStoragecacheInRange(hostType []string, excludes []string, rangeObjs []interface{}) (*SStoragecachedimage, error) {
 	storageCachedImage := StoragecachedimageManager.Query().SubQuery()
 	storage := StorageManager.Query().SubQuery()
 	hostStorage := HoststorageManager.Query().SubQuery()
@@ -635,7 +635,7 @@ func (cachedImage *SCachedimage) ChooseSourceStoragecacheInRange(hostType string
 		q = q.Filter(sqlchemy.NotIn(host.Field("id"), excludes))
 	}
 	if len(hostType) > 0 {
-		q = q.Filter(sqlchemy.Equals(host.Field("host_type"), hostType))
+		q = q.Filter(sqlchemy.In(host.Field("host_type"), hostType))
 	}
 
 	for _, rangeObj := range rangeObjs {
diff --git a/pkg/hostman/storageman/imagecache_local.go b/pkg/hostman/storageman/imagecache_local.go
@@ -211,6 +211,9 @@ func (l *SLocalImageCache) prepare(ctx context.Context, input api.CacheImageInpu
 
 	l.remoteFile = remotefile.NewRemoteFile(ctx, url,
 		l.GetPath(), false, input.Checksum, -1, nil, l.GetTmpPath(), input.SrcUrl)
+	if l.Manager.GetStorageType() == api.STORAGE_NFS {
+		l.remoteFile.NfsSetTargetStorageId(l.Manager.GetStorageId(), l.Manager.GetStoragePath())
+	}
 	return false, nil
 }
 
diff --git a/pkg/hostman/storageman/imagecachemanager_base.go b/pkg/hostman/storageman/imagecachemanager_base.go
@@ -53,6 +53,8 @@ type IImageCacheManger interface {
 
 	IsLocal() bool
 	GetStorageType() string
+	GetStorageId() string
+	GetStoragePath() string
 
 	// for diskhandler
 	PrefetchImageCache(ctx context.Context, data interface{}) (jsonutils.JSONObject, error)
diff --git a/pkg/hostman/storageman/imagecachemanager_local.go b/pkg/hostman/storageman/imagecachemanager_local.go
@@ -75,6 +75,20 @@ func (c *SLocalImageCacheManager) GetStorageType() string {
 	return c.storage.StorageType()
 }
 
+func (c *SLocalImageCacheManager) GetStorageId() string {
+	if c.storage == nil {
+		return ""
+	}
+	return c.storage.GetId()
+}
+
+func (c *SLocalImageCacheManager) GetStoragePath() string {
+	if c.storage == nil {
+		return ""
+	}
+	return c.storage.GetPath()
+}
+
 func (c *SLocalImageCacheManager) loadCache(ctx context.Context) {
 	if len(c.cachePath) == 0 {
 		return
diff --git a/pkg/hostman/storageman/imagecachemanager_lvm.go b/pkg/hostman/storageman/imagecachemanager_lvm.go
@@ -65,6 +65,14 @@ func (c *SLVMImageCacheManager) GetStorageType() string {
 	return c.storage.StorageType()
 }
 
+func (s *SLVMImageCacheManager) GetStorageId() string {
+	return s.storage.GetId()
+}
+
+func (s *SLVMImageCacheManager) GetStoragePath() string {
+	return s.storage.GetPath()
+}
+
 func (c *SLVMImageCacheManager) Lvmlockd() bool {
 	return c.lvmlockd
 }
diff --git a/pkg/hostman/storageman/imagecachemanager_rbd.go b/pkg/hostman/storageman/imagecachemanager_rbd.go
@@ -118,6 +118,14 @@ func (c *SRbdImageCacheManager) GetStorageType() string {
 	return c.storage.StorageType()
 }
 
+func (c *SRbdImageCacheManager) GetStorageId() string {
+	return c.storage.GetId()
+}
+
+func (c *SRbdImageCacheManager) GetStoragePath() string {
+	return c.storage.GetPath()
+}
+
 func (c *SRbdImageCacheManager) GetPath() string {
 	return c.Pool
 }
diff --git a/pkg/hostman/storageman/remotefile/nfsremotefile.go b/pkg/hostman/storageman/remotefile/nfsremotefile.go
@@ -0,0 +1,32 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package remotefile
+
+import (
+	"os"
+	"path"
+
+	"yunion.io/x/log"
+)
+
+type NfsRemoteFileInfo struct {
+	NfsImagePath string `json:"nfs_image_path"`
+}
+
+func (i *NfsRemoteFileInfo) nfsLinkImage(targetPath, storagePath string) error {
+	srcPath := path.Join(storagePath, i.NfsImagePath)
+	log.Infof("NfsRemoteFileInfo start link %s to %s", srcPath, targetPath)
+	return os.Link(srcPath, targetPath)
+}
diff --git a/pkg/hostman/storageman/remotefile/remotefile.go b/pkg/hostman/storageman/remotefile/remotefile.go
@@ -48,27 +48,30 @@ type SImageDesc struct {
 }
 
 type SRemoteFile struct {
-	ctx          context.Context
-	url          string
-	downloadUrl  string
-	localPath    string
-	tmpPath      string
-	preChksum    string
-	compress     bool
-	timeout      time.Duration
-	extraHeaders map[string]string
+	ctx                  context.Context
+	url                  string
+	downloadUrl          string
+	localPath            string
+	tmpPath              string
+	preChksum            string
+	compress             bool
+	timeout              time.Duration
+	extraHeaders         map[string]string
+	nfsTargetStorageId   string
+	nfsTargetStoragePath string
 
 	chksum string
 	format string
 	name   string
 
-	s3Info *S3RemoteFileInfo
+	s3Info  *S3RemoteFileInfo
+	nfsInfo *NfsRemoteFileInfo
 }
 
 func NewRemoteFile(
 	ctx context.Context, url, localPath string, compress bool,
 	PreChksum string, timeout int, extraHeaders map[string]string,
-	tmpPath string, downloadUrl string,
+	tmpPath, downloadUrl string,
 ) *SRemoteFile {
 	if timeout <= 0 {
 		timeout = 24 * 3600 //24 hours
@@ -90,6 +93,11 @@ func NewRemoteFile(
 	}
 }
 
+func (r *SRemoteFile) NfsSetTargetStorageId(storageId, storagePath string) {
+	r.nfsTargetStorageId = storageId
+	r.nfsTargetStoragePath = storagePath
+}
+
 func (r *SRemoteFile) GetFormat() string {
 	return r.format
 }
@@ -230,8 +238,12 @@ func (r *SRemoteFile) downloadS3(callback func(progress, progressMbps float64, t
 }
 
 func (r *SRemoteFile) downloadInternal(getData bool, preChksum string, callback func(progress, progressMbps float64, totalSizeMb int64)) error {
-	if getData && r.s3Info != nil {
-		return r.downloadS3(callback)
+	if getData {
+		if r.s3Info != nil {
+			return r.downloadS3(callback)
+		} else if r.nfsInfo != nil {
+			return r.nfsInfo.nfsLinkImage(r.tmpPath, r.nfsTargetStoragePath)
+		}
 	}
 
 	var header = http.Header{}
@@ -363,4 +375,12 @@ func (r *SRemoteFile) setProperties(header http.Header) {
 			r.s3Info.Bucket = s3Bucket
 		}
 	}
+	if nfsStorageId := header.Get("X-Image-Meta-Nfs_storage_id"); len(nfsStorageId) > 0 && r.nfsTargetStorageId == nfsStorageId {
+		if imgPath := header.Get("X-Image-Meta-Nfs_image_path"); len(imgPath) > 0 {
+			r.nfsInfo = &NfsRemoteFileInfo{
+				NfsImagePath: imgPath,
+			}
+
+		}
+	}
 }
diff --git a/pkg/image/models/images.go b/pkg/image/models/images.go
@@ -400,6 +400,11 @@ func (img *SImage) GetExtraDetailsHeaders(ctx context.Context, userCred mcclient
 		headers[fmt.Sprintf("%s%s", modules.IMAGE_META, "s3_info_sign_ver")] = options.Options.S3SignVersion
 	}
 
+	if strings.HasPrefix(img.Location, api.NfsPrefix) {
+		headers[fmt.Sprintf("%s%s", modules.IMAGE_META, "nfs_storage_id")] = options.Options.NfsStorageId
+		headers[fmt.Sprintf("%s%s", modules.IMAGE_META, "nfs_image_path")] = img.Location[len(api.NfsPrefix):]
+	}
+
 	return headers
 }
 
@@ -461,6 +466,8 @@ func (self *SImage) GetPath(format string) string {
 	path := filepath.Join(options.Options.FilesystemStoreDatadir, self.Id)
 	if options.Options.StorageDriver == api.IMAGE_STORAGE_DRIVER_S3 {
 		path = filepath.Join(options.Options.S3MountPoint, self.Id)
+	} else if options.Options.StorageDriver == api.IMAGE_STORAGE_DRIVER_NFS {
+		path = filepath.Join(options.Options.NfsMountPoint, api.NfsSubDirName, self.Id)
 	}
 	if len(format) > 0 {
 		path = fmt.Sprintf("%s.%s", path, format)
@@ -1180,6 +1187,8 @@ func (self *SImage) GetLocalLocation() string {
 		return self.Location[len(api.LocalFilePrefix):]
 	} else if strings.HasPrefix(self.Location, api.S3Prefix) {
 		return path.Join(options.Options.S3MountPoint, self.Location[len(api.S3Prefix):])
+	} else if strings.HasPrefix(self.Location, api.NfsPrefix) {
+		return path.Join(options.Options.NfsMountPoint, self.Location[len(api.NfsPrefix):])
 	} else {
 		return ""
 	}
@@ -1190,6 +1199,8 @@ func (self *SImage) GetPrefix() string {
 		return api.LocalFilePrefix
 	} else if strings.HasPrefix(self.Location, api.S3Prefix) {
 		return api.S3Prefix
+	} else if strings.HasPrefix(self.Location, api.NfsPrefix) {
+		return api.NfsPrefix
 	} else {
 		return api.LocalFilePrefix
 	}
@@ -1198,6 +1209,8 @@ func (self *SImage) GetPrefix() string {
 func (self *SImage) GetNewLocation(newLocalPath string) string {
 	if strings.HasPrefix(self.Location, api.S3Prefix) {
 		return api.S3Prefix + path.Base(newLocalPath)
+	} else if strings.HasPrefix(self.Location, api.NfsPrefix) {
+		return api.NfsPrefix + path.Join(api.NfsSubDirName, path.Base(newLocalPath))
 	} else {
 		return api.LocalFilePrefix + newLocalPath
 	}
diff --git a/pkg/image/models/storage.go b/pkg/image/models/storage.go
@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path"
 	"strings"
 
 	"yunion.io/x/pkg/errors"
@@ -33,6 +34,7 @@ import (
 
 var local IImageStorage = &LocalStorage{}
 var s3Instance IImageStorage = &S3Storage{}
+var nfsInstance IImageStorage = &NFSStorage{}
 var storage IImageStorage
 
 func GetStorage() IImageStorage {
@@ -43,6 +45,8 @@ func GetImage(ctx context.Context, location string) (int64, io.ReadCloser, error
 	switch {
 	case strings.HasPrefix(location, image.S3Prefix):
 		return s3Instance.GetImage(ctx, location[len(image.S3Prefix):])
+	case strings.HasPrefix(location, image.NfsPrefix):
+		return nfsInstance.GetImage(ctx, location[len(image.NfsPrefix):])
 	case strings.HasPrefix(location, image.LocalFilePrefix):
 		return local.GetImage(ctx, location[len(image.LocalFilePrefix):])
 	default:
@@ -54,6 +58,8 @@ func RemoveImage(ctx context.Context, location string) error {
 	switch {
 	case strings.HasPrefix(location, image.S3Prefix):
 		return s3Instance.RemoveImage(ctx, location[len(image.S3Prefix):])
+	case strings.HasPrefix(location, image.NfsPrefix):
+		return nfsInstance.RemoveImage(ctx, location[len(image.NfsPrefix):])
 	case strings.HasPrefix(location, image.LocalFilePrefix):
 		return local.RemoveImage(ctx, location[len(image.LocalFilePrefix):])
 	default:
@@ -65,6 +71,8 @@ func IsCheckStatusEnabled(img *SImage) bool {
 	switch {
 	case strings.HasPrefix(img.Location, image.S3Prefix):
 		return s3Instance.IsCheckStatusEnabled()
+	case strings.HasPrefix(img.Location, image.NfsPrefix):
+		return nfsInstance.IsCheckStatusEnabled()
 	case strings.HasPrefix(img.Location, image.LocalFilePrefix):
 		return local.IsCheckStatusEnabled()
 	default:
@@ -76,6 +84,8 @@ func Init(storageBackend string) {
 	switch storageBackend {
 	case image.IMAGE_STORAGE_DRIVER_LOCAL:
 		storage = &LocalStorage{}
+	case image.IMAGE_STORAGE_DRIVER_NFS:
+		storage = &NFSStorage{}
 	case image.IMAGE_STORAGE_DRIVER_S3:
 		storage = &S3Storage{}
 	default:
@@ -144,6 +154,41 @@ func (s *LocalStorage) RemoveImage(ctx context.Context, imagePath string) error
 	return os.Remove(imagePath)
 }
 
+type NFSStorage struct {
+	LocalStorage
+}
+
+func (s *NFSStorage) Type() string {
+	return image.IMAGE_STORAGE_DRIVER_NFS
+}
+
+func (s *NFSStorage) SaveImage(ctx context.Context, imagePath string, progresser func(saved int64)) (string, error) {
+	imageName := imagePathToName(imagePath)
+	imageNewpath := path.Join(options.Options.NfsMountPoint, image.NfsSubDirName, imageName)
+	out, err := procutils.NewRemoteCommandAsFarAsPossible("cp", imagePath, imageNewpath).Output()
+	if err != nil {
+		return "", errors.Wrapf(err, "failed to copy %s to %s: %s", imagePath, imageNewpath, out)
+	}
+
+	return fmt.Sprintf("%s%s", image.NfsPrefix, path.Join(image.NfsSubDirName, imageName)), nil
+}
+
+func (s *NFSStorage) ConvertImage(ctx context.Context, oimg *SImage, targetFormat string, progresser func(saved int64)) (*SConverImageInfo, error) {
+	location := oimg.GetPath(targetFormat)
+	img, err := oimg.getQemuImage()
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to image.getQemuImage")
+	}
+	nimg, err := img.Clone(location, qemuimgfmt.String2ImageFormat(targetFormat), true)
+	if err != nil {
+		return nil, errors.Wrap(err, "unable to img.Clone")
+	}
+	return &SConverImageInfo{
+		Location:  fmt.Sprintf("%s%s", image.NfsPrefix, location),
+		SizeBytes: nimg.ActualSizeBytes,
+	}, nil
+}
+
 type S3Storage struct{}
 
 func imagePathToName(imagePath string) string {
diff --git a/pkg/image/options/options.go b/pkg/image/options/options.go
@@ -48,7 +48,7 @@ type SImageOptions struct {
 
 	// DeployServerSocketPath string `help:"Deploy server listen socket path" default:"/var/run/onecloud/deploy.sock"`
 
-	StorageDriver string `help:"image backend storage" default:"local" choices:"s3|local"`
+	StorageDriver string `help:"image backend storage" default:"local" choices:"s3|local|nfs"`
 
 	S3MountPoint       string `help:"s3fs mount point" default:"/opt/cloud/workspace/data/glance/s3images"`
 	S3CheckImageStatus bool   `help:"Enable s3 check image status"`
@@ -58,6 +58,10 @@ type SImageOptions struct {
 
 	S3DirectDownload bool `help:"enable s3 direct download" default:"false"`
 
+	NfsStorageId    string `help:"region nfs storage id or name used as glance filesystem backend"`
+	NfsMountOptions string `help:"nfs mount options for glance filesystem backend"`
+	NfsMountPoint   string `help:"nfs mount point" default:"/opt/cloud/workspace/data/glance/nfsimages"`
+
 	ImageStreamWorkerCount int `help:"Image stream worker count" default:"10"`
 
 	VerifyImageStatusIntervalMinutes int `help:"verify image status periodically, default 15 minutes" default:"15"`
diff --git a/pkg/image/service/nfs.go b/pkg/image/service/nfs.go
diff --git a/pkg/image/service/service.go b/pkg/image/service/service.go

Original file line number	Diff line number	Diff line change
`@@ -220,7 +220,7 @@ func (self *SKVMHostDriver) CheckAndSetCacheImage(ctx context.Context, userCred`
`220`	`220`	`if srcHost != nil {`
`221`	`221`	`rangeObjs = append(rangeObjs, srcHost)`
`222`	`222`	`}`
`223`		`- srcHostCacheImage, err := cacheImage.ChooseSourceStoragecacheInRange(api.HOST_TYPE_HYPERVISOR, []string{host.Id}, rangeObjs)`
	`223`	`+ srcHostCacheImage, err := cacheImage.ChooseSourceStoragecacheInRange([]string{api.HOST_TYPE_HYPERVISOR, api.HOST_TYPE_CONTAINER}, []string{host.Id}, rangeObjs)`
`224`	`224`	`if err != nil {`
`225`	`225`	`return errors.Wrapf(err, "Choose source storagecache")`
`226`	`226`	`}`
Original file line number	Diff line number	Diff line change
`@@ -611,7 +611,7 @@ func (cachedImage *SCachedimage) addRefCount() {`
`611`	`611`	`}`
`612`	`612`	`}`
`613`	`613`
`614`		`-func (cachedImage SCachedimage) ChooseSourceStoragecacheInRange(hostType string, excludes []string, rangeObjs []interface{}) (SStoragecachedimage, error) {`
	`614`	`+func (cachedImage SCachedimage) ChooseSourceStoragecacheInRange(hostType []string, excludes []string, rangeObjs []interface{}) (SStoragecachedimage, error) {`
`615`	`615`	`storageCachedImage := StoragecachedimageManager.Query().SubQuery()`
`616`	`616`	`storage := StorageManager.Query().SubQuery()`
`617`	`617`	`hostStorage := HoststorageManager.Query().SubQuery()`
`@@ -635,7 +635,7 @@ func (cachedImage *SCachedimage) ChooseSourceStoragecacheInRange(hostType string`
`635`	`635`	`q = q.Filter(sqlchemy.NotIn(host.Field("id"), excludes))`
`636`	`636`	`}`
`637`	`637`	`if len(hostType) > 0 {`
`638`		`- q = q.Filter(sqlchemy.Equals(host.Field("host_type"), hostType))`
	`638`	`+ q = q.Filter(sqlchemy.In(host.Field("host_type"), hostType))`
`639`	`639`	`}`
`640`	`640`
`641`	`641`	`for _, rangeObj := range rangeObjs {`
Original file line number	Diff line number	Diff line change
`@@ -211,6 +211,9 @@ func (l *SLocalImageCache) prepare(ctx context.Context, input api.CacheImageInpu`
`211`	`211`
`212`	`212`	`l.remoteFile = remotefile.NewRemoteFile(ctx, url,`
`213`	`213`	`l.GetPath(), false, input.Checksum, -1, nil, l.GetTmpPath(), input.SrcUrl)`
	`214`	`+ if l.Manager.GetStorageType() == api.STORAGE_NFS {`
	`215`	`+ l.remoteFile.NfsSetTargetStorageId(l.Manager.GetStorageId(), l.Manager.GetStoragePath())`
	`216`	`+ }`
`214`	`217`	`return false, nil`
`215`	`218`	`}`
`216`	`219`
Original file line number	Diff line number	Diff line change
`@@ -65,6 +65,14 @@ func (c *SLVMImageCacheManager) GetStorageType() string {`
`65`	`65`	`return c.storage.StorageType()`
`66`	`66`	`}`
`67`	`67`
	`68`	`+func (s *SLVMImageCacheManager) GetStorageId() string {`
	`69`	`+ return s.storage.GetId()`
	`70`	`+}`
	`71`	`+`
	`72`	`+func (s *SLVMImageCacheManager) GetStoragePath() string {`
	`73`	`+ return s.storage.GetPath()`
	`74`	`+}`
	`75`	`+`
`68`	`76`	`func (c *SLVMImageCacheManager) Lvmlockd() bool {`
`69`	`77`	`return c.lvmlockd`
`70`	`78`	`}`
Original file line number	Diff line number	Diff line change
`@@ -118,6 +118,14 @@ func (c *SRbdImageCacheManager) GetStorageType() string {`
`118`	`118`	`return c.storage.StorageType()`
`119`	`119`	`}`
`120`	`120`
	`121`	`+func (c *SRbdImageCacheManager) GetStorageId() string {`
	`122`	`+ return c.storage.GetId()`
	`123`	`+}`
	`124`	`+`
	`125`	`+func (c *SRbdImageCacheManager) GetStoragePath() string {`
	`126`	`+ return c.storage.GetPath()`
	`127`	`+}`
	`128`	`+`
`121`	`129`	`func (c *SRbdImageCacheManager) GetPath() string {`
`122`	`130`	`return c.Pool`
`123`	`131`	`}`