feat(datastore): support backup and restore

Author: jialeicui

client/starwhale/base/client/models/models.py

@@ -410,6 +410,19 @@ class OrderByDesc(SwBaseModel):
     descending: Optional[bool] = None
 
 
+class BackupVo(SwBaseModel):
+    id: Optional[str] = None
+    created_at: Optional[int] = Field(None, alias='createdAt')
+    done_at: Optional[int] = Field(None, alias='doneAt')
+    approximate_size_bytes: Optional[int] = Field(None, alias='approximateSizeBytes')
+
+
+class ResponseMessageBackupVo(SwBaseModel):
+    code: str
+    message: str
+    data: BackupVo
+
+
 class ListTablesRequest(SwBaseModel):
     prefix: Optional[str] = None
     prefixes: Optional[List[str]] = Field(None)
@@ -1127,6 +1140,12 @@ class RuntimeSuggestionVo(SwBaseModel):
     runtimes: Optional[List[RuntimeVersionVo]] = None
 
 
+class ResponseMessageListBackupVo(SwBaseModel):
+    code: str
+    message: str
+    data: List[BackupVo]
+
+
 class UserRoleDeleteRequest(UserCheckPasswordRequest):
     pass
 

console/src/api/server/Api.ts

@@ -33,6 +33,7 @@ import {
     ICompleteUploadBlobData,
     IConfigRequest,
     IConsumeNextDataData,
+    ICreateBackupData,
     ICreateJobData,
     ICreateJobTemplateRequest,
     ICreateModelServingData,
@@ -49,6 +50,7 @@ import {
     IDatasetBuildRequest,
     IDatasetTagRequest,
     IDatasetUploadRequest,
+    IDeleteBackupData,
     IDeleteDatasetData,
     IDeleteDatasetVersionTagData,
     IDeleteModelData,
@@ -113,6 +115,7 @@ import {
     IJobModifyPinRequest,
     IJobModifyRequest,
     IJobRequest,
+    IListBackupsData,
     IListBuildRecordsData,
     IListDatasetData,
     IListDatasetTreeData,
@@ -179,6 +182,7 @@ import {
     IReleaseFtData,
     IRemoveJobData,
     IResourcePool,
+    IRestoreBackupData,
     IRevertDatasetRequest,
     IRevertDatasetVersionData,
     IRevertModelVersionData,
@@ -2776,6 +2780,90 @@ export class Api<SecurityDataType = unknown> {
             ...params,
         })
 
+    /**
+     * No description
+     *
+     * @tags data-store-controller
+     * @name ListBackups
+     * @request GET:/api/v1/datastore/maintain/backup
+     * @secure
+     * @response `200` `IListBackupsData` OK
+     */
+    listBackups = (params: RequestParams = {}) =>
+        this.http.request<IListBackupsData, any>({
+            path: `/api/v1/datastore/maintain/backup`,
+            method: 'GET',
+            secure: true,
+            ...params,
+        })
+
+    useListBackups = (params: RequestParams = {}) =>
+        useQuery(qs.stringify(['listBackups', params]), () => this.listBackups(params), {
+            enabled: [].every(Boolean),
+        })
+    /**
+     * No description
+     *
+     * @tags data-store-controller
+     * @name CreateBackup
+     * @request POST:/api/v1/datastore/maintain/backup
+     * @secure
+     * @response `200` `ICreateBackupData` OK
+     */
+    createBackup = (params: RequestParams = {}) =>
+        this.http.request<ICreateBackupData, any>({
+            path: `/api/v1/datastore/maintain/backup`,
+            method: 'POST',
+            secure: true,
+            ...params,
+        })
+
+    /**
+     * No description
+     *
+     * @tags data-store-controller
+     * @name DeleteBackup
+     * @request DELETE:/api/v1/datastore/maintain/backup
+     * @secure
+     * @response `200` `IDeleteBackupData` OK
+     */
+    deleteBackup = (
+        query: {
+            id: string
+        },
+        params: RequestParams = {}
+    ) =>
+        this.http.request<IDeleteBackupData, any>({
+            path: `/api/v1/datastore/maintain/backup`,
+            method: 'DELETE',
+            query: query,
+            secure: true,
+            ...params,
+        })
+
+    /**
+     * No description
+     *
+     * @tags data-store-controller
+     * @name RestoreBackup
+     * @request POST:/api/v1/datastore/maintain/backup/restore
+     * @secure
+     * @response `200` `IRestoreBackupData` OK
+     */
+    restoreBackup = (
+        query: {
+            id: string
+        },
+        params: RequestParams = {}
+    ) =>
+        this.http.request<IRestoreBackupData, any>({
+            path: `/api/v1/datastore/maintain/backup/restore`,
+            method: 'POST',
+            query: query,
+            secure: true,
+            ...params,
+        })
+
     /**
      * No description
      *

console/src/api/server/data-contracts.ts

@@ -548,6 +548,22 @@ export interface ITableQueryOperandDesc {
     bytesValue?: string
 }
 
+export interface IBackupVo {
+    id?: string
+    /** @format int64 */
+    createdAt?: number
+    /** @format int64 */
+    doneAt?: number
+    /** @format int64 */
+    approximateSizeBytes?: number
+}
+
+export interface IResponseMessageBackupVo {
+    code: string
+    message: string
+    data: IBackupVo
+}
+
 export interface IListTablesRequest {
     prefix?: string
     /** @uniqueItems true */
@@ -2145,6 +2161,12 @@ export interface IRuntimeSuggestionVo {
     runtimes?: IRuntimeVersionVo[]
 }
 
+export interface IResponseMessageListBackupVo {
+    code: string
+    message: string
+    data: IBackupVo[]
+}
+
 export interface IUserRoleDeleteRequest {
     currentUserPwd: string
 }
@@ -2359,6 +2381,14 @@ export type IQueryTableData = IResponseMessageRecordListVo['data']
 
 export type IQueryAndExportData = any
 
+export type IListBackupsData = IResponseMessageListBackupVo['data']
+
+export type ICreateBackupData = IResponseMessageBackupVo['data']
+
+export type IDeleteBackupData = IResponseMessageString['data']
+
+export type IRestoreBackupData = IResponseMessageString['data']
+
 export type IListTablesData = IResponseMessageTableNameListVo['data']
 
 export type IFlushData = IResponseMessageString['data']

server/controller/src/main/java/ai/starwhale/mlops/api/DataStoreController.java

@@ -18,6 +18,7 @@
 
 import ai.starwhale.mlops.api.protocol.Code;
 import ai.starwhale.mlops.api.protocol.ResponseMessage;
+import ai.starwhale.mlops.api.protocol.datastore.BackupVo;
 import ai.starwhale.mlops.api.protocol.datastore.ColumnDesc;
 import ai.starwhale.mlops.api.protocol.datastore.FlushRequest;
 import ai.starwhale.mlops.api.protocol.datastore.ListTablesRequest;
@@ -35,6 +36,7 @@
 import ai.starwhale.mlops.datastore.DataStoreScanRequest;
 import ai.starwhale.mlops.datastore.RecordList;
 import ai.starwhale.mlops.datastore.TableQueryFilter;
+import ai.starwhale.mlops.datastore.backup.Meta;
 import ai.starwhale.mlops.datastore.exporter.RecordsStreamingExporter;
 import ai.starwhale.mlops.datastore.impl.RecordDecoder;
 import ai.starwhale.mlops.exception.SwProcessException;
@@ -60,9 +62,12 @@
 import org.springframework.util.CollectionUtils;
 import org.springframework.util.StringUtils;
 import org.springframework.validation.annotation.Validated;
+import org.springframework.web.bind.annotation.DeleteMapping;
+import org.springframework.web.bind.annotation.GetMapping;
 import org.springframework.web.bind.annotation.PostMapping;
 import org.springframework.web.bind.annotation.RequestBody;
 import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
 import org.springframework.web.bind.annotation.RestController;
 
 @RestController
@@ -293,6 +298,49 @@ void scanAndExport(
         }
     }
 
+    @PostMapping(value = "/datastore/maintain/backup")
+    @PreAuthorize("hasAnyRole('OWNER')")
+    ResponseEntity<ResponseMessage<BackupVo>> createBackup() {
+        try {
+            var meta = this.dataStore.createBackup();
+            return ResponseEntity.ok(Code.success.asResponse(meta.toVo()));
+        } catch (IOException e) {
+            log.error("create backup failed", e);
+            throw new SwProcessException(ErrorType.SYSTEM, "create backup failed", e);
+        }
+    }
+
+    @GetMapping(value = "/datastore/maintain/backup")
+    @PreAuthorize("hasAnyRole('OWNER')")
+    ResponseEntity<ResponseMessage<List<BackupVo>>> listBackups() {
+        var backups = this.dataStore.listBackups().stream().map(Meta::toVo).collect(Collectors.toList());
+        return ResponseEntity.ok(Code.success.asResponse(backups));
+    }
+
+    @PostMapping(value = "/datastore/maintain/backup/restore")
+    @PreAuthorize("hasAnyRole('OWNER')")
+    ResponseEntity<ResponseMessage<String>> restoreBackup(@Valid @RequestParam String id) {
+        try {
+            this.dataStore.restoreWithBackup(id);
+        } catch (IOException e) {
+            log.error("restore backup failed", e);
+            throw new SwProcessException(ErrorType.SYSTEM, "restore backup failed", e);
+        }
+        return ResponseEntity.ok(Code.success.asResponse("success"));
+    }
+
+    @DeleteMapping(value = "/datastore/maintain/backup")
+    @PreAuthorize("hasAnyRole('OWNER')")
+    ResponseEntity<ResponseMessage<String>> deleteBackup(@Valid @RequestParam String id) {
+        try {
+            this.dataStore.deleteBackup(id);
+        } catch (IOException e) {
+            log.error("delete backup failed", e);
+            throw new SwProcessException(ErrorType.SYSTEM, "delete backup failed", e);
+        }
+        return ResponseEntity.ok(Code.success.asResponse("success"));
+    }
+
     private RecordList queryRecordList(QueryTableRequest request) {
         if (request.getTableName() == null) {
             throw new SwValidationException(

server/controller/src/main/java/ai/starwhale/mlops/api/protocol/datastore/BackupVo.java

@@ -0,0 +1,45 @@
+/*
+ * Copyright 2022 Starwhale, Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ai.starwhale.mlops.api.protocol.datastore;
+
+import lombok.AllArgsConstructor;
+import lombok.Builder;
+import lombok.Data;
+import lombok.NoArgsConstructor;
+
+@Data
+@Builder
+@NoArgsConstructor
+@AllArgsConstructor
+public class BackupVo {
+    String id;
+
+    /**
+     * The time when the backup is created
+     */
+    private Long createdAt;
+
+    /**
+     * The time when the backup is done
+     */
+    private Long doneAt;
+
+    /**
+     * The approximate size of the backup in bytes
+     */
+    private Long approximateSizeBytes;
+}