Added No VSS flag, closes #31

Signed-off-by: Tiziano Bacocco <tizbac2@gmail.com>

Author: tizbac

directorybackup/config.go

@@ -39,6 +39,7 @@ type Config struct {
 	BackupStreamName string      `json:"backupstreamname"`
 	PxarOut          string      `json:"pxarout"`
 	SMTP             *SMTPConfig `json:"smtp"`
+	UseVSS 			 bool        `json:"usevss"`
 }
 
 func (c *Config) valid() bool {
@@ -73,6 +74,7 @@ func loadConfig() *Config {
 	backupSourceDirFlag := flag.String("backupdir", "", "Backup source directory, must not be symlink")
 	backupStreamNameFlag := flag.String("backupstream", "", "Filename for stream backup")
 	pxarOutFlag := flag.String("pxarout", "", "Output PXAR archive for debug purposes (optional)")
+	noVSSFlag := flag.Bool("novss", false, "Disable VSS ( For filesystems that don't support it, for example veracrypt )")
 
 	mailHostFlag := flag.String("mail-host", "", "mail notification system: mail server host(optional)")
 	mailPortFlag := flag.String("mail-port", "", "mail notification system: mail server port(optional)")
@@ -89,7 +91,9 @@ func loadConfig() *Config {
 	// Parse command line flags
 	flag.Parse()
 
-	config := &Config{}
+	config := &Config{
+		UseVSS: true,
+	}
 	if *configFile != "" {
 		file, err := os.ReadFile(*configFile)
 		if err != nil {
@@ -134,6 +138,9 @@ func loadConfig() *Config {
 	if *pxarOutFlag != "" {
 		config.PxarOut = *pxarOutFlag
 	}
+	if *noVSSFlag {
+		config.UseVSS = false
+	}
 
 	initSmtpConfigIfNeeded := func() {
 		if config.SMTP == nil {

directorybackup/main.go

@@ -218,7 +218,7 @@ func main() {
 
 	begin := time.Now()
 	if cfg.BackupSourceDir != "" {
-		err = backup(client, newchunk, reusechunk, cfg.PxarOut, cfg.BackupSourceDir)
+		err = backup(client, newchunk, reusechunk, cfg.PxarOut, cfg.BackupSourceDir, cfg.UseVSS)
 	} else if cfg.BackupStreamName != "" {
 		sn := cfg.BackupStreamName
 		if !strings.HasSuffix(sn, ".didx") {
@@ -359,113 +359,122 @@ func backup_stream(client *pbscommon.PBSClient, newchunk, reusechunk *atomic.Uin
 	return client.Finish()
 }
 
-func backup(client *pbscommon.PBSClient, newchunk, reusechunk *atomic.Uint64, pxarOut string, backupdir string) error {
+func backup_real(client *pbscommon.PBSClient, newchunk, reusechunk *atomic.Uint64, pxarOut string, backupdir string) error {
+	client.Connect(false)
 	knownChunks := hashmap.New[string, bool]()
 
-	fmt.Printf("Starting backup of %s\n", backupdir)
-
-	err := snapshot.CreateVSSSnapshot(([]string{backupdir}), func(snaps map[string]snapshot.SnapShot) error {
-		k := maps.Keys(snaps)
-		k2 := slices.Collect(k)
-		SNAP := snaps[k2[0]]
-		backupdir = SNAP.FullPath
-		//Remove VSS snapshot on windows, on linux for now NOP
-
-		client.Connect(false)
-
-		archive := &pbscommon.PXARArchive{}
-		archive.ArchiveName = "backup.pxar.didx"
+	archive := &pbscommon.PXARArchive{}
+	archive.ArchiveName = "backup.pxar.didx"
 
-		previousDidx, err := client.DownloadPreviousToBytes(archive.ArchiveName)
-		if err != nil {
-			return err
-		}
+	previousDidx, err := client.DownloadPreviousToBytes(archive.ArchiveName)
+	if err != nil {
+		return err
+	}
 
-		fmt.Printf("Downloaded previous DIDX: %d bytes\n", len(previousDidx))
+	fmt.Printf("Downloaded previous DIDX: %d bytes\n", len(previousDidx))
 
-		/*f2, _ := os.Create("test.didx")
-		defer f2.Close()
+	/*f2, _ := os.Create("test.didx")
+	defer f2.Close()
 
-		f2.Write(previous_didx)*/
+	f2.Write(previous_didx)*/
 
-		/*
-			Here we download the previous dynamic index to figure out which chunks are the same of what
-			we are going to upload to avoid unnecessary traffic and compression cpu usage
-		*/
+	/*
+		Here we download the previous dynamic index to figure out which chunks are the same of what
+		we are going to upload to avoid unnecessary traffic and compression cpu usage
+	*/
 
-		if !bytes.HasPrefix(previousDidx, didxMagic) {
-			fmt.Printf("Previous index has wrong magic (%s)!\n", previousDidx[:8])
+	if !bytes.HasPrefix(previousDidx, didxMagic) {
+		fmt.Printf("Previous index has wrong magic (%s)!\n", previousDidx[:8])
 
-		} else {
-			//Header as per proxmox documentation is fixed size of 4096 bytes,
-			//then offset of type uint64 and sha256 digests follow , so 40 byte each record until EOF
-			previousDidx = previousDidx[4096:]
-			for i := 0; i*40 < len(previousDidx); i += 1 {
-				e := DidxEntry{}
-				e.offset = binary.LittleEndian.Uint64(previousDidx[i*40 : i*40+8])
-				e.digest = previousDidx[i*40+8 : i*40+40]
-				shahash := hex.EncodeToString(e.digest)
-				fmt.Printf("Previous: %s\n", shahash)
-				knownChunks.Set(shahash, true)
-			}
+	} else {
+		//Header as per proxmox documentation is fixed size of 4096 bytes,
+		//then offset of type uint64 and sha256 digests follow , so 40 byte each record until EOF
+		previousDidx = previousDidx[4096:]
+		for i := 0; i*40 < len(previousDidx); i += 1 {
+			e := DidxEntry{}
+			e.offset = binary.LittleEndian.Uint64(previousDidx[i*40 : i*40+8])
+			e.digest = previousDidx[i*40+8 : i*40+40]
+			shahash := hex.EncodeToString(e.digest)
+			fmt.Printf("Previous: %s\n", shahash)
+			knownChunks.Set(shahash, true)
 		}
+	}
 
-		fmt.Printf("Known chunks: %d!\n", knownChunks.Len())
-		f := &os.File{}
-		if pxarOut != "" {
-			f, err = os.Create(pxarOut)
-			if err != nil {
-				return err
-			}
-			defer f.Close()
+	fmt.Printf("Known chunks: %d!\n", knownChunks.Len())
+	f := &os.File{}
+	if pxarOut != "" {
+		f, err = os.Create(pxarOut)
+		if err != nil {
+			return err
 		}
-		/**/
+		defer f.Close()
+	}
+	/**/
 
-		pxarChunk := ChunkState{}
-		pxarChunk.Init(newchunk, reusechunk, knownChunks)
+	pxarChunk := ChunkState{}
+	pxarChunk.Init(newchunk, reusechunk, knownChunks)
 
-		pcat1Chunk := ChunkState{}
-		pcat1Chunk.Init(newchunk, reusechunk, knownChunks)
+	pcat1Chunk := ChunkState{}
+	pcat1Chunk.Init(newchunk, reusechunk, knownChunks)
 
-		pxarChunk.wrid, err = client.CreateDynamicIndex(archive.ArchiveName)
-		if err != nil {
-			return err
-		}
-		pcat1Chunk.wrid, err = client.CreateDynamicIndex("catalog.pcat1.didx")
-		if err != nil {
-			return err
+	pxarChunk.wrid, err = client.CreateDynamicIndex(archive.ArchiveName)
+	if err != nil {
+		return err
+	}
+	pcat1Chunk.wrid, err = client.CreateDynamicIndex("catalog.pcat1.didx")
+	if err != nil {
+		return err
+	}
+
+	archive.WriteCB = func(b []byte) {
+
+		if pxarOut != "" {
+			// TODO: error handling inside callback
+			f.Write(b)
 		}
 
-		archive.WriteCB = func(b []byte) {
+		pxarChunk.HandleData(b, client)
 
-			if pxarOut != "" {
-				// TODO: error handling inside callback
-				f.Write(b)
-			}
+		//
+	}
 
-			pxarChunk.HandleData(b, client)
+	archive.CatalogWriteCB = func(b []byte) {
+		pcat1Chunk.HandleData(b, client)
+	}
 
-			//
-		}
+	//This is the entry point of backup job which will start streaming with the PCAT and PXAR write callback
+	//Data to be hashed and eventuall uploaded
 
-		archive.CatalogWriteCB = func(b []byte) {
-			pcat1Chunk.HandleData(b, client)
-		}
+	archive.WriteDir(backupdir, "", true)
 
-		//This is the entry point of backup job which will start streaming with the PCAT and PXAR write callback
-		//Data to be hashed and eventuall uploaded
+	pxarChunk.Eof(client)
+	pcat1Chunk.Eof(client)
 
-		archive.WriteDir(backupdir, "", true)
+	err = client.UploadManifest()
+	if err != nil {
+		return err
+	}
+	return nil
+}
 
-		pxarChunk.Eof(client)
-		pcat1Chunk.Eof(client)
+func backup(client *pbscommon.PBSClient, newchunk, reusechunk *atomic.Uint64, pxarOut string, backupdir string, usevss bool) error {
+	
 
-		err = client.UploadManifest()
-		if err != nil {
-			return err
-		}
-		return nil
-	})
+	fmt.Printf("Starting backup of %s\n", backupdir)
+	var err error
+	if usevss {
+		err = snapshot.CreateVSSSnapshot(([]string{backupdir}), func(snaps map[string]snapshot.SnapShot) error {
+			k := maps.Keys(snaps)
+			k2 := slices.Collect(k)
+			SNAP := snaps[k2[0]]
+			backupdir = SNAP.FullPath
+			//Remove VSS snapshot on windows, on linux for now NOP
+			return backup_real(client, newchunk, reusechunk, pxarOut, backupdir)
+			
+		})
+	}else{
+		err = backup_real(client, newchunk, reusechunk, pxarOut, backupdir)
+	}
 
 	if err != nil {
 		return err

go.work

@@ -4,4 +4,5 @@ use ./clientcommon
 use ./pbscommon
 use ./directorybackup
 use ./snapshot
-use ./machinebackup
+use ./machinebackup
+use ./nbd

pbscommon/pbsapi.go

@@ -103,6 +103,52 @@ type PBSClient struct {
 var blobCompressedMagic = []byte{49, 185, 88, 66, 111, 182, 163, 127}
 var blobUncompressedMagic = []byte{66, 171, 56, 7, 190, 131, 112, 161}
 
+type SnapshotsResp struct {
+	Data []BackupManifest `json:"data"`
+}
+
+func (pbs *PBSClient) ListSnapshots() ([]BackupManifest, error) {
+	client := &http.Client{
+		Timeout: 10 * time.Second,
+	}
+	if pbs.Insecure {
+		tr := &http.Transport{
+		TLSClientConfig: &tls.Config{
+			InsecureSkipVerify: true,
+		},
+		
+	}
+	client.Transport = tr
+	}
+	
+	ret := make([]BackupManifest, 0)
+	var r SnapshotsResp
+	params := url.Values{}
+	params.Add("ns", pbs.Namespace)
+	fullURL := fmt.Sprintf("%s/api2/json/admin/datastore/%s/snapshots?%s", pbs.BaseURL, pbs.Datastore, params.Encode())
+	
+	req, err := http.NewRequest(http.MethodGet, fullURL, nil)
+	if err != nil {
+		return ret, err
+	}
+	req.Header.Set("Accept", "application/json")
+	req.Header.Add("Authorization", fmt.Sprintf("PBSAPIToken=%s:%s", pbs.AuthID, pbs.Secret))
+	resp, err := client.Do(req)
+	if err != nil {
+		return ret, err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+		body, _ := io.ReadAll(resp.Body)
+		return ret, fmt.Errorf("HTTP error: %d - %s", resp.StatusCode, string(body))
+	}
+	if err := json.NewDecoder(resp.Body).Decode(&r); err != nil {
+		return ret, err
+	}
+	return r.Data, nil
+
+}
+
 func (pbs *PBSClient) CreateFixedIndex(fic FixedIndexCreateReq) (uint64, error) {
 	jd, err := json.Marshal(fic)
 	if err != nil {
@@ -460,7 +506,7 @@ func (pbs *PBSClient) Connect(reader bool) {
 			calculatedFingerprint := sha256.Sum256(peerCert.Raw)
 
 			// Compare the calculated fingerprint with the expected one
-			if hex.EncodeToString(calculatedFingerprint[:]) != expectedFingerprint {
+			if hex.EncodeToString(calculatedFingerprint[:]) != expectedFingerprint && !pbs.Insecure {
 				return fmt.Errorf("certificate fingerprint does not match (%s,%s)", expectedFingerprint, hex.EncodeToString(calculatedFingerprint[:]))
 			}
 
@@ -576,6 +622,33 @@ func (pbs *PBSClient) DownloadPreviousToBytes(archivename string) ([]byte, error
 
 }
 
+func (pbs *PBSClient) DownloadToBytes(archivename string) ([]byte, error) { //In the future also download to tmp if index is extremely big...
+	q := &url.Values{}
+
+	q.Add("file-name", archivename)
+
+	req, err := http.NewRequest("GET", pbs.BaseURL+"/download?"+q.Encode(), nil)
+	req.Header.Add("Authorization", fmt.Sprintf("PBSAPIToken=%s:%s", pbs.AuthID, pbs.Secret))
+	if err != nil {
+		return nil, err
+	}
+	resp2, err := pbs.Client.Do(req)
+	if err != nil {
+		fmt.Println("Error making request:", err)
+		return nil, err
+	}
+	defer resp2.Body.Close()
+
+	ret, err := io.ReadAll(resp2.Body)
+
+	if err != nil {
+		return nil, err
+	}
+
+	return ret, nil
+
+}
+
 func (pbs *PBSClient) GetKnownSha265FromFIDX(archivename string) (*hashmap.Map[string, bool], error) {
 	data, err := pbs.DownloadPreviousToBytes(archivename)
 	if err != nil {