Add SSH/SFTP backend (#658)

* fix: file backend logging wrong error

* feat: add ssh/sftp storage backend

* fix: add new line after backup stats

* chore: add ssh docs & test

README.md

@@ -37,6 +37,12 @@ This project was inspired by the [duplicity project](http://duplicity.nongnu.org
   - Auth: Set the B2_ACCOUNT_ID and B2_ACCOUNT_KEY environmental variables to the appropiate values
   - [99.999999999% durability](https://help.backblaze.com/hc/en-us/articles/218485257-B2-Resiliency-Durability-and-Availability) - Using the Reed-Solomon erasure encoding
 - Local file path (file://[relative|/absolute]/local/path)
+- SSH/SFTP (ssh://)
+  - Auth: username & password, public key or ssh-agent.
+  - For username & password set the SSH_USERNAME and SSH_PASSWORD environment variables or use the url format: `ssh://username:password@example.org/remote/path`.
+  - For public key auth set the SSH_KEY_FILE environment variable. By default zfsbackup tries to use common key names from the users home directory.  
+  - ssh-agent auth is activated when SSH_AUTH_SOCK exists.
+  - By default zfsbackup also uses the known hosts file from the users home directory. To disable host key checking set SSH_KNOWN_HOSTS to `ignore`. You can also specify the path to your own known hosts file.
 
 ### Compression
 
@@ -224,7 +230,7 @@ Global Flags:
 - Make PGP cipher configurable.
 - Refactor
 - Test Coverage
-- Add more backends (e.g. SSH, SCP, etc.)
+- Add more backends
 - Add delete feature
 - Appease linters
 - Track intermediary snaps as part of backup jobs

backends/backends.go

@@ -84,6 +84,8 @@ func GetBackendForURI(uri string) (Backend, error) {
 		return &AzureBackend{}, nil
 	case B2BackendPrefix:
 		return &B2Backend{}, nil
+	case SSHBackendPrefix:
+		return &SSHBackend{}, nil
 	default:
 		return nil, ErrInvalidPrefix
 	}

backends/file_backend.go

@@ -94,7 +94,7 @@ func (f *FileBackend) Upload(ctx context.Context, vol *files.VolumeInfo) error {
 	_, err = io.Copy(w, vol)
 	if err != nil {
 		if closeErr := w.Close(); closeErr != nil {
-			log.AppLogger.Warningf("file backend: Error closing volume %s - %v", vol.ObjectName, err)
+			log.AppLogger.Warningf("file backend: Error closing volume %s - %v", vol.ObjectName, closeErr)
 		}
 		if deleteErr := os.Remove(destinationPath); deleteErr != nil {
 			log.AppLogger.Warningf("file backend: Error deleting failed upload file %s - %v", destinationPath, deleteErr)

backends/ssh_backend.go

@@ -0,0 +1,293 @@
+package backends
+
+import (
+	"context"
+	"errors"
+	"io"
+	"net"
+	"net/url"
+	"os"
+	"os/user"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/pkg/sftp"
+	"github.com/someone1/zfsbackup-go/files"
+	"github.com/someone1/zfsbackup-go/log"
+	"golang.org/x/crypto/ssh"
+	"golang.org/x/crypto/ssh/agent"
+	"golang.org/x/crypto/ssh/knownhosts"
+)
+
+// SSHBackendPrefix is the URI prefix used for the SSHBackend.
+const SSHBackendPrefix = "ssh"
+
+// SSHBackend provides a ssh/sftp storage option.
+type SSHBackend struct {
+	conf       *BackendConfig
+	sshClient  *ssh.Client
+	sftpClient *sftp.Client
+	remotePath string
+}
+
+// buildSshSigner reads the private key file at privateKeyPath and transforms it into a ssh.Signer,
+// using password to decrypt the key if required.
+func buildSshSigner(privateKeyPath string, password string) (ssh.Signer, error) {
+	privateKey, err := os.ReadFile(privateKeyPath)
+	if err != nil {
+		return nil, err
+	}
+
+	signer, err := ssh.ParsePrivateKey(privateKey)
+	_, isMissingPassword := err.(*ssh.PassphraseMissingError)
+	if isMissingPassword && password != "" {
+		signer, err = ssh.ParsePrivateKeyWithPassphrase(privateKey, []byte(password))
+	}
+
+	return signer, err
+}
+
+// buildAuthMethods builds ssh auth methods based on the provided password and private keys in the the users home directory.
+// To use a specific key instead of the default files set the env variable SSH_KEY_FILE.
+// buildAuthMethods also adds ssh-agent auth if the env variable SSH_AUTH_SOCK exists.
+func buildAuthMethods(userHomeDir string, password string) (sshAuths []ssh.AuthMethod, err error) {
+	sshAuthSock := os.Getenv("SSH_AUTH_SOCK")
+	if sshAuthSock != "" {
+		sshAgent, err := net.Dial("unix", sshAuthSock)
+		if err != nil {
+			return nil, err
+		}
+		sshAuths = append(sshAuths, ssh.PublicKeysCallback(agent.NewClient(sshAgent).Signers))
+	}
+
+	sshKeyFile := os.Getenv("SSH_KEY_FILE")
+	if sshKeyFile != "" {
+		signer, err := buildSshSigner(sshKeyFile, password)
+		if err != nil {
+			return nil, err
+		}
+		sshAuths = append(sshAuths, ssh.PublicKeys(signer))
+	} else {
+		signers := make([]ssh.Signer, 0)
+
+		defaultKeys := []string{
+			filepath.Join(userHomeDir, ".ssh/id_rsa"),
+			filepath.Join(userHomeDir, ".ssh/id_cdsa"),
+			filepath.Join(userHomeDir, ".ssh/id_ecdsa_sk"),
+			filepath.Join(userHomeDir, ".ssh/id_ed25519"),
+			filepath.Join(userHomeDir, ".ssh/id_ed25519_sk"),
+			filepath.Join(userHomeDir, ".ssh/id_dsa"),
+		}
+
+		for _, keyPath := range defaultKeys {
+			signer, err := buildSshSigner(keyPath, password)
+			if err != nil {
+				if !errors.Is(err, os.ErrNotExist) {
+					log.AppLogger.Warningf("ssh backend: Failed to use ssh key at %s - %v", keyPath, err)
+				}
+				continue
+			}
+			signers = append(signers, signer)
+		}
+		if len(signers) > 0 {
+			sshAuths = append(sshAuths, ssh.PublicKeys(signers...))
+		}
+	}
+
+	if password != "" {
+		sshAuths = append(sshAuths, ssh.Password(password))
+	}
+
+	return sshAuths, nil
+}
+
+// buildHostKeyCallback builds a ssh.HostKeyCallback that uses the known_hosts file from the users home directory.
+// Or from a custom file specified by the SSH_KNOWN_HOSTS env variable.
+// If SSH_KNOWN_HOSTS is set to "ignore" host key checking is disabled.
+func buildHostKeyCallback(userHomeDir string) (callback ssh.HostKeyCallback, err error) {
+	knownHostsFile := os.Getenv("SSH_KNOWN_HOSTS")
+	if knownHostsFile == "" {
+		knownHostsFile = filepath.Join(userHomeDir, ".ssh/known_hosts")
+	}
+	if knownHostsFile == "ignore" {
+		callback = ssh.InsecureIgnoreHostKey()
+	} else {
+		callback, err = knownhosts.New(knownHostsFile)
+	}
+	return callback, err
+}
+
+// Init will initialize the SSHBackend and verify the provided URI is valid and the target directory exists.
+func (s *SSHBackend) Init(ctx context.Context, conf *BackendConfig, opts ...Option) (err error) {
+	s.conf = conf
+
+	if !strings.HasPrefix(s.conf.TargetURI, SSHBackendPrefix+"://") {
+		return ErrInvalidURI
+	}
+
+	targetUrl, err := url.Parse(s.conf.TargetURI)
+	if err != nil {
+		log.AppLogger.Errorf("ssh backend: Error while parsing target uri %s - %v", s.conf.TargetURI, err)
+		return err
+	}
+
+	s.remotePath = strings.TrimSuffix(targetUrl.Path, "/")
+	if s.remotePath == "" && targetUrl.Path != "/" { // allow root path
+		log.AppLogger.Errorf("ssh backend: No remote path provided!")
+		return ErrInvalidURI
+	}
+
+	username := os.Getenv("SSH_USERNAME")
+	password := os.Getenv("SSH_PASSWORD")
+	if targetUrl.User != nil {
+		urlUsername := targetUrl.User.Username()
+		if urlUsername != "" {
+			username = urlUsername
+		}
+		urlPassword, _ := targetUrl.User.Password()
+		if urlPassword != "" {
+			password = urlPassword
+		}
+	}
+
+	userInfo, err := user.Current()
+	if err != nil {
+		return err
+	}
+	if username == "" {
+		username = userInfo.Username
+	}
+
+	sshAuths, err := buildAuthMethods(userInfo.HomeDir, password)
+	if err != nil {
+		return err
+	}
+
+	hostKeyCallback, err := buildHostKeyCallback(userInfo.HomeDir)
+	if err != nil {
+		return err
+	}
+
+	sshConfig := &ssh.ClientConfig{
+		User:            username,
+		Auth:            sshAuths,
+		HostKeyCallback: hostKeyCallback,
+		Timeout:         30 * time.Second,
+	}
+
+	hostname := targetUrl.Host
+	if !strings.Contains(hostname, ":") {
+		hostname = hostname + ":22"
+	}
+	s.sshClient, err = ssh.Dial("tcp", hostname, sshConfig)
+	if err != nil {
+		return err
+	}
+
+	s.sftpClient, err = sftp.NewClient(s.sshClient)
+	if err != nil {
+		return err
+	}
+
+	fi, err := s.sftpClient.Stat(s.remotePath)
+	if err != nil {
+		log.AppLogger.Errorf("ssh backend: Error while verifying remote path %s - %v", s.remotePath, err)
+		return err
+	}
+
+	if !fi.IsDir() {
+		log.AppLogger.Errorf("ssh backend: Provided remote path is not a directory!")
+		return ErrInvalidURI
+	}
+
+	return nil
+}
+
+// Upload will upload the provided VolumeInfo to the remote sftp server.
+func (s *SSHBackend) Upload(ctx context.Context, vol *files.VolumeInfo) error {
+	s.conf.MaxParallelUploadBuffer <- true
+	defer func() {
+		<-s.conf.MaxParallelUploadBuffer
+	}()
+
+	destinationPath := filepath.Join(s.remotePath, vol.ObjectName)
+	destinationDir := filepath.Dir(destinationPath)
+
+	if err := s.sftpClient.MkdirAll(destinationDir); err != nil {
+		log.AppLogger.Debugf("ssh backend: Could not create path %s due to error - %v", destinationDir, err)
+		return err
+	}
+
+	w, err := s.sftpClient.Create(destinationPath)
+	if err != nil {
+		log.AppLogger.Debugf("ssh backend: Could not create file %s due to error - %v", destinationPath, err)
+		return err
+	}
+
+	_, err = io.Copy(w, vol)
+	if err != nil {
+		if closeErr := w.Close(); closeErr != nil {
+			log.AppLogger.Warningf("ssh backend: Error closing volume %s - %v", vol.ObjectName, closeErr)
+		}
+		if deleteErr := os.Remove(destinationPath); deleteErr != nil {
+			log.AppLogger.Warningf("ssh backend: Error deleting failed upload file %s - %v", destinationPath, deleteErr)
+		}
+		log.AppLogger.Debugf("ssh backend: Error while copying volume %s - %v", vol.ObjectName, err)
+		return err
+	}
+
+	return w.Close()
+}
+
+// List will return a list of all files matching the provided prefix.
+func (s *SSHBackend) List(ctx context.Context, prefix string) ([]string, error) {
+	l := make([]string, 0, 1000)
+
+	w := s.sftpClient.Walk(s.remotePath)
+	for w.Step() {
+		if err := w.Err(); err != nil {
+			return l, err
+		}
+
+		trimmedPath := strings.TrimPrefix(w.Path(), s.remotePath+string(filepath.Separator))
+		if !w.Stat().IsDir() && strings.HasPrefix(trimmedPath, prefix) {
+			l = append(l, trimmedPath)
+		}
+	}
+
+	return l, nil
+}
+
+// Close will release any resources used by SSHBackend.
+func (s *SSHBackend) Close() (err error) {
+	if s.sftpClient != nil {
+		err = s.sftpClient.Close()
+		s.sftpClient = nil
+	}
+	if s.sshClient != nil {
+		sshErr := s.sshClient.Close()
+		if sshErr == nil && err == nil {
+			err = sshErr
+		}
+		s.sshClient = nil
+	}
+	return err
+}
+
+// PreDownload does nothing on this backend.
+func (s *SSHBackend) PreDownload(ctx context.Context, objects []string) error {
+	return nil
+}
+
+// Download will open the remote file for reading.
+func (s *SSHBackend) Download(ctx context.Context, filename string) (io.ReadCloser, error) {
+	return s.sftpClient.Open(filepath.Join(s.remotePath, filename))
+}
+
+// Delete will delete the given object from the provided path.
+func (s *SSHBackend) Delete(ctx context.Context, filename string) error {
+	return s.sftpClient.Remove(filepath.Join(s.remotePath, filename))
+}
+
+var _ Backend = (*SSHBackend)(nil)