diff --git a/.am.secret b/.am.secret new file mode 100644 index 00000000..b1e60f10 --- /dev/null +++ b/.am.secret @@ -0,0 +1,25 @@ +apiVersion: v1 +kind: Secret +metadata: + name: am-secret +type: Opaque +stringData: + enduro_am_user: "analyst" + enduro_am_api_key: "1w9ioGI2reeeeua8uPaiOee4Eiueev6u" + enduro_known_hosts: | + # enduro-temporal.ss.analyst.archivematica.net:22 SSH-2.0-OpenSSH_8.2p1 Ubuntu-4ubuntu0.9 + enduro-temporal.ss.analyst.archivematica.net ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDW98QbhNAq2bwazlJb57ptBgxhWELNVo4R4Mh6WEO9FJHUfOLc6bE6eKD65MsJ0BZ4utpe64M1rvH6OX5PVQPFW8IjDgl132UPzkMSAJlze+aZhVI7Xjwq/ppJyRS3EJNyZ5cUV2yv/pDZ8zmaq06EpWnpbwYj8fmvdXqYJAvX7iJ4DZ9dycEPmNnWMwlh4pv3EOHlGO7gFqo2hurbzZFroz9O7mG/msf0qO3F/IXjJT6kw8yvxJ9nBMymp8ErdsLKhMxgBRPyrO2HULRUsDchZk550DCPrOAIH8Dea+WFsmczhJhKYDRWqiwTGhmaXQT3qezlMwNxJJnIeErEVSJx3Ic7pzcF8ZjTy2QwiGEdcMgFSgQobx9ujiuc+C7HaXkx8eOely8j6fiFr2cM6PzuuxgmBomcNOJjAz0r1vEGlbDLy5t/D7cYk3zElarXbqn1q9cronD6MR0h46hs7fxh+Ki6nwf60oOioD0kQmkflm6CCYA4LBTNt3+g3K6tqyU= + # enduro-temporal.ss.analyst.archivematica.net:22 SSH-2.0-OpenSSH_8.2p1 Ubuntu-4ubuntu0.9 + enduro-temporal.ss.analyst.archivematica.net ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBIbi18z3t2gzGJfZLWGVWJLEEquuhPIxTcFDFE4N4j2rXt5FLHJS8zb/CSyWKjsBmGrspQz4xfTYSJJ0lc6+fww= + # enduro-temporal.ss.analyst.archivematica.net:22 SSH-2.0-OpenSSH_8.2p1 Ubuntu-4ubuntu0.9 + enduro-temporal.ss.analyst.archivematica.net ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIEYTs4txTlcU8iGQ6C9ilvITtZaW4S3mT32w+WvzMmC0 +data: + enduro_id_ed25519: | + LS0tLS1CRUdJTiBPUEVOU1NIIFBSSVZBVEUgS0VZLS0tLS0KYjNCbGJuTnphQzFyWlhrdGRqRUFB + QUFBQkc1dmJtVUFBQUFFYm05dVpRQUFBQUFBQUFBQkFBQUFNd0FBQUF0emMyZ3RaVwpReU5UVXhP + UUFBQUNEV1ZhSVR5d0xwTnh2STQwY0xHQUI3dWhmZDArc1JqNTc3VDNsNFU0dUdpd0FBQUppbXJG + Y3ZwcXhYCkx3QUFBQXR6YzJndFpXUXlOVFV4T1FBQUFDRFdWYUlUeXdMcE54dkk0MGNMR0FCN3Vo + ZmQwK3NSajU3N1QzbDRVNHVHaXcKQUFBRUFZRGlGdWh5d3FGNTBVMUFLTnhJY05NTnZTUWMwZE9R + a0piZ25uMzlxb0c5WlZvaFBMQXVrM0c4ampSd3NZQUh1NgpGOTNUNnhHUG52dFBlWGhUaTRhTEFB + QUFGV1JoZG1sa1FHRnlkR1ZtWVdOMGRXRnNMbU52YlE9PQotLS0tLUVORCBPUEVOU1NIIFBSSVZB + VEUgS0VZLS0tLS0K diff --git a/go.mod b/go.mod index 16f866cc..c8231548 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/aws/aws-sdk-go v1.45.25 github.com/coreos/go-oidc/v3 v3.7.0 github.com/cyphar/filepath-securejoin v0.2.4 + github.com/dolmen-go/contextio v1.0.0 github.com/fsnotify/fsnotify v1.6.0 github.com/gliderlabs/ssh v0.3.5 github.com/go-logr/logr v1.2.4 diff --git a/go.sum b/go.sum index 843f3a90..9d798a6f 100644 --- a/go.sum +++ b/go.sum @@ -907,6 +907,8 @@ github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5Xh github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dolmen-go/contextio v1.0.0 h1:bNfCo4gsRIhMeo6Z1ImXzkxZG81B6I5t2fUFJjphdAU= +github.com/dolmen-go/contextio v1.0.0/go.mod h1:cxc20xI7fOgsFHWgt+PenlDDnMcrvh7Ocuj5hEFIdEk= github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 h1:iFaUwBSo5Svw6L7HYpRu/0lE3e0BaElwnNO1qkNQxBY= github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5/go.mod h1:qssHWj60/X5sZFNxpG4HBPDHVqxNm4DfnCKgrbZOT+s= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= diff --git a/go.work.sum b/go.work.sum index fb2acf59..23ddc4be 100644 --- a/go.work.sum +++ b/go.work.sum @@ -193,7 +193,6 @@ github.com/alecthomas/kingpin/v2 v2.3.2/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HR github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/antihax/optional v1.0.0 h1:xK2lYat7ZLaVVcIuj82J8kIro4V6kDe0AUDFboUCwcg= github.com/apache/arrow/go/v10 v10.0.1 h1:n9dERvixoC/1JjDmBcs9FPaEryoANa2sCgVFo6ez9cI= github.com/apache/arrow/go/v11 v11.0.0 h1:hqauxvFQxww+0mEU/2XHG6LT7eZternCZq+A5Yly2uM= @@ -303,7 +302,6 @@ github.com/gabriel-vasile/mimetype v1.4.1 h1:TRWk7se+TOjCYgRth7+1/OYLNiRNIotknkF github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M= github.com/getkin/kin-openapi v0.114.0/go.mod h1:l5e9PaFUo9fyLJCPGQeXI2ML8c3P8BHOEV2VaAVf/pc= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= -github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= github.com/go-fonts/dejavu v0.1.0 h1:JSajPXURYqpr+Cu8U9bt8K+XcACIHWqWrvWCKyeFmVQ= github.com/go-fonts/latin-modern v0.2.0 h1:5/Tv1Ek/QCr20C6ZOz15vw3g7GELYL98KWr8Hgo+3vk= github.com/go-fonts/liberation v0.2.0 h1:jAkAWJP4S+OsrPLZM4/eC9iW7CtHy+HBXrEwZXWo5VM= @@ -344,7 +342,6 @@ github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2V github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A= github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI= github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 h1:DACJavvAHhabrF08vX0COfcOBJRhZ8lUbR+ZWIs0Y5g= -github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -380,8 +377,6 @@ github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMW github.com/hashicorp/consul/api v1.20.0/go.mod h1:nR64eD44KQ59Of/ECwt2vUmIK2DKsDzAwTmwmLl8Wpo= github.com/hashicorp/consul/api v1.25.1 h1:CqrdhYzc8XZuPnhIYZWH45toM0LB9ZeYr/gvpLVI3PE= github.com/hashicorp/consul/api v1.25.1/go.mod h1:iiLVwR/htV7mas/sy0O+XSuEnrdBUUydemjxcUrAt4g= -github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= -github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v1.2.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= @@ -466,6 +461,7 @@ github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= @@ -496,6 +492,7 @@ github.com/muesli/termenv v0.15.1 h1:UzuTb/+hhlBugQz28rpzey4ZuKcZ03MeKsoG7IJZIxs github.com/muesli/termenv v0.15.1/go.mod h1:HeAQPTzpfs016yGtA4g00CsdYnVLJvxsS4ANqrZs2sQ= github.com/mutecomm/go-sqlcipher/v4 v4.4.0 h1:sV1tWCWGAVlPhNGT95Q+z/txFxuhAYWwHD1afF5bMZg= github.com/mutecomm/go-sqlcipher/v4 v4.4.0/go.mod h1:PyN04SaWalavxRGH9E8ZftG6Ju7rsPrGmQRjrEaVpiY= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8 h1:P48LjvUQpTReR3TQRbxSeSBsMXzfK0uol7eRcr7VBYQ= github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8/go.mod h1:86wM1zFnC6/uDBfZGNwB65O+pR2OFi5q/YQaEUid1qA= @@ -652,7 +649,6 @@ go.uber.org/multierr v1.10.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN8 golang.org/x/arch v0.1.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20220517005047-85d78b3ac167/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= @@ -666,7 +662,6 @@ golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028 h1:4+4C/Iv2U4fMZBiMCc98MG golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20200421231249-e086a090c8fd/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.16.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= @@ -677,11 +672,9 @@ golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E= diff --git a/internal/sftp/client.go b/internal/sftp/client.go new file mode 100644 index 00000000..5050eec7 --- /dev/null +++ b/internal/sftp/client.go @@ -0,0 +1,17 @@ +package sftp + +import ( + "context" + "io" +) + +// A Client manages the transmission of data over SFTP. +// +// Implementations of the Client interface handle the connection details, +// authentication, and other intricacies associated with different SFTP +// servers and protocols. +type Client interface { + // Upload transfers data from the provided source reader to a specified + // destination on the SFTP server. + Upload(ctx context.Context, src io.Reader, dest string) (bytes int64, err error) +} diff --git a/internal/sftp/config.go b/internal/sftp/config.go index dc34dc9f..e3cb8438 100644 --- a/internal/sftp/config.go +++ b/internal/sftp/config.go @@ -1,17 +1,39 @@ package sftp -import "path/filepath" +import ( + "os" + "path/filepath" +) type Config struct { + // Host address, e.g. 127.0.0.1 (default), sftp.example.org. Host string + + // User name. + User string + + // Host port (default: 22). Port string + // Path to known_hosts file as per https://linux.die.net/man/8/sshd + // "SSH_KNOWN_HOSTS FILE FORMAT" (default: "$HOME/.ssh/known_hosts"). The + // known_hosts file must include the public key of the SFTP server for + // authentication to succeed. KnownHostsFile string - PrivateKey PrivateKey + + // Private key used for authentication. + PrivateKey PrivateKey + + // Default directory on SFTP server for file transfers. + RemoteDir string } type PrivateKey struct { - Path string + // Path to private key file used for authentication (default: + // "$HOME/.ssh/id_rsa") + Path string + + // Passphrase (if any) used to decrypt private key. Passphrase string } @@ -25,11 +47,16 @@ func (c *Config) SetDefaults() { c.Port = "22" } + home, err := os.UserHomeDir() + if err != nil { + return // Don't set default paths if homedir is unknown. + } + if c.KnownHostsFile == "" { - c.KnownHostsFile = filepath.Join("$HOME", ".ssh", "known_hosts") + c.KnownHostsFile = filepath.Join(home, ".ssh", "known_hosts") } if c.PrivateKey.Path == "" { - c.PrivateKey.Path = filepath.Join("$HOME", ".ssh", "id_rsa") + c.PrivateKey.Path = filepath.Join(home, ".ssh", "id_rsa") } } diff --git a/internal/sftp/goclient.go b/internal/sftp/goclient.go index cd7c685c..6952a9d7 100644 --- a/internal/sftp/goclient.go +++ b/internal/sftp/goclient.go @@ -1,11 +1,13 @@ package sftp import ( + "context" "errors" "fmt" "io" "os" + "github.com/dolmen-go/contextio" "github.com/pkg/sftp" "golang.org/x/crypto/ssh" ) @@ -18,7 +20,7 @@ type GoClient struct { sftp *sftp.Client } -var _ Service = (*GoClient)(nil) +var _ Client = (*GoClient)(nil) // NewGoClient returns a new GoSFTP client with the given configuration. func NewGoClient(cfg Config) *GoClient { @@ -29,8 +31,10 @@ func NewGoClient(cfg Config) *GoClient { // Upload writes the data from src to the remote file at dest and returns the // number of bytes written. A new SFTP connection is opened before writing, and -// closed when the upload is complete. -func (c *GoClient) Upload(src io.Reader, dest string) (int64, error) { +// closed when the upload is complete or cancelled. +// +// Upload is not thread safe. +func (c *GoClient) Upload(ctx context.Context, src io.Reader, dest string) (int64, error) { if err := c.dial(); err != nil { return 0, err } @@ -39,12 +43,15 @@ func (c *GoClient) Upload(src io.Reader, dest string) (int64, error) { // Note: Some SFTP servers don't support O_RDWR mode. w, err := c.sftp.OpenFile(dest, (os.O_WRONLY | os.O_CREATE | os.O_TRUNC)) if err != nil { - return 0, fmt.Errorf("SFTP: couldn't create remote file %q: %w", dest, err) + return 0, fmt.Errorf("SFTP: couldn't create remote file %q: %v", dest, err) } + defer w.Close() - bytes, err := io.Copy(w, src) + // Use contextio to stop the upload if a context cancellation signal is + // received. + bytes, err := io.Copy(contextio.NewWriter(ctx, w), contextio.NewReader(ctx, src)) if err != nil { - return 0, fmt.Errorf("SFTP: failed to write to %q: %w", dest, err) + return 0, fmt.Errorf("SFTP: failed to write to %q: %v", dest, err) } return bytes, nil @@ -54,17 +61,17 @@ func (c *GoClient) Upload(src io.Reader, dest string) (int64, error) { // When the clients are no longer needed, close() must be called to prevent // leaks. func (c *GoClient) dial() error { - sshc, err := SSHConnect(c.cfg) + var err error + + c.ssh, err = sshConnect(c.cfg) if err != nil { - return fmt.Errorf("SSH: %w", err) + return fmt.Errorf("SSH: %v", err) } - c.ssh = sshc - sftpc, err := sftp.NewClient(sshc) + c.sftp, err = sftp.NewClient(c.ssh) if err != nil { - return fmt.Errorf("Unable to start SFTP subsystem: %w", err) + return fmt.Errorf("Unable to start SFTP subsystem: %v", err) } - c.sftp = sftpc return nil } @@ -73,11 +80,16 @@ func (c *GoClient) dial() error { func (c *GoClient) close() error { var errs error - if err := c.sftp.Close(); err != nil { - errs = errors.Join(err, errs) + if c.sftp != nil { + if err := c.sftp.Close(); err != nil { + errs = errors.Join(err, errs) + } } - if err := c.ssh.Close(); err != nil { - errs = errors.Join(err, errs) + + if c.ssh != nil { + if err := c.ssh.Close(); err != nil { + errs = errors.Join(err, errs) + } } return errs diff --git a/internal/sftp/goclient_test.go b/internal/sftp/goclient_test.go index 3edce07e..321ddf50 100644 --- a/internal/sftp/goclient_test.go +++ b/internal/sftp/goclient_test.go @@ -2,9 +2,11 @@ package sftp_test import ( "bufio" + "context" "fmt" "io" "log" + "net" "os" "strings" "testing" @@ -19,6 +21,9 @@ import ( "github.com/artefactual-sdps/enduro/internal/sftp" ) +// ServerAddress is the test SFTP server address. +const serverAddress = "127.0.0.1:2222" + // PubkeyHandler handles checking the client's public key against the keys in // the authorized_keys file. func pubkeyHandler(ctx ssh.Context, key ssh.PublicKey) bool { @@ -84,11 +89,14 @@ func sftpHandler(sess ssh.Session) { } // StartSFTPServer starts a test SFTP server, and returns a pointer to the -// server. The caller must call Close() to shut down the server when done with -// it. -func startSFTPServer() (*ssh.Server, error) { +// server. +func startSFTPServer(t *testing.T, addr string) *ssh.Server { + t.Helper() + + var err error + srv := ssh.Server{ - Addr: "127.0.0.1:2222", + Addr: addr, Handler: func(s ssh.Session) { authorizedKey := gossh.MarshalAuthorizedKey(s.PublicKey()) io.WriteString(s, fmt.Sprintf("public key used by %s:\n", s.User())) @@ -102,26 +110,52 @@ func startSFTPServer() (*ssh.Server, error) { signer, err := hostKeySigner() if err != nil { - return nil, err + t.Fatalf("SFTP server: %v", err) } srv.AddHostKey(signer) + errCh := make(chan error, 1) go func() { - err = srv.ListenAndServe() + errCh <- srv.ListenAndServe() + }() + + // Wait for the server to be ready + func() { + for { + select { + case err := <-errCh: + t.Fatalf("Couldn't start SFTP server: %v", err) + default: + conn, err := net.DialTimeout("tcp", addr, 1*time.Second) + if err == nil { + conn.Close() + return + } + time.Sleep(10 * time.Millisecond) + } + } }() - return &srv, err + t.Cleanup(func() { srv.Close() }) + return &srv } func TestGoClient(t *testing.T) { - srv, err := startSFTPServer() + host, port, err := net.SplitHostPort(serverAddress) if err != nil { - t.Fatalf("Failed to start SFTP server: %v", err) + t.Fatalf("Bad serverAddress: %s", serverAddress) } - t.Cleanup(func() { srv.Close() }) - // Give the server 100ms to start. - time.Sleep(100 * time.Millisecond) + _ = startSFTPServer(t, serverAddress) + + // Start a listener on an open port and use the address to test a bad SFTP + // server address. + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("couldn't start listener: %v", err) + } + defer listener.Close() + badHost, badPort, _ := net.SplitHostPort(listener.Addr().String()) type results struct { Bytes int64 @@ -134,13 +168,12 @@ func TestGoClient(t *testing.T) { want results wantErr string } - for _, tc := range []test{ { name: "Uploads a file using a key with no passphrase", cfg: sftp.Config{ - Host: "127.0.0.1", - Port: "2222", + Host: host, + Port: port, KnownHostsFile: "./testdata/known_hosts", PrivateKey: sftp.PrivateKey{ Path: "./testdata/clientkeys/test_ed25519", @@ -154,8 +187,8 @@ func TestGoClient(t *testing.T) { { name: "Uploads a file using a key with a passphrase", cfg: sftp.Config{ - Host: "127.0.0.1", - Port: "2222", + Host: host, + Port: port, KnownHostsFile: "./testdata/known_hosts", PrivateKey: sftp.PrivateKey{ Path: "./testdata/clientkeys/test_pass_rsa", @@ -170,33 +203,36 @@ func TestGoClient(t *testing.T) { { name: "Errors when the key passphrase is wrong", cfg: sftp.Config{ - Host: "127.0.0.1", - Port: "2222", + Host: host, + Port: port, KnownHostsFile: "./testdata/known_hosts", PrivateKey: sftp.PrivateKey{ Path: "./testdata/clientkeys/test_pass_rsa", Passphrase: "wrong", }, }, - wantErr: "SSH: failed to parse private key with passphrase: x509: decryption password incorrect", + wantErr: "SSH: couldn't parse private key with passphrase: x509: decryption password incorrect", }, { name: "Errors when the SFTP server isn't there", cfg: sftp.Config{ - Host: "127.0.0.1", - Port: "2200", + Host: badHost, + Port: badPort, KnownHostsFile: "./testdata/known_hosts", PrivateKey: sftp.PrivateKey{ Path: "./testdata/clientkeys/test_ed25519", }, }, - wantErr: "SSH: failed to connect: dial tcp 127.0.0.1:2200: connect: connection refused", + wantErr: fmt.Sprintf( + "SSH: failed to connect: dial tcp %s:%s: connect: connection refused", + badHost, badPort, + ), }, { name: "Errors when the private key is not recognized", cfg: sftp.Config{ - Host: "127.0.0.1", - Port: "2222", + Host: host, + Port: port, KnownHostsFile: "./testdata/known_hosts", PrivateKey: sftp.PrivateKey{ Path: "./testdata/clientkeys/test_unk_ed25519", @@ -207,8 +243,8 @@ func TestGoClient(t *testing.T) { { name: "Errors when the host key is not in known_hosts", cfg: sftp.Config{ - Host: "127.0.0.1", - Port: "2222", + Host: host, + Port: port, KnownHostsFile: "./testdata/empty_file", PrivateKey: sftp.PrivateKey{ Path: "./testdata/clientkeys/test_ed25519", @@ -219,14 +255,14 @@ func TestGoClient(t *testing.T) { { name: "Errors when the known_hosts file doesn't exist", cfg: sftp.Config{ - Host: "127.0.0.1", - Port: "2222", + Host: host, + Port: port, KnownHostsFile: "./testdata/missing", PrivateKey: sftp.PrivateKey{ Path: "./testdata/clientkeys/test_ed25519", }, }, - wantErr: "SSH: couldn't parse known_hosts_file: open ./testdata/missing: no such file or directory", + wantErr: "SSH: couldn't parse known_hosts file: open testdata/missing: no such file or directory", }, } { tc := tc @@ -237,7 +273,7 @@ func TestGoClient(t *testing.T) { src := strings.NewReader("Testing 1-2-3") dest := tfs.NewDir(t, "sftp_test") - bytes, err := sftpc.Upload(src, dest.Join("test.txt")) + bytes, err := sftpc.Upload(context.Background(), src, dest.Join("test.txt")) if tc.wantErr != "" { assert.Error(t, err, tc.wantErr) diff --git a/internal/sftp/service.go b/internal/sftp/service.go deleted file mode 100644 index c0232e68..00000000 --- a/internal/sftp/service.go +++ /dev/null @@ -1,9 +0,0 @@ -package sftp - -import ( - "io" -) - -type Service interface { - Upload(src io.Reader, dest string) (int64, error) -} diff --git a/internal/sftp/ssh.go b/internal/sftp/ssh.go index 43572555..99004a11 100644 --- a/internal/sftp/ssh.go +++ b/internal/sftp/ssh.go @@ -4,23 +4,23 @@ import ( "fmt" "net" "os" + "path/filepath" "time" "golang.org/x/crypto/ssh" "golang.org/x/crypto/ssh/knownhosts" ) -// SSHConnect connects to an SSH server using the given configuration and +// sshConnect connects to an SSH server using the given configuration and // returns a client connection. // // Only private key authentication is currently supported, with or without a // passphrase. -func SSHConnect(cfg Config) (*ssh.Client, error) { +func sshConnect(cfg Config) (*ssh.Client, error) { // Load private key for authentication. - keyPath := os.ExpandEnv(cfg.PrivateKey.Path) - keyBytes, err := os.ReadFile(keyPath) // #nosec G304 -- File data is validated below + keyBytes, err := os.ReadFile(filepath.Clean(cfg.PrivateKey.Path)) // #nosec G304 -- File data is validated below if err != nil { - return nil, fmt.Errorf("failed to read private key: %w", err) + return nil, fmt.Errorf("couldn't read private key: %v", err) } // Create a signer from the private key, with or without a passphrase. @@ -28,19 +28,19 @@ func SSHConnect(cfg Config) (*ssh.Client, error) { if cfg.PrivateKey.Passphrase != "" { signer, err = ssh.ParsePrivateKeyWithPassphrase(keyBytes, []byte(cfg.PrivateKey.Passphrase)) if err != nil { - return nil, fmt.Errorf("failed to parse private key with passphrase: %w", err) + return nil, fmt.Errorf("couldn't parse private key with passphrase: %v", err) } } else { signer, err = ssh.ParsePrivateKey(keyBytes) if err != nil { - return nil, fmt.Errorf("failed to parse private key: %w", err) + return nil, fmt.Errorf("couldn't parse private key: %v", err) } } // Check that the host key is in the client's known_hosts file. - hostcallback, err := knownhosts.New(os.ExpandEnv(cfg.KnownHostsFile)) + hostcallback, err := knownhosts.New(filepath.Clean(cfg.KnownHostsFile)) if err != nil { - return nil, fmt.Errorf("couldn't parse known_hosts_file: %w", err) + return nil, fmt.Errorf("couldn't parse known_hosts file: %v", err) } // Configure the SSH client. @@ -56,7 +56,7 @@ func SSHConnect(cfg Config) (*ssh.Client, error) { address := net.JoinHostPort(cfg.Host, cfg.Port) conn, err := ssh.Dial("tcp", address, sshConfig) if err != nil { - return nil, fmt.Errorf("failed to connect: %w", err) + return nil, fmt.Errorf("failed to connect: %v", err) } return conn, nil