-
Notifications
You must be signed in to change notification settings - Fork 43
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
5 changed files
with
288 additions
and
26 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1 +1,83 @@ | ||
| package xhttp | ||
| package xhttp | ||
|
|
||
| // https://gist.github.com/cnu/026744b1e86c6d9e22313d06cba4c2e9 | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "io" | ||
| "net/http" | ||
| "os" | ||
| "strings" | ||
| ) | ||
|
|
||
| // WriteCounter counts the number of bytes written to it. By implementing the Write method, | ||
| // it is of the io.Writer interface and we can pass this into io.TeeReader() | ||
| // Every write to this writer, will print the progress of the file write. | ||
| type WriteCounter struct { | ||
| Total uint64 | ||
| } | ||
|
|
||
| func (wc *WriteCounter) Write(p []byte) (int, error) { | ||
| n := len(p) | ||
| wc.Total += uint64(n) | ||
| wc.PrintProgress() | ||
| return n, nil | ||
| } | ||
|
|
||
| func humanizeBytes(bytes uint64) string { | ||
| const unit = 1024 | ||
| if bytes < unit { | ||
| return fmt.Sprintf("%d B", bytes) | ||
| } | ||
| div, exp := int64(unit), 0 | ||
| for n := bytes / unit; n >= unit; n /= unit { | ||
| div *= unit | ||
| exp++ | ||
| } | ||
| return fmt.Sprintf("%.1f %cB", float64(bytes)/float64(div), "kMGTPE"[exp]) | ||
| } | ||
|
|
||
| // PrintProgress prints the progress of a file write | ||
| func (wc WriteCounter) PrintProgress() { | ||
| // Clear the line by using a character return to go back to the start and remove | ||
| // the remaining characters by filling it with spaces | ||
| fmt.Printf("\r%s", strings.Repeat(" ", 50)) | ||
|
|
||
| // Return again and print current status of download | ||
| // We use the humanize package to print the bytes in a meaningful way (e.g. 10 MB) | ||
| fmt.Printf("\rDownloading... %s complete", humanizeBytes(wc.Total)) | ||
| } | ||
|
|
||
| // DownloadFile will download a url and store it in local filepath. | ||
| // It writes to the destination file as it downloads it, without | ||
| // loading the entire file into memory. | ||
| // We pass an io.TeeReader into Copy() to report progress on the download. | ||
| func DownloadFile(url string, filepath string) error { | ||
|
|
||
| // Create the file with .tmp extension, so that we won't overwrite a | ||
| // file until it's downloaded fully | ||
| out, err := os.Create(filepath) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer out.Close() | ||
|
|
||
| // Get the data | ||
| resp, err := http.Get(url) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer resp.Body.Close() | ||
|
|
||
| // Create our bytes counter and pass it to be used alongside our writer | ||
| counter := &WriteCounter{} | ||
| _, err = io.Copy(out, io.TeeReader(resp.Body, counter)) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| // The progress use the same line so print a new line once it's finished downloading | ||
| fmt.Println() | ||
|
|
||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| package xzip | ||
|
|
||
| import ( | ||
| "archive/zip" | ||
| "fmt" | ||
| "io" | ||
| "os" | ||
| "path/filepath" | ||
| "strings" | ||
| ) | ||
| // Unzip, from https://stackoverflow.com/questions/20357223/easy-way-to-unzip-file-with-golang | ||
| func Unzip(srcFile, destDir string) error { | ||
| r, err := zip.OpenReader(srcFile) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer func() { | ||
| if err := r.Close(); err != nil { | ||
| panic(err) | ||
| } | ||
| }() | ||
|
|
||
| os.MkdirAll(destDir, 0755) | ||
|
|
||
| // Closure to address file descriptors issue with all the deferred .Close() methods | ||
| extractAndWriteFile := func(f *zip.File) error { | ||
| rc, err := f.Open() | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer func() { | ||
| if err := rc.Close(); err != nil { | ||
| panic(err) | ||
| } | ||
| }() | ||
|
|
||
| path := filepath.Join(destDir, f.Name) | ||
|
|
||
| // Check for ZipSlip (Directory traversal) | ||
| if !strings.HasPrefix(path, filepath.Clean(destDir)+string(os.PathSeparator)) { | ||
| return fmt.Errorf("illegal file path: %s", path) | ||
| } | ||
|
|
||
| if f.FileInfo().IsDir() { | ||
| os.MkdirAll(path, f.Mode()) | ||
| } else { | ||
| os.MkdirAll(filepath.Dir(path), f.Mode()) | ||
| f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer func() { | ||
| if err := f.Close(); err != nil { | ||
| panic(err) | ||
| } | ||
| }() | ||
|
|
||
| _, err = io.Copy(f, rc) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| for _, f := range r.File { | ||
| err := extractAndWriteFile(f) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| } | ||
|
|
||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters