Merge pull request #34 from cuducos/cli

Re-introducing chunk CLI with richer parameter options

cmd/chunk/main.go

···
 	"fmt"
 	"log"
 	"os"
+	"time"
 
 	"github.com/cuducos/chunk"
+	"github.com/spf13/cobra"
 )
 
-func main() {
-	chunk := chunk.DefaultDownloader()
-	prog := newProgress()
-	for status := range chunk.Download(os.Args[1:len(os.Args)]...) {
-		if status.Error != nil {
-			log.Fatal(status.Error)
+var rootCmd = &cobra.Command{
+	Use:   "chunk",
+	Short: "Download tool for slow and unstable servers",
+	Long:  "Download tool for slow and unstable servers using HTTP range requests, retries per HTTP request (not by file), prevents re-downloading the same content range and supports wait time to give servers time to recover.",
+	Run: func(cmd *cobra.Command, args []string) {
+		chunk := chunk.DefaultDownloader()
+		chunk.Timeout = timeoutChunk
+		chunk.ConcurrencyPerServer = concurrencyPerServer
+		chunk.MaxRetries = maxRetriesChunk
+		chunk.WaitRetry = waitBetweenRetries
+		chunk.ChunkSize = chunkSize
+		prog := newProgress()
+		for status := range chunk.Download(os.Args[1:len(os.Args)]...) {
+			if status.Error != nil {
+				log.Fatal(status.Error)
+			}
+			prog.update(status)
 		}
-		prog.update(status)
+		fmt.Printf("\r%s\nDownloaded to: %s", prog.String(), os.TempDir())
+	},
+}
+
+// Flags
+var (
+	timeoutChunk         time.Duration
+	concurrencyPerServer int
+	maxRetriesChunk      uint
+	chunkSize            int64
+	waitBetweenRetries   time.Duration
+)
+
+func init() {
+	rootCmd.Flags().DurationVarP(&timeoutChunk, "timeout", "t", chunk.DefaultTimeout, "timeout for the download of each chunk from each URL.")
+	rootCmd.Flags().UintVarP(&maxRetriesChunk, "max-retries", "r", chunk.DefaultMaxRetries, "maximum number of retries for each chunk.")
+	rootCmd.Flags().DurationVarP(&waitBetweenRetries, "wait-retry", "w", chunk.DefaultWaitRetry, "pause before retrying an HTTP request that has failed.")
+	rootCmd.Flags().Int64VarP(&chunkSize, "chunk-size", "s", chunk.DefaultChunkSize, "maximum size of each HTTP request done using the content range header.")
+	rootCmd.Flags().IntVarP(&concurrencyPerServer, "concurrency-per-server", "c", chunk.DefaultConcurrencyPerServer, "controls the max number of concurrent connections opened to the same server.")
+}
+
+func main() {
+	if err := rootCmd.Execute(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
 	}
-	fmt.Printf("\r%s\nDownloaded to: %s", prog.String(), os.TempDir())
 }

downloader.go

···
 )
 
 const (
-	DefaultTimeoutPerChunk               = 90 * time.Second
-	DefaultMaxParallelDownloadsPerServer = 8
-	DefaultMaxRetriesPerChunk            = 5
-	DefaultChunkSize                     = 8192
-	DefaultWaitBetweenRetries            = 0 * time.Minute
+	DefaultTimeout              = 90 * time.Second
+	DefaultConcurrencyPerServer = 8
+	DefaultMaxRetries           = 5
+	DefaultChunkSize            = 8192
+	DefaultWaitRetry            = 1 * time.Second
 )
 
 // DownloadStatus is the data propagated via the channel sent back to the user
···
 	// URL. A chunk is a part of a file requested using the content range HTTP
 	// header. Thus, this timeout is not the timeout for the each file or for
 	// the the download of every file).
-	TimeoutPerChunk time.Duration
+	Timeout time.Duration
 
-	// MaxParallelDownloadsPerServer controls how many requests are sent in
-	// parallel to the same server. If all the URLs are from the same server
-	// this is the total of parallel requests. If the user is downloading files
-	// from different servers (including different subdomains), this limit is
-	// applied to each server idependently.
-	MaxParallelDownloadsPerServer int
+	// MaxParallelDownloadsPerServer controls the max number of concurrent
+	// connections opened to the same server. If all the URLs are from the same
+	// server this is the total of concurrent connections. If the user is downloading
+	// files from different servers, this limit is applied to each server idependently.
+	ConcurrencyPerServer int
 
 	// MaxRetriesPerChunk is the maximum amount of retries for each HTTP request
 	// using the content range header that fails.
-	MaxRetriesPerChunk uint
+	MaxRetries uint
 
 	// ChunkSize is the maximum size of each HTTP request done using the
 	// content range header. There is no way to specify how many chunks a
···
 
 	// WaitBetweenRetries is an optional pause before retrying an HTTP request
 	// that has failed.
-	WaitBetweenRetries time.Duration
+	WaitRetry time.Duration
 }
 
 type chunk struct {
···
 }
 
 func (d *Downloader) downloadChunkWithTimeout(userCtx context.Context, u string, c chunk) ([]byte, error) {
-	ctx, cancel := context.WithTimeout(userCtx, d.TimeoutPerChunk) // need to propagate context, which might contain app-specific data.
+	ctx, cancel := context.WithTimeout(userCtx, d.Timeout) // need to propagate context, which might contain app-specific data.
 	defer cancel()
 	ch := make(chan []byte)
 	errs := make(chan error)
···
 			ch <- resp
 			return nil
 		},
-		retry.Attempts(d.MaxRetriesPerChunk),
-		retry.MaxDelay(d.WaitBetweenRetries),
+		retry.Attempts(d.MaxRetries),
+		retry.MaxDelay(d.WaitRetry),
 	)
 	if err != nil {
 		return 0, fmt.Errorf("error sending get http request to %s: %w", u, err)
···
 			ch <- b
 			return nil
 		},
-		retry.Attempts(d.MaxRetriesPerChunk),
-		retry.MaxDelay(d.WaitBetweenRetries),
+		retry.Attempts(d.MaxRetries),
+		retry.MaxDelay(d.WaitRetry),
 	)
 	if err != nil {
 		return nil, fmt.Errorf("error downloading %s: %w", u, err)
···
 // context can be used to stop all downloads in progress.
 func (d *Downloader) DownloadWithContext(ctx context.Context, urls ...string) <-chan DownloadStatus {
 	if d.client == nil {
-		d.client = newClient(d.MaxParallelDownloadsPerServer, d.TimeoutPerChunk)
+		d.client = newClient(d.ConcurrencyPerServer, d.Timeout)
 	}
 	ch := make(chan DownloadStatus, 2*len(urls)) // the first status will be the total file size (and or an error creating/trucating the file).
 	var wg sync.WaitGroup                        // this wait group is used to wait for all chunks (from all downloads) to finish.
···
 // the constants in this package for their values.
 func DefaultDownloader() *Downloader {
 	return &Downloader{
-		TimeoutPerChunk:               DefaultTimeoutPerChunk,
-		MaxParallelDownloadsPerServer: DefaultMaxParallelDownloadsPerServer,
-		MaxRetriesPerChunk:            DefaultMaxRetriesPerChunk,
-		ChunkSize:                     DefaultChunkSize,
-		WaitBetweenRetries:            DefaultWaitBetweenRetries,
-		client:                        newClient(DefaultMaxRetriesPerChunk, DefaultTimeoutPerChunk),
+		Timeout:              DefaultTimeout,
+		ConcurrencyPerServer: DefaultConcurrencyPerServer,
+		MaxRetries:           DefaultMaxRetries,
+		ChunkSize:            DefaultChunkSize,
+		WaitRetry:            DefaultWaitRetry,
+		client:               newClient(DefaultMaxRetries, DefaultTimeout),
 	}
 }
 

downloader_test.go

···
 			))
 			defer s.Close()
 			d := Downloader{
-				TimeoutPerChunk:               timeout,
-				MaxRetriesPerChunk:            4,
-				MaxParallelDownloadsPerServer: 1,
-				ChunkSize:                     1024,
-				WaitBetweenRetries:            0 * time.Second,
+				Timeout:              timeout,
+				MaxRetries:           4,
+				ConcurrencyPerServer: 1,
+				ChunkSize:            1024,
+				WaitRetry:            0 * time.Second,
 			}
 			ch := d.Download(s.URL)
 			<-ch // discard the first got (just the file size)
···
 			defer s.Close()
 
 			d := Downloader{
-				TimeoutPerChunk:               timeout,
-				MaxRetriesPerChunk:            4,
-				MaxParallelDownloadsPerServer: 1,
-				ChunkSize:                     1024,
-				WaitBetweenRetries:            0 * time.Second,
+				Timeout:              timeout,
+				MaxRetries:           4,
+				ConcurrencyPerServer: 1,
+				ChunkSize:            1024,
+				WaitRetry:            0 * time.Second,
 			}
 			ch := d.Download(s.URL)
 			<-ch // discard the first status (just the file size)
···
 	))
 	defer s.Close()
 	d := Downloader{
-		TimeoutPerChunk:               timeout,
-		MaxRetriesPerChunk:            4,
-		MaxParallelDownloadsPerServer: 1,
-		ChunkSize:                     1024,
-		WaitBetweenRetries:            0 * time.Second,
+		Timeout:              timeout,
+		MaxRetries:           4,
+		ConcurrencyPerServer: 1,
+		ChunkSize:            1024,
+		WaitRetry:            0 * time.Second,
 	}
 	userCtx, cancFunc := context.WithTimeout(context.Background(), userTimeout)
 	defer cancFunc()

go.mod

···
 
 go 1.19
 
-require github.com/avast/retry-go v3.0.0+incompatible
+require (
+	github.com/avast/retry-go v3.0.0+incompatible
+	github.com/spf13/cobra v1.6.1
+)
 
-require github.com/stretchr/testify v1.8.1 // indirect
+require (
+	github.com/inconshreveable/mousetrap v1.0.1 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/stretchr/testify v1.8.1 // indirect
+)

go.sum

···
 github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0=
 github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
+github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc=
+github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA=
+github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=