community
/
rclone
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

copyurl: add --auto-filename flag for using file name from url in destination path (#3451)

This commit is contained in:
Denis 2019-09-03 19:25:19 +03:00 committed by Nick Craig-Wood
parent 5932acfee3
commit b71ac141cc
6 changed files with 74 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
cmd/copyurl/copyurl.go
View File

@ -4,12 +4,18 @@ import (
"context"
"github.com/rclone/rclone/cmd"
"github.com/rclone/rclone/fs"
"github.com/rclone/rclone/fs/operations"
"github.com/spf13/cobra"
)
var (
autoFilename = false
)
func init() {
cmd.Root.AddCommand(commandDefintion)
commandDefintion.Flags().BoolVarP(&autoFilename, "auto-filename", "a", autoFilename, "Get the file name from the url and use it for destination file path")
}
var commandDefintion = &cobra.Command{
@ -18,13 +24,22 @@ var commandDefintion = &cobra.Command{
Long: `
Download urls content and copy it to destination
without saving it in tmp storage.
Setting --auto-filename flag will cause retrieving file name from url and using it in destination path.
`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(2, 2, command, args)
fsdst, dstFileName := cmd.NewFsDstFile(args[1:])
var dstFileName string
var fsdst fs.Fs
if autoFilename {
fsdst = cmd.NewFsDir(args[1:])
} else {
fsdst, dstFileName = cmd.NewFsDstFile(args[1:])
}
cmd.Run(true, true, command, func() error {
_, err := operations.CopyURL(context.Background(), fsdst, dstFileName, args[0])
_, err := operations.CopyURL(context.Background(), fsdst, dstFileName, args[0], autoFilename)
return err
})
},

3
docs/content/commands/rclone_copyurl.md
View File

@ -22,7 +22,8 @@ rclone copyurl https://example.com dest:path [flags]
### Options
```
-h, --help help for copyurl
-a, --auto-filename Get the file name from the url and use it for destination file path
-h, --help help for copyurl
```
See the [global flags page](/flags/) for global options not listed here.

10
fs/operations/operations.go
View File

@ -1586,7 +1586,7 @@ func RcatSize(ctx context.Context, fdst fs.Fs, dstFileName string, in io.ReadClo
}
// CopyURL copies the data from the url to (fdst, dstFileName)
func CopyURL(ctx context.Context, fdst fs.Fs, dstFileName string, url string) (dst fs.Object, err error) {
func CopyURL(ctx context.Context, fdst fs.Fs, dstFileName string, url string, dstFileNameFromURL bool) (dst fs.Object, err error) {
client := fshttp.NewClient(fs.Config)
resp, err := client.Get(url)
if err != nil {
@ -1596,6 +1596,14 @@ func CopyURL(ctx context.Context, fdst fs.Fs, dstFileName string, url string) (d
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return nil, errors.Errorf("CopyURL failed: %s", resp.Status)
}
if dstFileNameFromURL {
dstFileName = path.Base(resp.Request.URL.Path)
if dstFileName == "." || dstFileName == "/" {
return nil, errors.Errorf("CopyURL failed: file name wasn't found in url")
}
}
return RcatSize(ctx, fdst, dstFileName, resp.Body, resp.ContentLength, time.Now())
}

20
fs/operations/operations_test.go
View File

@ -706,15 +706,27 @@ func TestCopyURL(t *testing.T) {
ts := httptest.NewServer(handler)
defer ts.Close()
o, err := operations.CopyURL(context.Background(), r.Fremote, "file1", ts.URL)
o, err := operations.CopyURL(context.Background(), r.Fremote, "file1", ts.URL, false)
require.NoError(t, err)
assert.Equal(t, int64(len(contents)), o.Size())
fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, nil, fs.ModTimeNotSupported)
// Check auto file naming
status = 0
urlFileName := "filename.txt"
o, err = operations.CopyURL(context.Background(), r.Fremote, "", ts.URL+"/"+urlFileName, true)
require.NoError(t, err)
assert.Equal(t, int64(len(contents)), o.Size())
assert.Equal(t, urlFileName, o.Remote())
// Check auto file naming when url without file name
o, err = operations.CopyURL(context.Background(), r.Fremote, "file1", ts.URL, true)
require.Error(t, err)
// Check an error is returned for a 404
status = http.StatusNotFound
o, err = operations.CopyURL(context.Background(), r.Fremote, "file1", ts.URL)
o, err = operations.CopyURL(context.Background(), r.Fremote, "file1", ts.URL, false)
require.Error(t, err)
assert.Contains(t, err.Error(), "Not Found")
assert.Nil(t, o)
@ -730,10 +742,10 @@ func TestCopyURL(t *testing.T) {
tss := httptest.NewTLSServer(handler)
defer tss.Close()
o, err = operations.CopyURL(context.Background(), r.Fremote, "file2", tss.URL)
o, err = operations.CopyURL(context.Background(), r.Fremote, "file2", tss.URL, false)
require.NoError(t, err)
assert.Equal(t, int64(len(contents)), o.Size())
fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, file2}, nil, fs.ModTimeNotSupported)
fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, file2, fstest.NewItem(urlFileName, contents, t1)}, nil, fs.ModTimeNotSupported)
}
func TestMoveFile(t *testing.T) {

6
fs/operations/rc.go
View File

@ -149,7 +149,7 @@ func init() {
{name: "rmdirs", title: "Remove all the empty directories in the path", help: "- leaveRoot - boolean, set to true not to delete the root\n"},
{name: "delete", title: "Remove files in the path", noRemote: true},
{name: "deletefile", title: "Remove the single file pointed to"},
{name: "copyurl", title: "Copy the URL to the object", help: "- url - string, URL to read from\n"},
{name: "copyurl", title: "Copy the URL to the object", help: "- url - string, URL to read from\n - autoFilename - boolean, set to true to retrieve destination file name from url"},
{name: "cleanup", title: "Remove trashed files in the remote or path", noRemote: true},
} {
op := op
@ -214,7 +214,9 @@ func rcSingleCommand(ctx context.Context, in rc.Params, name string, noRemote bo
if err != nil {
return nil, err
}
_, err = CopyURL(ctx, f, remote, url)
autoFilename, _ := in.GetBool("autoFilename")
_, err = CopyURL(ctx, f, remote, url, autoFilename)
return nil, err
case "cleanup":
return nil, CleanUp(ctx, f)

30
fs/operations/rc_test.go
View File

@ -105,15 +105,37 @@ func TestRcCopyurl(t *testing.T) {
defer ts.Close()
in := rc.Params{
"fs": r.FremoteName,
"remote": "file1",
"url": ts.URL,
"fs": r.FremoteName,
"remote": "file1",
"url": ts.URL,
"autoFilename": false,
}
out, err := call.Fn(context.Background(), in)
require.NoError(t, err)
assert.Equal(t, rc.Params(nil), out)
fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, nil, fs.ModTimeNotSupported)
urlFileName := "filename.txt"
in = rc.Params{
"fs": r.FremoteName,
"remote": "",
"url": ts.URL + "/" + urlFileName,
"autoFilename": true,
}
out, err = call.Fn(context.Background(), in)
require.NoError(t, err)
assert.Equal(t, rc.Params(nil), out)
in = rc.Params{
"fs": r.FremoteName,
"remote": "",
"url": ts.URL,
"autoFilename": true,
}
out, err = call.Fn(context.Background(), in)
require.Error(t, err)
assert.Equal(t, rc.Params(nil), out)
fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, fstest.NewItem(urlFileName, contents, t1)}, nil, fs.ModTimeNotSupported)
}
// operations/delete: Remove files in the path