diff --git a/backend/onedrive/api/types.go b/backend/onedrive/api/types.go index 346829afa..53140819c 100644 --- a/backend/onedrive/api/types.go +++ b/backend/onedrive/api/types.go @@ -410,3 +410,28 @@ func (i *Item) GetParentReference() *ItemReference { func (i *Item) IsRemote() bool { return i.RemoteItem != nil } + +// User details for each version +type User struct { + Email string `json:"email"` + ID string `json:"id"` + DisplayName string `json:"displayName"` +} + +// LastModifiedBy for each version +type LastModifiedBy struct { + User User `json:"user"` +} + +// Version info +type Version struct { + ID string `json:"id"` + LastModifiedDateTime time.Time `json:"lastModifiedDateTime"` + Size int `json:"size"` + LastModifiedBy LastModifiedBy `json:"lastModifiedBy"` +} + +// VersionsResponse is returned from /versions +type VersionsResponse struct { + Versions []Version `json:"value"` +} diff --git a/backend/onedrive/onedrive.go b/backend/onedrive/onedrive.go index ea4007b79..13cb10aa0 100755 --- a/backend/onedrive/onedrive.go +++ b/backend/onedrive/onedrive.go @@ -14,6 +14,7 @@ import ( "path" "strconv" "strings" + "sync" "time" "github.com/pkg/errors" @@ -26,6 +27,8 @@ import ( "github.com/rclone/rclone/fs/config/obscure" "github.com/rclone/rclone/fs/fserrors" "github.com/rclone/rclone/fs/hash" + "github.com/rclone/rclone/fs/operations" + "github.com/rclone/rclone/fs/walk" "github.com/rclone/rclone/lib/atexit" "github.com/rclone/rclone/lib/dircache" "github.com/rclone/rclone/lib/encoder" @@ -1275,6 +1278,73 @@ func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, return result.Link.WebURL, nil } +// CleanUp deletes all the hidden files. +func (f *Fs) CleanUp(ctx context.Context) error { + token := make(chan struct{}, fs.Config.Checkers) + var wg sync.WaitGroup + err := walk.Walk(ctx, f, "", true, -1, func(path string, entries fs.DirEntries, err error) error { + err = entries.ForObjectError(func(obj fs.Object) error { + o, ok := obj.(*Object) + if !ok { + return errors.New("internal error: not a onedrive object") + } + wg.Add(1) + token <- struct{}{} + go func() { + defer func() { + <-token + wg.Done() + }() + err := o.deleteVersions(ctx) + if err != nil { + fs.Errorf(o, "Failed to remove versions: %v", err) + } + }() + return nil + }) + wg.Wait() + return err + }) + return err +} + +// Finds and removes any old versions for o +func (o *Object) deleteVersions(ctx context.Context) error { + opts := newOptsCall(o.id, "GET", "/versions") + var versions api.VersionsResponse + err := o.fs.pacer.Call(func() (bool, error) { + resp, err := o.fs.srv.CallJSON(ctx, &opts, nil, &versions) + return shouldRetry(resp, err) + }) + if err != nil { + return err + } + if len(versions.Versions) < 2 { + return nil + } + for _, version := range versions.Versions[1:] { + err = o.deleteVersion(ctx, version.ID) + if err != nil { + return err + } + } + return nil +} + +// Finds and removes any old versions for o +func (o *Object) deleteVersion(ctx context.Context, ID string) error { + if operations.SkipDestructive(ctx, fmt.Sprintf("%s of %s", ID, o.remote), "delete version") { + return nil + } + fs.Infof(o, "removing version %q", ID) + opts := newOptsCall(o.id, "DELETE", "/versions/"+ID) + opts.NoResponse = true + return o.fs.pacer.Call(func() (bool, error) { + resp, err := o.fs.srv.Call(ctx, &opts) + return shouldRetry(resp, err) + }) +} + // ------------------------------------------------------------ // Fs returns the parent Fs @@ -1840,6 +1910,7 @@ var ( _ fs.DirCacheFlusher = (*Fs)(nil) _ fs.Abouter = (*Fs)(nil) _ fs.PublicLinker = (*Fs)(nil) + _ fs.CleanUpper = (*Fs)(nil) _ fs.Object = (*Object)(nil) _ fs.MimeTyper = &Object{} _ fs.IDer = &Object{} diff --git a/docs/content/onedrive.md b/docs/content/onedrive.md index 8a6e3d12d..ab91acb18 100644 --- a/docs/content/onedrive.md +++ b/docs/content/onedrive.md @@ -331,6 +331,8 @@ This counts against a users quota. For example changing the modification time of a file creates a second version, so the file is using twice the space. +You can use the `rclone cleanup` command (see below) to remove old versions. + The `copy` is the only rclone command affected by this as we copy the file and then afterwards set the modification time to match the source file. @@ -359,6 +361,18 @@ Note: This will disable the creation of new file versions, but will not remove a 8. Use rclone to upload or modify files. (I also use the --no-update-modtime flag) 9. Restore the versioning settings after using rclone. (Optional) +### Cleanup + +OneDrive supports `rclone cleanup` which causes rclone to look through +every file under the path supplied and delete all version but the +current version. Because this involves traversing all the files, then +querying each file for versions it can be quite slow. Rclone does +`--checkers` tests in parallel. The command also supports `-i` which +is a great way to see what it would do. + + rclone cleanup -i remote:path/subdir # interactively remove all old version for path/subdir + rclone cleanup remote:path/subdir # unconditionally remove all old version for path/subdir + ### Troubleshooting ### #### Unexpected file size/hash differences on Sharepoint #### diff --git a/docs/content/overview.md b/docs/content/overview.md index 5cc94d156..f45a3bd88 100644 --- a/docs/content/overview.md +++ b/docs/content/overview.md @@ -336,7 +336,7 @@ operations more efficient. | Mega | Yes | No | Yes | Yes | Yes | No | No | No [#2178](https://github.com/rclone/rclone/issues/2178) | Yes | Yes | | Memory | No | Yes | No | No | No | Yes | Yes | No | No | No | | Microsoft Azure Blob Storage | Yes | Yes | No | No | No | Yes | Yes | No [#2178](https://github.com/rclone/rclone/issues/2178) | No | No | -| Microsoft OneDrive | Yes | Yes | Yes | Yes | No [#575](https://github.com/rclone/rclone/issues/575) | No | No | Yes | Yes | Yes | +| Microsoft OneDrive | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes | | OpenDrive | Yes | Yes | Yes | Yes | No | No | No | No | No | Yes | | OpenStack Swift | Yes † | Yes | No | No | No | Yes | Yes | No [#2178](https://github.com/rclone/rclone/issues/2178) | Yes | No | | pCloud | Yes | Yes | Yes | Yes | Yes | No | No | Yes | Yes | Yes |