community
/
rclone
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

about: complete other providers and re-work internals 

* Implement about for:
    * local, crypt, cache, drive, swift, hubic, onedrive, pcloud, dropbox
  * Implement `--json` and `---full` flag for `rclone about`
  * change About interface to return a Usage structure
  * Remove operations.About as it is too thin an interface
  * Implement Integration test

Relates to #1138 and #1564
This commit is contained in:
Nick Craig-Wood 2018-04-16 22:19:25 +01:00
parent 94e277d759
commit 1ac6dacf0f
16 changed files with 364 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
backend/cache/cache.go vendored
View File

@ -1414,14 +1414,11 @@ func (f *Fs) CleanUp() error {
}
// About gets quota information from the Fs
func (f *Fs) About() error {
f.CleanUpCache(false)
func (f *Fs) About() (*fs.Usage, error) {
do := f.Fs.Features().About
if do == nil {
return nil
return nil, errors.New("About not supported")
}
return do()
}

10
backend/crypt/crypt.go
View File

@ -452,6 +452,15 @@ func (f *Fs) CleanUp() error {
return do()
}
// About gets quota information from the Fs
func (f *Fs) About() (*fs.Usage, error) {
do := f.Fs.Features().About
if do == nil {
return nil, errors.New("About not supported")
}
return do()
}
// UnWrap returns the Fs that this Fs is wrapping
func (f *Fs) UnWrap() fs.Fs {
return f.Fs
@ -699,6 +708,7 @@ var (
_ fs.CleanUpper = (*Fs)(nil)
_ fs.UnWrapper = (*Fs)(nil)
_ fs.ListRer = (*Fs)(nil)
_ fs.Abouter = (*Fs)(nil)
_ fs.ObjectInfo = (*ObjectInfo)(nil)
_ fs.Object = (*Object)(nil)
_ fs.ObjectUnWrapper = (*Object)(nil)

28
backend/drive/drive.go
View File

@ -1051,7 +1051,7 @@ func (f *Fs) CleanUp() error {
}
// About gets quota information
func (f *Fs) About() error {
func (f *Fs) About() (*fs.Usage, error) {
var about *drive.About
var err error
err = f.pacer.Call(func() (bool, error) {
@ -1059,18 +1059,19 @@ func (f *Fs) About() error {
return shouldRetry(err)
})
if err != nil {
fs.Errorf(f, "Failed to get Drive storageQuota: %v", err)
return nil
return nil, errors.Wrap(err, "failed to get Drive storageQuota")
}
quota := float64(about.StorageQuota.Limit) / (1 << 30)
usagetotal := float64(about.StorageQuota.Usage) / (1 << 30)
usagedrive := float64(about.StorageQuota.UsageInDrive) / (1 << 30)
usagetrash := float64(about.StorageQuota.UsageInDriveTrash) / (1 << 30)
fmt.Printf("Quota: %.0f GiB | Used: %.1f GiB (Trash: %.1f GiB) | Available: %.1f GiB | Usage: %d%%\n",
quota, usagedrive, usagetrash, quota-usagedrive, int((usagedrive/quota)*100))
fmt.Printf("Space used in other Google services (such as Gmail): %.2f GiB\n",
usagetotal-usagedrive)
return nil
q := about.StorageQuota
usage := &fs.Usage{
Used: fs.NewUsageValue(q.UsageInDrive), // bytes in use
Trashed: fs.NewUsageValue(q.UsageInDriveTrash), // bytes in trash
Other: fs.NewUsageValue(q.Usage - q.UsageInDrive), // other usage eg gmail in drive
}
if q.Limit > 0 {
usage.Total = fs.NewUsageValue(q.Limit) // quota of bytes that can be used
usage.Free = fs.NewUsageValue(q.Limit - q.Usage) // bytes which can be uploaded before reaching the quota
}
return usage, nil
}
// Move src to this remote using server side move operations.
@ -1664,6 +1665,7 @@ var (
_ fs.PutUncheckeder = (*Fs)(nil)
_ fs.PublicLinker = (*Fs)(nil)
_ fs.MergeDirser = (*Fs)(nil)
_ fs.Abouter = (*Fs)(nil)
_ fs.Object = (*Object)(nil)
_ fs.MimeTyper = &Object{}
_ fs.MimeTyper = (*Object)(nil)
)

32
backend/dropbox/dropbox.go
View File

@ -33,6 +33,7 @@ import (
"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox"
"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/files"
"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/sharing"
"github.com/dropbox/dropbox-sdk-go-unofficial/dropbox/users"
"github.com/ncw/rclone/fs"
"github.com/ncw/rclone/fs/config"
"github.com/ncw/rclone/fs/config/flags"
@ -128,6 +129,7 @@ type Fs struct {
features *fs.Features // optional features
srv files.Client // the connection to the dropbox server
sharingClient sharing.Client // as above, but for generating sharing links
users users.Client // as above, but for accessing user information
slashRoot string // root with "/" prefix, lowercase
slashRootSlash string // root with "/" prefix and postfix, lowercase
pacer *pacer.Pacer // To pace the API calls
@ -209,11 +211,13 @@ func NewFs(name, root string) (fs.Fs, error) {
}
srv := files.New(config)
sharingClient := sharing.New(config)
users := users.New(config)
f := &Fs{
name: name,
srv: srv,
sharingClient: sharingClient,
users: users,
pacer: pacer.New().SetMinSleep(minSleep).SetMaxSleep(maxSleep).SetDecayConstant(decayConstant),
}
f.features = (&fs.Features{
@ -727,6 +731,33 @@ func (f *Fs) DirMove(src fs.Fs, srcRemote, dstRemote string) error {
return nil
}
// About gets quota information
func (f *Fs) About() (usage *fs.Usage, err error) {
var q *users.SpaceUsage
err = f.pacer.Call(func() (bool, error) {
q, err = f.users.GetSpaceUsage()
return shouldRetry(err)
})
if err != nil {
return nil, errors.Wrap(err, "about failed")
}
var total uint64
if q.Allocation != nil {
if q.Allocation.Individual != nil {
total += q.Allocation.Individual.Allocated
}
if q.Allocation.Team != nil {
total += q.Allocation.Team.Allocated
}
}
usage = &fs.Usage{
Total: fs.NewUsageValue(int64(total)), // quota of bytes that can be used
Used: fs.NewUsageValue(int64(q.Used)), // bytes in use
Free: fs.NewUsageValue(int64(total - q.Used)), // bytes which can be uploaded before reaching the quota
}
return usage, nil
}
// Hashes returns the supported hash sets.
func (f *Fs) Hashes() hash.Set {
return hash.Set(hash.Dropbox)
@ -1012,5 +1043,6 @@ var (
_ fs.Mover = (*Fs)(nil)
_ fs.PublicLinker = (*Fs)(nil)
_ fs.DirMover = (*Fs)(nil)
_ fs.Abouter = (*Fs)(nil)
_ fs.Object = (*Object)(nil)
)

29
backend/local/about_unix.go Normal file
View File

@ -0,0 +1,29 @@
// +build darwin dragonfly freebsd linux
package local
import (
"syscall"
"github.com/ncw/rclone/fs"
"github.com/pkg/errors"
)
// About gets quota information
func (f *Fs) About() (*fs.Usage, error) {
var s syscall.Statfs_t
err := syscall.Statfs(f.root, &s)
if err != nil {
return nil, errors.Wrap(err, "failed to read disk usage")
}
bs := int64(s.Bsize)
usage := &fs.Usage{
Total: fs.NewUsageValue(bs * int64(s.Blocks)), // quota of bytes that can be used
Used: fs.NewUsageValue(bs * int64(s.Blocks-s.Bfree)), // bytes in use
Free: fs.NewUsageValue(bs * int64(s.Bavail)), // bytes which can be uploaded before reaching the quota
}
return usage, nil
}
// check interface
var _ fs.Abouter = &Fs{}

36
backend/local/about_windows.go Normal file
View File

@ -0,0 +1,36 @@
// +build windows
package local
import (
"syscall"
"unsafe"
"github.com/ncw/rclone/fs"
"github.com/pkg/errors"
)
var getFreeDiskSpace = syscall.NewLazyDLL("kernel32.dll").NewProc("GetDiskFreeSpaceExW")
// About gets quota information
func (f *Fs) About() (*fs.Usage, error) {
var available, total, free int64
_, _, e1 := getFreeDiskSpace.Call(
uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(f.root))),
uintptr(unsafe.Pointer(&available)), // lpFreeBytesAvailable - for this user
uintptr(unsafe.Pointer(&total)), // lpTotalNumberOfBytes
uintptr(unsafe.Pointer(&free)), // lpTotalNumberOfFreeBytes
)
if e1 != syscall.Errno(0) {
return nil, errors.Wrap(e1, "failed to read disk usage")
}
usage := &fs.Usage{
Total: fs.NewUsageValue(total), // quota of bytes that can be used
Used: fs.NewUsageValue(total - free), // bytes in use
Free: fs.NewUsageValue(available), // bytes which can be uploaded before reaching the quota
}
return usage, nil
}
// check interface
var _ fs.Abouter = &Fs{}

8
backend/onedrive/api/types.go
View File

@ -50,10 +50,10 @@ type IdentitySet struct {
// Quota groups storage space quota-related information on OneDrive into a single structure.
type Quota struct {
Total int `json:"total"`
Used int `json:"used"`
Remaining int `json:"remaining"`
Deleted int `json:"deleted"`
Total int64 `json:"total"`
Used int64 `json:"used"`
Remaining int64 `json:"remaining"`
Deleted int64 `json:"deleted"`
State string `json:"state"` // normal | nearing | critical | exceeded
}

26
backend/onedrive/onedrive.go
View File

@ -922,6 +922,31 @@ func (f *Fs) DirCacheFlush() {
f.dirCache.ResetRoot()
}
// About gets quota information
func (f *Fs) About() (usage *fs.Usage, err error) {
var drive api.Drive
opts := rest.Opts{
Method: "GET",
Path: "",
}
var resp *http.Response
err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, nil, &drive)
return shouldRetry(resp, err)
})
if err != nil {
return nil, errors.Wrap(err, "about failed")
}
q := drive.Quota
usage = &fs.Usage{
Total: fs.NewUsageValue(q.Total), // quota of bytes that can be used
Used: fs.NewUsageValue(q.Used), // bytes in use
Trashed: fs.NewUsageValue(q.Deleted), // bytes in trash
Free: fs.NewUsageValue(q.Remaining), // bytes which can be uploaded before reaching the quota
}
return usage, nil
}
// Hashes returns the supported hash sets.
func (f *Fs) Hashes() hash.Set {
return hash.Set(hash.SHA1)
@ -1273,6 +1298,7 @@ var (
_ fs.Mover = (*Fs)(nil)
// _ fs.DirMover = (*Fs)(nil)
_ fs.DirCacheFlusher = (*Fs)(nil)
_ fs.Abouter = (*Fs)(nil)
_ fs.Object = (*Object)(nil)
_ fs.MimeTyper = &Object{}
)

32
backend/pcloud/api/types.go
View File

@ -151,3 +151,35 @@ type ChecksumFileResult struct {
Hashes
Metadata Item `json:"metadata"`
}
// UserInfo is returned from /userinfo
type UserInfo struct {
Error
Cryptosetup bool `json:"cryptosetup"`
Plan int `json:"plan"`
CryptoSubscription bool `json:"cryptosubscription"`
PublicLinkQuota int64 `json:"publiclinkquota"`
Email string `json:"email"`
UserID int `json:"userid"`
Result int `json:"result"`
Quota int64 `json:"quota"`
TrashRevretentionDays int `json:"trashrevretentiondays"`
Premium bool `json:"premium"`
PremiumLifetime bool `json:"premiumlifetime"`
EmailVerified bool `json:"emailverified"`
UsedQuota int64 `json:"usedquota"`
Language string `json:"language"`
Business bool `json:"business"`
CryptoLifetime bool `json:"cryptolifetime"`
Registered string `json:"registered"`
Journey struct {
Claimed bool `json:"claimed"`
Steps struct {
VerifyMail bool `json:"verifymail"`
UploadFile bool `json:"uploadfile"`
AutoUpload bool `json:"autoupload"`
DownloadApp bool `json:"downloadapp"`
DownloadDrive bool `json:"downloaddrive"`
} `json:"steps"`
} `json:"journey"`
}

25
backend/pcloud/pcloud.go
View File

@ -806,6 +806,30 @@ func (f *Fs) DirCacheFlush() {
f.dirCache.ResetRoot()
}
// About gets quota information
func (f *Fs) About() (usage *fs.Usage, err error) {
opts := rest.Opts{
Method: "POST",
Path: "/userinfo",
}
var resp *http.Response
var q api.UserInfo
err = f.pacer.Call(func() (bool, error) {
resp, err = f.srv.CallJSON(&opts, nil, &q)
err = q.Error.Update(err)
return shouldRetry(resp, err)
})
if err != nil {
return nil, errors.Wrap(err, "about failed")
}
usage = &fs.Usage{
Total: fs.NewUsageValue(q.Quota), // quota of bytes that can be used
Used: fs.NewUsageValue(q.UsedQuota), // bytes in use
Free: fs.NewUsageValue(q.Quota - q.UsedQuota), // bytes which can be uploaded before reaching the quota
}
return usage, nil
}
// Hashes returns the supported hash sets.
func (f *Fs) Hashes() hash.Set {
return hash.Set(hash.MD5 | hash.SHA1)
@ -1107,5 +1131,6 @@ var (
_ fs.Mover = (*Fs)(nil)
_ fs.DirMover = (*Fs)(nil)
_ fs.DirCacheFlusher = (*Fs)(nil)
_ fs.Abouter = (*Fs)(nil)
_ fs.Object = (*Object)(nil)
)

18
backend/swift/swift.go
View File

@ -498,6 +498,24 @@ func (f *Fs) ListR(dir string, callback fs.ListRCallback) (err error) {
return list.Flush()
}
// About gets quota information
func (f *Fs) About() (*fs.Usage, error) {
containers, err := f.c.ContainersAll(nil)
if err != nil {
return nil, errors.Wrap(err, "container listing failed")
}
var total, objects int64
for _, c := range containers {
total += c.Bytes
objects += c.Count
}
usage := &fs.Usage{
Used: fs.NewUsageValue(total), // bytes in use
Objects: fs.NewUsageValue(objects), // objects in use
}
return usage, nil
}
// Put the object into the container
//
// Copy the reader in to the new object which is returned

95
cmd/about/about.go
View File

@ -1,27 +1,112 @@
package about
import (
"encoding/json"
"fmt"
"os"
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/fs/operations"
"github.com/ncw/rclone/fs"
"github.com/pkg/errors"
"github.com/spf13/cobra"
)
var (
jsonOutput bool
fullOutput bool
)
func init() {
cmd.Root.AddCommand(commandDefintion)
cmd.Root.AddCommand(commandDefinition)
commandDefinition.Flags().BoolVar(&jsonOutput, "json", false, "Format output as JSON")
commandDefinition.Flags().BoolVar(&fullOutput, "full", false, "Full numbers instead of SI units")
}
var commandDefintion = &cobra.Command{
// printValue formats uv to be output
func printValue(what string, uv *int64) {
what += ":"
if uv == nil {
return
}
var val string
if fullOutput {
val = fmt.Sprintf("%d", *uv)
} else {
val = fs.SizeSuffix(*uv).String()
}
fmt.Printf("%-9s%v\n", what, val)
}
var commandDefinition = &cobra.Command{
Use: "about remote:",
Short: `Get quota information from the remote.`,
Long: `
Get quota information from the remote, like bytes used/free/quota and bytes
used in the trash. Not supported by all remotes.
This will print to stdout something like this:
Total: 17G
Used: 7.444G
Free: 1.315G
Trashed: 100.000M
Other: 8.241G
Where the fields are:
* Total: total size available.
* Used: total size used
* Free: total amount this user could upload.
* Trashed: total amount in the trash
* Other: total amount in other storage (eg Gmail, Google Photos)
* Objects: total number of objects in the storage
Note that not all the backends provide all the fields - they will be
missing if they are not known for that backend. Where it is known
that the value is unlimited the value will also be omitted.
Use the --full flag to see the numbers written out in full, eg
Total: 18253611008
Used: 7993453766
Free: 1411001220
Trashed: 104857602
Other: 8849156022
Use the --json flag for a computer readable output, eg
{
"total": 18253611008,
"used": 7993453766,
"trashed": 104857602,
"other": 8849156022,
"free": 1411001220
}
`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(1, 1, command, args)
fsrc := cmd.NewFsSrc(args)
f := cmd.NewFsSrc(args)
cmd.Run(true, false, command, func() error {
return operations.About(fsrc)
doAbout := f.Features().About
if doAbout == nil {
return errors.Errorf("%v doesn't support about", f)
}
u, err := doAbout()
if err != nil {
return errors.Wrap(err, "About call failed")
}
if jsonOutput {
out := json.NewEncoder(os.Stdout)
out.SetIndent("", "\t")
return out.Encode(u)
}
printValue("Total", u.Total)
printValue("Used", u.Used)
printValue("Free", u.Free)
printValue("Trashed", u.Trashed)
printValue("Other", u.Other)
printValue("Objects", u.Objects)
return nil
})
},
}

12
docs/content/overview.md
View File

@ -125,21 +125,21 @@ operations more efficient.
| Amazon S3 | No | Yes | No | No | No | Yes | Yes | No [#2178](https://github.com/ncw/rclone/issues/2178) | No |
| Backblaze B2 | No | No | No | No | Yes | Yes | Yes | No [#2178](https://github.com/ncw/rclone/issues/2178) | No |
| Box | Yes | Yes | Yes | Yes | No [#575](https://github.com/ncw/rclone/issues/575) | No | Yes | No [#2178](https://github.com/ncw/rclone/issues/2178) | No |
| Dropbox | Yes | Yes | Yes | Yes | No [#575](https://github.com/ncw/rclone/issues/575) | No | Yes | Yes | No |
| Dropbox | Yes | Yes | Yes | Yes | No [#575](https://github.com/ncw/rclone/issues/575) | No | Yes | Yes | Yes |
| FTP | No | No | Yes | Yes | No | No | Yes | No [#2178](https://github.com/ncw/rclone/issues/2178) | No |
| Google Cloud Storage | Yes | Yes | No | No | No | Yes | Yes | No [#2178](https://github.com/ncw/rclone/issues/2178) | No |
| Google Drive | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes |
| HTTP | No | No | No | No | No | No | No | No [#2178](https://github.com/ncw/rclone/issues/2178) | No |
| Hubic | Yes † | Yes | No | No | No | Yes | Yes | No [#2178](https://github.com/ncw/rclone/issues/2178) | No |
| Hubic | Yes † | Yes | No | No | No | Yes | Yes | No [#2178](https://github.com/ncw/rclone/issues/2178) | Yes |
| Microsoft Azure Blob Storage | Yes | Yes | No | No | No | Yes | No | No [#2178](https://github.com/ncw/rclone/issues/2178) | No |
| Microsoft OneDrive | Yes | Yes | Yes | No [#197](https://github.com/ncw/rclone/issues/197) | No [#575](https://github.com/ncw/rclone/issues/575) | No | No | No [#2178](https://github.com/ncw/rclone/issues/2178) | No |
| Openstack Swift | Yes † | Yes | No | No | No | Yes | Yes | No [#2178](https://github.com/ncw/rclone/issues/2178) | No |
| pCloud | Yes | Yes | Yes | Yes | Yes | No | No | No [#2178](https://github.com/ncw/rclone/issues/2178) | No |
| Microsoft OneDrive | Yes | Yes | Yes | No [#197](https://github.com/ncw/rclone/issues/197) | No [#575](https://github.com/ncw/rclone/issues/575) | No | No | No [#2178](https://github.com/ncw/rclone/issues/2178) | Yes |
| Openstack Swift | Yes † | Yes | No | No | No | Yes | Yes | No [#2178](https://github.com/ncw/rclone/issues/2178) | Yes |
| pCloud | Yes | Yes | Yes | Yes | Yes | No | No | No [#2178](https://github.com/ncw/rclone/issues/2178) | Yes |
| QingStor | No | Yes | No | No | No | Yes | No | No [#2178](https://github.com/ncw/rclone/issues/2178) | No |
| SFTP | No | No | Yes | Yes | No | No | Yes | No [#2178](https://github.com/ncw/rclone/issues/2178) | No |
| WebDAV | Yes | Yes | Yes | Yes | No | No | Yes ‡ | No [#2178](https://github.com/ncw/rclone/issues/2178) | No |
| Yandex Disk | Yes | No | No | No | Yes | Yes | Yes | No [#2178](https://github.com/ncw/rclone/issues/2178) | No |
| The local filesystem | Yes | No | Yes | Yes | No | No | Yes | No | No |
| The local filesystem | Yes | No | Yes | Yes | No | No | Yes | No | Yes |
### Purge ###

25
fs/fs.go
View File

@ -262,6 +262,25 @@ type ListRCallback func(entries DirEntries) error
// ListRFn is defines the call used to recursively list a directory
type ListRFn func(dir string, callback ListRCallback) error
// NewUsageValue makes a valid value
func NewUsageValue(value int64) *int64 {
p := new(int64)
*p = value
return p
}
// Usage is returned by the About call
//
// If a value is nil then it isn't supported by that backend
type Usage struct {
Total *int64 `json:"total,omitempty"` // quota of bytes that can be used
Used *int64 `json:"used,omitempty"` // bytes in use
Trashed *int64 `json:"trashed,omitempty"` // bytes in trash
Other *int64 `json:"other,omitempty"` // other usage eg gmail in drive
Free *int64 `json:"free,omitempty"` // bytes which can be uploaded before reaching the quota
Objects *int64 `json:"objects,omitempty"` // objects in the storage system
}
// Features describe the optional features of the Fs
type Features struct {
// Feature flags, whether Fs
@ -378,8 +397,8 @@ type Features struct {
// of listing recursively that doing a directory traversal.
ListR ListRFn
// Get quota information from the Fs
About func() error
// About gets quota information from the Fs
About func() (*Usage, error)
}
// Disable nil's out the named feature. If it isn't found then it
@ -718,7 +737,7 @@ type RangeSeeker interface {
// Abouter is an optional interface for Fs
type Abouter interface {
// About gets quota information from the Fs
About() error
About() (*Usage, error)
}
// ObjectsChan is a channel of Objects

13
fs/operations/operations.go
View File

@ -1049,19 +1049,6 @@ func CleanUp(f fs.Fs) error {
return doCleanUp()
}
// About gets quota information from the remote
func About(f fs.Fs) error {
doAbout := f.Features().About
if doAbout == nil {
return errors.Errorf("%v doesn't support about", f)
}
if fs.Config.DryRun {
fs.Logf(f, "Not running about as --dry-run set")
return nil
}
return doAbout()
}
// wrap a Reader and a Closer together into a ReadCloser
type readCloser struct {
io.Reader

17
fstest/fstests/fstests.go
View File

@ -1079,6 +1079,23 @@ func Run(t *testing.T, opt *Opt) {
file.Check(t, obj, remote.Precision())
})
// TestAbout tests the About optional interface
t.Run("TestObjectAbout", func(t *testing.T) {
skipIfNotOk(t)
// Check have About
doAbout := remote.Features().About
if doAbout == nil {
t.Skip("FS does not support About")
}
// Can't really check the output much!
usage, err := doAbout()
require.NoError(t, err)
require.NotNil(t, usage)
assert.NotEqual(t, int64(0), usage.Total)
})
// TestObjectPurge tests Purge
t.Run("TestObjectPurge", func(t *testing.T) {
skipIfNotOk(t)