accounting: Make checkers show what they are doing

Before this change, all types of checkers showed "checking" after the
file name despite the fact that not all of them were checking.

After this change, they can show

- checking
- deleting
- hashing
- importing
- listing
- merging
- moving
- renaming

See: https://forum.rclone.org/t/what-is-rclone-checking-during-a-purge/35931/
This commit is contained in:
Nick Craig-Wood 2023-02-06 10:30:22 +00:00
parent 34ce11d2be
commit 019a486d5b
10 changed files with 28 additions and 24 deletions

View File

@ -1221,7 +1221,7 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error {
fs.Errorf(object.Name, "Can't create object %v", err)
continue
}
tr := accounting.Stats(ctx).NewCheckingTransfer(oi)
tr := accounting.Stats(ctx).NewCheckingTransfer(oi, "deleting")
err = f.deleteByID(ctx, object.ID, object.Name)
checkErr(err)
tr.Done(ctx, err)
@ -1235,7 +1235,7 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error {
if err != nil {
fs.Errorf(object, "Can't create object %+v", err)
}
tr := accounting.Stats(ctx).NewCheckingTransfer(oi)
tr := accounting.Stats(ctx).NewCheckingTransfer(oi, "checking")
if oldOnly && last != remote {
// Check current version of the file
if object.Action == "hide" {

View File

@ -161,7 +161,7 @@ func (f *Fs) dbImport(ctx context.Context, hashName, sumRemote string, sticky bo
if err := o.putHashes(ctx, hashMap{hashType: hash}); err != nil {
fs.Errorf(nil, "%s: failed to import: %v", remote, err)
}
accounting.Stats(ctx).NewCheckingTransfer(obj).Done(ctx, err)
accounting.Stats(ctx).NewCheckingTransfer(obj, "importing").Done(ctx, err)
doneCount++
}
})

View File

@ -4542,7 +4542,7 @@ func (f *Fs) purge(ctx context.Context, dir string, oldOnly bool) error {
fs.Errorf(object, "Can't create object %+v", err)
return nil
}
tr := accounting.Stats(ctx).NewCheckingTransfer(oi)
tr := accounting.Stats(ctx).NewCheckingTransfer(oi, "checking")
// Work out whether the file is the current version or not
isCurrentVersion := !versioned || !version.Match(remote)
fs.Debugf(nil, "%q version %v", remote, version.Match(remote))

View File

@ -689,8 +689,8 @@ func (s *StatsInfo) RetryAfter() time.Time {
}
// NewCheckingTransfer adds a checking transfer to the stats, from the object.
func (s *StatsInfo) NewCheckingTransfer(obj fs.DirEntry) *Transfer {
tr := newCheckingTransfer(s, obj)
func (s *StatsInfo) NewCheckingTransfer(obj fs.DirEntry, what string) *Transfer {
tr := newCheckingTransfer(s, obj, what)
s.checking.add(tr)
return tr
}
@ -720,7 +720,7 @@ func (s *StatsInfo) NewTransfer(obj fs.DirEntry) *Transfer {
// NewTransferRemoteSize adds a transfer to the stats based on remote and size.
func (s *StatsInfo) NewTransferRemoteSize(remote string, size int64) *Transfer {
tr := newTransferRemoteSize(s, remote, size, false)
tr := newTransferRemoteSize(s, remote, size, false, "")
s.transferring.add(tr)
s.startAverageLoop()
return tr

View File

@ -50,6 +50,7 @@ type Transfer struct {
size int64
startedAt time.Time
checking bool
what string // what kind of transfer this is
// Protects all below
//
@ -63,22 +64,23 @@ type Transfer struct {
}
// newCheckingTransfer instantiates new checking of the object.
func newCheckingTransfer(stats *StatsInfo, obj fs.DirEntry) *Transfer {
return newTransferRemoteSize(stats, obj.Remote(), obj.Size(), true)
func newCheckingTransfer(stats *StatsInfo, obj fs.DirEntry, what string) *Transfer {
return newTransferRemoteSize(stats, obj.Remote(), obj.Size(), true, what)
}
// newTransfer instantiates new transfer.
func newTransfer(stats *StatsInfo, obj fs.DirEntry) *Transfer {
return newTransferRemoteSize(stats, obj.Remote(), obj.Size(), false)
return newTransferRemoteSize(stats, obj.Remote(), obj.Size(), false, "")
}
func newTransferRemoteSize(stats *StatsInfo, remote string, size int64, checking bool) *Transfer {
func newTransferRemoteSize(stats *StatsInfo, remote string, size int64, checking bool, what string) *Transfer {
tr := &Transfer{
stats: stats,
remote: remote,
size: size,
startedAt: time.Now(),
checking: checking,
what: what,
}
stats.AddTransfer(tr)
return tr

View File

@ -110,10 +110,14 @@ func (tm *transferMap) String(ctx context.Context, progress *inProgress, exclude
if acc := progress.get(tr.remote); acc != nil {
out = acc.String()
} else {
what := tr.what
if what == "" {
what = tm.name
}
out = fmt.Sprintf("%*s: %s",
ci.StatsFileNameLength,
shortenName(tr.remote, ci.StatsFileNameLength),
tm.name,
what,
)
}
stringList = append(stringList, " * "+out)

View File

@ -120,7 +120,7 @@ func (c *checkMarch) SrcOnly(src fs.DirEntry) (recurse bool) {
// check to see if two objects are identical using the check function
func (c *checkMarch) checkIdentical(ctx context.Context, dst, src fs.Object) (differ bool, noHash bool, err error) {
ci := fs.GetConfig(ctx)
tr := accounting.Stats(ctx).NewCheckingTransfer(src)
tr := accounting.Stats(ctx).NewCheckingTransfer(src, "checking")
defer func() {
tr.Done(ctx, err)
}()
@ -450,7 +450,7 @@ func (c *checkMarch) checkSum(ctx context.Context, obj fs.Object, download bool,
}
var err error
tr := accounting.Stats(ctx).NewCheckingTransfer(obj)
tr := accounting.Stats(ctx).NewCheckingTransfer(obj, "hashing")
defer tr.Done(ctx, err)
if !sumFound {

View File

@ -286,7 +286,7 @@ func dedupeFindDuplicateDirs(ctx context.Context, f fs.Fs) (duplicateDirs [][]*d
ci := fs.GetConfig(ctx)
err = walk.ListR(ctx, f, "", false, ci.MaxDepth, walk.ListAll, func(entries fs.DirEntries) error {
for _, entry := range entries {
tr := accounting.Stats(ctx).NewCheckingTransfer(entry)
tr := accounting.Stats(ctx).NewCheckingTransfer(entry, "merging")
remote := entry.Remote()
parentRemote := path.Dir(remote)
@ -438,7 +438,7 @@ func Deduplicate(ctx context.Context, f fs.Fs, mode DeduplicateMode, byHash bool
files := map[string][]fs.Object{}
err := walk.ListR(ctx, f, "", false, ci.MaxDepth, walk.ListObjects, func(entries fs.DirEntries) error {
entries.ForObject(func(o fs.Object) {
tr := accounting.Stats(ctx).NewCheckingTransfer(o)
tr := accounting.Stats(ctx).NewCheckingTransfer(o, "checking")
defer tr.Done(ctx, nil)
var remote string

View File

@ -544,7 +544,7 @@ func SameObject(src, dst fs.Object) bool {
// be nil.
func Move(ctx context.Context, fdst fs.Fs, dst fs.Object, remote string, src fs.Object) (newDst fs.Object, err error) {
ci := fs.GetConfig(ctx)
tr := accounting.Stats(ctx).NewCheckingTransfer(src)
tr := accounting.Stats(ctx).NewCheckingTransfer(src, "moving")
defer func() {
if err == nil {
accounting.Stats(ctx).Renames(1)
@ -633,7 +633,7 @@ func SuffixName(ctx context.Context, remote string) string {
// deleting
func DeleteFileWithBackupDir(ctx context.Context, dst fs.Object, backupDir fs.Fs) (err error) {
ci := fs.GetConfig(ctx)
tr := accounting.Stats(ctx).NewCheckingTransfer(dst)
tr := accounting.Stats(ctx).NewCheckingTransfer(dst, "deleting")
defer func() {
tr.Done(ctx, err)
}()
@ -938,7 +938,7 @@ func List(ctx context.Context, f fs.Fs, w io.Writer) error {
func ListLong(ctx context.Context, f fs.Fs, w io.Writer) error {
ci := fs.GetConfig(ctx)
return ListFn(ctx, f, func(o fs.Object) {
tr := accounting.Stats(ctx).NewCheckingTransfer(o)
tr := accounting.Stats(ctx).NewCheckingTransfer(o, "listing")
defer func() {
tr.Done(ctx, nil)
}()
@ -996,7 +996,7 @@ func hashSum(ctx context.Context, ht hash.Type, base64Encoded bool, downloadFlag
return "ERROR", fmt.Errorf("hasher returned an error: %w", err)
}
} else {
tr := accounting.Stats(ctx).NewCheckingTransfer(o)
tr := accounting.Stats(ctx).NewCheckingTransfer(o, "hashing")
defer func() {
tr.Done(ctx, err)
}()
@ -1929,7 +1929,6 @@ func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName str
_, err = Op(ctx, fdst, dstObj, dstFileName, srcObj)
} else {
tr := accounting.Stats(ctx).NewCheckingTransfer(srcObj)
if !cp {
if ci.IgnoreExisting {
fs.Debugf(srcObj, "Not removing source file as destination file exists and --ignore-existing is set")
@ -1937,7 +1936,6 @@ func moveOrCopyFile(ctx context.Context, fdst fs.Fs, fsrc fs.Fs, dstFileName str
err = DeleteFile(ctx, srcObj)
}
}
tr.Done(ctx, err)
}
return err
}

View File

@ -329,7 +329,7 @@ func (s *syncCopyMove) pairChecker(in *pipe, out *pipe, fraction int, wg *sync.W
}
src := pair.Src
var err error
tr := accounting.Stats(s.ctx).NewCheckingTransfer(src)
tr := accounting.Stats(s.ctx).NewCheckingTransfer(src, "checking")
// Check to see if can store this
if src.Storable() {
needTransfer := operations.NeedTransfer(s.ctx, pair.Dst, pair.Src)
@ -779,7 +779,7 @@ func (s *syncCopyMove) makeRenameMap() {
for obj := range in {
// only create hash for dst fs.Object if its size could match
if _, found := possibleSizes[obj.Size()]; found {
tr := accounting.Stats(s.ctx).NewCheckingTransfer(obj)
tr := accounting.Stats(s.ctx).NewCheckingTransfer(obj, "renaming")
hash := s.renameID(obj, s.trackRenamesStrategy, s.modifyWindow)
if hash != "" {