community
/
rclone
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ncdu: fix issue where dir size is summed when file sizes are -1 

Some backends may not provide size for all objects, and instead
return -1. Existing version included these in directory sums,
with strange results. With this commit rclone ncdu will consider
negative sizes as zero, but add a new prefix flag '~' with a
description that indicates the shown size is inaccurate.

Fixes #6084
This commit is contained in:
albertony 2022-04-05 15:46:56 +02:00
parent d77736c21a
commit 87c201c92a
2 changed files with 36 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
cmd/ncdu/ncdu.go
View File

@ -283,7 +283,7 @@ func (u *UI) biggestEntry() (biggest int64) {
return
}
for i := range u.entries {
size, _, _, _, _, _ := u.d.AttrI(u.sortPerm[i])
size, _, _, _, _, _, _ := u.d.AttrI(u.sortPerm[i])
if size > biggest {
biggest = size
}
@ -297,7 +297,7 @@ func (u *UI) hasEmptyDir() bool {
return false
}
for i := range u.entries {
_, count, isDir, _, _, _ := u.d.AttrI(u.sortPerm[i])
_, count, _, isDir, _, _, _ := u.d.AttrI(u.sortPerm[i])
if isDir && count == 0 {
return true
}
@ -343,7 +343,7 @@ func (u *UI) Draw() error {
if y >= h-1 {
break
}
size, count, isDir, readable, entriesHaveErrors, err := u.d.AttrI(u.sortPerm[n])
size, count, countUnknownSize, isDir, readable, entriesHaveErrors, err := u.d.AttrI(u.sortPerm[n])
fg := termbox.ColorWhite
if entriesHaveErrors {
fg = termbox.ColorYellow
@ -364,6 +364,10 @@ func (u *UI) Draw() error {
if !readable {
message = " [not read yet]"
}
if countUnknownSize > 0 {
message = fmt.Sprintf(" [%d of %d files have unknown size, size may be underestimated]", countUnknownSize, count)
fileFlag = '~'
}
if entriesHaveErrors {
message = " [some subdirectories could not be read, size may be underestimated]"
fileFlag = '.'
@ -383,7 +387,10 @@ func (u *UI) Draw() error {
}
var averageSize float64
if count > 0 {
averageSize = float64(size) / float64(count)
countForAverage := count - countUnknownSize
if countForAverage > 0 {
averageSize = float64(size) / float64(countForAverage)
}
}
if u.showDirAverageSize {
ss := operations.SizeStringField(int64(averageSize), u.humanReadable, 9) + " "
@ -559,8 +566,8 @@ type ncduSort struct {
// Less is part of sort.Interface.
func (ds *ncduSort) Less(i, j int) bool {
var iAvgSize, jAvgSize float64
isize, icount, _, _, _, _ := ds.d.AttrI(ds.sortPerm[i])
jsize, jcount, _, _, _, _ := ds.d.AttrI(ds.sortPerm[j])
isize, icount, _, _, _, _, _ := ds.d.AttrI(ds.sortPerm[i])
jsize, jcount, _, _, _, _, _ := ds.d.AttrI(ds.sortPerm[j])
iname, jname := ds.entries[ds.sortPerm[i]].Remote(), ds.entries[ds.sortPerm[j]].Remote()
if icount > 0 {
iAvgSize = float64(isize / icount)

30
cmd/ncdu/scan/scan.go
View File

@ -16,8 +16,9 @@ type Dir struct {
parent *Dir
path string
mu sync.Mutex
count int64
size int64
count int64
countUnknownSize int64
entries fs.DirEntries
dirs map[string]*Dir
readError error
@ -49,7 +50,13 @@ func newDir(parent *Dir, dirPath string, entries fs.DirEntries, err error) *Dir
for _, entry := range entries {
if o, ok := entry.(fs.Object); ok {
d.count++
d.size += o.Size()
size := o.Size()
if size < 0 {
// Some backends may return -1 because size of object is not known
d.countUnknownSize++
} else {
d.size += size
}
}
}
// Set my directory entry in parent
@ -62,8 +69,9 @@ func newDir(parent *Dir, dirPath string, entries fs.DirEntries, err error) *Dir
// Accumulate counts in parents
for ; parent != nil; parent = parent.parent {
parent.mu.Lock()
parent.count += d.count
parent.size += d.size
parent.count += d.count
parent.countUnknownSize += d.countUnknownSize
if d.readError != nil {
parent.entriesHaveErrors = true
}
@ -91,17 +99,24 @@ func (d *Dir) Remove(i int) {
// Call with d.mu held
func (d *Dir) remove(i int) {
size := d.entries[i].Size()
countUnknownSize := int64(0)
if size < 0 {
size = 0
countUnknownSize = 1
}
count := int64(1)
subDir, ok := d.getDir(i)
if ok {
size = subDir.size
count = subDir.count
countUnknownSize = subDir.countUnknownSize
delete(d.dirs, path.Base(subDir.path))
}
d.size -= size
d.count -= count
d.countUnknownSize -= countUnknownSize
d.entries = append(d.entries[:i], d.entries[i+1:]...)
dir := d
@ -111,6 +126,7 @@ func (d *Dir) remove(i int) {
parent.dirs[path.Base(dir.path)] = dir
parent.size -= size
parent.count -= count
parent.countUnknownSize -= countUnknownSize
dir = parent
parent.mu.Unlock()
}
@ -151,19 +167,19 @@ func (d *Dir) Attr() (size int64, count int64) {
}
// AttrI returns the size, count and flags for the i-th directory entry
func (d *Dir) AttrI(i int) (size int64, count int64, isDir bool, readable bool, entriesHaveErrors bool, err error) {
func (d *Dir) AttrI(i int) (size int64, count int64, countUnknownSize int64, isDir bool, readable bool, entriesHaveErrors bool, err error) {
d.mu.Lock()
defer d.mu.Unlock()
subDir, isDir := d.getDir(i)
if !isDir {
return d.entries[i].Size(), 0, false, true, d.entriesHaveErrors, d.readError
return d.entries[i].Size(), 0, 0, false, true, d.entriesHaveErrors, d.readError
}
if subDir == nil {
return 0, 0, true, false, false, nil
return 0, 0, 0, true, false, false, nil
}
size, count = subDir.Attr()
return size, count, true, true, subDir.entriesHaveErrors, subDir.readError
return size, count, subDir.countUnknownSize, true, true, subDir.entriesHaveErrors, subDir.readError
}
// Scan the Fs passed in, returning a root directory channel and an