rclone ncdu for exploring a remote with a text based user interface.

This commit is contained in:
Nick Craig-Wood 2017-04-01 20:53:44 +02:00
parent e31fc877e2
commit c34f11a92f
3 changed files with 710 additions and 0 deletions

View File

@ -30,6 +30,7 @@ import (
_ "github.com/ncw/rclone/cmd/mount"
_ "github.com/ncw/rclone/cmd/move"
_ "github.com/ncw/rclone/cmd/moveto"
_ "github.com/ncw/rclone/cmd/ncdu"
_ "github.com/ncw/rclone/cmd/obscure"
_ "github.com/ncw/rclone/cmd/purge"
_ "github.com/ncw/rclone/cmd/rmdir"

542
cmd/ncdu/ncdu.go Normal file
View File

@ -0,0 +1,542 @@
package ncdu
import (
"fmt"
"log"
"path"
"sort"
"strings"
"github.com/ncw/rclone/cmd"
"github.com/ncw/rclone/cmd/ncdu/scan"
"github.com/ncw/rclone/fs"
termbox "github.com/nsf/termbox-go"
"github.com/pkg/errors"
"github.com/spf13/cobra"
)
func init() {
cmd.Root.AddCommand(commandDefintion)
}
var commandDefintion = &cobra.Command{
Use: "ncdu remote:path",
Short: `Explore a remote with a text based user interface.`,
Long: `
This displays a text based user interface allowing the navigation of a
remote. It is most useful for answering the question - "What is using
all my disk space?".
To make the user interface it first scans the entire remote given and
builds an in memory representation. rclone ncdu can be used during
this scanning phase and you will see it building up the directory
structure as it goes along.
Here are the keys - press '?' to toggle the help on and off
` + strings.Join(helpText[1:], "\n ") + `
This an homage to the [ncdu tool](https://dev.yorhel.nl/ncdu) but for
rclone remotes. It is missing lots of features at the moment, most
importantly deleting files, but is useful as it stands.
`,
Run: func(command *cobra.Command, args []string) {
cmd.CheckArgs(1, 1, command, args)
fsrc := cmd.NewFsSrc(args)
cmd.Run(false, false, command, func() error {
return NewUI(fsrc).Show()
})
},
}
// help text
var helpText = []string{
"rclone ncdu",
" ↑,↓ or k,j to Move",
" →,l to enter",
" ←,h to return",
" c toggle counts",
" g toggle graph",
" n,s,C sort by name,size,count",
" ? to toggle help on and off",
" q/ESC/c-C to quit",
}
// UI contains the state of the user interface
type UI struct {
f fs.Fs // fs being displayed
fsName string // human name of Fs
root *scan.Dir // root directory
d *scan.Dir // current directory being displayed
path string // path of current directory
showBox bool // whether to show a box
boxText []string // text to show in box
entries fs.DirEntries // entries of current directory
sortPerm []int // order to display entries in after sorting
invSortPerm []int // inverse order
dirListHeight int // height of listing
listing bool // whether listing is in progress
showGraph bool // toggle showing graph
showCounts bool // toggle showing counts
sortByName int8 // +1 for normal, 0 for off, -1 for reverse
sortBySize int8
sortByCount int8
dirPosMap map[string]dirPos // store for directory positions
}
// Where we have got to in the directory listing
type dirPos struct {
entry int
offset int
}
// Print a string
func Print(x, y int, fg, bg termbox.Attribute, msg string) {
for _, c := range msg {
termbox.SetCell(x, y, c, fg, bg)
x++
}
}
// Printf a string
func Printf(x, y int, fg, bg termbox.Attribute, format string, args ...interface{}) {
s := fmt.Sprintf(format, args...)
Print(x, y, fg, bg, s)
}
// Line prints a string to given xmax, with given space
func Line(x, y, xmax int, fg, bg termbox.Attribute, spacer rune, msg string) {
for _, c := range msg {
termbox.SetCell(x, y, c, fg, bg)
x++
if x >= xmax {
return
}
}
for ; x < xmax; x++ {
termbox.SetCell(x, y, spacer, fg, bg)
}
}
// Linef a string
func Linef(x, y, xmax int, fg, bg termbox.Attribute, spacer rune, format string, args ...interface{}) {
s := fmt.Sprintf(format, args...)
Line(x, y, xmax, fg, bg, spacer, s)
}
// Box the u.boxText onto the screen
func (u *UI) Box() {
w, h := termbox.Size()
// Find dimensions of text
boxWidth := 10
for _, s := range u.boxText {
if len(s) > boxWidth && len(s) < w-4 {
boxWidth = len(s)
}
}
boxHeight := len(u.boxText)
// position
x := (w - boxWidth) / 2
y := (h - boxHeight) / 2
xmax := x + boxWidth
// draw text
fg, bg := termbox.ColorRed, termbox.ColorWhite
for i, s := range u.boxText {
Line(x, y+i, xmax, fg, bg, ' ', s)
fg = termbox.ColorBlack
}
// FIXME draw a box around
}
// find the biggest entry in the current listing
func (u *UI) biggestEntry() (biggest int64) {
if u.d == nil {
return
}
for i := range u.entries {
size, _, _, _ := u.d.AttrI(u.sortPerm[i])
if size > biggest {
biggest = size
}
}
return
}
// Draw the current screen
func (u *UI) Draw() error {
w, h := termbox.Size()
u.dirListHeight = h - 3
// Plot
err := termbox.Clear(termbox.ColorDefault, termbox.ColorDefault)
if err != nil {
return errors.Wrap(err, "failed to clear screen")
}
// Header line
Linef(0, 0, w, termbox.ColorBlack, termbox.ColorWhite, ' ', "rclone ncdu %s - use the arrow keys to navigate, press ? for help", fs.Version)
// Directory line
Linef(0, 1, w, termbox.ColorWhite, termbox.ColorBlack, '-', "-- %s ", u.path)
// graphs
const (
graphBars = 10
graph = "########## "
)
// Directory listing
if u.d != nil {
y := 2
perBar := u.biggestEntry() / graphBars
if perBar == 0 {
perBar = 1
}
dirPos := u.dirPosMap[u.path]
for i, j := range u.sortPerm[dirPos.offset:] {
entry := u.entries[j]
n := i + dirPos.offset
if y >= h-1 {
break
}
fg := termbox.ColorWhite
bg := termbox.ColorBlack
if n == dirPos.entry {
fg, bg = bg, fg
}
size, count, isDir, readable := u.d.AttrI(u.sortPerm[n])
mark := ' '
if isDir {
mark = '/'
}
message := ""
if !readable {
message = " [not read yet]"
}
extras := ""
if u.showCounts {
if count > 0 {
extras += fmt.Sprintf("%8v ", fs.SizeSuffix(count))
} else {
extras += " "
}
}
if u.showGraph {
bars := (size + perBar/2 - 1) / perBar
// clip if necessary - only happens during startup
if bars > 10 {
bars = 10
} else if bars < 0 {
bars = 0
}
extras += "[" + graph[graphBars-bars:2*graphBars-bars] + "] "
}
Linef(0, y, w, fg, bg, ' ', "%8v %s%c%s%s", fs.SizeSuffix(size), extras, mark, path.Base(entry.Remote()), message)
y++
}
}
// Footer
if u.d == nil {
Line(0, h-1, w, termbox.ColorBlack, termbox.ColorWhite, ' ', "Waiting for root directory...")
} else {
message := ""
if u.listing {
message = " [listing in progress]"
}
size, count := u.d.Attr()
Linef(0, h-1, w, termbox.ColorBlack, termbox.ColorWhite, ' ', "Total usage: %v, Objects: %d%s", fs.SizeSuffix(size), count, message)
}
// Show the box on top if requred
if u.showBox {
u.Box()
}
err = termbox.Flush()
if err != nil {
return errors.Wrap(err, "failed to flush screen")
}
return nil
}
// Move the cursor this many spaces adjusting the viewport as necessary
func (u *UI) move(d int) {
if u.d == nil {
return
}
absD := d
if d < 0 {
absD = -d
}
entries := len(u.entries)
// Fetch current dirPos
dirPos := u.dirPosMap[u.path]
dirPos.entry += d
// check entry in range
if dirPos.entry < 0 {
dirPos.entry = 0
} else if dirPos.entry >= entries {
dirPos.entry = entries - 1
}
// check cursor still on screen
p := dirPos.entry - dirPos.offset // where dirPos.entry appears on the screen
if p < 0 {
dirPos.offset -= absD
} else if p >= u.dirListHeight {
dirPos.offset += absD
}
// check dirPos.offset in bounds
if dirPos.offset < 0 {
dirPos.offset = 0
} else if dirPos.offset >= entries {
dirPos.offset = entries - 1
}
// write dirPos back for later
u.dirPosMap[u.path] = dirPos
}
// Sort by the configured sort method
type ncduSort struct {
sortPerm []int
entries fs.DirEntries
d *scan.Dir
u *UI
}
// Less is part of sort.Interface.
func (ds *ncduSort) Less(i, j int) bool {
isize, icount, _, _ := ds.d.AttrI(ds.sortPerm[i])
jsize, jcount, _, _ := ds.d.AttrI(ds.sortPerm[j])
iname, jname := ds.entries[ds.sortPerm[i]].Remote(), ds.entries[ds.sortPerm[j]].Remote()
switch {
case ds.u.sortByName < 0:
return iname > jname
case ds.u.sortByName > 0:
break
case ds.u.sortBySize < 0:
if isize != jsize {
return isize < jsize
}
case ds.u.sortBySize > 0:
if isize != jsize {
return isize > jsize
}
case ds.u.sortByCount < 0:
if icount != jcount {
return icount < jcount
}
case ds.u.sortByCount > 0:
if icount != jcount {
return icount > jcount
}
}
// if everything equal, sort by name
return iname < jname
}
// Swap is part of sort.Interface.
func (ds *ncduSort) Swap(i, j int) {
ds.sortPerm[i], ds.sortPerm[j] = ds.sortPerm[j], ds.sortPerm[i]
}
// Len is part of sort.Interface.
func (ds *ncduSort) Len() int {
return len(ds.sortPerm)
}
// sort the permutation map of the current directory
func (u *UI) sortCurrentDir() {
u.sortPerm = u.sortPerm[:0]
for i := range u.entries {
u.sortPerm = append(u.sortPerm, i)
}
data := ncduSort{
sortPerm: u.sortPerm,
entries: u.entries,
d: u.d,
u: u,
}
sort.Sort(&data)
if len(u.invSortPerm) < len(u.sortPerm) {
u.invSortPerm = make([]int, len(u.sortPerm))
}
for i, j := range u.sortPerm {
u.invSortPerm[j] = i
}
}
// setCurrentDir sets the current directory
func (u *UI) setCurrentDir(d *scan.Dir) {
u.d = d
u.entries = d.Entries()
u.path = path.Join(u.fsName, d.Path())
u.sortCurrentDir()
}
// enters the current entry
func (u *UI) enter() {
if u.d == nil {
return
}
dirPos := u.dirPosMap[u.path]
d, _ := u.d.GetDir(u.sortPerm[dirPos.entry])
if d == nil {
return
}
u.setCurrentDir(d)
}
// up goes up to the parent directory
func (u *UI) up() {
if u.d == nil {
return
}
parent := u.d.Parent()
if parent != nil {
u.setCurrentDir(parent)
}
}
// popupBox shows a box with the text in
func (u *UI) popupBox(text []string) {
u.boxText = text
u.showBox = true
}
// togglePopupBox shows a box with the text in
func (u *UI) togglePopupBox(text []string) {
if u.showBox {
u.showBox = false
} else {
u.popupBox(text)
}
}
// toggle the sorting for the flag passed in
func (u *UI) toggleSort(sortType *int8) {
old := *sortType
u.sortBySize = 0
u.sortByCount = 0
u.sortByName = 0
if old == 0 {
*sortType = 1
} else {
*sortType = -old
}
u.sortCurrentDir()
}
// NewUI creates a new user interface for ncdu on f
func NewUI(f fs.Fs) *UI {
return &UI{
f: f,
path: "Waiting for root...",
dirListHeight: 20, // updated in Draw
fsName: f.Name() + ":" + f.Root(),
showGraph: true,
showCounts: false,
sortByName: 0, // +1 for normal, 0 for off, -1 for reverse
sortBySize: 1,
sortByCount: 0,
dirPosMap: make(map[string]dirPos),
}
}
// Show shows the user interface
func (u *UI) Show() error {
err := termbox.Init()
if err != nil {
log.Fatal(err)
}
defer termbox.Close()
// scan the disk in the background
u.listing = true
rootChan, errChan, updated := scan.Scan(u.f)
// Poll the events into a channel
events := make(chan termbox.Event)
doneWithEvent := make(chan bool)
go func() {
for {
events <- termbox.PollEvent()
<-doneWithEvent
}
}()
// Main loop, waiting for events and channels
outer:
for {
//Reset()
err := u.Draw()
if err != nil {
return errors.Wrap(err, "draw failed")
}
var root *scan.Dir
select {
case root = <-rootChan:
u.root = root
u.setCurrentDir(root)
case err := <-errChan:
if err != nil {
return errors.Wrap(err, "ncdu directory listing")
}
u.listing = false
case <-updated:
// redraw
// might want to limit updates per second
u.sortCurrentDir()
case ev := <-events:
doneWithEvent <- true
if ev.Type == termbox.EventKey {
switch ev.Key + termbox.Key(ev.Ch) {
case termbox.KeyEsc, termbox.KeyCtrlC, 'q':
if u.showBox {
u.showBox = false
} else {
break outer
}
case termbox.KeyArrowDown, 'j':
u.move(1)
case termbox.KeyArrowUp, 'k':
u.move(-1)
case termbox.KeyPgdn, '-', '_':
u.move(u.dirListHeight)
case termbox.KeyPgup, '=', '+':
u.move(-u.dirListHeight)
case termbox.KeyArrowLeft, 'h':
u.up()
case termbox.KeyArrowRight, 'l', termbox.KeyEnter:
u.enter()
case 'c':
u.showCounts = !u.showCounts
case 'g':
u.showGraph = !u.showGraph
case 'n':
u.toggleSort(&u.sortByName)
case 's':
u.toggleSort(&u.sortBySize)
case 'C':
u.toggleSort(&u.sortByCount)
case '?':
u.togglePopupBox(helpText)
}
}
}
// listen to key presses, etc
}
return nil
}

167
cmd/ncdu/scan/scan.go Normal file
View File

@ -0,0 +1,167 @@
// Package scan does concurrent scanning of an Fs building up a directory tree.
package scan
import (
"path"
"sync"
"github.com/ncw/rclone/fs"
"github.com/pkg/errors"
)
// Dir represents a directory found in the remote
type Dir struct {
parent *Dir
path string
mu sync.Mutex
count int64
size int64
complete bool
entries fs.DirEntries
dirs map[string]*Dir
offset int // current listing offset
entry int // current listing entry
}
// Parent returns the directory above this one
func (d *Dir) Parent() *Dir {
// no locking needed since these are write once in newDir()
return d.parent
}
// Path returns the position of the dir in the filesystem
func (d *Dir) Path() string {
// no locking needed since these are write once in newDir()
return d.path
}
// make a new directory
func newDir(parent *Dir, dirPath string, entries fs.DirEntries) *Dir {
d := &Dir{
parent: parent,
path: dirPath,
entries: entries,
dirs: make(map[string]*Dir),
}
// Count size in this dir
for _, entry := range entries {
if o, ok := entry.(fs.Object); ok {
d.count++
d.size += o.Size()
}
}
// Set my directory entry in parent
if parent != nil {
parent.mu.Lock()
leaf := path.Base(dirPath)
d.parent.dirs[leaf] = d
parent.mu.Unlock()
}
// Accumulate counts in parents
for ; parent != nil; parent = parent.parent {
parent.mu.Lock()
parent.count += d.count
parent.size += d.size
parent.mu.Unlock()
}
return d
}
// Entries returns a copy of the entries in the directory
func (d *Dir) Entries() fs.DirEntries {
return append(fs.DirEntries(nil), d.entries...)
}
// gets the directory of the i-th entry
//
// returns nil if it is a file
// returns a flag as to whether is directory or not
//
// Call with d.mu held
func (d *Dir) getDir(i int) (subDir *Dir, isDir bool) {
obj := d.entries[i]
dir, ok := obj.(*fs.Dir)
if !ok {
return nil, false
}
leaf := path.Base(dir.Remote())
subDir = d.dirs[leaf]
return subDir, true
}
// GetDir returns the Dir of the i-th entry
//
// returns nil if it is a file
// returns a flag as to whether is directory or not
func (d *Dir) GetDir(i int) (subDir *Dir, isDir bool) {
d.mu.Lock()
defer d.mu.Unlock()
return d.getDir(i)
}
// Attr returns the size and count for the directory
func (d *Dir) Attr() (size int64, count int64) {
d.mu.Lock()
defer d.mu.Unlock()
return d.size, d.count
}
// AttrI returns the size, count and flags for the i-th directory entry
func (d *Dir) AttrI(i int) (size int64, count int64, isDir bool, readable bool) {
d.mu.Lock()
defer d.mu.Unlock()
subDir, isDir := d.getDir(i)
if !isDir {
return d.entries[i].Size(), 0, false, true
}
if subDir == nil {
return 0, 0, true, false
}
size, count = subDir.Attr()
return size, count, true, true
}
// Scan the Fs passed in, returning a root directory channel and an
// error channel
func Scan(f fs.Fs) (chan *Dir, chan error, chan struct{}) {
root := make(chan *Dir, 1)
errChan := make(chan error, 1)
updated := make(chan struct{}, 1)
go func() {
parents := map[string]*Dir{}
err := fs.Walk(f, "", false, fs.Config.MaxDepth, func(dirPath string, entries fs.DirEntries, err error) error {
if err != nil {
return err // FIXME mark directory as errored instead of aborting
}
var parent *Dir
if dirPath != "" {
parentPath := path.Dir(dirPath)
if parentPath == "." {
parentPath = ""
}
var ok bool
parent, ok = parents[parentPath]
if !ok {
errChan <- errors.Errorf("couldn't find parent for %q", dirPath)
}
}
d := newDir(parent, dirPath, entries)
parents[dirPath] = d
if dirPath == "" {
root <- d
}
// Mark updated
select {
case updated <- struct{}{}:
default:
break
}
return nil
})
if err != nil {
errChan <- errors.Wrap(err, "ncdu listing failed")
}
errChan <- nil
}()
return root, errChan, updated
}