83 lines
1.9 KiB
Go
83 lines
1.9 KiB
Go
// Listing utility functions for fses which use dircache
|
|
|
|
package dircache
|
|
|
|
import (
|
|
"sync"
|
|
|
|
"github.com/ncw/rclone/fs"
|
|
)
|
|
|
|
// ListDirJob describe a directory listing that needs to be done
|
|
type ListDirJob struct {
|
|
DirID string
|
|
Path string
|
|
Depth int
|
|
}
|
|
|
|
// ListDirer describes the interface necessary to use ListDir
|
|
type ListDirer interface {
|
|
// ListDir reads the directory specified by the job into out, returning any more jobs
|
|
ListDir(out fs.ListOpts, job ListDirJob) (jobs []ListDirJob, err error)
|
|
}
|
|
|
|
// listDir lists the directory using a recursive list from the root
|
|
//
|
|
// It does this in parallel, calling f.ListDir to do the actual reading
|
|
func listDir(f ListDirer, out fs.ListOpts, dirID string, path string) {
|
|
// Start some directory listing go routines
|
|
var wg sync.WaitGroup // sync closing of go routines
|
|
var traversing sync.WaitGroup // running directory traversals
|
|
buffer := out.Buffer()
|
|
in := make(chan ListDirJob, buffer)
|
|
for i := 0; i < buffer; i++ {
|
|
wg.Add(1)
|
|
go func() {
|
|
defer wg.Done()
|
|
for job := range in {
|
|
jobs, err := f.ListDir(out, job)
|
|
if err != nil {
|
|
out.SetError(err)
|
|
fs.Debug(f, "Error reading %s: %s", path, err)
|
|
} else {
|
|
traversing.Add(len(jobs))
|
|
go func() {
|
|
// Now we have traversed this directory, send these
|
|
// jobs off for traversal in the background
|
|
for _, job := range jobs {
|
|
in <- job
|
|
}
|
|
}()
|
|
}
|
|
traversing.Done()
|
|
}
|
|
}()
|
|
}
|
|
|
|
// Start the process
|
|
traversing.Add(1)
|
|
in <- ListDirJob{DirID: dirID, Path: path, Depth: out.Level() - 1}
|
|
traversing.Wait()
|
|
close(in)
|
|
wg.Wait()
|
|
}
|
|
|
|
// List walks the path returning iles and directories into out
|
|
func (dc *DirCache) List(f ListDirer, out fs.ListOpts, dir string) {
|
|
defer out.Finished()
|
|
err := dc.FindRoot(false)
|
|
if err != nil {
|
|
out.SetError(err)
|
|
return
|
|
}
|
|
id, err := dc.FindDir(dir, false)
|
|
if err != nil {
|
|
out.SetError(err)
|
|
return
|
|
}
|
|
if dir != "" {
|
|
dir += "/"
|
|
}
|
|
listDir(f, out, id, dir)
|
|
}
|