From 5355881332a913c0cd2a5ae1c46f0151701924e9 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 16 Mar 2017 22:37:56 +0000 Subject: [PATCH] local: fix unnormalised unicode causing problems reading directories #1212 --- local/local.go | 75 ++++++++++++++++++++++++++++-------- local/local_internal_test.go | 20 ++++++++++ 2 files changed, 80 insertions(+), 15 deletions(-) create mode 100644 local/local_internal_test.go diff --git a/local/local.go b/local/local.go index 0d7ac810e..0695f8ba4 100644 --- a/local/local.go +++ b/local/local.go @@ -59,7 +59,8 @@ type Fs struct { warned map[string]struct{} // whether we have warned about this string nounc bool // Skip UNC conversion on Windows // do os.Lstat or os.Stat - lstat func(name string) (os.FileInfo, error) + lstat func(name string) (os.FileInfo, error) + dirNames *mapper // directory name mapping } // Object represents a local filesystem object @@ -79,11 +80,12 @@ func NewFs(name, root string) (fs.Fs, error) { nounc := fs.ConfigFileGet(name, "nounc") f := &Fs{ - name: name, - warned: make(map[string]struct{}), - nounc: nounc == "true", - dev: devUnset, - lstat: os.Lstat, + name: name, + warned: make(map[string]struct{}), + nounc: nounc == "true", + dev: devUnset, + lstat: os.Lstat, + dirNames: newMapper(), } f.root = f.cleanPath(root) f.features = (&fs.Features{CaseInsensitive: f.caseInsensitive()}).Fill(f) @@ -136,8 +138,12 @@ func (f *Fs) caseInsensitive() bool { } // newObject makes a half completed Object -func (f *Fs) newObject(remote string) *Object { - dstPath := f.cleanPath(filepath.Join(f.root, remote)) +// +// if dstPath is empty then it is made from remote +func (f *Fs) newObject(remote, dstPath string) *Object { + if dstPath == "" { + dstPath = f.cleanPath(filepath.Join(f.root, remote)) + } remote = f.cleanRemote(remote) return &Object{ fs: f, @@ -149,8 +155,8 @@ func (f *Fs) newObject(remote string) *Object { // Return an Object from a path // // May return nil if an error occurred -func (f *Fs) newObjectWithInfo(remote string, info os.FileInfo) (fs.Object, error) { - o := f.newObject(remote) +func (f *Fs) newObjectWithInfo(remote, dstPath string, info os.FileInfo) (fs.Object, error) { + o := f.newObject(remote, dstPath) if info != nil { o.info = info } else { @@ -171,7 +177,7 @@ func (f *Fs) newObjectWithInfo(remote string, info os.FileInfo) (fs.Object, erro // NewObject finds the Object at remote. If it can't be found // it returns the error ErrorObjectNotFound. func (f *Fs) NewObject(remote string) (fs.Object, error) { - return f.newObjectWithInfo(remote, nil) + return f.newObjectWithInfo(remote, "", nil) } // listArgs is the arguments that a new list takes @@ -227,7 +233,7 @@ func (f *Fs) list(out fs.ListOpts, remote string, dirpath string, level int) (su // are kind of a souped up symlink. Unix doesn't have directories which are symlinks. if (mode&os.ModeSymlink) == 0 && out.IncludeDirectory(newRemote) && f.dev == readDevice(fi) { dir := &fs.Dir{ - Name: f.cleanRemote(newRemote), + Name: f.dirNames.Save(newRemote, f.cleanRemote(newRemote)), When: fi.ModTime(), Bytes: 0, Count: 0, @@ -240,7 +246,7 @@ func (f *Fs) list(out fs.ListOpts, remote string, dirpath string, level int) (su } } } else { - fso, err := f.newObjectWithInfo(newRemote, fi) + fso, err := f.newObjectWithInfo(newRemote, newPath, fi) if err != nil { out.SetError(err) return nil @@ -259,6 +265,7 @@ func (f *Fs) list(out fs.ListOpts, remote string, dirpath string, level int) (su // Ignores everything which isn't Storable, eg links etc func (f *Fs) List(out fs.ListOpts, dir string) { defer out.Finished() + dir = f.dirNames.Load(dir) root := f.cleanPath(filepath.Join(f.root, dir)) dir = f.cleanRemote(dir) _, err := os.Stat(root) @@ -323,11 +330,49 @@ func (f *Fs) cleanRemote(name string) string { return name } +// mapper maps raw to cleaned directory names +type mapper struct { + mu sync.RWMutex // mutex to protect the below + m map[string]string // map of un-normalised directory names +} + +func newMapper() *mapper { + return &mapper{ + m: make(map[string]string), + } +} + +// Lookup a directory name to make a local name (reverses +// cleanDirName) +// +// FIXME this is temporary before we make a proper Directory object +func (m *mapper) Load(in string) string { + m.mu.RLock() + out, ok := m.m[in] + m.mu.RUnlock() + if ok { + return out + } + return in +} + +// Cleans a directory name recording if it needed to be altered +// +// FIXME this is temporary before we make a proper Directory object +func (m *mapper) Save(in, out string) string { + if in != out { + m.mu.Lock() + m.m[out] = in + m.mu.Unlock() + } + return out +} + // Put the Object to the local filesystem func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) { remote := src.Remote() // Temporary Object under construction - info filled in by Update() - o := f.newObject(remote) + o := f.newObject(remote, "") err := o.Update(in, src) if err != nil { return nil, err @@ -454,7 +499,7 @@ func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) { } // Temporary Object under construction - dstObj := f.newObject(remote) + dstObj := f.newObject(remote, "") // Check it is a file if it exists err := dstObj.lstat() diff --git a/local/local_internal_test.go b/local/local_internal_test.go new file mode 100644 index 000000000..f8160cf98 --- /dev/null +++ b/local/local_internal_test.go @@ -0,0 +1,20 @@ +package local + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMapper(t *testing.T) { + m := newMapper() + assert.Equal(t, m.m, map[string]string{}) + assert.Equal(t, "potato", m.Save("potato", "potato")) + assert.Equal(t, m.m, map[string]string{}) + assert.Equal(t, "-r'áö", m.Save("-r?'a´o¨", "-r'áö")) + assert.Equal(t, m.m, map[string]string{ + "-r'áö": "-r?'a´o¨", + }) + assert.Equal(t, "potato", m.Load("potato")) + assert.Equal(t, "-r?'a´o¨", m.Load("-r'áö")) +}