From 8a185deefa33d6085504b9dd79c5167109f88f31 Mon Sep 17 00:00:00 2001 From: Nick Craig-Wood Date: Thu, 3 Aug 2017 14:31:55 +0100 Subject: [PATCH] qingstor: Fixes before merge * use rclone's http.Client for bwlimit, logging, etc * remove extraneous fmt.Sprintf from logging * fix icon in docs * add docs about --fast-list * hoist md5 regexp compilation out of function * create container if necessary on server side copy * keep note of whether the container has been deleted * build constraint not to compile for plan9 --- docs/content/qingstor.md | 9 ++- docs/layouts/chrome/navbar.html | 2 +- fstest/fstests/gen_tests.go | 2 +- qingstor/qingstor.go | 74 +++++++++++++--------- qingstor/qingstor_test.go | 108 +++++++++++++++++--------------- 5 files changed, 110 insertions(+), 85 deletions(-) diff --git a/docs/content/qingstor.md b/docs/content/qingstor.md index 8fb814960..7fd66a5a9 100644 --- a/docs/content/qingstor.md +++ b/docs/content/qingstor.md @@ -4,7 +4,7 @@ description: "Rclone docs for QingStor Object Storage" date: "2017-06-26" --- - QingStor + QingStor --------------------------------------- Paths are specified as `remote:bucket` (or `remote:` for the `lsd` @@ -119,6 +119,12 @@ files in the bucket. rclone sync /home/local/directory remote:bucket +### --fast-list ### + +This remote supports `--fast-list` which allows you to use fewer +transactions in exchange for more memory. See the [rclone +docs](/docs/#fast-list) for more details. + ### Multipart uploads ### rclone supports multipart uploads with QingStor which means that it can @@ -134,6 +140,7 @@ you will get an error, `incorrect zone, the bucket is not in 'XXX' zone`. ### Authentication ### + There are two ways to supply `rclone` with a set of QingStor credentials. In order of precedence: diff --git a/docs/layouts/chrome/navbar.html b/docs/layouts/chrome/navbar.html index 59f94b838..68d97d2b2 100644 --- a/docs/layouts/chrome/navbar.html +++ b/docs/layouts/chrome/navbar.html @@ -61,7 +61,7 @@
  • HTTP
  • Hubic
  • Microsoft OneDrive
    • -
  • QingStor
    • +
  • QingStor
  • Openstack Swift
  • SFTP
  • Yandex Disk
    • diff --git a/fstest/fstests/gen_tests.go b/fstest/fstests/gen_tests.go index 171688b1e..59fc48ba5 100644 --- a/fstest/fstests/gen_tests.go +++ b/fstest/fstests/gen_tests.go @@ -162,6 +162,6 @@ func main() { generateTestProgram(t, fns, "Sftp") generateTestProgram(t, fns, "FTP") generateTestProgram(t, fns, "Box") - generateTestProgram(t, fns, "QingStor") + generateTestProgram(t, fns, "QingStor", buildConstraint("!plan9")) log.Printf("Done") } diff --git a/qingstor/qingstor.go b/qingstor/qingstor.go index 275ef6f36..4555fa044 100644 --- a/qingstor/qingstor.go +++ b/qingstor/qingstor.go @@ -1,5 +1,8 @@ // Package qingstor provides an interface to QingStor object storage // Home: https://www.qingcloud.com/ + +// +build !plan9 + package qingstor import ( @@ -89,14 +92,15 @@ func timestampToTime(tp int64) time.Time { // Fs represents a remote qingstor server type Fs struct { - name string // The name of the remote - zone string // The zone we are working on - bucket string // The bucket we are working on - bucketOK bool // true if we have created the bucket - bucketMtx sync.Mutex // mutex to protect bucket - root string // The root is a subdir, is a special object - features *fs.Features // optional features - svc *qs.Service // The connection to the qingstor server + name string // The name of the remote + zone string // The zone we are working on + bucket string // The bucket we are working on + bucketOKMu sync.Mutex // mutex to protect bucketOK and bucketDeleted + bucketOK bool // true if we have created the bucket + bucketDeleted bool // true if we have deleted the bucket + root string // The root is a subdir, is a special object + features *fs.Features // optional features + svc *qs.Service // The connection to the qingstor server } // Object describes a qingstor object @@ -206,6 +210,7 @@ func qsServiceConnection(name string) (*qs.Service, error) { cf.Host = host cf.Port = port cf.ConnectionRetries = connectionRetries + cf.Connection = fs.Config.Client() svc, _ := qs.Init(cf) @@ -320,6 +325,10 @@ func (f *Fs) Put(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOption) (fs. // // If it isn't possible then return fs.ErrorCantCopy func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) { + err := f.Mkdir("") + if err != nil { + return nil, err + } srcObj, ok := src.(*Object) if !ok { fs.Debugf(src, "Can't copy - not same remote type") @@ -329,7 +338,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) { key := f.root + remote source := path.Join("/"+srcFs.bucket, srcFs.root+srcObj.remote) - fs.Debugf(f, fmt.Sprintf("Copied, source key is: %s, and dst key is: %s", source, key)) + fs.Debugf(f, "Copied, source key is: %s, and dst key is: %s", source, key) req := qs.PutObjectInput{ XQSCopySource: &source, } @@ -340,7 +349,7 @@ func (f *Fs) Copy(src fs.Object, remote string) (fs.Object, error) { } _, err = bucketInit.PutObject(key, &req) if err != nil { - fs.Debugf(f, fmt.Sprintf("Copied Faild, API Error: %s", err)) + fs.Debugf(f, "Copied Faild, API Error: %v", err) return nil, err } return f.NewObject(remote) @@ -612,18 +621,20 @@ func (f *Fs) dirExists() (bool, error) { // Mkdir creates the bucket if it doesn't exist func (f *Fs) Mkdir(dir string) error { - f.bucketMtx.Lock() - defer f.bucketMtx.Unlock() + f.bucketOKMu.Lock() + defer f.bucketOKMu.Unlock() if f.bucketOK { return nil } - exists, err := f.dirExists() - if err == nil { - f.bucketOK = exists - } - if err != nil || exists { - return err + if !f.bucketDeleted { + exists, err := f.dirExists() + if err == nil { + f.bucketOK = exists + } + if err != nil || exists { + return err + } } bucketInit, err := f.svc.Bucket(f.bucket, f.zone) @@ -639,6 +650,7 @@ func (f *Fs) Mkdir(dir string) error { if err == nil { f.bucketOK = true + f.bucketDeleted = false } return err @@ -668,8 +680,8 @@ func (f *Fs) dirIsEmpty() (bool, error) { // Rmdir delete a bucket func (f *Fs) Rmdir(dir string) error { - f.bucketMtx.Lock() - defer f.bucketMtx.Unlock() + f.bucketOKMu.Lock() + defer f.bucketOKMu.Unlock() if f.root != "" || dir != "" { return nil } @@ -679,11 +691,11 @@ func (f *Fs) Rmdir(dir string) error { return err } if !isEmpty { - fs.Debugf(f, fmt.Sprintf("The bucket %s you tried to delete not empty.", f.bucket)) + fs.Debugf(f, "The bucket %s you tried to delete not empty.", f.bucket) return errors.New("BucketNotEmpty: The bucket you tried to delete is not empty") } - fs.Debugf(f, fmt.Sprintf("Tried to delete the bucket %s", f.bucket)) + fs.Debugf(f, "Tried to delete the bucket %s", f.bucket) bucketInit, err := f.svc.Bucket(f.bucket, f.zone) if err != nil { return err @@ -691,6 +703,7 @@ func (f *Fs) Rmdir(dir string) error { _, err = bucketInit.Delete() if err == nil { f.bucketOK = false + f.bucketDeleted = true } return err } @@ -705,10 +718,10 @@ func (o *Object) readMetaData() (err error) { } key := o.fs.root + o.remote - fs.Debugf(o, fmt.Sprintf("Read metadata of key: %s", key)) + fs.Debugf(o, "Read metadata of key: %s", key) resp, err := bucketInit.HeadObject(key, &qs.HeadObjectInput{}) if err != nil { - fs.Debugf(o, fmt.Sprintf("Read metadata faild, API Error: %s", err)) + fs.Debugf(o, "Read metadata faild, API Error: %v", err) if e, ok := err.(*qsErr.QingStorError); ok { if e.StatusCode == http.StatusNotFound { return fs.ErrorObjectNotFound @@ -834,10 +847,10 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio defer func() { if err != nil { - fs.Errorf(o, fmt.Sprintf("Create Object Faild, API ERROR: %s", err)) + fs.Errorf(o, "Create Object Faild, API ERROR: %v", err) // Abort Upload when init success and upload failed if uploadID != nil { - fs.Debugf(o, fmt.Sprintf("Abort Upload Multipart, upload_id: %s, objectParts: %s", *uploadID, objectParts)) + fs.Debugf(o, "Abort Upload Multipart, upload_id: %s, objectParts: %s", *uploadID, objectParts) abortReq := qs.AbortMultipartUploadInput{ UploadID: uploadID, } @@ -846,7 +859,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio } }() - fs.Debugf(o, fmt.Sprintf("Initiate Upload Multipart, key: %s", key)) + fs.Debugf(o, "Initiate Upload Multipart, key: %s", key) mimeType := fs.MimeType(src) initReq := qs.InitiateMultipartUploadInput{ ContentType: &mimeType, @@ -877,7 +890,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio ContentLength: &size, Body: buffer, } - fs.Debugf(o, fmt.Sprintf("Upload Multipart, upload_id: %s, part_number: %d", *uploadID, number)) + fs.Debugf(o, "Upload Multipart, upload_id: %s, part_number: %d", *uploadID, number) _, err = bucketInit.UploadMultipart(key, &req) if err != nil { return err @@ -891,7 +904,7 @@ func (o *Object) Update(in io.Reader, src fs.ObjectInfo, options ...fs.OpenOptio } // Complete Multipart Upload - fs.Debugf(o, fmt.Sprintf("Complete Upload Multipart, upload_id: %s, objectParts: %d", *uploadID, objectParts)) + fs.Debugf(o, "Complete Upload Multipart, upload_id: %s, objectParts: %d", *uploadID, objectParts) completeReq := qs.CompleteMultipartUploadInput{ UploadID: uploadID, ObjectParts: objectParts, @@ -923,10 +936,11 @@ func (o *Object) Fs() fs.Info { return o.fs } +var matchMd5 = regexp.MustCompile(`^[0-9a-f]{32}$`) + // Hash returns the selected checksum of the file // If no checksum is available it returns "" func (o *Object) Hash(t fs.HashType) (string, error) { - var matchMd5 = regexp.MustCompile(`^[0-9a-f]{32}$`) if t != fs.HashMD5 { return "", fs.ErrHashUnsupported } diff --git a/qingstor/qingstor_test.go b/qingstor/qingstor_test.go index d6ded0639..90411cb2d 100644 --- a/qingstor/qingstor_test.go +++ b/qingstor/qingstor_test.go @@ -2,6 +2,9 @@ // // Automatically generated - DO NOT EDIT // Regenerate with: make gen_tests + +// +build !plan9 + package qingstor_test import ( @@ -18,55 +21,56 @@ func TestSetup(t *testing.T) { } // Generic tests for the Fs -func TestInit(t *testing.T) { fstests.TestInit(t) } -func TestFsString(t *testing.T) { fstests.TestFsString(t) } -func TestFsName(t *testing.T) { fstests.TestFsName(t) } -func TestFsRoot(t *testing.T) { fstests.TestFsRoot(t) } -func TestFsRmdirEmpty(t *testing.T) { fstests.TestFsRmdirEmpty(t) } -func TestFsRmdirNotFound(t *testing.T) { fstests.TestFsRmdirNotFound(t) } -func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } -func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) } -func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } -func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } -func TestFsListRDirEmpty(t *testing.T) { fstests.TestFsListRDirEmpty(t) } -func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) } -func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } -func TestFsPutError(t *testing.T) { fstests.TestFsPutError(t) } -func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } -func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) } -func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } -func TestFsListRDirFile2(t *testing.T) { fstests.TestFsListRDirFile2(t) } -func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } -func TestFsListRDirRoot(t *testing.T) { fstests.TestFsListRDirRoot(t) } -func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) } -func TestFsListRSubdir(t *testing.T) { fstests.TestFsListRSubdir(t) } -func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) } -func TestFsListRLevel2(t *testing.T) { fstests.TestFsListRLevel2(t) } -func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } -func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) } -func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } -func TestFsNewObjectDir(t *testing.T) { fstests.TestFsNewObjectDir(t) } -func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) } -func TestFsMove(t *testing.T) { fstests.TestFsMove(t) } -func TestFsDirMove(t *testing.T) { fstests.TestFsDirMove(t) } -func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) } -func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) } -func TestFsDirChangeNotify(t *testing.T) { fstests.TestFsDirChangeNotify(t) } -func TestObjectString(t *testing.T) { fstests.TestObjectString(t) } -func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) } -func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) } -func TestObjectHashes(t *testing.T) { fstests.TestObjectHashes(t) } -func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) } -func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) } -func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } -func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } -func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } -func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) } -func TestObjectPartialRead(t *testing.T) { fstests.TestObjectPartialRead(t) } -func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } -func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } -func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) } -func TestFsIsFileNotFound(t *testing.T) { fstests.TestFsIsFileNotFound(t) } -func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) } -func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) } -func TestFinalise(t *testing.T) { fstests.TestFinalise(t) } +func TestInit(t *testing.T) { fstests.TestInit(t) } +func TestFsString(t *testing.T) { fstests.TestFsString(t) } +func TestFsName(t *testing.T) { fstests.TestFsName(t) } +func TestFsRoot(t *testing.T) { fstests.TestFsRoot(t) } +func TestFsRmdirEmpty(t *testing.T) { fstests.TestFsRmdirEmpty(t) } +func TestFsRmdirNotFound(t *testing.T) { fstests.TestFsRmdirNotFound(t) } +func TestFsMkdir(t *testing.T) { fstests.TestFsMkdir(t) } +func TestFsMkdirRmdirSubdir(t *testing.T) { fstests.TestFsMkdirRmdirSubdir(t) } +func TestFsListEmpty(t *testing.T) { fstests.TestFsListEmpty(t) } +func TestFsListDirEmpty(t *testing.T) { fstests.TestFsListDirEmpty(t) } +func TestFsListRDirEmpty(t *testing.T) { fstests.TestFsListRDirEmpty(t) } +func TestFsNewObjectNotFound(t *testing.T) { fstests.TestFsNewObjectNotFound(t) } +func TestFsPutFile1(t *testing.T) { fstests.TestFsPutFile1(t) } +func TestFsPutError(t *testing.T) { fstests.TestFsPutError(t) } +func TestFsPutFile2(t *testing.T) { fstests.TestFsPutFile2(t) } +func TestFsUpdateFile1(t *testing.T) { fstests.TestFsUpdateFile1(t) } +func TestFsListDirFile2(t *testing.T) { fstests.TestFsListDirFile2(t) } +func TestFsListRDirFile2(t *testing.T) { fstests.TestFsListRDirFile2(t) } +func TestFsListDirRoot(t *testing.T) { fstests.TestFsListDirRoot(t) } +func TestFsListRDirRoot(t *testing.T) { fstests.TestFsListRDirRoot(t) } +func TestFsListSubdir(t *testing.T) { fstests.TestFsListSubdir(t) } +func TestFsListRSubdir(t *testing.T) { fstests.TestFsListRSubdir(t) } +func TestFsListLevel2(t *testing.T) { fstests.TestFsListLevel2(t) } +func TestFsListRLevel2(t *testing.T) { fstests.TestFsListRLevel2(t) } +func TestFsListFile1(t *testing.T) { fstests.TestFsListFile1(t) } +func TestFsNewObject(t *testing.T) { fstests.TestFsNewObject(t) } +func TestFsListFile1and2(t *testing.T) { fstests.TestFsListFile1and2(t) } +func TestFsNewObjectDir(t *testing.T) { fstests.TestFsNewObjectDir(t) } +func TestFsCopy(t *testing.T) { fstests.TestFsCopy(t) } +func TestFsMove(t *testing.T) { fstests.TestFsMove(t) } +func TestFsDirMove(t *testing.T) { fstests.TestFsDirMove(t) } +func TestFsRmdirFull(t *testing.T) { fstests.TestFsRmdirFull(t) } +func TestFsPrecision(t *testing.T) { fstests.TestFsPrecision(t) } +func TestFsDirChangeNotify(t *testing.T) { fstests.TestFsDirChangeNotify(t) } +func TestObjectString(t *testing.T) { fstests.TestObjectString(t) } +func TestObjectFs(t *testing.T) { fstests.TestObjectFs(t) } +func TestObjectRemote(t *testing.T) { fstests.TestObjectRemote(t) } +func TestObjectHashes(t *testing.T) { fstests.TestObjectHashes(t) } +func TestObjectModTime(t *testing.T) { fstests.TestObjectModTime(t) } +func TestObjectMimeType(t *testing.T) { fstests.TestObjectMimeType(t) } +func TestObjectSetModTime(t *testing.T) { fstests.TestObjectSetModTime(t) } +func TestObjectSize(t *testing.T) { fstests.TestObjectSize(t) } +func TestObjectOpen(t *testing.T) { fstests.TestObjectOpen(t) } +func TestObjectOpenSeek(t *testing.T) { fstests.TestObjectOpenSeek(t) } +func TestObjectPartialRead(t *testing.T) { fstests.TestObjectPartialRead(t) } +func TestObjectUpdate(t *testing.T) { fstests.TestObjectUpdate(t) } +func TestObjectStorable(t *testing.T) { fstests.TestObjectStorable(t) } +func TestFsIsFile(t *testing.T) { fstests.TestFsIsFile(t) } +func TestFsIsFileNotFound(t *testing.T) { fstests.TestFsIsFileNotFound(t) } +func TestObjectRemove(t *testing.T) { fstests.TestObjectRemove(t) } +func TestFsPutUnknownLengthFile(t *testing.T) { fstests.TestFsPutUnknownLengthFile(t) } +func TestObjectPurge(t *testing.T) { fstests.TestObjectPurge(t) } +func TestFinalise(t *testing.T) { fstests.TestFinalise(t) }