storj: implement public link

This commit is contained in:
Kaloyan Raev 2023-01-10 19:40:04 +02:00 committed by GitHub
parent 98fa93f6d1
commit 1cafc12e8c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 75 additions and 9 deletions

View File

@ -23,6 +23,7 @@ import (
"golang.org/x/text/unicode/norm" "golang.org/x/text/unicode/norm"
"storj.io/uplink" "storj.io/uplink"
"storj.io/uplink/edge"
) )
const ( const (
@ -161,6 +162,7 @@ var (
_ fs.PutStreamer = &Fs{} _ fs.PutStreamer = &Fs{}
_ fs.Mover = &Fs{} _ fs.Mover = &Fs{}
_ fs.Copier = &Fs{} _ fs.Copier = &Fs{}
_ fs.PublicLinker = &Fs{}
) )
// NewFs creates a filesystem backed by Storj. // NewFs creates a filesystem backed by Storj.
@ -545,7 +547,7 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
defer func() { defer func() {
if err != nil { if err != nil {
aerr := upload.Abort() aerr := upload.Abort()
if aerr != nil { if aerr != nil && !errors.Is(aerr, uplink.ErrUploadDone) {
fs.Errorf(f, "cp input ./%s %+v: %+v", src.Remote(), options, aerr) fs.Errorf(f, "cp input ./%s %+v: %+v", src.Remote(), options, aerr)
} }
} }
@ -560,6 +562,16 @@ func (f *Fs) Put(ctx context.Context, in io.Reader, src fs.ObjectInfo, options .
_, err = io.Copy(upload, in) _, err = io.Copy(upload, in)
if err != nil { if err != nil {
if errors.Is(err, uplink.ErrBucketNotFound) {
// Rclone assumes the backend will create the bucket if not existing yet.
// Here we create the bucket and return a retry error for rclone to retry the upload.
_, err = f.project.EnsureBucket(ctx, bucketName)
if err != nil {
return nil, err
}
return nil, fserrors.RetryError(errors.New("bucket was not available, now created, the upload must be retried"))
}
err = fserrors.RetryError(err) err = fserrors.RetryError(err)
fs.Errorf(f, "cp input ./%s %+v: %+v\n", src.Remote(), options, err) fs.Errorf(f, "cp input ./%s %+v: %+v\n", src.Remote(), options, err)
@ -761,3 +773,55 @@ func (f *Fs) Copy(ctx context.Context, src fs.Object, remote string) (fs.Object,
// Return the new object // Return the new object
return newObjectFromUplink(f, remote, newObject), nil return newObjectFromUplink(f, remote, newObject), nil
} }
// PublicLink generates a public link to the remote path (usually readable by anyone)
func (f *Fs) PublicLink(ctx context.Context, remote string, expire fs.Duration, unlink bool) (string, error) {
bucket, key := f.absolute(remote)
if len(bucket) == 0 {
return "", errors.New("path must be specified")
}
// Rclone requires that a link is only generated if the remote path exists
if len(key) == 0 {
_, err := f.project.StatBucket(ctx, bucket)
if err != nil {
return "", err
}
} else {
_, err := f.project.StatObject(ctx, bucket, key)
if err != nil {
if !errors.Is(err, uplink.ErrObjectNotFound) {
return "", err
}
// No object found, check if there is such a prefix
iter := f.project.ListObjects(ctx, bucket, &uplink.ListObjectsOptions{Prefix: key + "/"})
if iter.Err() != nil {
return "", iter.Err()
}
if !iter.Next() {
return "", err
}
}
}
sharedPrefix := uplink.SharePrefix{Bucket: bucket, Prefix: key}
permission := uplink.ReadOnlyPermission()
if expire.IsSet() {
permission.NotAfter = time.Now().Add(time.Duration(expire))
}
sharedAccess, err := f.access.Share(permission, sharedPrefix)
if err != nil {
return "", fmt.Errorf("sharing access to object failed: %w", err)
}
creds, err := (&edge.Config{
AuthServiceAddress: "auth.storjshare.io:7777",
}).RegisterAccess(ctx, sharedAccess, &edge.RegisterAccessOptions{Public: true})
if err != nil {
return "", fmt.Errorf("creating public link failed: %w", err)
}
return edge.JoinShareURL("https://link.storjshare.io", creds.AccessKeyID, bucket, key, nil)
}

View File

@ -504,7 +504,7 @@ upon backend-specific capabilities.
| Sia | No | No | No | No | No | No | Yes | No | No | Yes | | Sia | No | No | No | No | No | No | Yes | No | No | Yes |
| SMB | No | No | Yes | Yes | No | No | Yes | No | No | Yes | | SMB | No | No | Yes | Yes | No | No | Yes | No | No | Yes |
| SugarSync | Yes | Yes | Yes | Yes | No | No | Yes | Yes | No | Yes | | SugarSync | Yes | Yes | Yes | Yes | No | No | Yes | Yes | No | Yes |
| Storj | Yes † | Yes | Yes | No | No | Yes | Yes | No | No | No | | Storj | Yes † | Yes | Yes | No | No | Yes | Yes | Yes | No | No |
| Uptobox | No | Yes | Yes | Yes | No | No | No | No | No | No | | Uptobox | No | Yes | Yes | Yes | No | No | No | No | No | No |
| WebDAV | Yes | Yes | Yes | Yes | No | No | Yes ‡ | No | Yes | Yes | | WebDAV | Yes | Yes | Yes | Yes | No | No | Yes ‡ | No | Yes | Yes |
| Yandex Disk | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes | Yes | | Yandex Disk | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes | Yes |

View File

@ -1734,8 +1734,10 @@ func Run(t *testing.T, opt *Opt) {
// ensure sub remote isn't empty // ensure sub remote isn't empty
buf := bytes.NewBufferString("somecontent") buf := bytes.NewBufferString("somecontent")
obji := object.NewStaticObjectInfo("somefile", time.Now(), int64(buf.Len()), true, nil, nil) obji := object.NewStaticObjectInfo("somefile", time.Now(), int64(buf.Len()), true, nil, nil)
_, err = subRemote.Put(ctx, buf, obji) retry(t, "Put", func() error {
require.NoError(t, err) _, err := subRemote.Put(ctx, buf, obji)
return err
})
link4, err := wrapPublicLinkFunc(subRemote.Features().PublicLink)(ctx, "", expiry, false) link4, err := wrapPublicLinkFunc(subRemote.Features().PublicLink)(ctx, "", expiry, false)
require.NoError(t, err, "Sharing root in a sub-remote should work") require.NoError(t, err, "Sharing root in a sub-remote should work")