quatrix: add backend to support Quatrix

Co-authored-by: Volodymyr Kit <v.kit@maytech.net>
This commit is contained in:
Oksana 2023-09-08 16:31:29 +03:00 committed by GitHub
parent 578c75cb1e
commit 628ff8e524
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 1821 additions and 0 deletions

View File

@ -75,6 +75,7 @@ Rclone *("rsync for cloud storage")* is a command-line program to sync files and
* put.io [:page_facing_up:](https://rclone.org/putio/)
* QingStor [:page_facing_up:](https://rclone.org/qingstor/)
* Qiniu Cloud Object Storage (Kodo) [:page_facing_up:](https://rclone.org/s3/#qiniu)
* Quatrix [:page_facing_up:](https://rclone.org/quatrix/)
* Rackspace Cloud Files [:page_facing_up:](https://rclone.org/swift/)
* RackCorp Object Storage [:page_facing_up:](https://rclone.org/s3/#RackCorp)
* Scaleway [:page_facing_up:](https://rclone.org/s3/#scaleway)

View File

@ -41,6 +41,7 @@ import (
_ "github.com/rclone/rclone/backend/protondrive"
_ "github.com/rclone/rclone/backend/putio"
_ "github.com/rclone/rclone/backend/qingstor"
_ "github.com/rclone/rclone/backend/quatrix"
_ "github.com/rclone/rclone/backend/s3"
_ "github.com/rclone/rclone/backend/seafile"
_ "github.com/rclone/rclone/backend/sftp"

View File

@ -0,0 +1,182 @@
// Package api provides types used by the Quatrix API.
package api
import (
"strconv"
"time"
)
// OverwriteOnCopyMode is a conflict resolve mode during copy. Files with conflicting names will be overwritten
const OverwriteOnCopyMode = "overwrite"
// ProfileInfo is a profile info about quota
type ProfileInfo struct {
UserUsed int64 `json:"user_used"`
UserLimit int64 `json:"user_limit"`
AccUsed int64 `json:"acc_used"`
AccLimit int64 `json:"acc_limit"`
}
// IDList is a general object that contains list of ids
type IDList struct {
IDs []string `json:"ids"`
}
// DeleteParams is the request to delete object
type DeleteParams struct {
IDs []string `json:"ids"`
DeletePermanently bool `json:"delete_permanently"`
}
// FileInfoParams is the request to get object's (file or directory) info
type FileInfoParams struct {
ParentID string `json:"parent_id,omitempty"`
Path string `json:"path"`
}
// FileInfo is the response to get object's (file or directory) info
type FileInfo struct {
FileID string `json:"file_id"`
ParentID string `json:"parent_id"`
Src string `json:"src"`
Type string `json:"type"`
}
// IsFile returns true if object is a file
// false otherwise
func (fi *FileInfo) IsFile() bool {
if fi == nil {
return false
}
return fi.Type == "F"
}
// IsDir returns true if object is a directory
// false otherwise
func (fi *FileInfo) IsDir() bool {
if fi == nil {
return false
}
return fi.Type == "D" || fi.Type == "S" || fi.Type == "T"
}
// CreateDirParams is the request to create a directory
type CreateDirParams struct {
Target string `json:"target,omitempty"`
Name string `json:"name"`
Resolve bool `json:"resolve"`
}
// File represent metadata about object in Quatrix (file or directory)
type File struct {
ID string `json:"id"`
Created JSONTime `json:"created"`
Modified JSONTime `json:"modified"`
Name string `json:"name"`
ParentID string `json:"parent_id"`
Size int64 `json:"size"`
ModifiedMS JSONTime `json:"modified_ms"`
Type string `json:"type"`
Operations int `json:"operations"`
SubType string `json:"sub_type"`
Content []File `json:"content"`
}
// IsFile returns true if object is a file
// false otherwise
func (f *File) IsFile() bool {
if f == nil {
return false
}
return f.Type == "F"
}
// IsDir returns true if object is a directory
// false otherwise
func (f *File) IsDir() bool {
if f == nil {
return false
}
return f.Type == "D" || f.Type == "S" || f.Type == "T"
}
// SetMTimeParams is the request to set modification time for object
type SetMTimeParams struct {
ID string `json:"id,omitempty"`
MTime JSONTime `json:"mtime"`
}
// JSONTime provides methods to marshal/unmarshal time.Time as Unix time
type JSONTime time.Time
// MarshalJSON returns time representation in Unix time
func (u JSONTime) MarshalJSON() ([]byte, error) {
return []byte(strconv.FormatFloat(float64(time.Time(u).UTC().UnixNano())/1e9, 'f', 6, 64)), nil
}
// UnmarshalJSON sets time from Unix time representation
func (u *JSONTime) UnmarshalJSON(data []byte) error {
f, err := strconv.ParseFloat(string(data), 64)
if err != nil {
return err
}
t := JSONTime(time.Unix(0, int64(f*1e9)))
*u = t
return nil
}
// String returns Unix time representation of time as string
func (u JSONTime) String() string {
return strconv.FormatInt(time.Time(u).UTC().Unix(), 10)
}
// DownloadLinkResponse is the response to download-link request
type DownloadLinkResponse struct {
ID string `json:"id"`
}
// UploadLinkParams is the request to get upload-link
type UploadLinkParams struct {
Name string `json:"name"`
ParentID string `json:"parent_id"`
Resolve bool `json:"resolve"`
}
// UploadLinkResponse is the response to upload-link request
type UploadLinkResponse struct {
Name string `json:"name"`
FileID string `json:"file_id"`
ParentID string `json:"parent_id"`
UploadKey string `json:"upload_key"`
}
// UploadFinalizeResponse is the response to finalize file method
type UploadFinalizeResponse struct {
FileID string `json:"id"`
ParentID string `json:"parent_id"`
Modified int64 `json:"modified"`
FileSize int64 `json:"size"`
}
// FileModifyParams is the request to get modify file link
type FileModifyParams struct {
ID string `json:"id"`
Truncate int64 `json:"truncate"`
}
// FileCopyMoveOneParams is the request to do server-side copy and move
// can be used for file or directory
type FileCopyMoveOneParams struct {
ID string `json:"file_id"`
Target string `json:"target_id"`
Name string `json:"name"`
MTime JSONTime `json:"mtime"`
Resolve bool `json:"resolve"`
ResolveMode string `json:"resolve_mode"`
}

1254
backend/quatrix/quatrix.go Normal file

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,17 @@
// Test Quatrix filesystem interface
package quatrix_test
import (
"testing"
"github.com/rclone/rclone/backend/quatrix"
"github.com/rclone/rclone/fstest/fstests"
)
// TestIntegration runs integration tests against the remote
func TestIntegration(t *testing.T) {
fstests.Run(t, &fstests.Opt{
RemoteName: "TestQuatrix:",
NilObject: (*quatrix.Object)(nil),
})
}

View File

@ -0,0 +1,108 @@
package quatrix
import (
"sync"
"time"
"github.com/rclone/rclone/fs"
)
// UploadMemoryManager dynamically calculates every chunk size for the transfer and increases or decreases it
// depending on the upload speed. This makes general upload time smaller, because transfers that are faster
// does not have to wait for the slower ones until they finish upload.
type UploadMemoryManager struct {
m sync.Mutex
useDynamicSize bool
shared int64
reserved int64
effectiveTime time.Duration
fileUsage map[string]int64
}
// NewUploadMemoryManager is a constructor for UploadMemoryManager
func NewUploadMemoryManager(ci *fs.ConfigInfo, opt *Options) *UploadMemoryManager {
useDynamicSize := true
sharedMemory := int64(opt.MaximalSummaryChunkSize) - int64(opt.MinimalChunkSize)*int64(ci.Transfers)
if sharedMemory <= 0 {
sharedMemory = 0
useDynamicSize = false
}
return &UploadMemoryManager{
useDynamicSize: useDynamicSize,
shared: sharedMemory,
reserved: int64(opt.MinimalChunkSize),
effectiveTime: time.Duration(opt.EffectiveUploadTime),
fileUsage: map[string]int64{},
}
}
// Consume -- decide amount of memory to consume
func (u *UploadMemoryManager) Consume(fileID string, neededMemory int64, speed float64) int64 {
if !u.useDynamicSize {
if neededMemory < u.reserved {
return neededMemory
}
return u.reserved
}
u.m.Lock()
defer u.m.Unlock()
borrowed, found := u.fileUsage[fileID]
if found {
u.shared += borrowed
borrowed = 0
}
defer func() { u.fileUsage[fileID] = borrowed }()
effectiveChunkSize := int64(speed * u.effectiveTime.Seconds())
if effectiveChunkSize < u.reserved {
effectiveChunkSize = u.reserved
}
if neededMemory < effectiveChunkSize {
effectiveChunkSize = neededMemory
}
if effectiveChunkSize <= u.reserved {
return effectiveChunkSize
}
toBorrow := effectiveChunkSize - u.reserved
if toBorrow <= u.shared {
u.shared -= toBorrow
borrowed = toBorrow
return effectiveChunkSize
}
borrowed = u.shared
u.shared = 0
return borrowed + u.reserved
}
// Return returns consumed memory for the previous chunk upload to the memory pool
func (u *UploadMemoryManager) Return(fileID string) {
if !u.useDynamicSize {
return
}
u.m.Lock()
defer u.m.Unlock()
borrowed, found := u.fileUsage[fileID]
if !found {
return
}
u.shared += borrowed
delete(u.fileUsage, fileID)
}

View File

@ -61,6 +61,7 @@ docs = [
"opendrive.md",
"oracleobjectstorage.md",
"qingstor.md",
"quatrix.md",
"sia.md",
"swift.md",
"pcloud.md",

View File

@ -160,6 +160,7 @@ WebDAV or S3, that work out of the box.)
{{< provider name="put.io" home="https://put.io/" config="/putio/" >}}
{{< provider name="QingStor" home="https://www.qingcloud.com/products/storage" config="/qingstor/" >}}
{{< provider name="Qiniu Cloud Object Storage (Kodo)" home="https://www.qiniu.com/en/products/kodo" config="/s3/#qiniu" >}}
{{< provider name="Quatrix by Maytech" home="https://www.maytech.net/products/quatrix-business" config="/quatrix/" >}}
{{< provider name="Rackspace Cloud Files" home="https://www.rackspace.com/cloud/files" config="/swift/" >}}
{{< provider name="rsync.net" home="https://rsync.net/products/rclone.html" config="/sftp/#rsync-net" >}}
{{< provider name="Scaleway" home="https://www.scaleway.com/object-storage/" config="/s3/#scaleway" >}}

View File

@ -67,6 +67,7 @@ See the following for detailed instructions for
* [premiumize.me](/premiumizeme/)
* [put.io](/putio/)
* [QingStor](/qingstor/)
* [Quatrix by Maytech](/quatrix/)
* [Seafile](/seafile/)
* [SFTP](/sftp/)
* [Sia](/sia/)

View File

@ -48,6 +48,7 @@ Here is an overview of the major features of each cloud storage system.
| premiumize.me | - | - | Yes | No | R | - |
| put.io | CRC-32 | R/W | No | Yes | R | - |
| QingStor | MD5 | - ⁹ | No | No | R/W | - |
| Quatrix by Maytech | - | R/W | No | No | - | - |
| Seafile | - | - | No | No | - | - |
| SFTP | MD5, SHA1 ² | R/W | Depends | No | - | - |
| Sia | - | - | No | No | - | - |
@ -499,6 +500,7 @@ upon backend-specific capabilities.
| premiumize.me | Yes | No | Yes | Yes | No | No | No | No | Yes | Yes | Yes |
| put.io | Yes | No | Yes | Yes | Yes | No | Yes | No | No | Yes | Yes |
| QingStor | No | Yes | No | No | Yes | Yes | No | No | No | No | No |
| Quatrix by Maytech | Yes | Yes | Yes | Yes | No | No | No | No | No | Yes | Yes |
| Seafile | Yes | Yes | Yes | Yes | Yes | Yes | Yes | No | Yes | Yes | Yes |
| SFTP | No | No | Yes | Yes | No | No | Yes | No | No | Yes | Yes |
| Sia | No | No | No | No | No | No | Yes | No | No | No | Yes |

249
docs/content/quatrix.md Normal file
View File

@ -0,0 +1,249 @@
---
title: "Quatrix"
description: "Rclone docs for Quatrix"
versionIntroduced: "v1.63.2"
---
# {{< icon "fas fa-shield-alt" >}} Quatrix
Quatrix by Maytech is [Quatrix Secure Compliant File Sharing | Maytech](https://www.maytech.net/products/quatrix-business).
Paths are specified as `remote:path`
Paths may be as deep as required, e.g., `remote:directory/subdirectory`.
The initial setup for Quatrix involves getting an API Key from Quatrix. You can get the API key in the user's profile at `https://<account>/profile/api-keys`
or with the help of the API - https://docs.maytech.net/quatrix/quatrix-api/api-explorer#/API-Key/post_api_key_create.
See complete Swagger documentation for Quatrix - https://docs.maytech.net/quatrix/quatrix-api/api-explorer
## Configuration
Here is an example of how to make a remote called `remote`. First run:
rclone config
This will guide you through an interactive setup process:
```
No remotes found, make a new one?
n) New remote
s) Set configuration password
q) Quit config
n/s/q> n
name> remote
Type of storage to configure.
Choose a number from below, or type in your own value
[snip]
XX / Quatrix by Maytech
\ "quatrix"
[snip]
Storage> quatrix
API key for accessing Quatrix account.
api_key> your_api_key
Host name of Quatrix account.
host> example.quatrix.it
--------------------
[remote]
api_key = your_api_key
host = example.quatrix.it
--------------------
y) Yes this is OK
e) Edit this remote
d) Delete this remote
y/e/d> y
```
Once configured you can then use `rclone` like this,
List directories in top level of your Quatrix
rclone lsd remote:
List all the files in your Quatrix
rclone ls remote:
To copy a local directory to an Quatrix directory called backup
rclone copy /home/source remote:backup
### API key validity
API Key is created with no expiration date. It will be valid until you delete or deactivate it in your account.
After disabling, the API Key can be enabled back. If the API Key was deleted and a new key was created, you can
update it in rclone config. The same happens if the hostname was changed.
```
$ rclone config
Current remotes:
Name Type
==== ====
remote quatrix
e) Edit existing remote
n) New remote
d) Delete remote
r) Rename remote
c) Copy remote
s) Set configuration password
q) Quit config
e/n/d/r/c/s/q> e
Choose a number from below, or type in an existing value
1 > remote
remote> remote
--------------------
[remote]
type = quatrix
host = some_host.quatrix.it
api_key = your_api_key
--------------------
Edit remote
Option api_key.
API key for accessing Quatrix account
Enter a string value. Press Enter for the default (your_api_key)
api_key>
Option host.
Host name of Quatrix account
Enter a string value. Press Enter for the default (some_host.quatrix.it).
--------------------
[remote]
type = quatrix
host = some_host.quatrix.it
api_key = your_api_key
--------------------
y) Yes this is OK
e) Edit this remote
d) Delete this remote
y/e/d> y
```
### Modified time and hashes
Quatrix allows modification times to be set on objects accurate to 1 microsecond.
These will be used to detect whether objects need syncing or not.
Quatrix does not support hashes, so you cannot use the `--checksum` flag.
### Restricted filename characters
File names in Quatrix are case sensitive and have limitations like the maximum length of a filename is 255, and the minimum length is 1. A file name cannot be equal to `.` or `..` nor contain `/` , `\` or non-printable ascii.
### Transfers
For files above 50 MiB rclone will use a chunked transfer. Rclone will upload up to `--transfers` chunks at the same time (shared among all multipart uploads).
Chunks are buffered in memory, and the minimal chunk size is 10_000_000 bytes by default, and it can be changed in the advanced configuration, so increasing `--transfers` will increase the memory use.
The chunk size has a maximum size limit, which is set to 100_000_000 bytes by default and can be changed in the advanced configuration.
The size of the uploaded chunk will dynamically change depending on the upload speed.
The total memory use equals the number of transfers multiplied by the minimal chunk size.
In case there's free memory allocated for the upload (which equals the difference of `maximal_summary_chunk_size` and `minimal_chunk_size` * `transfers`),
the chunk size may increase in case of high upload speed. As well as it can decrease in case of upload speed problems.
If no free memory is available, all chunks will equal `minimal_chunk_size`.
### Deleting files
Files you delete with rclone will end up in Trash and be stored there for 30 days.
Quatrix also provides an API to permanently delete files and an API to empty the Trash so that you can remove files permanently from your account.
{{< rem autogenerated options start" - DO NOT EDIT - instead edit fs.RegInfo in backend/quatrix/quatrix.go then run make backenddocs" >}}
### Standard options
Here are the Standard options specific to quatrix (Quatrix by Maytech).
#### --quatrix-api-key
API key for accessing Quatrix account
Properties:
- Config: api_key
- Env Var: RCLONE_QUATRIX_API_KEY
- Type: string
- Required: true
#### --quatrix-host
Host name of Quatrix account
Properties:
- Config: host
- Env Var: RCLONE_QUATRIX_HOST
- Type: string
- Required: true
### Advanced options
Here are the Advanced options specific to quatrix (Quatrix by Maytech).
#### --quatrix-encoding
The encoding for the backend.
See the [encoding section in the overview](/overview/#encoding) for more info.
Properties:
- Config: encoding
- Env Var: RCLONE_QUATRIX_ENCODING
- Type: MultiEncoder
- Default: Slash,BackSlash,Del,Ctl,InvalidUtf8,Dot
#### --quatrix-effective-upload-time
Wanted upload time for one chunk
Properties:
- Config: effective_upload_time
- Env Var: RCLONE_QUATRIX_EFFECTIVE_UPLOAD_TIME
- Type: string
- Default: "4s"
#### --quatrix-minimal-chunk-size
The minimal size for one chunk
Properties:
- Config: minimal_chunk_size
- Env Var: RCLONE_QUATRIX_MINIMAL_CHUNK_SIZE
- Type: SizeSuffix
- Default: 9.537Mi
#### --quatrix-maximal-summary-chunk-size
The maximal summary for all chunks. It should not be less than 'transfers'*'minimal_chunk_size'
Properties:
- Config: maximal_summary_chunk_size
- Env Var: RCLONE_QUATRIX_MAXIMAL_SUMMARY_CHUNK_SIZE
- Type: SizeSuffix
- Default: 95.367Mi
#### --quatrix-hard-delete
Delete files permanently rather than putting them into the trash.
Properties:
- Config: hard_delete
- Env Var: RCLONE_QUATRIX_HARD_DELETE
- Type: bool
- Default: false
{{< rem autogenerated options stop >}}
## Storage usage
The storage usage in Quatrix is restricted to the account during the purchase. You can restrict any user with a smaller storage limit.
The account limit is applied if the user has no custom storage limit. Once you've reached the limit, the upload of files will fail.
This can be fixed by freeing up the space or increasing the quota.
## Server-side operations
Quatrix supports server-side operations (copy and move). In case of conflict, files are overwritten during server-side operation.

View File

@ -90,6 +90,7 @@
<a class="dropdown-item" href="/pikpak/"><i class="fa fa-cloud fa-fw"></i> PikPak</a>
<a class="dropdown-item" href="/premiumizeme/"><i class="fa fa-user fa-fw"></i> premiumize.me</a>
<a class="dropdown-item" href="/putio/"><i class="fas fa-parking fa-fw"></i> put.io</a>
<a class="dropdown-item" href="/quatrix/"><i class="fas fa-shield-alt"></i> Quatrix</a>
<a class="dropdown-item" href="/seafile/"><i class="fa fa-server fa-fw"></i> Seafile</a>
<a class="dropdown-item" href="/sftp/"><i class="fa fa-server fa-fw"></i> SFTP</a>
<a class="dropdown-item" href="/sia/"><i class="fa fa-globe fa-fw"></i> Sia</a>

View File

@ -434,3 +434,6 @@ backends:
- TestIntegration/FsMkdir/FsEncoding/trailing_LF
- TestIntegration/FsMkdir/FsEncoding/leading_HT
- TestIntegration/FsMkdir/FsPutFiles/FsPutStream/0
- backend: "quatrix"
remote: "TestQuatrix:"
fastlist: false