diff --git a/backend/ftp/ftp.go b/backend/ftp/ftp.go index 330596e31..b5de006e5 100644 --- a/backend/ftp/ftp.go +++ b/backend/ftp/ftp.go @@ -98,6 +98,11 @@ to an encrypted one. Cannot be used in combination with implicit FTP.`, Help: "Disable using MLSD even if server advertises support.", Default: false, Advanced: true, + }, { + Name: "writing_mdtm", + Help: "Use MDTM to set modification time (VsFtpd quirk)", + Default: false, + Advanced: true, }, { Name: "idle_timeout", Default: fs.Duration(60 * time.Second), @@ -169,6 +174,7 @@ type Options struct { SkipVerifyTLSCert bool `config:"no_check_certificate"` DisableEPSV bool `config:"disable_epsv"` DisableMLSD bool `config:"disable_mlsd"` + WritingMDTM bool `config:"writing_mdtm"` IdleTimeout fs.Duration `config:"idle_timeout"` CloseTimeout fs.Duration `config:"close_timeout"` ShutTimeout fs.Duration `config:"shut_timeout"` @@ -192,6 +198,9 @@ type Fs struct { tokens *pacer.TokenDispenser tlsConf *tls.Config pacer *fs.Pacer // pacer for FTP connections + fGetTime bool // true if the ftp library accepts GetTime + fSetTime bool // true if the ftp library accepts SetTime + fLstTime bool // true if the List call returns precise time } // Object describes an FTP file @@ -206,6 +215,7 @@ type FileInfo struct { Name string Size uint64 ModTime time.Time + precise bool // true if the time is precise IsDir bool } @@ -320,6 +330,9 @@ func (f *Fs) ftpConnection(ctx context.Context) (c *ftp.ServerConn, err error) { if f.opt.ShutTimeout != 0 && f.opt.ShutTimeout != fs.DurationOff { ftpConfig = append(ftpConfig, ftp.DialWithShutTimeout(time.Duration(f.opt.ShutTimeout))) } + if f.opt.WritingMDTM { + ftpConfig = append(ftpConfig, ftp.DialWithWritingMDTM(true)) + } if f.ci.Dump&(fs.DumpHeaders|fs.DumpBodies|fs.DumpRequests|fs.DumpResponses) != 0 { ftpConfig = append(ftpConfig, ftp.DialWithDebugOutput(&debugLog{auth: f.ci.Dump&fs.DumpAuth != 0})) } @@ -491,6 +504,12 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (ff fs.Fs if err != nil { return nil, errors.Wrap(err, "NewFs") } + f.fGetTime = c.IsGetTimeSupported() + f.fSetTime = c.IsSetTimeSupported() + f.fLstTime = c.IsTimePreciseInList() + if !f.fLstTime && f.fGetTime { + f.features.SlowModTime = true + } f.putFtpConnection(&c, nil) if root != "" { // Check to see if the root actually an existing file @@ -609,13 +628,12 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (o fs.Object, err err fs: f, remote: remote, } - info := &FileInfo{ + o.info = &FileInfo{ Name: remote, Size: entry.Size, ModTime: entry.Time, + precise: f.fLstTime, } - o.info = info - return o, nil } return nil, fs.ErrorObjectNotFound @@ -710,6 +728,7 @@ func (f *Fs) List(ctx context.Context, dir string) (entries fs.DirEntries, err e Name: newremote, Size: object.Size, ModTime: object.Time, + precise: f.fLstTime, } o.info = info entries = append(entries, o) @@ -723,8 +742,18 @@ func (f *Fs) Hashes() hash.Set { return 0 } -// Precision shows Modified Time not supported +// Precision shows whether modified time is supported or not depending on the +// FTP server capabilities, namely whether FTP server: +// - accepts the MDTM command to get file time (fGetTime) +// or supports MLSD returning precise file time in the list (fLstTime) +// - accepts the MFMT command to set file time (fSetTime) +// or non-standard form of the MDTM command (fSetTime, too) +// used by VsFtpd for the same purpose (WritingMDTM) +// See "mdtm_write" in https://security.appspot.com/vsftpd/vsftpd_conf.html func (f *Fs) Precision() time.Duration { + if (f.fGetTime || f.fLstTime) && f.fSetTime { + return time.Second + } return fs.ModTimeNotSupported } @@ -776,6 +805,7 @@ func (f *Fs) getInfo(ctx context.Context, remote string) (fi *FileInfo, err erro Name: remote, Size: file.Size, ModTime: file.Time, + precise: f.fLstTime, IsDir: file.Type == ftp.EntryTypeFolder, } return info, nil @@ -961,12 +991,41 @@ func (o *Object) Size() int64 { // ModTime returns the modification time of the object func (o *Object) ModTime(ctx context.Context) time.Time { + if !o.info.precise && o.fs.fGetTime { + c, err := o.fs.getFtpConnection(ctx) + if err == nil { + path := path.Join(o.fs.root, o.remote) + path = o.fs.opt.Enc.FromStandardPath(path) + modTime, err := c.GetTime(path) + if err == nil && o.info != nil { + o.info.ModTime = modTime + o.info.precise = true + } + o.fs.putFtpConnection(&c, err) + } + } return o.info.ModTime } // SetModTime sets the modification time of the object func (o *Object) SetModTime(ctx context.Context, modTime time.Time) error { - return nil + if !o.fs.fSetTime { + fs.Errorf(o.fs, "SetModTime is not supported") + return nil + } + c, err := o.fs.getFtpConnection(ctx) + if err != nil { + return err + } + path := path.Join(o.fs.root, o.remote) + path = o.fs.opt.Enc.FromStandardPath(path) + err = c.SetTime(path, modTime.In(time.UTC)) + if err == nil && o.info != nil { + o.info.ModTime = modTime + o.info.precise = true + } + o.fs.putFtpConnection(&c, err) + return err } // Storable returns a boolean as to whether this object is storable @@ -1108,6 +1167,9 @@ func (o *Object) Update(ctx context.Context, in io.Reader, src fs.ObjectInfo, op return errors.Wrap(err, "update stor") } o.fs.putFtpConnection(&c, nil) + if err = o.SetModTime(ctx, src.ModTime(ctx)); err != nil { + return errors.Wrap(err, "SetModTime") + } o.info, err = o.fs.getInfo(ctx, path) if err != nil { return errors.Wrap(err, "update getinfo") diff --git a/backend/ftp/ftp_internal_test.go b/backend/ftp/ftp_internal_test.go index ada0e9fb1..173ffe1cd 100644 --- a/backend/ftp/ftp_internal_test.go +++ b/backend/ftp/ftp_internal_test.go @@ -90,9 +90,26 @@ func (f *Fs) testUploadTimeout(t *testing.T) { } } +// rclone must support precise time with ProFtpd and PureFtpd out of the box. +// The VsFtpd server does not support the MFMT command to set file time like +// other servers but by default supports the MDTM command in the non-standard +// two-argument form for the same purpose. +// See "mdtm_write" in https://security.appspot.com/vsftpd/vsftpd_conf.html +func (f *Fs) testTimePrecision(t *testing.T) { + name := f.Name() + if pos := strings.Index(name, "{"); pos != -1 { + name = name[:pos] + } + switch name { + case "TestFTPProftpd", "TestFTPPureftpd", "TestFTPVsftpd": + assert.LessOrEqual(t, f.Precision(), time.Second) + } +} + // InternalTest dispatches all internal tests func (f *Fs) InternalTest(t *testing.T) { t.Run("UploadTimeout", f.testUploadTimeout) + t.Run("TimePrecision", f.testTimePrecision) } var _ fstests.InternalTester = (*Fs)(nil) diff --git a/docs/content/ftp.md b/docs/content/ftp.md index 8a1d0ddf6..2b791a905 100644 --- a/docs/content/ftp.md +++ b/docs/content/ftp.md @@ -246,6 +246,15 @@ Disable using MLSD even if server advertises support - Type: bool - Default: false +#### --ftp-writing-mdtm + +Use MDTM to set modification time (VsFtpd quirk) + +- Config: writing_mdtm +- Env Var: RCLONE_FTP_WRITING_MDTM +- Type: bool +- Default: false + #### --ftp-idle-timeout Max time before closing idle connections @@ -298,9 +307,6 @@ Rclone's FTP implementation is not compatible with `active` mode as [the library it uses doesn't support it](https://github.com/jlaffaye/ftp/issues/29). This will likely never be supported due to security concerns. -Modified times are not supported. Times you see on the FTP server -through rclone are those of upload. - Rclone's FTP backend does not support any checksums but can compare file sizes. @@ -324,3 +330,23 @@ Rclone's FTP backend could support server-side move but does not at present. The `ftp_proxy` environment variable is not currently supported. + +#### Modified time + +File modification time (timestamps) is supported to 1 second resolution +for major FTP servers: ProFTPd, PureFTPd, VsFTPd, and FileZilla FTP server. +The `VsFTPd` server has non-standard implementation of time related protocol +commands and needs a special configuration setting: `writing_mdtm = true`. + +Support for precise file time with other FTP servers varies depending on what +protocol extensions they advertise. If all the `MLSD`, `MDTM` and `MFTM` +extensions are present, rclone will use them together to provide precise time. +Otherwise the times you see on the FTP server through rclone are those of the +last file upload. + +You can use the following command to check whether rclone can use precise time +with your FTP server: `rclone backend features your_ftp_remote:` (the trailing +colon is important). Look for the number in the line tagged by `Precision` +designating the remote time precision expressed as nanoseconds. A value of +`1000000000` means that file time precision of 1 second is available. +A value of `3153600000000000000` (or another large number) means "unsupported". diff --git a/fstest/testserver/init.d/TestFTPVsftpd b/fstest/testserver/init.d/TestFTPVsftpd index abfa8bf50..d20e637df 100755 --- a/fstest/testserver/init.d/TestFTPVsftpd +++ b/fstest/testserver/init.d/TestFTPVsftpd @@ -18,6 +18,7 @@ start() { echo host=$(docker_ip) echo user=$USER echo pass=$(rclone obscure $PASS) + echo writing_mdtm=true echo encoding=Ctl,LeftPeriod,Slash echo _connect=$(docker_ip):21 }