From 38a4d50e73060e1b486de6dc0caba68fa0a0c0a0 Mon Sep 17 00:00:00 2001 From: Gary Kim Date: Wed, 26 Feb 2020 16:34:32 +0800 Subject: [PATCH] rcd: Add Prometheus metrics support - fixes #3858 Signed-off-by: Gary Kim --- docs/content/rc.md | 7 +++ fs/accounting/prometheus.go | 94 +++++++++++++++++++++++++++++++++ fs/accounting/stats.go | 21 +++++--- fs/rc/rc.go | 1 + fs/rc/rcflags/rcflags.go | 1 + fs/rc/rcserver/rcserver.go | 17 +++++- fs/rc/rcserver/rcserver_test.go | 59 ++++++++++++++++++++- 7 files changed, 189 insertions(+), 11 deletions(-) create mode 100644 fs/accounting/prometheus.go diff --git a/docs/content/rc.md b/docs/content/rc.md index bfae78e53..d698732e0 100644 --- a/docs/content/rc.md +++ b/docs/content/rc.md @@ -9,6 +9,7 @@ date: "2018-03-05" If rclone is run with the `--rc` flag then it starts an http server which can be used to remote control rclone. + If you just want to run a remote control then see the [rcd command](/commands/rclone_rcd/). **NB** this is experimental and everything here is subject to change! @@ -85,6 +86,12 @@ style. Default Off. +### --rc-enable-metrics + +Enable OpenMetrics/Prometheus compatible endpoint at `/metrics`. + +Default Off. + ### --rc-web-gui Set this flag to serve the default web gui on the same port as rclone. diff --git a/fs/accounting/prometheus.go b/fs/accounting/prometheus.go new file mode 100644 index 000000000..535f99e6a --- /dev/null +++ b/fs/accounting/prometheus.go @@ -0,0 +1,94 @@ +package accounting + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +var namespace = "rclone_" + +// RcloneCollector is a Prometheus collector for Rclone +type RcloneCollector struct { + bytesTransferred *prometheus.Desc + transferSpeed *prometheus.Desc + numOfErrors *prometheus.Desc + numOfCheckFiles *prometheus.Desc + transferredFiles *prometheus.Desc + deletes *prometheus.Desc + fatalError *prometheus.Desc + retryError *prometheus.Desc +} + +// NewRcloneCollector make a new RcloneCollector +func NewRcloneCollector() *RcloneCollector { + return &RcloneCollector{ + bytesTransferred: prometheus.NewDesc(namespace+"bytes_transferred_total", + "Total transferred bytes since the start of the Rclone process", + nil, nil, + ), + transferSpeed: prometheus.NewDesc(namespace+"speed", + "Average speed in bytes/sec since the start of the Rclone process", + nil, nil, + ), + numOfErrors: prometheus.NewDesc(namespace+"errors_total", + "Number of errors thrown", + nil, nil, + ), + numOfCheckFiles: prometheus.NewDesc(namespace+"checked_files_total", + "Number of checked files", + nil, nil, + ), + transferredFiles: prometheus.NewDesc(namespace+"files_transferred_total", + "Number of transferred files", + nil, nil, + ), + deletes: prometheus.NewDesc(namespace+"files_deleted_total", + "Total number of files deleted", + nil, nil, + ), + fatalError: prometheus.NewDesc(namespace+"fatal_error", + "Whether a fatal error has occurred", + nil, nil, + ), + retryError: prometheus.NewDesc(namespace+"retry_error", + "Whether there has been an error that will be retried", + nil, nil, + ), + } +} + +// Describe is part of the Collector interface: https://godoc.org/github.com/prometheus/client_golang/prometheus#Collector +func (c *RcloneCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- c.bytesTransferred + ch <- c.transferSpeed + ch <- c.numOfErrors + ch <- c.numOfCheckFiles + ch <- c.transferredFiles + ch <- c.deletes + ch <- c.fatalError + ch <- c.retryError +} + +// Collect is part of the Collector interface: https://godoc.org/github.com/prometheus/client_golang/prometheus#Collector +func (c *RcloneCollector) Collect(ch chan<- prometheus.Metric) { + s := GlobalStats() + s.mu.RLock() + + ch <- prometheus.MustNewConstMetric(c.bytesTransferred, prometheus.CounterValue, float64(s.bytes)) + ch <- prometheus.MustNewConstMetric(c.transferSpeed, prometheus.GaugeValue, s.Speed()) + ch <- prometheus.MustNewConstMetric(c.numOfErrors, prometheus.CounterValue, float64(s.errors)) + ch <- prometheus.MustNewConstMetric(c.numOfCheckFiles, prometheus.CounterValue, float64(s.checks)) + ch <- prometheus.MustNewConstMetric(c.transferredFiles, prometheus.CounterValue, float64(s.transfers)) + ch <- prometheus.MustNewConstMetric(c.deletes, prometheus.CounterValue, float64(s.deletes)) + ch <- prometheus.MustNewConstMetric(c.fatalError, prometheus.GaugeValue, bool2Float(s.fatalError)) + ch <- prometheus.MustNewConstMetric(c.retryError, prometheus.GaugeValue, bool2Float(s.retryError)) + + s.mu.RUnlock() +} + +// bool2Float is a small function to convert a boolean into a float64 value that can be used for Prometheus +func bool2Float(e bool) float64 { + if e { + return 1 + } + return 0 +} diff --git a/fs/accounting/stats.go b/fs/accounting/stats.go index a61b88886..39831edf0 100644 --- a/fs/accounting/stats.go +++ b/fs/accounting/stats.go @@ -56,13 +56,7 @@ func NewStats() *StatsInfo { func (s *StatsInfo) RemoteStats() (out rc.Params, err error) { out = make(rc.Params) s.mu.RLock() - dt := s.totalDuration() - dtSeconds := dt.Seconds() - speed := 0.0 - if dt > 0 { - speed = float64(s.bytes) / dtSeconds - } - out["speed"] = speed + out["speed"] = s.Speed() out["bytes"] = s.bytes out["errors"] = s.errors out["fatalError"] = s.fatalError @@ -70,7 +64,7 @@ func (s *StatsInfo) RemoteStats() (out rc.Params, err error) { out["checks"] = s.checks out["transfers"] = s.transfers out["deletes"] = s.deletes - out["elapsedTime"] = dtSeconds + out["elapsedTime"] = s.totalDuration().Seconds() s.mu.RUnlock() if !s.checking.empty() { var c []string @@ -101,6 +95,17 @@ func (s *StatsInfo) RemoteStats() (out rc.Params, err error) { return out, nil } +// Speed returns the average speed of the transfer in bytes/second +func (s *StatsInfo) Speed() float64 { + dt := s.totalDuration() + dtSeconds := dt.Seconds() + speed := 0.0 + if dt > 0 { + speed = float64(s.bytes) / dtSeconds + } + return speed +} + func (s *StatsInfo) transferRemoteStats(name string) rc.Params { s.mu.RLock() defer s.mu.RUnlock() diff --git a/fs/rc/rc.go b/fs/rc/rc.go index 2ac58022f..57d9f18da 100644 --- a/fs/rc/rc.go +++ b/fs/rc/rc.go @@ -29,6 +29,7 @@ type Options struct { WebGUINoOpenBrowser bool // set to disable auto opening browser WebGUIFetchURL string // set the default url for fetching webgui AccessControlAllowOrigin string // set the access control for CORS configuration + EnableMetrics bool // set to disable prometheus metrics on /metrics JobExpireDuration time.Duration JobExpireInterval time.Duration } diff --git a/fs/rc/rcflags/rcflags.go b/fs/rc/rcflags/rcflags.go index 40aaadf07..acae13535 100644 --- a/fs/rc/rcflags/rcflags.go +++ b/fs/rc/rcflags/rcflags.go @@ -26,6 +26,7 @@ func AddFlags(flagSet *pflag.FlagSet) { flags.BoolVarP(flagSet, &Opt.WebGUINoOpenBrowser, "rc-web-gui-no-open-browser", "", false, "Don't open the browser automatically") flags.StringVarP(flagSet, &Opt.WebGUIFetchURL, "rc-web-fetch-url", "", "https://api.github.com/repos/rclone/rclone-webui-react/releases/latest", "URL to fetch the releases for webgui.") flags.StringVarP(flagSet, &Opt.AccessControlAllowOrigin, "rc-allow-origin", "", "", "Set the allowed origin for CORS.") + flags.BoolVarP(flagSet, &Opt.EnableMetrics, "rc-enable-metrics", "", false, "Enable prometheus metrics on /metrics") flags.DurationVarP(flagSet, &Opt.JobExpireDuration, "rc-job-expire-duration", "", Opt.JobExpireDuration, "expire finished async jobs older than this value") flags.DurationVarP(flagSet, &Opt.JobExpireInterval, "rc-job-expire-interval", "", Opt.JobExpireInterval, "interval to check for expired async jobs") httpflags.AddFlagsPrefix(flagSet, "rc-", &Opt.HTTPOptions) diff --git a/fs/rc/rcserver/rcserver.go b/fs/rc/rcserver/rcserver.go index 4a2b5c71e..bf0e4db90 100644 --- a/fs/rc/rcserver/rcserver.go +++ b/fs/rc/rcserver/rcserver.go @@ -16,9 +16,14 @@ import ( "strings" "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/skratchdot/open-golang/open" + "github.com/rclone/rclone/cmd/serve/httplib" "github.com/rclone/rclone/cmd/serve/httplib/serve" "github.com/rclone/rclone/fs" + "github.com/rclone/rclone/fs/accounting" "github.com/rclone/rclone/fs/cache" "github.com/rclone/rclone/fs/config" "github.com/rclone/rclone/fs/list" @@ -26,9 +31,16 @@ import ( "github.com/rclone/rclone/fs/rc/jobs" "github.com/rclone/rclone/fs/rc/rcflags" "github.com/rclone/rclone/lib/random" - "github.com/skratchdot/open-golang/open" ) +var promHandler http.Handler + +func init() { + rcloneCollector := accounting.NewRcloneCollector() + prometheus.MustRegister(rcloneCollector) + promHandler = promhttp.Handler() +} + // Start the remote control server if configured // // If the server wasn't configured the *Server returned may be nil @@ -335,6 +347,9 @@ func (s *Server) handleGet(w http.ResponseWriter, r *http.Request, path string) // Serve /[fs]/remote files s.serveRemote(w, r, match[2], match[1]) return + case path == "metrics" && s.opt.EnableMetrics: + promHandler.ServeHTTP(w, r) + return case path == "*" && s.opt.Serve: // Serve /* as the remote listing s.serveRoot(w, r) diff --git a/fs/rc/rcserver/rcserver_test.go b/fs/rc/rcserver/rcserver_test.go index 626ca7849..415e92fb2 100644 --- a/fs/rc/rcserver/rcserver_test.go +++ b/fs/rc/rcserver/rcserver_test.go @@ -12,10 +12,12 @@ import ( "testing" "time" - _ "github.com/rclone/rclone/backend/local" - "github.com/rclone/rclone/fs/rc" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + _ "github.com/rclone/rclone/backend/local" + "github.com/rclone/rclone/fs/accounting" + "github.com/rclone/rclone/fs/rc" ) const ( @@ -481,6 +483,59 @@ func TestMethods(t *testing.T) { testServer(t, tests, &opt) } +func TestMetrics(t *testing.T) { + stats := accounting.GlobalStats() + tests := makeMetricsTestCases(stats) + opt := newTestOpt() + opt.EnableMetrics = true + testServer(t, tests, &opt) + + // Test changing a couple options + stats.Bytes(500) + stats.Deletes(30) + stats.Errors(2) + stats.Bytes(324) + + tests = makeMetricsTestCases(stats) + testServer(t, tests, &opt) +} + +func makeMetricsTestCases(stats *accounting.StatsInfo) (tests []testRun) { + tests = []testRun{{ + Name: "Bytes Transferred Metric", + URL: "/metrics", + Method: "GET", + Status: http.StatusOK, + Contains: regexp.MustCompile(fmt.Sprintf("rclone_bytes_transferred_total %d", stats.GetBytes())), + }, { + Name: "Checked Files Metric", + URL: "/metrics", + Method: "GET", + Status: http.StatusOK, + Contains: regexp.MustCompile(fmt.Sprintf("rclone_checked_files_total %d", stats.GetChecks())), + }, { + Name: "Errors Metric", + URL: "/metrics", + Method: "GET", + Status: http.StatusOK, + Contains: regexp.MustCompile(fmt.Sprintf("rclone_errors_total %d", stats.GetErrors())), + }, { + Name: "Deleted Files Metric", + URL: "/metrics", + Method: "GET", + Status: http.StatusOK, + Contains: regexp.MustCompile(fmt.Sprintf("rclone_files_deleted_total %d", stats.Deletes(0))), + }, { + Name: "Files Transferred Metric", + URL: "/metrics", + Method: "GET", + Status: http.StatusOK, + Contains: regexp.MustCompile(fmt.Sprintf("rclone_files_transferred_total %d", stats.GetTransfers())), + }, + } + return +} + var matchRemoteDirListing = regexp.MustCompile(`List of all rclone remotes.`) func TestServingRoot(t *testing.T) {