diff --git a/config/config.go b/config/config.go index 50b0193aa..fc1698941 100644 --- a/config/config.go +++ b/config/config.go @@ -31,6 +31,7 @@ import ( relayv2 "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/relay" "github.com/libp2p/go-libp2p/p2p/protocol/holepunch" "github.com/libp2p/go-libp2p/p2p/transport/quicreuse" + "github.com/prometheus/client_golang/prometheus" ma "github.com/multiformats/go-multiaddr" madns "github.com/multiformats/go-multiaddr-dns" @@ -117,6 +118,9 @@ type Config struct { EnableHolePunching bool HolePunchingOptions []holepunch.Option + + DisableMetrics bool + PrometheusRegisterer prometheus.Registerer } func (cfg *Config) makeSwarm(enableMetrics bool) (*swarm.Swarm, error) { @@ -168,7 +172,8 @@ func (cfg *Config) makeSwarm(enableMetrics bool) (*swarm.Swarm, error) { opts = append(opts, swarm.WithMultiaddrResolver(cfg.MultiaddrResolver)) } if enableMetrics { - opts = append(opts, swarm.WithMetricsTracer(swarm.NewMetricsTracer())) + opts = append(opts, + swarm.WithMetricsTracer(swarm.NewMetricsTracer(swarm.WithRegisterer(cfg.PrometheusRegisterer)))) } // TODO: Make the swarm implementation configurable. return swarm.NewSwarm(pid, cfg.Peerstore, opts...) @@ -279,22 +284,24 @@ func (cfg *Config) addTransports(h host.Host) error { // // This function consumes the config. Do not reuse it (really!). func (cfg *Config) NewNode() (host.Host, error) { - swrm, err := cfg.makeSwarm(true) + swrm, err := cfg.makeSwarm(!cfg.DisableMetrics) if err != nil { return nil, err } h, err := bhost.NewHost(swrm, &bhost.HostOpts{ - ConnManager: cfg.ConnManager, - AddrsFactory: cfg.AddrsFactory, - NATManager: cfg.NATManager, - EnablePing: !cfg.DisablePing, - UserAgent: cfg.UserAgent, - ProtocolVersion: cfg.ProtocolVersion, - EnableHolePunching: cfg.EnableHolePunching, - HolePunchingOptions: cfg.HolePunchingOptions, - EnableRelayService: cfg.EnableRelayService, - RelayServiceOpts: cfg.RelayServiceOpts, + ConnManager: cfg.ConnManager, + AddrsFactory: cfg.AddrsFactory, + NATManager: cfg.NATManager, + EnablePing: !cfg.DisablePing, + UserAgent: cfg.UserAgent, + ProtocolVersion: cfg.ProtocolVersion, + EnableHolePunching: cfg.EnableHolePunching, + HolePunchingOptions: cfg.HolePunchingOptions, + EnableRelayService: cfg.EnableRelayService, + RelayServiceOpts: cfg.RelayServiceOpts, + EnableMetrics: !cfg.DisableMetrics, + PrometheusRegisterer: cfg.PrometheusRegisterer, }) if err != nil { swrm.Close() @@ -354,7 +361,11 @@ func (cfg *Config) NewNode() (host.Host, error) { autonat.UsingAddresses(func() []ma.Multiaddr { return addrF(h.AllAddrs()) }), - autonat.WithMetricsTracer(autonat.NewMetricsTracer()), + } + if !cfg.DisableMetrics { + autonatOpts = append(autonatOpts, + autonat.WithMetricsTracer( + autonat.NewMetricsTracer(autonat.WithRegisterer(cfg.PrometheusRegisterer)))) } if cfg.AutoNATConfig.ThrottleInterval != 0 { autonatOpts = append(autonatOpts, diff --git a/defaults.go b/defaults.go index 72f618a77..c0ed6698a 100644 --- a/defaults.go +++ b/defaults.go @@ -16,6 +16,7 @@ import ( "github.com/libp2p/go-libp2p/p2p/transport/tcp" ws "github.com/libp2p/go-libp2p/p2p/transport/websocket" webtransport "github.com/libp2p/go-libp2p/p2p/transport/webtransport" + "github.com/prometheus/client_golang/prometheus" "github.com/multiformats/go-multiaddr" madns "github.com/multiformats/go-multiaddr-dns" @@ -129,6 +130,11 @@ var DefaultMultiaddrResolver = func(cfg *Config) error { return cfg.Apply(MultiaddrResolver(madns.DefaultResolver)) } +// DefaultPrometheusRegisterer configures libp2p to use the default registerer +var DefaultPrometheusRegisterer = func(cfg *Config) error { + return cfg.Apply(PrometheusRegisterer(prometheus.DefaultRegisterer)) +} + // Complete list of default options and when to fallback on them. // // Please *DON'T* specify default options any other way. Putting this all here @@ -181,6 +187,10 @@ var defaults = []struct { fallback: func(cfg *Config) bool { return cfg.MultiaddrResolver == nil }, opt: DefaultMultiaddrResolver, }, + { + fallback: func(cfg *Config) bool { return !cfg.DisableMetrics && cfg.PrometheusRegisterer == nil }, + opt: DefaultPrometheusRegisterer, + }, } // Defaults configures libp2p to use the default options. Can be combined with diff --git a/options.go b/options.go index c97bc9c8e..1809ec44b 100644 --- a/options.go +++ b/options.go @@ -27,6 +27,7 @@ import ( relayv2 "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/relay" "github.com/libp2p/go-libp2p/p2p/protocol/holepunch" "github.com/libp2p/go-libp2p/p2p/transport/quicreuse" + "github.com/prometheus/client_golang/prometheus" ma "github.com/multiformats/go-multiaddr" madns "github.com/multiformats/go-multiaddr-dns" @@ -548,3 +549,28 @@ func WithDialTimeout(t time.Duration) Option { return nil } } + +// DisableMetrics configures libp2p to disable prometheus metrics +func DisableMetrics() Option { + return func(cfg *Config) error { + cfg.DisableMetrics = true + return nil + } +} + +// PrometheusRegisterer configures libp2p to use reg as the Registerer for all metrics subsystems +func PrometheusRegisterer(reg prometheus.Registerer) Option { + return func(cfg *Config) error { + if cfg.DisableMetrics { + return errors.New("cannot set registerer when metrics are disabled") + } + if cfg.PrometheusRegisterer != nil { + return errors.New("registerer already set") + } + if reg == nil { + return errors.New("registerer cannot be nil") + } + cfg.PrometheusRegisterer = reg + return nil + } +} diff --git a/p2p/host/autonat/metrics.go b/p2p/host/autonat/metrics.go index a13459823..4207d4e7d 100644 --- a/p2p/host/autonat/metrics.go +++ b/p2p/host/autonat/metrics.go @@ -1,7 +1,6 @@ package autonat import ( - "sync" "time" "github.com/libp2p/go-libp2p/core/network" @@ -58,20 +57,15 @@ var ( Help: "Time of next probe", }, ) -) - -var initMetricsOnce sync.Once - -func initMetrics(reg prometheus.Registerer) { - reg.MustRegister( + collectors = []prometheus.Collector{ reachabilityStatus, reachabilityStatusConfidence, receivedDialResponseTotal, outgoingDialResponseTotal, outgoingDialRefusedTotal, nextProbeTimestamp, - ) -} + } +) type MetricsTracer interface { ReachabilityStatus(status network.Reachability) @@ -107,14 +101,32 @@ const ( no_valid_address = "no valid address" ) +type metricsTracer struct{} + +var _ MetricsTracer = &metricsTracer{} + type metricsTracerSetting struct { reg prometheus.Registerer } -type metricsTracer struct { +type MetricsTracerOption func(*metricsTracerSetting) + +func WithRegisterer(reg prometheus.Registerer) MetricsTracerOption { + return func(s *metricsTracerSetting) { + if reg != nil { + s.reg = reg + } + } } -var _ MetricsTracer = &metricsTracer{} +func NewMetricsTracer(opts ...MetricsTracerOption) MetricsTracer { + setting := &metricsTracerSetting{reg: prometheus.DefaultRegisterer} + for _, opt := range opts { + opt(setting) + } + metricshelper.RegisterCollectors(setting.reg, collectors...) + return &metricsTracer{} +} func (mt *metricsTracer) ReachabilityStatus(status network.Reachability) { reachabilityStatus.Set(float64(status)) @@ -148,20 +160,3 @@ func (mt *metricsTracer) OutgoingDialRefused(reason string) { func (mt *metricsTracer) NextProbeTime(t time.Time) { nextProbeTimestamp.Set(float64(t.Unix())) } - -type MetricsTracerOption = func(*metricsTracerSetting) - -func MustRegisterWith(reg prometheus.Registerer) MetricsTracerOption { - return func(s *metricsTracerSetting) { - s.reg = reg - } -} - -func NewMetricsTracer(opts ...MetricsTracerOption) MetricsTracer { - settings := &metricsTracerSetting{reg: prometheus.DefaultRegisterer} - for _, opt := range opts { - opt(settings) - } - initMetricsOnce.Do(func() { initMetrics(settings.reg) }) - return &metricsTracer{} -} diff --git a/p2p/host/basic/basic_host.go b/p2p/host/basic/basic_host.go index a269a4111..707178522 100644 --- a/p2p/host/basic/basic_host.go +++ b/p2p/host/basic/basic_host.go @@ -27,6 +27,7 @@ import ( "github.com/libp2p/go-libp2p/p2p/protocol/holepunch" "github.com/libp2p/go-libp2p/p2p/protocol/identify" "github.com/libp2p/go-libp2p/p2p/protocol/ping" + "github.com/prometheus/client_golang/prometheus" "github.com/libp2p/go-netroute" @@ -151,19 +152,32 @@ type HostOpts struct { EnableHolePunching bool // HolePunchingOptions are options for the hole punching service HolePunchingOptions []holepunch.Option + + // EnableMetrics enables the metrics subsystems + EnableMetrics bool + // PrometheusRegisterer is the PrometheusRegisterer used for metrics + PrometheusRegisterer prometheus.Registerer } // NewHost constructs a new *BasicHost and activates it by attaching its stream and connection handlers to the given inet.Network. func NewHost(n network.Network, opts *HostOpts) (*BasicHost, error) { - eventBus := eventbus.NewBus(eventbus.WithMetricsTracer(eventbus.NewMetricsTracer())) + if opts == nil { + opts = &HostOpts{} + } + + var eventBus event.Bus + if opts.EnableMetrics { + eventBus = eventbus.NewBus( + eventbus.WithMetricsTracer(eventbus.NewMetricsTracer(eventbus.WithRegisterer(opts.PrometheusRegisterer)))) + } else { + eventBus = eventbus.NewBus() + } + psManager, err := pstoremanager.NewPeerstoreManager(n.Peerstore(), eventBus) if err != nil { return nil, err } hostCtx, cancel := context.WithCancel(context.Background()) - if opts == nil { - opts = &HostOpts{} - } h := &BasicHost{ network: n, @@ -223,23 +237,22 @@ func NewHost(n network.Network, opts *HostOpts) (*BasicHost, error) { h.mux = opts.MultistreamMuxer } + idOpts := []identify.Option{ + identify.UserAgent(opts.UserAgent), + identify.ProtocolVersion(opts.ProtocolVersion), + } + // we can't set this as a default above because it depends on the *BasicHost. if h.disableSignedPeerRecord { - h.ids, err = identify.NewIDService( - h, - identify.UserAgent(opts.UserAgent), - identify.ProtocolVersion(opts.ProtocolVersion), - identify.DisableSignedPeerRecord(), - identify.WithMetricsTracer(identify.NewMetricsTracer()), - ) - } else { - h.ids, err = identify.NewIDService( - h, - identify.UserAgent(opts.UserAgent), - identify.ProtocolVersion(opts.ProtocolVersion), - identify.WithMetricsTracer(identify.NewMetricsTracer()), - ) + idOpts = append(idOpts, identify.DisableSignedPeerRecord()) } + if opts.EnableMetrics { + idOpts = append(idOpts, + identify.WithMetricsTracer( + identify.NewMetricsTracer(identify.WithRegisterer(opts.PrometheusRegisterer)))) + } + + h.ids, err = identify.NewIDService(h, idOpts...) if err != nil { return nil, fmt.Errorf("failed to create Identify service: %s", err) } diff --git a/p2p/host/eventbus/basic_metrics.go b/p2p/host/eventbus/basic_metrics.go index 3c306b6ab..8e7b1e88d 100644 --- a/p2p/host/eventbus/basic_metrics.go +++ b/p2p/host/eventbus/basic_metrics.go @@ -3,7 +3,6 @@ package eventbus import ( "reflect" "strings" - "sync" "github.com/libp2p/go-libp2p/p2p/metricshelper" @@ -53,6 +52,13 @@ var ( }, []string{"subscriber_name"}, ) + collectors = []prometheus.Collector{ + eventsEmitted, + totalSubscribers, + subscriberQueueLength, + subscriberQueueFull, + subscriberEventQueued, + } ) // MetricsTracer tracks metrics for the eventbus subsystem @@ -81,30 +87,26 @@ type metricsTracer struct{} var _ MetricsTracer = &metricsTracer{} -type MetricsTracerOption = func(*metricsTracerSetting) - type metricsTracerSetting struct { reg prometheus.Registerer } -var initMetricsOnce sync.Once - -func initMetrics(reg prometheus.Registerer) { - reg.MustRegister(eventsEmitted, totalSubscribers, subscriberQueueLength, subscriberQueueFull, subscriberEventQueued) -} +type MetricsTracerOption func(*metricsTracerSetting) -func MustRegisterWith(reg prometheus.Registerer) MetricsTracerOption { +func WithRegisterer(reg prometheus.Registerer) MetricsTracerOption { return func(s *metricsTracerSetting) { - s.reg = reg + if reg != nil { + s.reg = reg + } } } func NewMetricsTracer(opts ...MetricsTracerOption) MetricsTracer { - settings := &metricsTracerSetting{reg: prometheus.DefaultRegisterer} + setting := &metricsTracerSetting{reg: prometheus.DefaultRegisterer} for _, opt := range opts { - opt(settings) + opt(setting) } - initMetricsOnce.Do(func() { initMetrics(settings.reg) }) + metricshelper.RegisterCollectors(setting.reg, collectors...) return &metricsTracer{} } diff --git a/p2p/metricshelper/registerer.go b/p2p/metricshelper/registerer.go new file mode 100644 index 000000000..99027c0db --- /dev/null +++ b/p2p/metricshelper/registerer.go @@ -0,0 +1,20 @@ +package metricshelper + +import ( + "errors" + + "github.com/prometheus/client_golang/prometheus" +) + +// RegisterCollectors registers the collectors with reg ignoring +// reregistration error and panics on any other error +func RegisterCollectors(reg prometheus.Registerer, collectors ...prometheus.Collector) { + for _, c := range collectors { + err := reg.Register(c) + if err != nil { + if ok := errors.As(err, &prometheus.AlreadyRegisteredError{}); !ok { + panic(err) + } + } + } +} diff --git a/p2p/metricshelper/registerer_test.go b/p2p/metricshelper/registerer_test.go new file mode 100644 index 000000000..b4c550376 --- /dev/null +++ b/p2p/metricshelper/registerer_test.go @@ -0,0 +1,32 @@ +package metricshelper + +import ( + "testing" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/stretchr/testify/require" +) + +func TestRegisterCollectors(t *testing.T) { + reg := prometheus.NewRegistry() + c1 := prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "counter", + }, + ) + c2 := prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: "test", + Name: "gauge", + }, + ) + // c3 == c1 + c3 := prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "counter", + }, + ) + require.NotPanics(t, func() { RegisterCollectors(reg, c1, c2) }) + require.NotPanics(t, func() { RegisterCollectors(reg, c3) }, "should not panic on duplicate registration") +} diff --git a/p2p/net/swarm/swarm_metrics.go b/p2p/net/swarm/swarm_metrics.go index 6af2cf919..c1c83c3f0 100644 --- a/p2p/net/swarm/swarm_metrics.go +++ b/p2p/net/swarm/swarm_metrics.go @@ -5,7 +5,6 @@ import ( "errors" "net" "strings" - "sync" "time" "github.com/libp2p/go-libp2p/core/crypto" @@ -70,14 +69,16 @@ var ( }, []string{"transport", "security", "muxer", "ip_version"}, ) + collectors = []prometheus.Collector{ + connsOpened, + keyTypes, + connsClosed, + dialError, + connDuration, + connHandshakeLatency, + } ) -var initMetricsOnce sync.Once - -func initMetrics() { - prometheus.MustRegister(connsOpened, keyTypes, connsClosed, dialError, connDuration, connHandshakeLatency) -} - type MetricsTracer interface { OpenedConnection(network.Direction, crypto.PubKey, network.ConnectionState, ma.Multiaddr) ClosedConnection(network.Direction, time.Duration, network.ConnectionState, ma.Multiaddr) @@ -89,8 +90,26 @@ type metricsTracer struct{} var _ MetricsTracer = &metricsTracer{} -func NewMetricsTracer() *metricsTracer { - initMetricsOnce.Do(initMetrics) +type metricsTracerSetting struct { + reg prometheus.Registerer +} + +type MetricsTracerOption func(*metricsTracerSetting) + +func WithRegisterer(reg prometheus.Registerer) MetricsTracerOption { + return func(s *metricsTracerSetting) { + if reg != nil { + s.reg = reg + } + } +} + +func NewMetricsTracer(opts ...MetricsTracerOption) MetricsTracer { + setting := &metricsTracerSetting{reg: prometheus.DefaultRegisterer} + for _, opt := range opts { + opt(setting) + } + metricshelper.RegisterCollectors(setting.reg, collectors...) return &metricsTracer{} } diff --git a/p2p/protocol/identify/metrics.go b/p2p/protocol/identify/metrics.go index f6a40f432..f8ffc6ddd 100644 --- a/p2p/protocol/identify/metrics.go +++ b/p2p/protocol/identify/metrics.go @@ -1,8 +1,6 @@ package identify import ( - "sync" - "github.com/libp2p/go-libp2p/core/event" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/p2p/metricshelper" @@ -37,14 +35,13 @@ var ( }, []string{"dir"}, ) + collectors = []prometheus.Collector{ + pushesTriggered, + identify, + identifyPush, + } ) -var initMetricsOnce sync.Once - -func initMetrics() { - prometheus.MustRegister(pushesTriggered, identify, identifyPush) -} - type MetricsTracer interface { TriggeredPushes(event any) Identify(network.Direction) @@ -55,8 +52,26 @@ type metricsTracer struct{} var _ MetricsTracer = &metricsTracer{} -func NewMetricsTracer() MetricsTracer { - initMetricsOnce.Do(initMetrics) +type metricsTracerSetting struct { + reg prometheus.Registerer +} + +type MetricsTracerOption func(*metricsTracerSetting) + +func WithRegisterer(reg prometheus.Registerer) MetricsTracerOption { + return func(s *metricsTracerSetting) { + if reg != nil { + s.reg = reg + } + } +} + +func NewMetricsTracer(opts ...MetricsTracerOption) MetricsTracer { + setting := &metricsTracerSetting{reg: prometheus.DefaultRegisterer} + for _, opt := range opts { + opt(setting) + } + metricshelper.RegisterCollectors(setting.reg, collectors...) return &metricsTracer{} }