From 4ea04c23b3f9ef455d7e67b3fd7831f81566beaa Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 17 Oct 2018 17:03:59 +0300 Subject: [PATCH 01/36] identify: implement identify push protocol --- p2p/protocol/identify/id.go | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/p2p/protocol/identify/id.go b/p2p/protocol/identify/id.go index 2d32d1f41..d49bdf28a 100644 --- a/p2p/protocol/identify/id.go +++ b/p2p/protocol/identify/id.go @@ -3,6 +3,7 @@ package identify import ( "context" "sync" + "time" pb "github.com/libp2p/go-libp2p/p2p/protocol/identify/pb" @@ -23,6 +24,9 @@ var log = logging.Logger("net/identify") // ID is the protocol.ID of the Identify Service. const ID = "/ipfs/id/1.0.0" +// IDPush is the protocol.ID of the Identify push protocol +const IDPush = "/ipfs/id/push/1.0.0" + // LibP2PVersion holds the current protocol version for a client running this code // TODO(jbenet): fix the versioning mess. const LibP2PVersion = "ipfs/0.1.0" @@ -60,6 +64,7 @@ func NewIDService(h host.Host) *IDService { currid: make(map[inet.Conn]chan struct{}), } h.SetStreamHandler(ID, s.requestHandler) + h.SetStreamHandler(IDPush, s.pushHandler) h.Network().Notify((*netNotifiee)(s)) return s } @@ -138,6 +143,24 @@ func (ids *IDService) responseHandler(s inet.Stream) { go inet.FullClose(s) } +func (ids *IDService) pushHandler(s inet.Stream) { + ids.responseHandler(s) +} + +func (ids *IDService) Push() { + for _, p := range ids.Host.Network().Peers() { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + s, err := ids.Host.NewStream(ctx, p, IDPush) + cancel() + if err != nil { + log.Debugf("error opening push stream: %s", err.Error()) + continue + } + + ids.requestHandler(s) + } +} + func (ids *IDService) populateMessage(mes *pb.Identify, c inet.Conn) { // set protocols this node is currently handling From a89e74a1553db4b110cd2bf55759f2529752be59 Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 17 Oct 2018 17:04:46 +0300 Subject: [PATCH 02/36] basic host: export AddrsFactory, provide method to push identify --- p2p/host/basic/basic_host.go | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/p2p/host/basic/basic_host.go b/p2p/host/basic/basic_host.go index d1bd0d653..b5d2108b9 100644 --- a/p2p/host/basic/basic_host.go +++ b/p2p/host/basic/basic_host.go @@ -59,10 +59,11 @@ type BasicHost struct { ids *identify.IDService pings *ping.PingService natmgr NATManager - addrs AddrsFactory maResolver *madns.Resolver cmgr ifconnmgr.ConnManager + AddrsFactory AddrsFactory + negtimeout time.Duration proc goprocess.Process @@ -106,11 +107,11 @@ type HostOpts struct { // NewHost constructs a new *BasicHost and activates it by attaching its stream and connection handlers to the given inet.Network. func NewHost(ctx context.Context, net inet.Network, opts *HostOpts) (*BasicHost, error) { h := &BasicHost{ - network: net, - mux: msmux.NewMultistreamMuxer(), - negtimeout: DefaultNegotiationTimeout, - addrs: DefaultAddrsFactory, - maResolver: madns.DefaultResolver, + network: net, + mux: msmux.NewMultistreamMuxer(), + negtimeout: DefaultNegotiationTimeout, + AddrsFactory: DefaultAddrsFactory, + maResolver: madns.DefaultResolver, } h.proc = goprocessctx.WithContextAndTeardown(ctx, func() error { @@ -136,7 +137,7 @@ func NewHost(ctx context.Context, net inet.Network, opts *HostOpts) (*BasicHost, } if opts.AddrsFactory != nil { - h.addrs = opts.AddrsFactory + h.AddrsFactory = opts.AddrsFactory } if opts.NATManager != nil { @@ -256,6 +257,11 @@ func (h *BasicHost) newStreamHandler(s inet.Stream) { go handle(protoID, s) } +// PushIdentify pushes an identify update through the identify push protocol +func (h *BasicHost) PushIdentify() { + h.ids.Push() +} + // ID returns the (local) peer.ID associated with this Host func (h *BasicHost) ID() peer.ID { return h.Network().LocalPeer() @@ -474,7 +480,7 @@ func (h *BasicHost) ConnManager() ifconnmgr.ConnManager { // Addrs returns listening addresses that are safe to announce to the network. // The output is the same as AllAddrs, but processed by AddrsFactory. func (h *BasicHost) Addrs() []ma.Multiaddr { - return h.addrs(h.AllAddrs()) + return h.AddrsFactory(h.AllAddrs()) } // mergeAddrs merges input address lists, leave only unique addresses From a5858912d6da151ea5c95ba62742bbdbb9ad3ac6 Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 17 Oct 2018 17:05:15 +0300 Subject: [PATCH 03/36] autorelay hosts --- p2p/host/relay/autorelay.go | 252 ++++++++++++++++++++++++++++++++++++ p2p/host/relay/log.go | 7 + p2p/host/relay/relay.go | 23 ++++ 3 files changed, 282 insertions(+) create mode 100644 p2p/host/relay/autorelay.go create mode 100644 p2p/host/relay/log.go create mode 100644 p2p/host/relay/relay.go diff --git a/p2p/host/relay/autorelay.go b/p2p/host/relay/autorelay.go new file mode 100644 index 000000000..09a9e9884 --- /dev/null +++ b/p2p/host/relay/autorelay.go @@ -0,0 +1,252 @@ +package relay + +import ( + "context" + "fmt" + "math/rand" + "sync" + "time" + + basic "github.com/libp2p/go-libp2p/p2p/host/basic" + + autonat "github.com/libp2p/go-libp2p-autonat" + discovery "github.com/libp2p/go-libp2p-discovery" + host "github.com/libp2p/go-libp2p-host" + inet "github.com/libp2p/go-libp2p-net" + peer "github.com/libp2p/go-libp2p-peer" + pstore "github.com/libp2p/go-libp2p-peerstore" + ma "github.com/multiformats/go-multiaddr" + manet "github.com/multiformats/go-multiaddr-net" +) + +var DesiredRelays = 3 + +// AutoRelayHost is a Host that uses relays for connectivity when a NAT is detected. +type AutoRelayHost struct { + *basic.BasicHost + discover discovery.Discoverer + autonat autonat.AutoNAT + addrsF basic.AddrsFactory + + disconnect chan struct{} + + mx sync.Mutex + relays map[peer.ID]pstore.PeerInfo + addrs []ma.Multiaddr +} + +func NewAutoRelayHost(ctx context.Context, bhost *basic.BasicHost, discover discovery.Discoverer) *AutoRelayHost { + autonat := autonat.NewAutoNAT(ctx, bhost) + h := &AutoRelayHost{ + BasicHost: bhost, + discover: discover, + autonat: autonat, + addrsF: bhost.AddrsFactory, + relays: make(map[peer.ID]pstore.PeerInfo), + disconnect: make(chan struct{}, 1), + } + bhost.AddrsFactory = h.hostAddrs + bhost.Network().Notify(h) + go h.background(ctx) + return h +} + +func (h *AutoRelayHost) hostAddrs(addrs []ma.Multiaddr) []ma.Multiaddr { + h.mx.Lock() + defer h.mx.Unlock() + if h.addrs != nil && h.autonat.Status() == autonat.NATStatusPrivate { + return h.addrs + } else { + return h.addrsF(addrs) + } +} + +func (h *AutoRelayHost) background(ctx context.Context) { + select { + case <-time.After(autonat.AutoNATBootDelay + 30*time.Second): + case <-ctx.Done(): + return + } + + for { + wait := autonat.AutoNATRefreshInterval + switch h.autonat.Status() { + case autonat.NATStatusUnknown: + wait = autonat.AutoNATRetryInterval + case autonat.NATStatusPublic: + case autonat.NATStatusPrivate: + h.findRelays(ctx) + } + + select { + case <-h.disconnect: + // invalidate addrs + h.mx.Lock() + h.addrs = nil + h.mx.Unlock() + case <-time.After(wait): + case <-ctx.Done(): + return + } + } +} + +func (h *AutoRelayHost) findRelays(ctx context.Context) { + h.mx.Lock() + if len(h.relays) >= DesiredRelays { + h.mx.Unlock() + return + } + need := DesiredRelays - len(h.relays) + h.mx.Unlock() + + limit := 20 + for ; need > limit; limit *= 2 { + } + + dctx, cancel := context.WithTimeout(ctx, 60*time.Second) + pis, err := discovery.FindPeers(dctx, h.discover, "/libp2p/relay", limit) + cancel() + if err != nil { + log.Debugf("error discovering relays: %s", err.Error()) + return + } + + // TODO better relay selection strategy; this just selects random relays + // but we should probably use ping latency as the selection metric + shuffleRelays(pis) + + update := 0 + + for _, pi := range pis { + h.mx.Lock() + if _, ok := h.relays[pi.ID]; ok { + h.mx.Unlock() + continue + } + h.mx.Unlock() + + cctx, cancel := context.WithTimeout(ctx, 60*time.Second) + err = h.Connect(cctx, pi) + cancel() + if err != nil { + log.Debugf("error connecting to relay %s: %s", pi.ID, err.Error()) + continue + } + + log.Debugf("connected to relay %s", pi.ID) + h.mx.Lock() + h.relays[pi.ID] = pi + h.mx.Unlock() + + update++ + need-- + if need == 0 { + break + } + } + + if update > 0 || h.addrs == nil { + h.updateAddrs() + } +} + +func (h *AutoRelayHost) updateAddrs() { + h.doUpdateAddrs() + h.PushIdentify() +} + +func (h *AutoRelayHost) doUpdateAddrs() { + h.mx.Lock() + defer h.mx.Unlock() + + addrs := h.addrsF(h.AllAddrs()) + raddrs := make([]ma.Multiaddr, 0, len(addrs)+len(h.relays)) + + // remove our public addresses from the list and replace them by just the public IP + for _, addr := range addrs { + if manet.IsPublicAddr(addr) { + ip, err := addr.ValueForProtocol(ma.P_IP4) + if err == nil { + pub, err := ma.NewMultiaddr(fmt.Sprintf("/ip4/%s", ip)) + if err != nil { + panic(err) + } + + if !containsAddr(raddrs, pub) { + raddrs = append(raddrs, pub) + } + continue + } + + ip, err = addr.ValueForProtocol(ma.P_IP6) + if err == nil { + pub, err := ma.NewMultiaddr(fmt.Sprintf("/ip6/%s", ip)) + if err != nil { + panic(err) + } + if !containsAddr(raddrs, pub) { + raddrs = append(raddrs, pub) + } + continue + } + } else { + raddrs = append(raddrs, addr) + } + } + + circuit, err := ma.NewMultiaddr("/p2p-circuit") + if err != nil { + panic(err) + } + + for _, pi := range h.relays { + for _, addr := range pi.Addrs { + if !manet.IsPrivateAddr(addr) { + pub := addr.Encapsulate(circuit) + raddrs = append(raddrs, pub) + } + } + } + + h.addrs = raddrs +} + +func shuffleRelays(pis []pstore.PeerInfo) { + for i := range pis { + j := rand.Intn(i + 1) + pis[i], pis[j] = pis[j], pis[i] + } +} + +func containsAddr(lst []ma.Multiaddr, addr ma.Multiaddr) bool { + for _, xaddr := range lst { + if xaddr.Equal(addr) { + return true + } + } + return false +} + +// notify +func (h *AutoRelayHost) Listen(inet.Network, ma.Multiaddr) {} +func (h *AutoRelayHost) ListenClose(inet.Network, ma.Multiaddr) {} +func (h *AutoRelayHost) Connected(inet.Network, inet.Conn) {} + +func (h *AutoRelayHost) Disconnected(_ inet.Network, c inet.Conn) { + p := c.RemotePeer() + h.mx.Lock() + defer h.mx.Unlock() + if _, ok := h.relays[p]; ok { + delete(h.relays, p) + select { + case h.disconnect <- struct{}{}: + default: + } + } +} + +func (h *AutoRelayHost) OpenedStream(inet.Network, inet.Stream) {} +func (h *AutoRelayHost) ClosedStream(inet.Network, inet.Stream) {} + +var _ host.Host = (*AutoRelayHost)(nil) diff --git a/p2p/host/relay/log.go b/p2p/host/relay/log.go new file mode 100644 index 000000000..9671dc760 --- /dev/null +++ b/p2p/host/relay/log.go @@ -0,0 +1,7 @@ +package relay + +import ( + logging "github.com/ipfs/go-log" +) + +var log = logging.Logger("relay") diff --git a/p2p/host/relay/relay.go b/p2p/host/relay/relay.go new file mode 100644 index 000000000..95b5079c4 --- /dev/null +++ b/p2p/host/relay/relay.go @@ -0,0 +1,23 @@ +package relay + +import ( + "context" + + discovery "github.com/libp2p/go-libp2p-discovery" + host "github.com/libp2p/go-libp2p-host" +) + +// RelayHost is a Host that provides Relay services. +type RelayHost struct { + host.Host + advertise discovery.Advertiser +} + +// New constructs a new RelayHost +func NewRelayHost(ctx context.Context, host host.Host, advertise discovery.Advertiser) *RelayHost { + h := &RelayHost{Host: host, advertise: advertise} + discovery.Advertise(ctx, advertise, "/libp2p/relay") + return h +} + +var _ host.Host = (*RelayHost)(nil) From 9795a01ba31444b1a3807d5b10f14e2cf229c505 Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 17 Oct 2018 17:13:55 +0300 Subject: [PATCH 04/36] configurable boot delay for autorelay --- p2p/host/relay/autorelay.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/p2p/host/relay/autorelay.go b/p2p/host/relay/autorelay.go index 09a9e9884..164b72be4 100644 --- a/p2p/host/relay/autorelay.go +++ b/p2p/host/relay/autorelay.go @@ -19,7 +19,11 @@ import ( manet "github.com/multiformats/go-multiaddr-net" ) -var DesiredRelays = 3 +var ( + DesiredRelays = 3 + + BootDelay = 90 * time.Second +) // AutoRelayHost is a Host that uses relays for connectivity when a NAT is detected. type AutoRelayHost struct { @@ -63,7 +67,7 @@ func (h *AutoRelayHost) hostAddrs(addrs []ma.Multiaddr) []ma.Multiaddr { func (h *AutoRelayHost) background(ctx context.Context) { select { - case <-time.After(autonat.AutoNATBootDelay + 30*time.Second): + case <-time.After(autonat.AutoNATBootDelay + BootDelay): case <-ctx.Done(): return } From 5c623f269fbf0391dd684f3593f9c4a95ee5936f Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 17 Oct 2018 17:19:13 +0300 Subject: [PATCH 05/36] name the autorelay logger as such --- p2p/host/relay/log.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/host/relay/log.go b/p2p/host/relay/log.go index 9671dc760..eca0fa45a 100644 --- a/p2p/host/relay/log.go +++ b/p2p/host/relay/log.go @@ -4,4 +4,4 @@ import ( logging "github.com/ipfs/go-log" ) -var log = logging.Logger("relay") +var log = logging.Logger("autorelay") From d24fe6ae9161e8e6ce475fe8e7554d2e2c9fdbcb Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 17 Oct 2018 18:24:54 +0300 Subject: [PATCH 06/36] update libp2p.New constructor to construct relayed/routed hosts --- config/config.go | 46 +++++++++++++++++++++++++++++++++++++++++++--- options.go | 11 +++++++++++ 2 files changed, 54 insertions(+), 3 deletions(-) diff --git a/config/config.go b/config/config.go index dd3d3996b..aa06705c4 100644 --- a/config/config.go +++ b/config/config.go @@ -5,10 +5,13 @@ import ( "fmt" bhost "github.com/libp2p/go-libp2p/p2p/host/basic" + relay "github.com/libp2p/go-libp2p/p2p/host/relay" + routed "github.com/libp2p/go-libp2p/p2p/host/routed" logging "github.com/ipfs/go-log" circuit "github.com/libp2p/go-libp2p-circuit" crypto "github.com/libp2p/go-libp2p-crypto" + discovery "github.com/libp2p/go-libp2p-discovery" host "github.com/libp2p/go-libp2p-host" ifconnmgr "github.com/libp2p/go-libp2p-interface-connmgr" pnet "github.com/libp2p/go-libp2p-interface-pnet" @@ -16,6 +19,7 @@ import ( inet "github.com/libp2p/go-libp2p-net" peer "github.com/libp2p/go-libp2p-peer" pstore "github.com/libp2p/go-libp2p-peerstore" + routing "github.com/libp2p/go-libp2p-routing" swarm "github.com/libp2p/go-libp2p-swarm" tptu "github.com/libp2p/go-libp2p-transport-upgrader" filter "github.com/libp2p/go-maddr-filter" @@ -31,6 +35,13 @@ type AddrsFactory = bhost.AddrsFactory // NATManagerC is a NATManager constructor. type NATManagerC func(inet.Network) bhost.NATManager +type Routing interface { + routing.ContentRouting + routing.PeerRouting +} + +type RoutingC func(host.Host) (Routing, error) + // Config describes a set of settings for a libp2p node // // This is *not* a stable interface. Use the options defined in the root @@ -58,6 +69,8 @@ type Config struct { Reporter metrics.Reporter DisablePing bool + + Routing RoutingC } // NewNode constructs a new libp2p Host from the Config. @@ -99,8 +112,8 @@ func (cfg *Config) NewNode(ctx context.Context) (host.Host, error) { swrm.Filters = cfg.Filters } - // TODO: make host implementation configurable. - h, err := bhost.NewHost(ctx, swrm, &bhost.HostOpts{ + var h host.Host + h, err = bhost.NewHost(ctx, swrm, &bhost.HostOpts{ ConnManager: cfg.ConnManager, AddrsFactory: cfg.AddrsFactory, NATManager: cfg.NATManager, @@ -158,7 +171,34 @@ func (cfg *Config) NewNode(ctx context.Context) (host.Host, error) { return nil, err } - // TODO: Configure routing (it's a pain to setup). + if cfg.Routing != nil { + router, err := cfg.Routing(h) + if err != nil { + h.Close() + return nil, err + } + + if cfg.Relay { + discovery := discovery.NewRoutingDiscovery(router) + + hop := false + for _, opt := range cfg.RelayOpts { + if opt == circuit.OptHop { + hop = true + break + } + } + + if hop { + h = relay.NewRelayHost(swrm.Context(), h.(*bhost.BasicHost), discovery) + } else { + h = relay.NewAutoRelayHost(swrm.Context(), h.(*bhost.BasicHost), discovery) + } + } + + h = routed.Wrap(h, router) + } + // TODO: Bootstrapping. return h, nil diff --git a/options.go b/options.go index 4ab4f39c7..23c9f7d22 100644 --- a/options.go +++ b/options.go @@ -260,6 +260,17 @@ func Ping(enable bool) Option { } } +// Routing will configure libp2p to use routing. +func Routing(rt config.RoutingC) Option { + return func(cfg *Config) error { + if cfg.Routing != nil { + return fmt.Errorf("cannot specified multiple routing options") + } + cfg.Routing = rt + return nil + } +} + // NoListenAddrs will configure libp2p to not listen by default. // // This will both clear any configured listen addrs and prevent libp2p from From 6bdfcac0b4984c773cb36761df4f13dc4e3a5de6 Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 17 Oct 2018 23:53:28 +0300 Subject: [PATCH 07/36] use AllAddrs as the address factory in autonat --- p2p/host/relay/autorelay.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/host/relay/autorelay.go b/p2p/host/relay/autorelay.go index 164b72be4..409a5e25f 100644 --- a/p2p/host/relay/autorelay.go +++ b/p2p/host/relay/autorelay.go @@ -40,7 +40,7 @@ type AutoRelayHost struct { } func NewAutoRelayHost(ctx context.Context, bhost *basic.BasicHost, discover discovery.Discoverer) *AutoRelayHost { - autonat := autonat.NewAutoNAT(ctx, bhost) + autonat := autonat.NewAutoNAT(ctx, bhost, bhost.AllAddrs) h := &AutoRelayHost{ BasicHost: bhost, discover: discover, From 0ff739334949f534b4b57307a8bb3beb98c3ad6d Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 18 Oct 2018 11:32:35 +0300 Subject: [PATCH 08/36] fix announced relay address --- p2p/host/relay/autorelay.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/p2p/host/relay/autorelay.go b/p2p/host/relay/autorelay.go index 409a5e25f..18858795b 100644 --- a/p2p/host/relay/autorelay.go +++ b/p2p/host/relay/autorelay.go @@ -199,12 +199,12 @@ func (h *AutoRelayHost) doUpdateAddrs() { } } - circuit, err := ma.NewMultiaddr("/p2p-circuit") - if err != nil { - panic(err) - } - for _, pi := range h.relays { + circuit, err := ma.NewMultiaddr(fmt.Sprintf("/ipfs/%s/p2p-circuit", pi.ID.Pretty())) + if err != nil { + panic(err) + } + for _, addr := range pi.Addrs { if !manet.IsPrivateAddr(addr) { pub := addr.Encapsulate(circuit) From 477c19aac12bb1217e3a2c0e855ca373e60f4e1c Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 18 Oct 2018 11:48:13 +0300 Subject: [PATCH 09/36] reduce boot delay to 60s --- p2p/host/relay/autorelay.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/host/relay/autorelay.go b/p2p/host/relay/autorelay.go index 18858795b..aaae496ac 100644 --- a/p2p/host/relay/autorelay.go +++ b/p2p/host/relay/autorelay.go @@ -22,7 +22,7 @@ import ( var ( DesiredRelays = 3 - BootDelay = 90 * time.Second + BootDelay = 60 * time.Second ) // AutoRelayHost is a Host that uses relays for connectivity when a NAT is detected. From 7fabe39290271ea6b8a2b52db1140e20d6563a14 Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 18 Oct 2018 12:00:22 +0300 Subject: [PATCH 10/36] parallel identify push --- p2p/protocol/identify/id.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/p2p/protocol/identify/id.go b/p2p/protocol/identify/id.go index d49bdf28a..5991fe695 100644 --- a/p2p/protocol/identify/id.go +++ b/p2p/protocol/identify/id.go @@ -149,15 +149,17 @@ func (ids *IDService) pushHandler(s inet.Stream) { func (ids *IDService) Push() { for _, p := range ids.Host.Network().Peers() { - ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) - s, err := ids.Host.NewStream(ctx, p, IDPush) - cancel() - if err != nil { - log.Debugf("error opening push stream: %s", err.Error()) - continue - } + go func(p peer.ID) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + s, err := ids.Host.NewStream(ctx, p, IDPush) + if err != nil { + log.Debugf("error opening push stream: %s", err.Error()) + return + } - ids.requestHandler(s) + ids.requestHandler(s) + }(p) } } From c6d1eebf15dae80ea5684566f2fa8bcbd5647c0e Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 18 Oct 2018 16:47:40 +0300 Subject: [PATCH 11/36] autorelay test --- p2p/host/relay/autorelay_test.go | 169 +++++++++++++++++++++++++++++++ 1 file changed, 169 insertions(+) create mode 100644 p2p/host/relay/autorelay_test.go diff --git a/p2p/host/relay/autorelay_test.go b/p2p/host/relay/autorelay_test.go new file mode 100644 index 000000000..f4e2d3a4e --- /dev/null +++ b/p2p/host/relay/autorelay_test.go @@ -0,0 +1,169 @@ +package relay_test + +import ( + "context" + "net" + "sync" + "testing" + "time" + + libp2p "github.com/libp2p/go-libp2p" + config "github.com/libp2p/go-libp2p/config" + relay "github.com/libp2p/go-libp2p/p2p/host/relay" + + ggio "github.com/gogo/protobuf/io" + cid "github.com/ipfs/go-cid" + autonat "github.com/libp2p/go-libp2p-autonat" + autonatpb "github.com/libp2p/go-libp2p-autonat/pb" + circuit "github.com/libp2p/go-libp2p-circuit" + host "github.com/libp2p/go-libp2p-host" + inet "github.com/libp2p/go-libp2p-net" + peer "github.com/libp2p/go-libp2p-peer" + pstore "github.com/libp2p/go-libp2p-peerstore" + routing "github.com/libp2p/go-libp2p-routing" + manet "github.com/multiformats/go-multiaddr-net" +) + +// test specific parameters +func init() { + autonat.AutoNATIdentifyDelay = 10 * time.Millisecond + autonat.AutoNATBootDelay = 1 * time.Second + relay.BootDelay = 1 * time.Second + manet.Private4 = []*net.IPNet{} +} + +// mock routing +type mockRoutingTable struct { + mx sync.Mutex + providers map[string]map[peer.ID]pstore.PeerInfo +} + +type mockRouting struct { + h host.Host + tab *mockRoutingTable +} + +func newMockRoutingTable() *mockRoutingTable { + return &mockRoutingTable{providers: make(map[string]map[peer.ID]pstore.PeerInfo)} +} + +func newMockRouting(h host.Host, tab *mockRoutingTable) *mockRouting { + return &mockRouting{h: h, tab: tab} +} + +func (m *mockRouting) FindPeer(ctx context.Context, p peer.ID) (pstore.PeerInfo, error) { + return pstore.PeerInfo{}, routing.ErrNotFound +} + +func (m *mockRouting) Provide(ctx context.Context, cid cid.Cid, bcast bool) error { + m.tab.mx.Lock() + defer m.tab.mx.Unlock() + + pmap, ok := m.tab.providers[cid.String()] + if !ok { + pmap = make(map[peer.ID]pstore.PeerInfo) + m.tab.providers[cid.String()] = pmap + } + + pmap[m.h.ID()] = pstore.PeerInfo{ID: m.h.ID(), Addrs: m.h.Addrs()} + + return nil +} + +func (m *mockRouting) FindProvidersAsync(ctx context.Context, cid cid.Cid, limit int) <-chan pstore.PeerInfo { + ch := make(chan pstore.PeerInfo) + go func() { + defer close(ch) + m.tab.mx.Lock() + defer m.tab.mx.Unlock() + + pmap, ok := m.tab.providers[cid.String()] + if !ok { + return + } + + for _, pi := range pmap { + select { + case ch <- pi: + case <-ctx.Done(): + return + } + } + }() + + return ch +} + +// mock autonat +func makeAutoNATServicePrivate(ctx context.Context, t *testing.T) host.Host { + h, err := libp2p.New(ctx) + if err != nil { + t.Fatal(err) + } + h.SetStreamHandler(autonat.AutoNATProto, sayAutoNATPrivate) + return h +} + +func sayAutoNATPrivate(s inet.Stream) { + defer s.Close() + w := ggio.NewDelimitedWriter(s) + res := autonatpb.Message{ + Type: autonatpb.Message_DIAL_RESPONSE.Enum(), + DialResponse: newDialResponseError(autonatpb.Message_E_DIAL_ERROR, "no dialable addresses"), + } + w.WriteMsg(&res) +} + +func newDialResponseError(status autonatpb.Message_ResponseStatus, text string) *autonatpb.Message_DialResponse { + dr := new(autonatpb.Message_DialResponse) + dr.Status = status.Enum() + dr.StatusText = &text + return dr +} + +// connector +func connect(t *testing.T, a, b host.Host) { + pinfo := pstore.PeerInfo{ID: a.ID(), Addrs: a.Addrs()} + err := b.Connect(context.Background(), pinfo) + if err != nil { + t.Fatal(err) + } +} + +// and the actual test! +func TestAutoRelay(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + mtab := newMockRoutingTable() + makeRouting := func(h host.Host) (config.Routing, error) { + mr := newMockRouting(h, mtab) + return mr, nil + } + + h1 := makeAutoNATServicePrivate(ctx, t) + _, err := libp2p.New(ctx, libp2p.EnableRelay(circuit.OptHop), libp2p.Routing(makeRouting)) + if err != nil { + t.Fatal(err) + } + h3, err := libp2p.New(ctx, libp2p.EnableRelay(), libp2p.Routing(makeRouting)) + if err != nil { + t.Fatal(err) + } + + connect(t, h1, h3) + time.Sleep(3 * time.Second) + + haveRelay := false + for _, addr := range h3.Addrs() { + _, err := addr.ValueForProtocol(circuit.P_CIRCUIT) + if err != nil { + haveRelay = true + break + } + } + + if !haveRelay { + t.Fatal("No relay addrs advertised") + } +} From f6c38c9d9191eb5a431136cc473e995b478783e9 Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 18 Oct 2018 17:17:49 +0300 Subject: [PATCH 12/36] filter unspecific relay address --- p2p/host/relay/autorelay.go | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/p2p/host/relay/autorelay.go b/p2p/host/relay/autorelay.go index aaae496ac..13020793c 100644 --- a/p2p/host/relay/autorelay.go +++ b/p2p/host/relay/autorelay.go @@ -10,6 +10,7 @@ import ( basic "github.com/libp2p/go-libp2p/p2p/host/basic" autonat "github.com/libp2p/go-libp2p-autonat" + _ "github.com/libp2p/go-libp2p-circuit" discovery "github.com/libp2p/go-libp2p-discovery" host "github.com/libp2p/go-libp2p-host" inet "github.com/libp2p/go-libp2p-net" @@ -23,8 +24,18 @@ var ( DesiredRelays = 3 BootDelay = 60 * time.Second + + unspecificRelay ma.Multiaddr ) +func init() { + var err error + unspecificRelay, err = ma.NewMultiaddr("/p2p-circuit") + if err != nil { + panic(err) + } +} + // AutoRelayHost is a Host that uses relays for connectivity when a NAT is detected. type AutoRelayHost struct { *basic.BasicHost @@ -61,7 +72,7 @@ func (h *AutoRelayHost) hostAddrs(addrs []ma.Multiaddr) []ma.Multiaddr { if h.addrs != nil && h.autonat.Status() == autonat.NATStatusPrivate { return h.addrs } else { - return h.addrsF(addrs) + return filterUnspecificRelay(h.addrsF(addrs)) } } @@ -164,7 +175,7 @@ func (h *AutoRelayHost) doUpdateAddrs() { h.mx.Lock() defer h.mx.Unlock() - addrs := h.addrsF(h.AllAddrs()) + addrs := filterUnspecificRelay(h.addrsF(h.AllAddrs())) raddrs := make([]ma.Multiaddr, 0, len(addrs)+len(h.relays)) // remove our public addresses from the list and replace them by just the public IP @@ -216,6 +227,17 @@ func (h *AutoRelayHost) doUpdateAddrs() { h.addrs = raddrs } +func filterUnspecificRelay(addrs []ma.Multiaddr) []ma.Multiaddr { + res := make([]ma.Multiaddr, 0, len(addrs)) + for _, addr := range addrs { + if addr.Equal(unspecificRelay) { + continue + } + res = append(res, addr) + } + return res +} + func shuffleRelays(pis []pstore.PeerInfo) { for i := range pis { j := rand.Intn(i + 1) From 8d108dddcb24d9efce386c63df444b7a04160335 Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 18 Oct 2018 18:22:49 +0300 Subject: [PATCH 13/36] import go-libp2p-discovery and go-libp2p-autonat --- package.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/package.json b/package.json index c83cc672a..52c3f84b4 100644 --- a/package.json +++ b/package.json @@ -226,6 +226,18 @@ "hash": "QmdxUuburamoF6zF9qjeQC4WYcWGbWuRmdLacMEsW8ioD8", "name": "gogo-protobuf", "version": "0.0.0" + }, + { + "author": "vyzo", + "hash": "QmYwn471zWPm7dLaEtNWMUj9ytMDFwnpFy1uMQDK3BZ4xj", + "name": "go-libp2p-discovery", + "version": "1.0.0" + }, + { + "author": "vyzo", + "hash": "QmNtEW59XTrreF9U4NAjYUmixEJ56HnfiJCj1p97bFnah2", + "name": "go-libp2p-autonat", + "version": "1.0.0" } ], "gxVersion": "0.4.0", From 8faf5403c544a7064d1e78605e9d4dcc9d7acb48 Mon Sep 17 00:00:00 2001 From: vyzo Date: Fri, 19 Oct 2018 15:54:17 +0300 Subject: [PATCH 14/36] fix typo --- options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/options.go b/options.go index 23c9f7d22..0015c8081 100644 --- a/options.go +++ b/options.go @@ -264,7 +264,7 @@ func Ping(enable bool) Option { func Routing(rt config.RoutingC) Option { return func(cfg *Config) error { if cfg.Routing != nil { - return fmt.Errorf("cannot specified multiple routing options") + return fmt.Errorf("cannot specify multiple routing options") } cfg.Routing = rt return nil From b62c83bd5cf1fdcc9507a87ca3a1f6f4b4b70f34 Mon Sep 17 00:00:00 2001 From: vyzo Date: Fri, 19 Oct 2018 15:56:45 +0300 Subject: [PATCH 15/36] better limit adjustment for relay discovery --- p2p/host/relay/autorelay.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/p2p/host/relay/autorelay.go b/p2p/host/relay/autorelay.go index 13020793c..8d8302da3 100644 --- a/p2p/host/relay/autorelay.go +++ b/p2p/host/relay/autorelay.go @@ -116,7 +116,8 @@ func (h *AutoRelayHost) findRelays(ctx context.Context) { h.mx.Unlock() limit := 20 - for ; need > limit; limit *= 2 { + if need > limit/2 { + limit = 2 * need } dctx, cancel := context.WithTimeout(ctx, 60*time.Second) From ee69383c2c49aabb23e790f61e679c91f869f29f Mon Sep 17 00:00:00 2001 From: vyzo Date: Sat, 20 Oct 2018 08:29:20 +0300 Subject: [PATCH 16/36] extend autorelay test to verify connectivity --- p2p/host/relay/autorelay_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/p2p/host/relay/autorelay_test.go b/p2p/host/relay/autorelay_test.go index f4e2d3a4e..5f34880d0 100644 --- a/p2p/host/relay/autorelay_test.go +++ b/p2p/host/relay/autorelay_test.go @@ -21,6 +21,7 @@ import ( peer "github.com/libp2p/go-libp2p-peer" pstore "github.com/libp2p/go-libp2p-peerstore" routing "github.com/libp2p/go-libp2p-routing" + ma "github.com/multiformats/go-multiaddr" manet "github.com/multiformats/go-multiaddr-net" ) @@ -150,10 +151,12 @@ func TestAutoRelay(t *testing.T) { if err != nil { t.Fatal(err) } + h4, err := libp2p.New(ctx, libp2p.EnableRelay()) connect(t, h1, h3) time.Sleep(3 * time.Second) + // verify that we advertise relay addrs haveRelay := false for _, addr := range h3.Addrs() { _, err := addr.ValueForProtocol(circuit.P_CIRCUIT) @@ -166,4 +169,18 @@ func TestAutoRelay(t *testing.T) { if !haveRelay { t.Fatal("No relay addrs advertised") } + + // check that we can connect through the relay + var raddrs []ma.Multiaddr + for _, addr := range h3.Addrs() { + _, err := addr.ValueForProtocol(circuit.P_CIRCUIT) + if err != nil { + raddrs = append(raddrs, addr) + } + } + + err = h4.Connect(ctx, pstore.PeerInfo{h3.ID(), raddrs}) + if err != nil { + t.Fatal(err) + } } From dcb8fd28134ac92b18a3cc205a1251efa0e0d272 Mon Sep 17 00:00:00 2001 From: vyzo Date: Sat, 20 Oct 2018 20:53:56 +0300 Subject: [PATCH 17/36] fix inverted address selection logic in test --- p2p/host/relay/autorelay_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/p2p/host/relay/autorelay_test.go b/p2p/host/relay/autorelay_test.go index 5f34880d0..38e7f518c 100644 --- a/p2p/host/relay/autorelay_test.go +++ b/p2p/host/relay/autorelay_test.go @@ -160,7 +160,7 @@ func TestAutoRelay(t *testing.T) { haveRelay := false for _, addr := range h3.Addrs() { _, err := addr.ValueForProtocol(circuit.P_CIRCUIT) - if err != nil { + if err == nil { haveRelay = true break } @@ -174,7 +174,7 @@ func TestAutoRelay(t *testing.T) { var raddrs []ma.Multiaddr for _, addr := range h3.Addrs() { _, err := addr.ValueForProtocol(circuit.P_CIRCUIT) - if err != nil { + if err == nil { raddrs = append(raddrs, addr) } } From 89aef893e6f3bc1c7abe7be89dc165b14338a7e2 Mon Sep 17 00:00:00 2001 From: vyzo Date: Sat, 20 Oct 2018 21:01:57 +0300 Subject: [PATCH 18/36] don't adveretise unspecific relay addrs in RelayHost --- p2p/host/relay/relay.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/p2p/host/relay/relay.go b/p2p/host/relay/relay.go index 95b5079c4..17aa09cd4 100644 --- a/p2p/host/relay/relay.go +++ b/p2p/host/relay/relay.go @@ -3,21 +3,34 @@ package relay import ( "context" + basic "github.com/libp2p/go-libp2p/p2p/host/basic" + discovery "github.com/libp2p/go-libp2p-discovery" host "github.com/libp2p/go-libp2p-host" + ma "github.com/multiformats/go-multiaddr" ) // RelayHost is a Host that provides Relay services. type RelayHost struct { - host.Host + *basic.BasicHost advertise discovery.Advertiser + addrsF basic.AddrsFactory } // New constructs a new RelayHost -func NewRelayHost(ctx context.Context, host host.Host, advertise discovery.Advertiser) *RelayHost { - h := &RelayHost{Host: host, advertise: advertise} +func NewRelayHost(ctx context.Context, bhost *basic.BasicHost, advertise discovery.Advertiser) *RelayHost { + h := &RelayHost{ + BasicHost: bhost, + addrsF: bhost.AddrsFactory, + advertise: advertise, + } + bhost.AddrsFactory = h.hostAddrs discovery.Advertise(ctx, advertise, "/libp2p/relay") return h } +func (h *RelayHost) hostAddrs(addrs []ma.Multiaddr) []ma.Multiaddr { + return filterUnspecificRelay(h.addrsF(addrs)) +} + var _ host.Host = (*RelayHost)(nil) From a7e1bf0f15ffad372b07313769af6a542fd2f086 Mon Sep 17 00:00:00 2001 From: vyzo Date: Mon, 22 Oct 2018 21:20:31 +0300 Subject: [PATCH 19/36] call the routing interface BasicRouting, alias to top level type --- config/config.go | 4 ++-- libp2p.go | 3 +++ p2p/host/relay/autorelay_test.go | 3 +-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/config/config.go b/config/config.go index aa06705c4..3d06fdca0 100644 --- a/config/config.go +++ b/config/config.go @@ -35,12 +35,12 @@ type AddrsFactory = bhost.AddrsFactory // NATManagerC is a NATManager constructor. type NATManagerC func(inet.Network) bhost.NATManager -type Routing interface { +type BasicRouting interface { routing.ContentRouting routing.PeerRouting } -type RoutingC func(host.Host) (Routing, error) +type RoutingC func(host.Host) (BasicRouting, error) // Config describes a set of settings for a libp2p node // diff --git a/libp2p.go b/libp2p.go index ad026ad4b..6c928c315 100644 --- a/libp2p.go +++ b/libp2p.go @@ -15,6 +15,9 @@ type Config = config.Config // (`libp2p.New`). type Option = config.Option +// BasicRouting is the combination of PeerRouting and ContentRouting +type BasicRouting = config.BasicRouting + // ChainOptions chains multiple options into a single option. func ChainOptions(opts ...Option) Option { return func(cfg *Config) error { diff --git a/p2p/host/relay/autorelay_test.go b/p2p/host/relay/autorelay_test.go index 38e7f518c..ff4d85071 100644 --- a/p2p/host/relay/autorelay_test.go +++ b/p2p/host/relay/autorelay_test.go @@ -8,7 +8,6 @@ import ( "time" libp2p "github.com/libp2p/go-libp2p" - config "github.com/libp2p/go-libp2p/config" relay "github.com/libp2p/go-libp2p/p2p/host/relay" ggio "github.com/gogo/protobuf/io" @@ -137,7 +136,7 @@ func TestAutoRelay(t *testing.T) { defer cancel() mtab := newMockRoutingTable() - makeRouting := func(h host.Host) (config.Routing, error) { + makeRouting := func(h host.Host) (libp2p.BasicRouting, error) { mr := newMockRouting(h, mtab) return mr, nil } From 2993fd9a688251c95240572c1c477fb5b5077900 Mon Sep 17 00:00:00 2001 From: vyzo Date: Tue, 23 Oct 2018 12:57:09 +0300 Subject: [PATCH 20/36] add autorelay documentation --- p2p/host/relay/doc.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 p2p/host/relay/doc.go diff --git a/p2p/host/relay/doc.go b/p2p/host/relay/doc.go new file mode 100644 index 000000000..8e642b2ad --- /dev/null +++ b/p2p/host/relay/doc.go @@ -0,0 +1,28 @@ +/* +The relay package contains host implementations that automatically +advertise relay addresses when the presence of NAT is detected. This +feature is dubbed `autorelay`. + +System Components: +- AutoNATService instances -- see https://github.com/libp2p/go-libp2p-autonat-svc +- One or more relays, instances of `RelayHost` +- The autorelayed hosts, instances of `AutoRelayHost`. + +How it works: +- `AutoNATService` instances are instantiated in the + bootstrappers (or other well known publicly reachable hosts) + +- `RelayHost`s are constructed with + `libp2p.New(libp2p.EnableRelay(circuit.OptHop), libp2p.Routing(makeDHT))`. + They provide Relay Hop services, and advertise through the DHT + in the `/libp2p/relay` namespace + +- `AutoRelayHost`s are constructed with `libp2p.New(libp2p.Routing(makeDHT))` + They passively discover autonat service instances and test dialability of + their listen address set through them. When the presence of NAT is detected, + they discover relays through the DHT, connect to some of them and begin + advertising relay addresses. The new set of addresses is propagated to + connected peers through the `identify/push` protocol. + +*/ +package relay From cc2dd220289c8eb7ba5146ffc6c370b331b8cd40 Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 24 Oct 2018 10:27:42 +0300 Subject: [PATCH 21/36] make randezvous key a constant --- p2p/host/relay/autorelay.go | 6 +++++- p2p/host/relay/relay.go | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/p2p/host/relay/autorelay.go b/p2p/host/relay/autorelay.go index 8d8302da3..f506a0baa 100644 --- a/p2p/host/relay/autorelay.go +++ b/p2p/host/relay/autorelay.go @@ -20,6 +20,10 @@ import ( manet "github.com/multiformats/go-multiaddr-net" ) +const ( + RelayRenezvous = "/libp2p/relay" +) + var ( DesiredRelays = 3 @@ -121,7 +125,7 @@ func (h *AutoRelayHost) findRelays(ctx context.Context) { } dctx, cancel := context.WithTimeout(ctx, 60*time.Second) - pis, err := discovery.FindPeers(dctx, h.discover, "/libp2p/relay", limit) + pis, err := discovery.FindPeers(dctx, h.discover, RelayRenezvous, limit) cancel() if err != nil { log.Debugf("error discovering relays: %s", err.Error()) diff --git a/p2p/host/relay/relay.go b/p2p/host/relay/relay.go index 17aa09cd4..8ba99d5ed 100644 --- a/p2p/host/relay/relay.go +++ b/p2p/host/relay/relay.go @@ -25,7 +25,7 @@ func NewRelayHost(ctx context.Context, bhost *basic.BasicHost, advertise discove advertise: advertise, } bhost.AddrsFactory = h.hostAddrs - discovery.Advertise(ctx, advertise, "/libp2p/relay") + discovery.Advertise(ctx, advertise, RelayRenezvous) return h } From 67aba4d50149406de36e5cc79b381b16ea7debfe Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 24 Oct 2018 10:38:15 +0300 Subject: [PATCH 22/36] move relay selection strategy out of line --- p2p/host/relay/autorelay.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/p2p/host/relay/autorelay.go b/p2p/host/relay/autorelay.go index f506a0baa..9e2352710 100644 --- a/p2p/host/relay/autorelay.go +++ b/p2p/host/relay/autorelay.go @@ -132,9 +132,7 @@ func (h *AutoRelayHost) findRelays(ctx context.Context) { return } - // TODO better relay selection strategy; this just selects random relays - // but we should probably use ping latency as the selection metric - shuffleRelays(pis) + pis = h.selectRelays(pis) update := 0 @@ -171,6 +169,13 @@ func (h *AutoRelayHost) findRelays(ctx context.Context) { } } +func (h *AutoRelayHost) selectRelays(pis []pstore.PeerInfo) []pstore.PeerInfo { + // TODO better relay selection strategy; this just selects random relays + // but we should probably use ping latency as the selection metric + shuffleRelays(pis) + return pis +} + func (h *AutoRelayHost) updateAddrs() { h.doUpdateAddrs() h.PushIdentify() From 4f90393629b0cb984e9ae84820174e4918efcb5f Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 24 Oct 2018 10:41:43 +0300 Subject: [PATCH 23/36] add a comment --- p2p/host/relay/autorelay.go | 1 + 1 file changed, 1 insertion(+) diff --git a/p2p/host/relay/autorelay.go b/p2p/host/relay/autorelay.go index 9e2352710..26140b022 100644 --- a/p2p/host/relay/autorelay.go +++ b/p2p/host/relay/autorelay.go @@ -220,6 +220,7 @@ func (h *AutoRelayHost) doUpdateAddrs() { } } + // add relay specific addrs to the list for _, pi := range h.relays { circuit, err := ma.NewMultiaddr(fmt.Sprintf("/ipfs/%s/p2p-circuit", pi.ID.Pretty())) if err != nil { From 84400e22f2b6fc9b8346d578badfef1956c38f0c Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 24 Oct 2018 16:17:49 +0300 Subject: [PATCH 24/36] fix typo --- p2p/host/relay/autorelay.go | 4 ++-- p2p/host/relay/relay.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/p2p/host/relay/autorelay.go b/p2p/host/relay/autorelay.go index 26140b022..664f82d68 100644 --- a/p2p/host/relay/autorelay.go +++ b/p2p/host/relay/autorelay.go @@ -21,7 +21,7 @@ import ( ) const ( - RelayRenezvous = "/libp2p/relay" + RelayRendezvous = "/libp2p/relay" ) var ( @@ -125,7 +125,7 @@ func (h *AutoRelayHost) findRelays(ctx context.Context) { } dctx, cancel := context.WithTimeout(ctx, 60*time.Second) - pis, err := discovery.FindPeers(dctx, h.discover, RelayRenezvous, limit) + pis, err := discovery.FindPeers(dctx, h.discover, RelayRendezvous, limit) cancel() if err != nil { log.Debugf("error discovering relays: %s", err.Error()) diff --git a/p2p/host/relay/relay.go b/p2p/host/relay/relay.go index 8ba99d5ed..69b673707 100644 --- a/p2p/host/relay/relay.go +++ b/p2p/host/relay/relay.go @@ -25,7 +25,7 @@ func NewRelayHost(ctx context.Context, bhost *basic.BasicHost, advertise discove advertise: advertise, } bhost.AddrsFactory = h.hostAddrs - discovery.Advertise(ctx, advertise, RelayRenezvous) + discovery.Advertise(ctx, advertise, RelayRendezvous) return h } From e96605d1a34a56a63ef5dc94a0d5b9dd86ed6d60 Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 24 Oct 2018 16:44:02 +0300 Subject: [PATCH 25/36] use /p2p multiaddr --- p2p/host/relay/autorelay.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/host/relay/autorelay.go b/p2p/host/relay/autorelay.go index 664f82d68..088a477f3 100644 --- a/p2p/host/relay/autorelay.go +++ b/p2p/host/relay/autorelay.go @@ -222,7 +222,7 @@ func (h *AutoRelayHost) doUpdateAddrs() { // add relay specific addrs to the list for _, pi := range h.relays { - circuit, err := ma.NewMultiaddr(fmt.Sprintf("/ipfs/%s/p2p-circuit", pi.ID.Pretty())) + circuit, err := ma.NewMultiaddr(fmt.Sprintf("/p2p/%s/p2p-circuit", pi.ID.Pretty())) if err != nil { panic(err) } From 4be7ada37bcdeadd2cab8a3010fe4f1939d05aa5 Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 24 Oct 2018 16:58:06 +0300 Subject: [PATCH 26/36] tag relay connections --- p2p/host/relay/autorelay.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/p2p/host/relay/autorelay.go b/p2p/host/relay/autorelay.go index 088a477f3..01451ceb2 100644 --- a/p2p/host/relay/autorelay.go +++ b/p2p/host/relay/autorelay.go @@ -157,6 +157,9 @@ func (h *AutoRelayHost) findRelays(ctx context.Context) { h.relays[pi.ID] = pi h.mx.Unlock() + // tag the connection as very important + h.ConnManager().TagPeer(pi.ID, "relay", 42) + update++ need-- if need == 0 { From fdfa2246991938f733ccef87e52b59a6d8bfb797 Mon Sep 17 00:00:00 2001 From: vyzo Date: Wed, 24 Oct 2018 17:09:09 +0300 Subject: [PATCH 27/36] document doUpdateAddrs --- p2p/host/relay/autorelay.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/p2p/host/relay/autorelay.go b/p2p/host/relay/autorelay.go index 01451ceb2..6ea85bed4 100644 --- a/p2p/host/relay/autorelay.go +++ b/p2p/host/relay/autorelay.go @@ -184,6 +184,15 @@ func (h *AutoRelayHost) updateAddrs() { h.PushIdentify() } +// This function updates our NATed advertised addrs (h.addrs) +// The public addrs are rewritten so that they only retain the public IP part; they +// become undialable but are useful as a hint to the dialer to determine whether or not +// to dial private addrs. +// The non-public addrs are included verbatim so that peers behind the same NAT/firewall +// can still dial us directly. +// On top of those, we add the relay-specific addrs for the relays to which we are +// connected. For each non-private relay addr, we encapsulate the p2p-circuit addr +// through which we can be dialed. func (h *AutoRelayHost) doUpdateAddrs() { h.mx.Lock() defer h.mx.Unlock() From 292b8a12801d51d6be50e2f7aff81405d5759526 Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 25 Oct 2018 11:49:17 +0300 Subject: [PATCH 28/36] update gx deps --- package.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 52c3f84b4..0bfb05796 100644 --- a/package.json +++ b/package.json @@ -229,15 +229,15 @@ }, { "author": "vyzo", - "hash": "QmYwn471zWPm7dLaEtNWMUj9ytMDFwnpFy1uMQDK3BZ4xj", + "hash": "QmfWLPvbWEHjMbEsLwCzg3igdeijJHDWhLth3k9E3imdEL", "name": "go-libp2p-discovery", - "version": "1.0.0" + "version": "1.0.1" }, { "author": "vyzo", - "hash": "QmNtEW59XTrreF9U4NAjYUmixEJ56HnfiJCj1p97bFnah2", + "hash": "QmUn8mtaf4tTFwKnFRzkNYYLc8XEo3yz6qBfp5ShVB1HYZ", "name": "go-libp2p-autonat", - "version": "1.0.0" + "version": "1.0.1" } ], "gxVersion": "0.4.0", From c2d846a7c38e1685babd19a77d7cc75c9ffdca98 Mon Sep 17 00:00:00 2001 From: vyzo Date: Thu, 25 Oct 2018 12:00:00 +0300 Subject: [PATCH 29/36] fix go vet issue --- p2p/host/relay/autorelay_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/host/relay/autorelay_test.go b/p2p/host/relay/autorelay_test.go index ff4d85071..241e33f65 100644 --- a/p2p/host/relay/autorelay_test.go +++ b/p2p/host/relay/autorelay_test.go @@ -178,7 +178,7 @@ func TestAutoRelay(t *testing.T) { } } - err = h4.Connect(ctx, pstore.PeerInfo{h3.ID(), raddrs}) + err = h4.Connect(ctx, pstore.PeerInfo{ID: h3.ID(), Addrs: raddrs}) if err != nil { t.Fatal(err) } From 7b324b14b5702044c118d07c3a41af1621b89652 Mon Sep 17 00:00:00 2001 From: vyzo Date: Fri, 26 Oct 2018 13:58:35 +0300 Subject: [PATCH 30/36] remove BasicRouting interface; use PeerRouting and upcast for discovery --- config/config.go | 36 +++++++++++++++----------------- libp2p.go | 3 --- p2p/host/relay/autorelay_test.go | 2 +- 3 files changed, 18 insertions(+), 23 deletions(-) diff --git a/config/config.go b/config/config.go index 3d06fdca0..c50350946 100644 --- a/config/config.go +++ b/config/config.go @@ -35,12 +35,7 @@ type AddrsFactory = bhost.AddrsFactory // NATManagerC is a NATManager constructor. type NATManagerC func(inet.Network) bhost.NATManager -type BasicRouting interface { - routing.ContentRouting - routing.PeerRouting -} - -type RoutingC func(host.Host) (BasicRouting, error) +type RoutingC func(host.Host) (routing.PeerRouting, error) // Config describes a set of settings for a libp2p node // @@ -178,21 +173,24 @@ func (cfg *Config) NewNode(ctx context.Context) (host.Host, error) { return nil, err } - if cfg.Relay { - discovery := discovery.NewRoutingDiscovery(router) - - hop := false - for _, opt := range cfg.RelayOpts { - if opt == circuit.OptHop { - hop = true - break + crouter, ok := router.(routing.ContentRouting) + if ok { + if cfg.Relay { + discovery := discovery.NewRoutingDiscovery(crouter) + + hop := false + for _, opt := range cfg.RelayOpts { + if opt == circuit.OptHop { + hop = true + break + } } - } - if hop { - h = relay.NewRelayHost(swrm.Context(), h.(*bhost.BasicHost), discovery) - } else { - h = relay.NewAutoRelayHost(swrm.Context(), h.(*bhost.BasicHost), discovery) + if hop { + h = relay.NewRelayHost(swrm.Context(), h.(*bhost.BasicHost), discovery) + } else { + h = relay.NewAutoRelayHost(swrm.Context(), h.(*bhost.BasicHost), discovery) + } } } diff --git a/libp2p.go b/libp2p.go index 6c928c315..ad026ad4b 100644 --- a/libp2p.go +++ b/libp2p.go @@ -15,9 +15,6 @@ type Config = config.Config // (`libp2p.New`). type Option = config.Option -// BasicRouting is the combination of PeerRouting and ContentRouting -type BasicRouting = config.BasicRouting - // ChainOptions chains multiple options into a single option. func ChainOptions(opts ...Option) Option { return func(cfg *Config) error { diff --git a/p2p/host/relay/autorelay_test.go b/p2p/host/relay/autorelay_test.go index 241e33f65..a3ac7dd29 100644 --- a/p2p/host/relay/autorelay_test.go +++ b/p2p/host/relay/autorelay_test.go @@ -136,7 +136,7 @@ func TestAutoRelay(t *testing.T) { defer cancel() mtab := newMockRoutingTable() - makeRouting := func(h host.Host) (libp2p.BasicRouting, error) { + makeRouting := func(h host.Host) (routing.PeerRouting, error) { mr := newMockRouting(h, mtab) return mr, nil } From 0dfca3ba5d068b178a3431caaa1e26b52b7be0a8 Mon Sep 17 00:00:00 2001 From: vyzo Date: Fri, 26 Oct 2018 14:06:19 +0300 Subject: [PATCH 31/36] increase test AutoNATIdentifyDelay to 100ms macosx seems to fail intermittently, and that race is the likely culprit. --- p2p/host/relay/autorelay_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/p2p/host/relay/autorelay_test.go b/p2p/host/relay/autorelay_test.go index a3ac7dd29..bd738f5ce 100644 --- a/p2p/host/relay/autorelay_test.go +++ b/p2p/host/relay/autorelay_test.go @@ -26,7 +26,7 @@ import ( // test specific parameters func init() { - autonat.AutoNATIdentifyDelay = 10 * time.Millisecond + autonat.AutoNATIdentifyDelay = 100 * time.Millisecond autonat.AutoNATBootDelay = 1 * time.Second relay.BootDelay = 1 * time.Second manet.Private4 = []*net.IPNet{} From 69144bd78bcc0868b063bc61462ae6a4eb9a0f1c Mon Sep 17 00:00:00 2001 From: vyzo Date: Fri, 26 Oct 2018 15:05:20 +0300 Subject: [PATCH 32/36] extend autorelay test to verify pushing of relay addrs also check that we don't initially advertise any. --- p2p/host/relay/autorelay_test.go | 40 +++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/p2p/host/relay/autorelay_test.go b/p2p/host/relay/autorelay_test.go index bd738f5ce..ce50ea8c0 100644 --- a/p2p/host/relay/autorelay_test.go +++ b/p2p/host/relay/autorelay_test.go @@ -152,16 +152,33 @@ func TestAutoRelay(t *testing.T) { } h4, err := libp2p.New(ctx, libp2p.EnableRelay()) + // verify that we don't advertise relay addrs initially + for _, addr := range h3.Addrs() { + _, err := addr.ValueForProtocol(circuit.P_CIRCUIT) + if err == nil { + t.Fatal("relay addr advertised before auto detection") + } + } + + // connect to AutoNAT and let detection/discovery work its magic connect(t, h1, h3) time.Sleep(3 * time.Second) - // verify that we advertise relay addrs + // verify that we now advertise relay addrs (but not unspecific relay addrs) + unspecificRelay, err := ma.NewMultiaddr("/p2p-circuit") + if err != nil { + t.Fatal(err) + } + haveRelay := false for _, addr := range h3.Addrs() { + if addr.Equal(unspecificRelay) { + t.Fatal("unspecific relay addr advertised") + } + _, err := addr.ValueForProtocol(circuit.P_CIRCUIT) if err == nil { haveRelay = true - break } } @@ -169,7 +186,7 @@ func TestAutoRelay(t *testing.T) { t.Fatal("No relay addrs advertised") } - // check that we can connect through the relay + // verify that we can connect through the relay var raddrs []ma.Multiaddr for _, addr := range h3.Addrs() { _, err := addr.ValueForProtocol(circuit.P_CIRCUIT) @@ -182,4 +199,21 @@ func TestAutoRelay(t *testing.T) { if err != nil { t.Fatal(err) } + + // verify that we have pushed relay addrs to connected peers + haveRelay = false + for _, addr := range h1.Peerstore().Addrs(h3.ID()) { + if addr.Equal(unspecificRelay) { + t.Fatal("unspecific relay addr advertised") + } + + _, err := addr.ValueForProtocol(circuit.P_CIRCUIT) + if err == nil { + haveRelay = true + } + } + + if !haveRelay { + t.Fatal("No relay addrs pushed") + } } From 5d8988f3ee0d2ce8621972edbfc6ad090782fc6d Mon Sep 17 00:00:00 2001 From: vyzo Date: Tue, 30 Oct 2018 11:08:02 +0200 Subject: [PATCH 33/36] add comment about the unstable nature of BasicHost.PushIdentify --- p2p/host/basic/basic_host.go | 1 + 1 file changed, 1 insertion(+) diff --git a/p2p/host/basic/basic_host.go b/p2p/host/basic/basic_host.go index b5d2108b9..13452e2f9 100644 --- a/p2p/host/basic/basic_host.go +++ b/p2p/host/basic/basic_host.go @@ -258,6 +258,7 @@ func (h *BasicHost) newStreamHandler(s inet.Stream) { } // PushIdentify pushes an identify update through the identify push protocol +// Warning: this interface is unstable and may disappear in the future. func (h *BasicHost) PushIdentify() { h.ids.Push() } From 5a1c09c4afe061e4a5913b66985315e043b8c665 Mon Sep 17 00:00:00 2001 From: vyzo Date: Fri, 2 Nov 2018 11:39:51 +0200 Subject: [PATCH 34/36] use advertised addrs for autonat dial back, not all addrs --- p2p/host/relay/autorelay.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/p2p/host/relay/autorelay.go b/p2p/host/relay/autorelay.go index 6ea85bed4..e97ce8832 100644 --- a/p2p/host/relay/autorelay.go +++ b/p2p/host/relay/autorelay.go @@ -55,15 +55,14 @@ type AutoRelayHost struct { } func NewAutoRelayHost(ctx context.Context, bhost *basic.BasicHost, discover discovery.Discoverer) *AutoRelayHost { - autonat := autonat.NewAutoNAT(ctx, bhost, bhost.AllAddrs) h := &AutoRelayHost{ BasicHost: bhost, discover: discover, - autonat: autonat, addrsF: bhost.AddrsFactory, relays: make(map[peer.ID]pstore.PeerInfo), disconnect: make(chan struct{}, 1), } + h.autonat = autonat.NewAutoNAT(ctx, bhost, h.baseAddrs) bhost.AddrsFactory = h.hostAddrs bhost.Network().Notify(h) go h.background(ctx) @@ -80,6 +79,10 @@ func (h *AutoRelayHost) hostAddrs(addrs []ma.Multiaddr) []ma.Multiaddr { } } +func (h *AutoRelayHost) baseAddrs() []ma.Multiaddr { + return filterUnspecificRelay(h.addrsF(h.AllAddrs())) +} + func (h *AutoRelayHost) background(ctx context.Context) { select { case <-time.After(autonat.AutoNATBootDelay + BootDelay): @@ -197,7 +200,7 @@ func (h *AutoRelayHost) doUpdateAddrs() { h.mx.Lock() defer h.mx.Unlock() - addrs := filterUnspecificRelay(h.addrsF(h.AllAddrs())) + addrs := h.baseAddrs() raddrs := make([]ma.Multiaddr, 0, len(addrs)+len(h.relays)) // remove our public addresses from the list and replace them by just the public IP From 4cb5d00e2cf34d9194c1414c2fb4f735b9cfbebc Mon Sep 17 00:00:00 2001 From: vyzo Date: Sun, 4 Nov 2018 10:55:17 +0200 Subject: [PATCH 35/36] gx update --- package.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 0bfb05796..65d8edec2 100644 --- a/package.json +++ b/package.json @@ -229,15 +229,15 @@ }, { "author": "vyzo", - "hash": "QmfWLPvbWEHjMbEsLwCzg3igdeijJHDWhLth3k9E3imdEL", + "hash": "QmT8eNT96scr6wcrARzyxrCcsBAPKn9GkEx2TeXZniHiMP", "name": "go-libp2p-discovery", - "version": "1.0.1" + "version": "1.0.2" }, { "author": "vyzo", - "hash": "QmUn8mtaf4tTFwKnFRzkNYYLc8XEo3yz6qBfp5ShVB1HYZ", + "hash": "QmU2XQcwPxikg1jZqDBMFgLQVan7zjT2HtQWrWui3vbUVS", "name": "go-libp2p-autonat", - "version": "1.0.1" + "version": "1.0.2" } ], "gxVersion": "0.4.0", From a309f096067c578402a4fdaec133160a9abb1351 Mon Sep 17 00:00:00 2001 From: vyzo Date: Tue, 6 Nov 2018 09:50:25 +0200 Subject: [PATCH 36/36] Add note to relay docs about internal interface instability --- p2p/host/relay/doc.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/p2p/host/relay/doc.go b/p2p/host/relay/doc.go index 8e642b2ad..c0a587811 100644 --- a/p2p/host/relay/doc.go +++ b/p2p/host/relay/doc.go @@ -3,6 +3,8 @@ The relay package contains host implementations that automatically advertise relay addresses when the presence of NAT is detected. This feature is dubbed `autorelay`. +Warning: the internal interfaces are unstable. + System Components: - AutoNATService instances -- see https://github.com/libp2p/go-libp2p-autonat-svc - One or more relays, instances of `RelayHost`