Browse Source

feat: libp2phttp `/http-path` (#2850)

* Bump go-multiaddr dep

* Add support for http-path

* Support redirects

* Don't split at p2p, split at P_HTTP_PATH

* fixup

* Fill in host if missing

* mod tidy

* Fix test

* Add MultiaddrURIRedirect

* Only alloc err once
set-protocol-fix
Marco Munizaga 4 months ago
committed by GitHub
parent
commit
d84736f704
No known key found for this signature in database GPG Key ID: B5690EEEBB952194
  1. 2
      go.mod
  2. 4
      go.sum
  3. 131
      p2p/http/libp2phttp.go
  4. 191
      p2p/http/libp2phttp_test.go
  5. 2
      test-plans/go.mod
  6. 4
      test-plans/go.sum

2
go.mod

@ -34,7 +34,7 @@ require (
github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b
github.com/mr-tron/base58 v1.2.0
github.com/multiformats/go-base32 v0.1.0
github.com/multiformats/go-multiaddr v0.12.4
github.com/multiformats/go-multiaddr v0.13.0
github.com/multiformats/go-multiaddr-dns v0.3.1
github.com/multiformats/go-multiaddr-fmt v0.1.0
github.com/multiformats/go-multibase v0.2.0

4
go.sum

@ -235,8 +235,8 @@ github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9
github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4=
github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo=
github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4=
github.com/multiformats/go-multiaddr v0.12.4 h1:rrKqpY9h+n80EwhhC/kkcunCZZ7URIF8yN1WEUt2Hvc=
github.com/multiformats/go-multiaddr v0.12.4/go.mod h1:sBXrNzucqkFJhvKOiwwLyqamGa/P5EIXNPLovyhQCII=
github.com/multiformats/go-multiaddr v0.13.0 h1:BCBzs61E3AGHcYYTv8dqRH43ZfyrqM8RXVPT8t13tLQ=
github.com/multiformats/go-multiaddr v0.13.0/go.mod h1:sBXrNzucqkFJhvKOiwwLyqamGa/P5EIXNPLovyhQCII=
github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A=
github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk=
github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E=

131
p2p/http/libp2phttp.go

@ -227,7 +227,10 @@ var ErrNoListeners = errors.New("nothing to listen on")
func (h *Host) setupListeners(listenerErrCh chan error) error {
for _, addr := range h.ListenAddrs {
parsedAddr := parseMultiaddr(addr)
parsedAddr, err := parseMultiaddr(addr)
if err != nil {
return err
}
// resolve the host
ipaddr, err := net.ResolveIPAddr("ip", parsedAddr.host)
if err != nil {
@ -477,9 +480,61 @@ func (rt *streamRoundTripper) RoundTrip(r *http.Request) (*http.Response, error)
}
resp.Body = &streamReadCloser{resp.Body, s}
locUrl, err := resp.Location()
if err == nil {
// Location url in response. Is this a multiaddr uri? and is it relative?
// If it's relative we want to convert it to an absolute multiaddr uri
// so that the next request knows how to reach the endpoint.
if locUrl.Scheme == "multiaddr" && resp.Request.URL.Scheme == "multiaddr" {
// Check if it's a relative URI and turn it into an absolute one
u, err := relativeMultiaddrURIToAbs(resp.Request.URL, locUrl)
if err == nil {
// It was a relative URI and we were able to convert it to an absolute one
// Update the location header to be an absolute multiaddr uri
resp.Header.Set("Location", u.String())
}
}
}
return resp, nil
}
var errNotRelative = errors.New("not relative")
// relativeMultiaddrURIToAbs takes a relative multiaddr URI and turns it into an
// absolute one. Useful, for example, when a server gives us a relative URI for a redirect.
// It allows the following request (the one after redirected) to reach the correct server.
func relativeMultiaddrURIToAbs(original *url.URL, relative *url.URL) (*url.URL, error) {
// Is this a relative uri? We know if it is because non-relative URI's of the form:
// "multiaddr:/ip4/1.2.3.4/tcp/9899" when parsed by Go's url package will have url.OmitHost == true
// But if it is relative (just a path to an http resource e.g. /here-instead)
// a redirect will inherit the multiaddr scheme, but set url.OmitHost == false. It will also stringify as something like
// multiaddr://here-instead.
if relative.OmitHost {
// Not relative (at least we can't tell). Nothing we can do here
return nil, errNotRelative
}
originalStr := original.RawPath
if originalStr == "" {
originalStr = original.Path
}
originalMa, err := ma.NewMultiaddr(originalStr)
if err != nil {
return nil, errors.New("original uri is not a multiaddr")
}
relativePathComponent, err := ma.NewComponent("http-path", relative.Path)
if err != nil {
return nil, errors.New("relative path is not a valid http-path")
}
withoutPath, _ := ma.SplitFunc(originalMa, func(c ma.Component) bool {
return c.Protocol().Code == ma.P_HTTP_PATH
})
withNewPath := withoutPath.Encapsulate(relativePathComponent)
return url.Parse("multiaddr:" + withNewPath.String())
}
// roundTripperForSpecificServer is an http.RoundTripper targets a specific server. Still reuses the underlying RoundTripper for the requests.
// The underlying RoundTripper MUST be an HTTP Transport.
type roundTripperForSpecificServer struct {
@ -657,12 +712,23 @@ func (h *Host) RoundTrip(r *http.Request) (*http.Response, error) {
return nil, err
}
addr, isHTTP := normalizeHTTPMultiaddr(addr)
parsed, err := parseMultiaddr(addr)
if err != nil {
return nil, err
}
if isHTTP {
parsed := parseMultiaddr(addr)
scheme := "http"
if parsed.useHTTPS {
scheme = "https"
}
u := url.URL{
Scheme: scheme,
Host: parsed.host + ":" + parsed.port,
Path: parsed.httpPath,
}
r.URL = &u
h.initDefaultRT()
rt := h.DefaultClientRoundTripper
if parsed.sni != parsed.host {
@ -676,13 +742,6 @@ func (h *Host) RoundTrip(r *http.Request) (*http.Response, error) {
rt.TLSClientConfig.ServerName = parsed.sni
}
// TODO add http-path support
url := url.URL{
Scheme: scheme,
Host: parsed.host + ":" + parsed.port,
}
r.URL = &url
return rt.RoundTrip(r)
}
@ -690,14 +749,23 @@ func (h *Host) RoundTrip(r *http.Request) (*http.Response, error) {
return nil, fmt.Errorf("can not do HTTP over streams. Missing StreamHost")
}
addr, pid := peer.SplitAddr(addr)
if pid == "" {
if parsed.peer == "" {
return nil, fmt.Errorf("no peer ID in multiaddr")
}
h.StreamHost.Peerstore().AddAddrs(pid, []ma.Multiaddr{addr}, peerstore.TempAddrTTL)
withoutHTTPPath, _ := ma.SplitFunc(addr, func(c ma.Component) bool {
return c.Protocol().Code == ma.P_HTTP_PATH
})
h.StreamHost.Peerstore().AddAddrs(parsed.peer, []ma.Multiaddr{withoutHTTPPath}, peerstore.TempAddrTTL)
// Set the Opaque field to the http-path so that the HTTP request only makes
// a reference to that path and not the whole multiaddr uri
r.URL.Opaque = parsed.httpPath
if r.Host == "" {
// Fill in the host if it's not already set
r.Host = parsed.host + ":" + parsed.port
}
srt := streamRoundTripper{
server: pid,
server: parsed.peer,
skipAddAddrs: true,
httpHost: h,
h: h.StreamHost,
@ -750,7 +818,10 @@ func (h *Host) NewConstrainedRoundTripper(server peer.AddrInfo, opts ...RoundTri
// Currently the HTTP transport can not authenticate peer IDs.
if !options.serverMustAuthenticatePeerID && len(httpAddrs) > 0 && (options.preferHTTPTransport || (firstAddrIsHTTP && !existingStreamConn)) {
parsed := parseMultiaddr(httpAddrs[0])
parsed, err := parseMultiaddr(httpAddrs[0])
if err != nil {
return nil, err
}
scheme := "http"
if parsed.useHTTPS {
scheme = "https"
@ -791,15 +862,18 @@ func (h *Host) NewConstrainedRoundTripper(server peer.AddrInfo, opts ...RoundTri
return &streamRoundTripper{h: h.StreamHost, server: server.ID, serverAddrs: nonHTTPAddrs, httpHost: h}, nil
}
type httpMultiaddr struct {
type explodedMultiaddr struct {
useHTTPS bool
host string
port string
sni string
httpPath string
peer peer.ID
}
func parseMultiaddr(addr ma.Multiaddr) httpMultiaddr {
out := httpMultiaddr{}
func parseMultiaddr(addr ma.Multiaddr) (explodedMultiaddr, error) {
out := explodedMultiaddr{}
var err error
ma.ForEach(addr, func(c ma.Component) bool {
switch c.Protocol().Code {
case ma.P_IP4, ma.P_IP6, ma.P_DNS, ma.P_DNS4, ma.P_DNS6:
@ -810,15 +884,27 @@ func parseMultiaddr(addr ma.Multiaddr) httpMultiaddr {
out.useHTTPS = true
case ma.P_SNI:
out.sni = c.Value()
case ma.P_HTTP_PATH:
out.httpPath, err = url.QueryUnescape(c.Value())
if err == nil && out.httpPath[0] != '/' {
out.httpPath = "/" + out.httpPath
}
case ma.P_P2P:
out.peer, err = peer.Decode(c.Value())
}
return out.host == "" || out.port == "" || !out.useHTTPS || out.sni == ""
// stop if there is an error, otherwise iterate over all components in case this is a circuit address
return err == nil
})
if out.useHTTPS && out.sni == "" {
out.sni = out.host
}
return out
if out.httpPath == "" {
out.httpPath = "/"
}
return out, err
}
var httpComponent, _ = ma.NewComponent("http", "")
@ -839,6 +925,9 @@ func normalizeHTTPMultiaddr(addr ma.Multiaddr) (ma.Multiaddr, bool) {
}
return false
})
if beforeHTTPS == nil || !isHTTPMultiaddr {
return addr, false
}
if afterIncludingHTTPS == nil {
// No HTTPS component, just return the original
@ -918,7 +1007,7 @@ func (h *Host) getAndStorePeerMetadata(ctx context.Context, roundtripper http.Ro
}
func requestPeerMeta(ctx context.Context, roundtripper http.RoundTripper, wellKnownResource string) (PeerMeta, error) {
req, err := http.NewRequest("GET", wellKnownResource, nil)
req, err := http.NewRequestWithContext(ctx, "GET", wellKnownResource, nil)
if err != nil {
return nil, err
}

191
p2p/http/libp2phttp_test.go

@ -28,6 +28,7 @@ import (
"github.com/libp2p/go-libp2p/core/peer"
libp2phttp "github.com/libp2p/go-libp2p/p2p/http"
httpping "github.com/libp2p/go-libp2p/p2p/http/ping"
libp2pquic "github.com/libp2p/go-libp2p/p2p/transport/quic"
ma "github.com/multiformats/go-multiaddr"
"github.com/stretchr/testify/require"
)
@ -769,30 +770,33 @@ func TestHTTPHostAsRoundTripper(t *testing.T) {
}
serverHttpHost.SetHTTPHandlerAtPath("/hello", "/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path != "/" {
w.WriteHeader(http.StatusNotFound)
return
}
w.Write([]byte("hello"))
}))
// Uncomment when we get the http-path changes in go-multiaddr
// // Different protocol.ID and mounted at a different path
// serverHttpHost.SetHTTPHandlerAtPath("/hello-again", "/hello", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// w.Write([]byte("hello"))
// }))
// Different protocol.ID and mounted at a different path
serverHttpHost.SetHTTPHandlerAtPath("/hello-again", "/hello2", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("hello"))
}))
go serverHttpHost.Serve()
defer serverHttpHost.Close()
testCases := []string{
// Version that has an http-path. Will uncomment when we get the http-path changes in go-multiaddr
// "multiaddr:" + serverHost.Addrs()[0].String() + "/http-path/hello",
}
httpPathSuffix := "/http-path/hello2"
var testCases []string
for _, a := range serverHttpHost.Addrs() {
if _, err := a.ValueForProtocol(ma.P_HTTP); err == nil {
testCases = append(testCases, "multiaddr:"+a.String())
testCases = append(testCases, "multiaddr:"+a.String()+httpPathSuffix)
serverPort, err := a.ValueForProtocol(ma.P_TCP)
require.NoError(t, err)
testCases = append(testCases, "http://127.0.0.1:"+serverPort)
} else {
testCases = append(testCases, "multiaddr:"+a.String()+"/p2p/"+serverHost.ID().String())
testCases = append(testCases, "multiaddr:"+a.String()+"/p2p/"+serverHost.ID().String()+httpPathSuffix)
}
}
@ -806,6 +810,7 @@ func TestHTTPHostAsRoundTripper(t *testing.T) {
t.Run(tc, func(t *testing.T) {
resp, err := client.Get(tc)
require.NoError(t, err)
require.Equal(t, http.StatusOK, resp.StatusCode)
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
require.NoError(t, err)
@ -823,3 +828,171 @@ func TestHTTPHostAsRoundTripperFailsWhenNoStreamHostPresent(t *testing.T) {
require.Error(t, err)
require.ErrorContains(t, err, "Missing StreamHost")
}
// TestRedirects tests a client being redirected through multiple HTTP redirects
func TestRedirects(t *testing.T) {
serverHost, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/udp/0/quic-v1"))
require.NoError(t, err)
serverHttpHost := libp2phttp.Host{
StreamHost: serverHost,
InsecureAllowHTTP: true,
ListenAddrs: []ma.Multiaddr{ma.StringCast("/ip4/127.0.0.1/tcp/0/http")},
}
go serverHttpHost.Serve()
defer serverHttpHost.Close()
serverHttpHost.SetHTTPHandlerAtPath("/redirect-1/0.0.1", "/a", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Location", "/b/")
w.WriteHeader(http.StatusMovedPermanently)
}))
serverHttpHost.SetHTTPHandlerAtPath("/redirect-2/0.0.1", "/b", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Location", "/c/")
w.WriteHeader(http.StatusMovedPermanently)
}))
serverHttpHost.SetHTTPHandlerAtPath("/redirect-3/0.0.1", "/c", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Location", "/d/")
w.WriteHeader(http.StatusMovedPermanently)
}))
serverHttpHost.SetHTTPHandlerAtPath("/redirect-4/0.0.1", "/d", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Write([]byte("hello"))
}))
clientStreamHost, err := libp2p.New(libp2p.NoListenAddrs, libp2p.Transport(libp2pquic.NewTransport))
require.NoError(t, err)
client := http.Client{Transport: &libp2phttp.Host{StreamHost: clientStreamHost}}
type testCase struct {
initialURI string
expectedURI string
}
var testCases []testCase
for _, a := range serverHttpHost.Addrs() {
if _, err := a.ValueForProtocol(ma.P_HTTP); err == nil {
port, err := a.ValueForProtocol(ma.P_TCP)
require.NoError(t, err)
u := fmt.Sprintf("multiaddr:%s/http-path/a%%2f", a)
f := fmt.Sprintf("http://127.0.0.1:%s/d/", port)
testCases = append(testCases, testCase{u, f})
} else {
u := fmt.Sprintf("multiaddr:%s/p2p/%s/http-path/a%%2f", a, serverHost.ID())
f := fmt.Sprintf("multiaddr:%s/p2p/%s/http-path/%%2Fd%%2F", a, serverHost.ID())
testCases = append(testCases, testCase{u, f})
}
}
for _, tc := range testCases {
t.Run(tc.initialURI, func(t *testing.T) {
resp, err := client.Get(tc.initialURI)
require.NoError(t, err)
defer resp.Body.Close()
require.Equal(t, http.StatusOK, resp.StatusCode)
body, err := io.ReadAll(resp.Body)
require.NoError(t, err)
require.Equal(t, "hello", string(body))
finalReqURL := *resp.Request.URL
finalReqURL.Opaque = "" // Clear the opaque so we can compare the URI
require.Equal(t, tc.expectedURI, finalReqURL.String())
})
}
}
// TestMultiaddrURIRedirect tests that we can redirect using a multiaddr URI. We
// redirect from the http transport to the stream based transport
func TestMultiaddrURIRedirect(t *testing.T) {
serverHost, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/udp/0/quic-v1"))
require.NoError(t, err)
serverHttpHost := libp2phttp.Host{
StreamHost: serverHost,
InsecureAllowHTTP: true,
ListenAddrs: []ma.Multiaddr{ma.StringCast("/ip4/127.0.0.1/tcp/0/http")},
}
go serverHttpHost.Serve()
defer serverHttpHost.Close()
var httpMultiaddr ma.Multiaddr
var streamMultiaddr ma.Multiaddr
for _, a := range serverHttpHost.Addrs() {
if _, err := a.ValueForProtocol(ma.P_HTTP); err == nil {
httpMultiaddr = a
} else {
streamMultiaddr = a
}
}
require.NotNil(t, httpMultiaddr)
require.NotNil(t, streamMultiaddr)
// Redirect to a whole other transport!
serverHttpHost.SetHTTPHandlerAtPath("/redirect-1/0.0.1", "/a", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Location", fmt.Sprintf("multiaddr:%s/p2p/%s/http-path/b", streamMultiaddr, serverHost.ID()))
w.WriteHeader(http.StatusMovedPermanently)
}))
serverHttpHost.SetHTTPHandlerAtPath("/redirect-2/0.0.1", "/b", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}))
clientStreamHost, err := libp2p.New(libp2p.NoListenAddrs, libp2p.Transport(libp2pquic.NewTransport))
require.NoError(t, err)
client := http.Client{Transport: &libp2phttp.Host{StreamHost: clientStreamHost}}
resp, err := client.Get(fmt.Sprintf("multiaddr:%s/http-path/a", httpMultiaddr))
require.NoError(t, err)
require.Equal(t, http.StatusOK, resp.StatusCode)
require.True(t, strings.HasPrefix(resp.Request.URL.RawPath, streamMultiaddr.String()), "expected redirect to stream transport")
}
func TestImpliedHostIsSet(t *testing.T) {
serverHost, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/udp/0/quic-v1"))
require.NoError(t, err)
serverHttpHost := libp2phttp.Host{
StreamHost: serverHost,
InsecureAllowHTTP: true,
ListenAddrs: []ma.Multiaddr{ma.StringCast("/ip4/127.0.0.1/tcp/0/http")},
}
go serverHttpHost.Serve()
defer serverHttpHost.Close()
serverHttpHost.SetHTTPHandlerAtPath("/hi", "/", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasPrefix(r.Host, "localhost") && r.URL.Path == "/" {
w.WriteHeader(http.StatusOK)
return
}
w.WriteHeader(http.StatusNotFound)
}))
clientStreamHost, err := libp2p.New(libp2p.NoListenAddrs, libp2p.Transport(libp2pquic.NewTransport))
require.NoError(t, err)
client := http.Client{Transport: &libp2phttp.Host{StreamHost: clientStreamHost}}
type testCase struct {
uri string
}
var testCases []testCase
for _, a := range serverHttpHost.Addrs() {
if _, err := a.ValueForProtocol(ma.P_HTTP); err == nil {
port, err := a.ValueForProtocol(ma.P_TCP)
require.NoError(t, err)
u := fmt.Sprintf("multiaddr:/dns/localhost/tcp/%s/http", port)
testCases = append(testCases, testCase{u})
} else {
port, err := a.ValueForProtocol(ma.P_UDP)
require.NoError(t, err)
u := fmt.Sprintf("multiaddr:/dns/localhost/udp/%s/quic-v1/p2p/%s", port, serverHost.ID())
testCases = append(testCases, testCase{u})
}
}
for _, tc := range testCases {
t.Run(tc.uri, func(t *testing.T) {
resp, err := client.Get(tc.uri)
require.NoError(t, err)
defer resp.Body.Close()
require.Equal(t, http.StatusOK, resp.StatusCode)
})
}
}

2
test-plans/go.mod

@ -5,7 +5,7 @@ go 1.21
require (
github.com/go-redis/redis/v8 v8.11.5
github.com/libp2p/go-libp2p v0.0.0
github.com/multiformats/go-multiaddr v0.12.4
github.com/multiformats/go-multiaddr v0.13.0
)
require (

4
test-plans/go.sum

@ -187,8 +187,8 @@ github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9
github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4=
github.com/multiformats/go-multiaddr v0.1.1/go.mod h1:aMKBKNEYmzmDmxfX88/vz+J5IU55txyt0p4aiWVohjo=
github.com/multiformats/go-multiaddr v0.2.0/go.mod h1:0nO36NvPpyV4QzvTLi/lafl2y95ncPj0vFwVF6k6wJ4=
github.com/multiformats/go-multiaddr v0.12.4 h1:rrKqpY9h+n80EwhhC/kkcunCZZ7URIF8yN1WEUt2Hvc=
github.com/multiformats/go-multiaddr v0.12.4/go.mod h1:sBXrNzucqkFJhvKOiwwLyqamGa/P5EIXNPLovyhQCII=
github.com/multiformats/go-multiaddr v0.13.0 h1:BCBzs61E3AGHcYYTv8dqRH43ZfyrqM8RXVPT8t13tLQ=
github.com/multiformats/go-multiaddr v0.13.0/go.mod h1:sBXrNzucqkFJhvKOiwwLyqamGa/P5EIXNPLovyhQCII=
github.com/multiformats/go-multiaddr-dns v0.3.1 h1:QgQgR+LQVt3NPTjbrLLpsaT2ufAA2y0Mkk+QRVJbW3A=
github.com/multiformats/go-multiaddr-dns v0.3.1/go.mod h1:G/245BRQ6FJGmryJCrOuTdB37AMA5AMOVuO6NY3JwTk=
github.com/multiformats/go-multiaddr-fmt v0.1.0 h1:WLEFClPycPkp4fnIzoFoV9FVd49/eQsuaL3/CWe167E=

Loading…
Cancel
Save