From 6aa49188108af8538e4eafc552b207f5924cdde5 Mon Sep 17 00:00:00 2001
From: John Howard <howardjohn@google.com>
Date: Mon, 27 Jun 2022 09:36:58 -0700
Subject: [PATCH 1/4] hbone: initial echo server/client implementation

---
 pkg/config/protocol/instance.go             |   2 +
 pkg/hbone/README.md                         |  71 ++++++
 pkg/hbone/dialer.go                         | 173 ++++++++++++++
 pkg/hbone/dialer_test.go                    |  91 ++++++++
 pkg/hbone/server.go                         |  77 +++++++
 pkg/hbone/util.go                           | 112 +++++++++
 pkg/test/echo/cmd/client/main.go            |  39 +++-
 pkg/test/echo/cmd/server/main.go            |  13 +-
 pkg/test/echo/proto/echo.pb.go              | 241 ++++++++++++++++----
 pkg/test/echo/proto/echo.proto              |  20 ++
 pkg/test/echo/server/endpoint/hbone.go      |  99 ++++++++
 pkg/test/echo/server/endpoint/http.go       |   8 +-
 pkg/test/echo/server/endpoint/instance.go   |   2 +
 pkg/test/echo/server/forwarder/config.go    | 104 +++++++++
 pkg/test/echo/server/forwarder/grpc.go      |   3 +-
 pkg/test/echo/server/forwarder/http.go      |   9 +-
 pkg/test/echo/server/forwarder/tcp.go       |   6 +-
 pkg/test/echo/server/forwarder/tls.go       |   3 +-
 pkg/test/echo/server/forwarder/util.go      |  13 +-
 pkg/test/echo/server/forwarder/websocket.go |   4 +-
 pkg/test/echo/server/instance.go            |   1 +
 21 files changed, 1026 insertions(+), 65 deletions(-)
 create mode 100644 pkg/hbone/README.md
 create mode 100644 pkg/hbone/dialer.go
 create mode 100644 pkg/hbone/dialer_test.go
 create mode 100644 pkg/hbone/server.go
 create mode 100644 pkg/hbone/util.go
 create mode 100644 pkg/test/echo/server/endpoint/hbone.go

diff --git a/pkg/config/protocol/instance.go b/pkg/config/protocol/instance.go
index 276f0864b8c0..e6811db19a47 100644
--- a/pkg/config/protocol/instance.go
+++ b/pkg/config/protocol/instance.go
@@ -54,6 +54,8 @@ const (
 	Redis Instance = "Redis"
 	// MySQL declares that the port carries MySQL traffic.
 	MySQL Instance = "MySQL"
+	// HBONE declares that the port carries HBONE traffic.
+	HBONE Instance = "HBONE"
 	// Unsupported - value to signify that the protocol is unsupported.
 	Unsupported Instance = "UnsupportedProtocol"
 )
diff --git a/pkg/hbone/README.md b/pkg/hbone/README.md
new file mode 100644
index 000000000000..94a501fce433
--- /dev/null
+++ b/pkg/hbone/README.md
@@ -0,0 +1,71 @@
+# HTTP Based Overlay Network (HBONE)
+
+HTTP Based Overlay Network (HBONE) is the protocol used by Istio for communication between workloads in the mesh.
+At a high level, the protocol consists of tunneling TCP connections over HTTP/2 CONNECT, over mTLS.
+
+## Specification
+
+TODO
+
+## Implementations
+
+### Clients
+
+#### CLI
+
+A CLI client is available using the `client` binary.
+
+Usage examples:
+
+```shell
+go install ./pkg/test/echo/cmd/client
+# Send request to 127.0.0.1:8080 (Note only IPs are supported) via an HBONE proxy on port 15008
+client --hbone-client-cert tests/testdata/certs/cert.crt --hbone-client-key tests/testdata/certs/cert.key \
+  http://127.0.0.1:8080 \
+  --hbone 127.0.0.1:15008
+```
+
+#### Golang
+
+An (unstable) library to make HBONE connections is available at `pkg/hbone`.
+
+Usage example:
+
+```go
+d := hbone.NewDialer(hbone.Config{
+    ProxyAddress: "1.2.3.4:15008",
+    Headers: map[string][]string{
+        "some-addition-metadata": {"test-value"},
+    },
+    TLS:          nil, // TLS is strongly recommended in real world
+})
+client, _ := d.Dial("tcp", testAddr)
+client.Write([]byte("hello world"))
+```
+
+### Server
+
+#### Server CLI
+
+A CLI client is available using the `server` binary.
+
+Usage examples:
+
+```shell
+go install ./pkg/test/echo/cmd/server
+# Serve on port 15008 (default) with TLS
+server --tls 15008 --crt tests/testdata/certs/cert.crt --key tests/testdata/certs/cert.key
+```
+
+#### Server Golang Library
+
+An (unstable) library to run an HBONE server is available at `pkg/hbone`.
+
+Usage example:
+
+```go
+s := hbone.NewServer()
+// TLS is strongly recommended in real world
+l, _ := net.Listen("tcp", "0.0.0.0:15008")
+s.Serve(l)
+```
diff --git a/pkg/hbone/dialer.go b/pkg/hbone/dialer.go
new file mode 100644
index 000000000000..8fe7c7205cf7
--- /dev/null
+++ b/pkg/hbone/dialer.go
@@ -0,0 +1,173 @@
+// Copyright Istio Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hbone
+
+import (
+	"context"
+	"crypto/tls"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"strings"
+	"sync"
+	"time"
+
+	"golang.org/x/net/http2"
+	"golang.org/x/net/proxy"
+
+	"istio.io/istio/security/pkg/pki/util"
+	istiolog "istio.io/pkg/log"
+)
+
+var log = istiolog.RegisterScope("hbone", "", 0)
+
+type Config struct {
+	ProxyAddress string
+	Headers      http.Header
+	TLS          *tls.Config
+}
+
+type Dialer interface {
+	proxy.Dialer
+	proxy.ContextDialer
+}
+
+func NewDialer(cfg Config) Dialer {
+	return &dialer{cfg: cfg}
+}
+
+type dialer struct {
+	cfg Config
+}
+
+func (d dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
+	if network != "tcp" {
+		return net.Dial(network, address)
+	}
+	// TODO: use context
+	c, s := net.Pipe()
+	go func() {
+		err := proxyTo(s, d.cfg, address)
+		log.Infof("Tunnel complete: %v", err)
+	}()
+	return c, nil
+}
+
+func (d dialer) Dial(network, address string) (c net.Conn, err error) {
+	return d.DialContext(context.Background(), network, address)
+}
+
+func proxyTo(conn io.ReadWriteCloser, req Config, address string) error {
+	defer conn.Close()
+	t0 := time.Now()
+
+	url := "http://" + req.ProxyAddress
+	if req.TLS != nil {
+		url = "https://" + req.ProxyAddress
+	}
+	// Setup a pipe. We could just pass `conn` to `http.NewRequest`, but this has a few issues:
+	// * Less visibility into i/o
+	// * http will call conn.Close, which will close before we want to (finished writing response).
+	pr, pw := io.Pipe()
+	r, err := http.NewRequest("CONNECT", url, pr)
+	if err != nil {
+		return fmt.Errorf("new request: %v", err)
+	}
+	r.Host = address
+
+	var transport *http2.Transport
+	if req.TLS != nil {
+		transport = &http2.Transport{
+			TLSClientConfig: req.TLS,
+		}
+	} else {
+		transport = &http2.Transport{
+			// For h2c
+			AllowHTTP: true,
+			DialTLS: func(network, addr string, cfg *tls.Config) (net.Conn, error) {
+				return net.Dial(network, addr)
+			},
+		}
+	}
+
+	// Initiate CONNECT.
+	log.Infof("initiate CONNECT to %v via %v", r.Host, url)
+
+	resp, err := transport.RoundTrip(r)
+	if err != nil {
+		return fmt.Errorf("round trip: %v", err)
+	}
+	defer resp.Body.Close()
+	var remoteID string
+	if resp.TLS != nil && len(resp.TLS.PeerCertificates) > 0 {
+		ids, _ := util.ExtractIDs(resp.TLS.PeerCertificates[0].Extensions)
+		if len(ids) > 0 {
+			remoteID = ids[0]
+		}
+	}
+	log.WithLabels("host", r.Host, "remote", remoteID).Info("CONNECT established")
+	wg := sync.WaitGroup{}
+	wg.Add(1)
+	go func() {
+		// handle upstream (mtp server) --> downstream (app)
+		copyBuffered(conn, resp.Body, log.WithLabels("name", "body to conn"))
+		wg.Done()
+	}()
+	// Copy from conn into the pipe, which will then be sent as part of the request
+	// handle upstream (mtp server) <-- downstream (app)
+	copyBuffered(pw, conn, log.WithLabels("name", "conn to pipe"))
+
+	wg.Wait()
+	log.Info("stream closed in ", time.Since(t0))
+	return nil
+}
+
+// TLSDialWithDialer is an implementation of tls.DialWithDialer that accepts a generic Dialer
+func TLSDialWithDialer(dialer Dialer, network, addr string, config *tls.Config) (*tls.Conn, error) {
+	return tlsDial(context.Background(), dialer, network, addr, config)
+}
+
+func tlsDial(ctx context.Context, netDialer Dialer, network, addr string, config *tls.Config) (*tls.Conn, error) {
+	rawConn, err := netDialer.DialContext(ctx, network, addr)
+	if err != nil {
+		return nil, err
+	}
+
+	colonPos := strings.LastIndex(addr, ":")
+	if colonPos == -1 {
+		colonPos = len(addr)
+	}
+	hostname := addr[:colonPos]
+
+	if config == nil {
+		config = &tls.Config{}
+	}
+	// If no ServerName is set, infer the ServerName
+	// from the hostname we're connecting to.
+	if config.ServerName == "" {
+		// Make a copy to avoid polluting argument or default.
+		c := config.Clone()
+		c.ServerName = hostname
+		config = c
+	}
+
+	conn := tls.Client(rawConn, config)
+	if err := conn.HandshakeContext(ctx); err != nil {
+		rawConn.Close()
+		return nil, err
+	}
+	return conn, nil
+}
diff --git a/pkg/hbone/dialer_test.go b/pkg/hbone/dialer_test.go
new file mode 100644
index 000000000000..39c9434b8be4
--- /dev/null
+++ b/pkg/hbone/dialer_test.go
@@ -0,0 +1,91 @@
+// Copyright Istio Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hbone
+
+import (
+	"net"
+	"testing"
+)
+
+func newTCPServer(t testing.TB, data string) string {
+	n, err := net.Listen("tcp", "127.0.0.1:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("opened listener on %v", n.Addr().String())
+	go func() {
+		for {
+			c, err := n.Accept()
+			if err != nil {
+				t.Log(err)
+				return
+			}
+			t.Log("accepted connection")
+			c.Write([]byte(data))
+			c.Close()
+		}
+	}()
+	t.Cleanup(func() {
+		n.Close()
+	})
+	return n.Addr().String()
+}
+
+func TestDialer(t *testing.T) {
+	testAddr := newTCPServer(t, "hello")
+	proxy := newHBONEServer(t)
+	d := NewDialer(Config{
+		ProxyAddress: proxy,
+		Headers: map[string][]string{
+			"some-addition-metadata": {"test-value"},
+		},
+		TLS: nil, // No TLS for simplification
+	})
+	client, err := d.Dial("tcp", testAddr)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer client.Close()
+
+	go func() {
+		n, err := client.Write([]byte("hello world"))
+		t.Logf("wrote %v/%v", n, err)
+	}()
+
+	buf := make([]byte, 8)
+	n, err := client.Read(buf)
+	if err != nil {
+		t.Fatalf("err with %v: %v", n, err)
+	}
+	if string(buf[:n]) != "hello" {
+		t.Fatalf("got unexpected buffer: %v", string(buf[:n]))
+	}
+	t.Logf("Read %v", string(buf[:n]))
+}
+
+func newHBONEServer(t *testing.T) string {
+	s := NewServer()
+	l, err := net.Listen("tcp", "0.0.0.0:0")
+	if err != nil {
+		t.Fatal(err)
+	}
+	go func() {
+		_ = s.Serve(l)
+	}()
+	t.Cleanup(func() {
+		_ = l.Close()
+	})
+	return l.Addr().String()
+}
diff --git a/pkg/hbone/server.go b/pkg/hbone/server.go
new file mode 100644
index 000000000000..24f04f1df545
--- /dev/null
+++ b/pkg/hbone/server.go
@@ -0,0 +1,77 @@
+// Copyright Istio Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hbone
+
+import (
+	"context"
+	"net"
+	"net/http"
+	"sync"
+	"time"
+
+	"golang.org/x/net/http2"
+	"golang.org/x/net/http2/h2c"
+)
+
+func NewServer() *http.Server {
+	// Need to set this to allow timeout on the read header
+	h1 := &http.Transport{
+		ExpectContinueTimeout: 3 * time.Second,
+	}
+	h2, _ := http2.ConfigureTransports(h1)
+	h2.ReadIdleTimeout = 10 * time.Minute // TODO: much larger to support long-lived connections
+	h2.AllowHTTP = true
+	h2.StrictMaxConcurrentStreams = false
+	h2Server := &http2.Server{}
+	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		t0 := time.Now()
+		if r.Method == http.MethodConnect {
+			log.WithLabels("host", r.Host).Info("Received CONNECT")
+			// Send headers back immediately so we can start getting the body
+			w.(http.Flusher).Flush()
+			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+			defer cancel()
+
+			dst, err := (&net.Dialer{}).DialContext(ctx, "tcp", r.Host)
+			if err != nil {
+				w.WriteHeader(http.StatusServiceUnavailable)
+				log.Errorf("failed to dial upstream: %v", err)
+				return
+			}
+			log.Infof("Connected to %v", r.Host)
+			w.WriteHeader(http.StatusOK)
+
+			wg := sync.WaitGroup{}
+			wg.Add(1)
+			go func() {
+				// downstream (hbone client) <-- upstream (app)
+				copyBuffered(w, dst, log.WithLabels("name", "dst to w"))
+				r.Body.Close()
+				wg.Done()
+			}()
+			// downstream (hbone client) --> upstream (app)
+			copyBuffered(dst, r.Body, log.WithLabels("name", "body to dst"))
+			wg.Wait()
+			log.Infof("connection closed in %v", time.Since(t0))
+		} else {
+			log.Errorf("non-CONNECT: %v", r.Method)
+			w.WriteHeader(http.StatusMethodNotAllowed)
+		}
+	})
+	hs := &http.Server{
+		Handler: h2c.NewHandler(handler, h2Server),
+	}
+	return hs
+}
diff --git a/pkg/hbone/util.go b/pkg/hbone/util.go
new file mode 100644
index 000000000000..f321ee0590b5
--- /dev/null
+++ b/pkg/hbone/util.go
@@ -0,0 +1,112 @@
+// Copyright Istio Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hbone
+
+import (
+	"io"
+	"net"
+	"net/http"
+	"sync"
+	"time"
+
+	istiolog "istio.io/pkg/log"
+)
+
+// createBuffer to get a buffer. io.Copy uses 32k.
+// experimental use shows ~20k max read with Firefox.
+var bufferPoolCopy = sync.Pool{New: func() interface{} {
+	return make([]byte, 0, 32*1024)
+}}
+
+func copyBuffered(dst io.Writer, src io.Reader, log *istiolog.Scope) {
+	buf1 := bufferPoolCopy.Get().([]byte)
+	// nolint: staticcheck
+	defer bufferPoolCopy.Put(buf1)
+	bufCap := cap(buf1)
+	buf := buf1[0:bufCap:bufCap]
+
+	// For netstack: src is a gonet.Conn, doesn't implement WriterTo. Dst is a net.TcpConn - and implements ReadFrom.
+	// CopyBuffered is the actual implementation of Copy and CopyBuffer.
+	// if buf is nil, one is allocated.
+	// Duplicated from io
+
+	// This will prevent stats from working.
+	// If the reader has a WriteTo method, use it to do the copy.
+	// Avoids an allocation and a copy.
+	//if wt, ok := src.(io.WriterTo); ok {
+	//	return wt.WriteTo(dst)
+	//}
+	// Similarly, if the writer has a ReadFrom method, use it to do the copy.
+	//if rt, ok := dst.(io.ReaderFrom); ok {
+	//	return rt.ReadFrom(src)
+	//}
+	for {
+		if srcc, ok := src.(net.Conn); ok {
+			// Best effort
+			_ = srcc.SetReadDeadline(time.Now().Add(15 * time.Minute))
+		}
+		nr, err := src.Read(buf)
+		log.Debugf("read %v/%v", nr, err)
+		if nr > 0 { // before dealing with the read error
+			nw, ew := dst.Write(buf[0:nr])
+			log.Debugf("write %v/%v", nw, ew)
+			if f, ok := dst.(http.Flusher); ok {
+				f.Flush()
+			}
+			if nr != nw { // Should not happen
+				ew = io.ErrShortWrite
+			}
+			if ew != nil {
+				return
+			}
+		}
+		if err != nil {
+			// read is already closed - we need to close out
+			_ = closeWriter(dst)
+			return
+		}
+	}
+}
+
+// CloseWriter is one of possible interfaces implemented by Out to send a FIN, without closing
+// the input. Some writers only do this when Close is called.
+type CloseWriter interface {
+	CloseWrite() error
+}
+
+func closeWriter(dst io.Writer) error {
+	if cw, ok := dst.(CloseWriter); ok {
+		return cw.CloseWrite()
+	}
+	if c, ok := dst.(io.Closer); ok {
+		return c.Close()
+	}
+	if rw, ok := dst.(http.ResponseWriter); ok {
+		// Server side HTTP stream. For client side, FIN can be sent by closing the pipe (or
+		// request body). For server, the FIN will be sent when the handler returns - but
+		// this only happen after request is completed and body has been read. If server wants
+		// to send FIN first - while still reading the body - we are in trouble.
+
+		// That means HTTP2 TCP servers provide no way to send a FIN from server, without
+		// having the request fully read.
+
+		// This works for H2 with the current library - but very tricky, if not set as trailer.
+		rw.Header().Set("X-Close", "0")
+		rw.(http.Flusher).Flush()
+		return nil
+	}
+	log.Infof("Server out not Closer nor CloseWriter nor ResponseWriter: %v", dst)
+	return nil
+}
diff --git a/pkg/test/echo/cmd/client/main.go b/pkg/test/echo/cmd/client/main.go
index 29cf0d8d6c1e..1a260144ee61 100644
--- a/pkg/test/echo/cmd/client/main.go
+++ b/pkg/test/echo/cmd/client/main.go
@@ -56,11 +56,18 @@ var (
 	followRedirects         bool
 	newConnectionPerRequest bool
 	forceDNSLookup          bool
-	clientCert              string
-	clientKey               string
+
+	clientCert string
+	clientKey  string
 
 	caFile string
 
+	hboneAddress    string
+	hboneHeaders    []string
+	hboneClientCert string
+	hboneClientKey  string
+	hboneCaFile     string
+
 	loggingOptions = log.DefaultOptions()
 
 	rootCmd = &cobra.Command{
@@ -151,6 +158,13 @@ func init() {
 	rootCmd.PersistentFlags().StringSliceVarP(&alpn, "alpn", "", nil, "alpn to set")
 	rootCmd.PersistentFlags().StringVarP(&serverName, "server-name", "", serverName, "server name to set")
 
+	rootCmd.PersistentFlags().StringVar(&hboneAddress, "hbone", "", "address to send HBONE request to")
+	rootCmd.PersistentFlags().StringSliceVarP(&hboneHeaders, "hbone-header", "M", hboneHeaders,
+		"A list of http headers for HBONE connection (use Host for authority) - 'name: value', following curl syntax")
+	rootCmd.PersistentFlags().StringVar(&hboneCaFile, "hbone-ca", "", "CA root cert file used for the HBONE request")
+	rootCmd.PersistentFlags().StringVar(&hboneClientCert, "hbone-client-cert", "", "client certificate file used for the HBONE request")
+	rootCmd.PersistentFlags().StringVar(&hboneClientKey, "hbone-client-key", "", "client certificate key file used for the HBONE request")
+
 	loggingOptions.AttachCobraFlags(rootCmd)
 
 	cmd.AddFlags(rootCmd)
@@ -185,6 +199,27 @@ func getRequest(url string) (*proto.ForwardEchoRequest, error) {
 		NewConnectionPerRequest: newConnectionPerRequest,
 		ForceDNSLookup:          forceDNSLookup,
 	}
+	if len(hboneAddress) > 0 {
+		request.Hbone = &proto.HBONE{
+			Address:            hboneAddress,
+			CertFile:           hboneClientCert,
+			KeyFile:            hboneClientKey,
+			CaCertFile:         hboneCaFile,
+			InsecureSkipVerify: false,
+		}
+		for _, header := range hboneHeaders {
+			parts := strings.SplitN(header, ":", 2)
+			// require name:value format
+			if len(parts) != 2 {
+				return nil, fmt.Errorf("invalid header format: %q (want name:value)", header)
+			}
+
+			request.Hbone.Headers = append(request.Hbone.Headers, &proto.Header{
+				Key:   parts[0],
+				Value: strings.Trim(parts[1], " "),
+			})
+		}
+	}
 
 	if expectSet {
 		request.ExpectedResponse = &wrappers.StringValue{Value: expect}
diff --git a/pkg/test/echo/cmd/server/main.go b/pkg/test/echo/cmd/server/main.go
index 30bc9d7a017e..2a948b18faab 100644
--- a/pkg/test/echo/cmd/server/main.go
+++ b/pkg/test/echo/cmd/server/main.go
@@ -37,6 +37,7 @@ var (
 	grpcPorts        []int
 	tcpPorts         []int
 	tlsPorts         []int
+	hbonePorts       []int
 	instanceIPPorts  []int
 	localhostIPPorts []int
 	serverFirstPorts []int
@@ -59,7 +60,7 @@ var (
 		Long:              `Echo application for testing Istio E2E`,
 		PersistentPreRunE: configureLogging,
 		Run: func(cmd *cobra.Command, args []string) {
-			ports := make(common.PortList, len(httpPorts)+len(grpcPorts)+len(tcpPorts))
+			ports := make(common.PortList, len(httpPorts)+len(grpcPorts)+len(tcpPorts)+len(hbonePorts))
 			tlsByPort := map[int]bool{}
 			for _, p := range tlsPorts {
 				tlsByPort[p] = true
@@ -104,6 +105,15 @@ var (
 				}
 				portIndex++
 			}
+			for i, p := range hbonePorts {
+				ports[portIndex] = &common.Port{
+					Name:     "hbone-" + strconv.Itoa(i),
+					Protocol: protocol.HBONE,
+					Port:     p,
+					TLS:      tlsByPort[p],
+				}
+				portIndex++
+			}
 			instanceIPByPort := map[int]struct{}{}
 			for _, p := range instanceIPPorts {
 				instanceIPByPort[p] = struct{}{}
@@ -154,6 +164,7 @@ func init() {
 	rootCmd.PersistentFlags().IntSliceVar(&httpPorts, "port", []int{8080}, "HTTP/1.1 ports")
 	rootCmd.PersistentFlags().IntSliceVar(&grpcPorts, "grpc", []int{7070}, "GRPC ports")
 	rootCmd.PersistentFlags().IntSliceVar(&tcpPorts, "tcp", []int{9090}, "TCP ports")
+	rootCmd.PersistentFlags().IntSliceVar(&hbonePorts, "hbone", []int{}, "HBONE ports")
 	rootCmd.PersistentFlags().IntSliceVar(&tlsPorts, "tls", []int{}, "Ports that are using TLS. These must be defined as http/grpc/tcp.")
 	rootCmd.PersistentFlags().IntSliceVar(&instanceIPPorts, "bind-ip", []int{}, "Ports that are bound to INSTANCE_IP rather than wildcard IP.")
 	rootCmd.PersistentFlags().IntSliceVar(&localhostIPPorts, "bind-localhost", []int{}, "Ports that are bound to localhost rather than wildcard IP.")
diff --git a/pkg/test/echo/proto/echo.pb.go b/pkg/test/echo/proto/echo.pb.go
index ff2935fb0638..3c05745b144f 100644
--- a/pkg/test/echo/proto/echo.pb.go
+++ b/pkg/test/echo/proto/echo.pb.go
@@ -232,6 +232,8 @@ type ForwardEchoRequest struct {
 	NewConnectionPerRequest bool `protobuf:"varint,22,opt,name=newConnectionPerRequest,proto3" json:"newConnectionPerRequest,omitempty"`
 	// If set, each request will force a DNS lookup. Only applies if newConnectionPerRequest is set.
 	ForceDNSLookup bool `protobuf:"varint,23,opt,name=forceDNSLookup,proto3" json:"forceDNSLookup,omitempty"`
+	// HBONE communication settings. If provided, requests will be tunnelled.
+	Hbone *HBONE `protobuf:"bytes,24,opt,name=hbone,proto3" json:"hbone,omitempty"`
 }
 
 func (x *ForwardEchoRequest) Reset() {
@@ -427,6 +429,129 @@ func (x *ForwardEchoRequest) GetForceDNSLookup() bool {
 	return false
 }
 
+func (x *ForwardEchoRequest) GetHbone() *HBONE {
+	if x != nil {
+		return x.Hbone
+	}
+	return nil
+}
+
+type HBONE struct {
+	state         protoimpl.MessageState
+	sizeCache     protoimpl.SizeCache
+	unknownFields protoimpl.UnknownFields
+
+	Address string    `protobuf:"bytes,9,opt,name=address,proto3" json:"address,omitempty"`
+	Headers []*Header `protobuf:"bytes,1,rep,name=headers,proto3" json:"headers,omitempty"`
+	// If non-empty, make the request with the corresponding cert and key.
+	Cert string `protobuf:"bytes,2,opt,name=cert,proto3" json:"cert,omitempty"`
+	Key  string `protobuf:"bytes,3,opt,name=key,proto3" json:"key,omitempty"`
+	// If non-empty, verify the server CA
+	CaCert string `protobuf:"bytes,4,opt,name=caCert,proto3" json:"caCert,omitempty"`
+	// If non-empty, make the request with the corresponding cert and key file.
+	CertFile string `protobuf:"bytes,5,opt,name=certFile,proto3" json:"certFile,omitempty"`
+	KeyFile  string `protobuf:"bytes,6,opt,name=keyFile,proto3" json:"keyFile,omitempty"`
+	// If non-empty, verify the server CA with the ca cert file.
+	CaCertFile string `protobuf:"bytes,7,opt,name=caCertFile,proto3" json:"caCertFile,omitempty"`
+	// Skip verifying peer's certificate.
+	InsecureSkipVerify bool `protobuf:"varint,8,opt,name=insecureSkipVerify,proto3" json:"insecureSkipVerify,omitempty"`
+}
+
+func (x *HBONE) Reset() {
+	*x = HBONE{}
+	if protoimpl.UnsafeEnabled {
+		mi := &file_test_echo_proto_echo_proto_msgTypes[4]
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		ms.StoreMessageInfo(mi)
+	}
+}
+
+func (x *HBONE) String() string {
+	return protoimpl.X.MessageStringOf(x)
+}
+
+func (*HBONE) ProtoMessage() {}
+
+func (x *HBONE) ProtoReflect() protoreflect.Message {
+	mi := &file_test_echo_proto_echo_proto_msgTypes[4]
+	if protoimpl.UnsafeEnabled && x != nil {
+		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+		if ms.LoadMessageInfo() == nil {
+			ms.StoreMessageInfo(mi)
+		}
+		return ms
+	}
+	return mi.MessageOf(x)
+}
+
+// Deprecated: Use HBONE.ProtoReflect.Descriptor instead.
+func (*HBONE) Descriptor() ([]byte, []int) {
+	return file_test_echo_proto_echo_proto_rawDescGZIP(), []int{4}
+}
+
+func (x *HBONE) GetAddress() string {
+	if x != nil {
+		return x.Address
+	}
+	return ""
+}
+
+func (x *HBONE) GetHeaders() []*Header {
+	if x != nil {
+		return x.Headers
+	}
+	return nil
+}
+
+func (x *HBONE) GetCert() string {
+	if x != nil {
+		return x.Cert
+	}
+	return ""
+}
+
+func (x *HBONE) GetKey() string {
+	if x != nil {
+		return x.Key
+	}
+	return ""
+}
+
+func (x *HBONE) GetCaCert() string {
+	if x != nil {
+		return x.CaCert
+	}
+	return ""
+}
+
+func (x *HBONE) GetCertFile() string {
+	if x != nil {
+		return x.CertFile
+	}
+	return ""
+}
+
+func (x *HBONE) GetKeyFile() string {
+	if x != nil {
+		return x.KeyFile
+	}
+	return ""
+}
+
+func (x *HBONE) GetCaCertFile() string {
+	if x != nil {
+		return x.CaCertFile
+	}
+	return ""
+}
+
+func (x *HBONE) GetInsecureSkipVerify() bool {
+	if x != nil {
+		return x.InsecureSkipVerify
+	}
+	return false
+}
+
 type Alpn struct {
 	state         protoimpl.MessageState
 	sizeCache     protoimpl.SizeCache
@@ -438,7 +563,7 @@ type Alpn struct {
 func (x *Alpn) Reset() {
 	*x = Alpn{}
 	if protoimpl.UnsafeEnabled {
-		mi := &file_test_echo_proto_echo_proto_msgTypes[4]
+		mi := &file_test_echo_proto_echo_proto_msgTypes[5]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		ms.StoreMessageInfo(mi)
 	}
@@ -451,7 +576,7 @@ func (x *Alpn) String() string {
 func (*Alpn) ProtoMessage() {}
 
 func (x *Alpn) ProtoReflect() protoreflect.Message {
-	mi := &file_test_echo_proto_echo_proto_msgTypes[4]
+	mi := &file_test_echo_proto_echo_proto_msgTypes[5]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -464,7 +589,7 @@ func (x *Alpn) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use Alpn.ProtoReflect.Descriptor instead.
 func (*Alpn) Descriptor() ([]byte, []int) {
-	return file_test_echo_proto_echo_proto_rawDescGZIP(), []int{4}
+	return file_test_echo_proto_echo_proto_rawDescGZIP(), []int{5}
 }
 
 func (x *Alpn) GetValue() []string {
@@ -485,7 +610,7 @@ type ForwardEchoResponse struct {
 func (x *ForwardEchoResponse) Reset() {
 	*x = ForwardEchoResponse{}
 	if protoimpl.UnsafeEnabled {
-		mi := &file_test_echo_proto_echo_proto_msgTypes[5]
+		mi := &file_test_echo_proto_echo_proto_msgTypes[6]
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		ms.StoreMessageInfo(mi)
 	}
@@ -498,7 +623,7 @@ func (x *ForwardEchoResponse) String() string {
 func (*ForwardEchoResponse) ProtoMessage() {}
 
 func (x *ForwardEchoResponse) ProtoReflect() protoreflect.Message {
-	mi := &file_test_echo_proto_echo_proto_msgTypes[5]
+	mi := &file_test_echo_proto_echo_proto_msgTypes[6]
 	if protoimpl.UnsafeEnabled && x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 		if ms.LoadMessageInfo() == nil {
@@ -511,7 +636,7 @@ func (x *ForwardEchoResponse) ProtoReflect() protoreflect.Message {
 
 // Deprecated: Use ForwardEchoResponse.ProtoReflect.Descriptor instead.
 func (*ForwardEchoResponse) Descriptor() ([]byte, []int) {
-	return file_test_echo_proto_echo_proto_rawDescGZIP(), []int{5}
+	return file_test_echo_proto_echo_proto_rawDescGZIP(), []int{6}
 }
 
 func (x *ForwardEchoResponse) GetOutput() []string {
@@ -536,7 +661,7 @@ var file_test_echo_proto_echo_proto_rawDesc = []byte{
 	0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x30, 0x0a, 0x06, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72,
 	0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b,
 	0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28,
-	0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0xf9, 0x05, 0x0a, 0x12, 0x46, 0x6f, 0x72,
+	0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x9d, 0x06, 0x0a, 0x12, 0x46, 0x6f, 0x72,
 	0x77, 0x61, 0x72, 0x64, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12,
 	0x14, 0x0a, 0x05, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05,
 	0x63, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x71, 0x70, 0x73, 0x18, 0x02, 0x20, 0x01,
@@ -584,23 +709,42 @@ var file_test_echo_proto_echo_proto_rawDesc = []byte{
 	0x6f, 0x6e, 0x50, 0x65, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x26, 0x0a, 0x0e,
 	0x66, 0x6f, 0x72, 0x63, 0x65, 0x44, 0x4e, 0x53, 0x4c, 0x6f, 0x6f, 0x6b, 0x75, 0x70, 0x18, 0x17,
 	0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x66, 0x6f, 0x72, 0x63, 0x65, 0x44, 0x4e, 0x53, 0x4c, 0x6f,
-	0x6f, 0x6b, 0x75, 0x70, 0x22, 0x1c, 0x0a, 0x04, 0x41, 0x6c, 0x70, 0x6e, 0x12, 0x14, 0x0a, 0x05,
-	0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c,
-	0x75, 0x65, 0x22, 0x2d, 0x0a, 0x13, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x45, 0x63, 0x68,
-	0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x75, 0x74,
-	0x70, 0x75, 0x74, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06, 0x6f, 0x75, 0x74, 0x70, 0x75,
-	0x74, 0x32, 0x88, 0x01, 0x0a, 0x0f, 0x45, 0x63, 0x68, 0x6f, 0x54, 0x65, 0x73, 0x74, 0x53, 0x65,
-	0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x2f, 0x0a, 0x04, 0x45, 0x63, 0x68, 0x6f, 0x12, 0x12, 0x2e,
-	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
-	0x74, 0x1a, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65,
-	0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x44, 0x0a, 0x0b, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72,
-	0x64, 0x45, 0x63, 0x68, 0x6f, 0x12, 0x19, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x6f,
-	0x72, 0x77, 0x61, 0x72, 0x64, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
-	0x1a, 0x1a, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64,
-	0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x1f, 0x0a, 0x0d,
-	0x69, 0x6f, 0x2e, 0x69, 0x73, 0x74, 0x69, 0x6f, 0x2e, 0x74, 0x65, 0x73, 0x74, 0x42, 0x04, 0x45,
-	0x63, 0x68, 0x6f, 0x5a, 0x08, 0x2e, 0x2e, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70,
-	0x72, 0x6f, 0x74, 0x6f, 0x33,
+	0x6f, 0x6b, 0x75, 0x70, 0x12, 0x22, 0x0a, 0x05, 0x68, 0x62, 0x6f, 0x6e, 0x65, 0x18, 0x18, 0x20,
+	0x01, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x48, 0x42, 0x4f, 0x4e,
+	0x45, 0x52, 0x05, 0x68, 0x62, 0x6f, 0x6e, 0x65, 0x22, 0x8e, 0x02, 0x0a, 0x05, 0x48, 0x42, 0x4f,
+	0x4e, 0x45, 0x12, 0x18, 0x0a, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x18, 0x09, 0x20,
+	0x01, 0x28, 0x09, 0x52, 0x07, 0x61, 0x64, 0x64, 0x72, 0x65, 0x73, 0x73, 0x12, 0x27, 0x0a, 0x07,
+	0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0d, 0x2e,
+	0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, 0x52, 0x07, 0x68, 0x65,
+	0x61, 0x64, 0x65, 0x72, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x63, 0x65, 0x72, 0x74, 0x18, 0x02, 0x20,
+	0x01, 0x28, 0x09, 0x52, 0x04, 0x63, 0x65, 0x72, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79,
+	0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x16, 0x0a, 0x06, 0x63,
+	0x61, 0x43, 0x65, 0x72, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x61, 0x43,
+	0x65, 0x72, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x65, 0x72, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x18,
+	0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x63, 0x65, 0x72, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x12,
+	0x18, 0x0a, 0x07, 0x6b, 0x65, 0x79, 0x46, 0x69, 0x6c, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09,
+	0x52, 0x07, 0x6b, 0x65, 0x79, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x63, 0x61, 0x43,
+	0x65, 0x72, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x63,
+	0x61, 0x43, 0x65, 0x72, 0x74, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x2e, 0x0a, 0x12, 0x69, 0x6e, 0x73,
+	0x65, 0x63, 0x75, 0x72, 0x65, 0x53, 0x6b, 0x69, 0x70, 0x56, 0x65, 0x72, 0x69, 0x66, 0x79, 0x18,
+	0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x12, 0x69, 0x6e, 0x73, 0x65, 0x63, 0x75, 0x72, 0x65, 0x53,
+	0x6b, 0x69, 0x70, 0x56, 0x65, 0x72, 0x69, 0x66, 0x79, 0x22, 0x1c, 0x0a, 0x04, 0x41, 0x6c, 0x70,
+	0x6e, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09,
+	0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x2d, 0x0a, 0x13, 0x46, 0x6f, 0x72, 0x77, 0x61,
+	0x72, 0x64, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16,
+	0x0a, 0x06, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x06,
+	0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x32, 0x88, 0x01, 0x0a, 0x0f, 0x45, 0x63, 0x68, 0x6f, 0x54,
+	0x65, 0x73, 0x74, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x2f, 0x0a, 0x04, 0x45, 0x63,
+	0x68, 0x6f, 0x12, 0x12, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x63, 0x68, 0x6f, 0x52,
+	0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45,
+	0x63, 0x68, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x44, 0x0a, 0x0b, 0x46,
+	0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x45, 0x63, 0x68, 0x6f, 0x12, 0x19, 0x2e, 0x70, 0x72, 0x6f,
+	0x74, 0x6f, 0x2e, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65,
+	0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1a, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x46, 0x6f,
+	0x72, 0x77, 0x61, 0x72, 0x64, 0x45, 0x63, 0x68, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73,
+	0x65, 0x42, 0x1f, 0x0a, 0x0d, 0x69, 0x6f, 0x2e, 0x69, 0x73, 0x74, 0x69, 0x6f, 0x2e, 0x74, 0x65,
+	0x73, 0x74, 0x42, 0x04, 0x45, 0x63, 0x68, 0x6f, 0x5a, 0x08, 0x2e, 0x2e, 0x2f, 0x70, 0x72, 0x6f,
+	0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
 }
 
 var (
@@ -615,29 +759,32 @@ func file_test_echo_proto_echo_proto_rawDescGZIP() []byte {
 	return file_test_echo_proto_echo_proto_rawDescData
 }
 
-var file_test_echo_proto_echo_proto_msgTypes = make([]protoimpl.MessageInfo, 6)
+var file_test_echo_proto_echo_proto_msgTypes = make([]protoimpl.MessageInfo, 7)
 var file_test_echo_proto_echo_proto_goTypes = []interface{}{
 	(*EchoRequest)(nil),            // 0: proto.EchoRequest
 	(*EchoResponse)(nil),           // 1: proto.EchoResponse
 	(*Header)(nil),                 // 2: proto.Header
 	(*ForwardEchoRequest)(nil),     // 3: proto.ForwardEchoRequest
-	(*Alpn)(nil),                   // 4: proto.Alpn
-	(*ForwardEchoResponse)(nil),    // 5: proto.ForwardEchoResponse
-	(*wrapperspb.StringValue)(nil), // 6: google.protobuf.StringValue
+	(*HBONE)(nil),                  // 4: proto.HBONE
+	(*Alpn)(nil),                   // 5: proto.Alpn
+	(*ForwardEchoResponse)(nil),    // 6: proto.ForwardEchoResponse
+	(*wrapperspb.StringValue)(nil), // 7: google.protobuf.StringValue
 }
 var file_test_echo_proto_echo_proto_depIdxs = []int32{
 	2, // 0: proto.ForwardEchoRequest.headers:type_name -> proto.Header
-	4, // 1: proto.ForwardEchoRequest.alpn:type_name -> proto.Alpn
-	6, // 2: proto.ForwardEchoRequest.expectedResponse:type_name -> google.protobuf.StringValue
-	0, // 3: proto.EchoTestService.Echo:input_type -> proto.EchoRequest
-	3, // 4: proto.EchoTestService.ForwardEcho:input_type -> proto.ForwardEchoRequest
-	1, // 5: proto.EchoTestService.Echo:output_type -> proto.EchoResponse
-	5, // 6: proto.EchoTestService.ForwardEcho:output_type -> proto.ForwardEchoResponse
-	5, // [5:7] is the sub-list for method output_type
-	3, // [3:5] is the sub-list for method input_type
-	3, // [3:3] is the sub-list for extension type_name
-	3, // [3:3] is the sub-list for extension extendee
-	0, // [0:3] is the sub-list for field type_name
+	5, // 1: proto.ForwardEchoRequest.alpn:type_name -> proto.Alpn
+	7, // 2: proto.ForwardEchoRequest.expectedResponse:type_name -> google.protobuf.StringValue
+	4, // 3: proto.ForwardEchoRequest.hbone:type_name -> proto.HBONE
+	2, // 4: proto.HBONE.headers:type_name -> proto.Header
+	0, // 5: proto.EchoTestService.Echo:input_type -> proto.EchoRequest
+	3, // 6: proto.EchoTestService.ForwardEcho:input_type -> proto.ForwardEchoRequest
+	1, // 7: proto.EchoTestService.Echo:output_type -> proto.EchoResponse
+	6, // 8: proto.EchoTestService.ForwardEcho:output_type -> proto.ForwardEchoResponse
+	7, // [7:9] is the sub-list for method output_type
+	5, // [5:7] is the sub-list for method input_type
+	5, // [5:5] is the sub-list for extension type_name
+	5, // [5:5] is the sub-list for extension extendee
+	0, // [0:5] is the sub-list for field type_name
 }
 
 func init() { file_test_echo_proto_echo_proto_init() }
@@ -695,7 +842,7 @@ func file_test_echo_proto_echo_proto_init() {
 			}
 		}
 		file_test_echo_proto_echo_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} {
-			switch v := v.(*Alpn); i {
+			switch v := v.(*HBONE); i {
 			case 0:
 				return &v.state
 			case 1:
@@ -707,6 +854,18 @@ func file_test_echo_proto_echo_proto_init() {
 			}
 		}
 		file_test_echo_proto_echo_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} {
+			switch v := v.(*Alpn); i {
+			case 0:
+				return &v.state
+			case 1:
+				return &v.sizeCache
+			case 2:
+				return &v.unknownFields
+			default:
+				return nil
+			}
+		}
+		file_test_echo_proto_echo_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} {
 			switch v := v.(*ForwardEchoResponse); i {
 			case 0:
 				return &v.state
@@ -725,7 +884,7 @@ func file_test_echo_proto_echo_proto_init() {
 			GoPackagePath: reflect.TypeOf(x{}).PkgPath(),
 			RawDescriptor: file_test_echo_proto_echo_proto_rawDesc,
 			NumEnums:      0,
-			NumMessages:   6,
+			NumMessages:   7,
 			NumExtensions: 0,
 			NumServices:   1,
 		},
diff --git a/pkg/test/echo/proto/echo.proto b/pkg/test/echo/proto/echo.proto
index 7e931b13b99a..7d488b266cc8 100644
--- a/pkg/test/echo/proto/echo.proto
+++ b/pkg/test/echo/proto/echo.proto
@@ -82,6 +82,26 @@ message ForwardEchoRequest {
   bool newConnectionPerRequest = 22;
   // If set, each request will force a DNS lookup. Only applies if newConnectionPerRequest is set.
   bool forceDNSLookup = 23;
+
+  // HBONE communication settings. If provided, requests will be tunnelled.
+  HBONE hbone = 24;
+}
+
+message HBONE {
+  string address = 9;
+  repeated Header headers = 1;
+  // If non-empty, make the request with the corresponding cert and key.
+  string cert = 2;
+  string key = 3;
+  // If non-empty, verify the server CA
+  string caCert = 4;
+  // If non-empty, make the request with the corresponding cert and key file.
+  string certFile = 5;
+  string keyFile = 6;
+  // If non-empty, verify the server CA with the ca cert file.
+  string caCertFile = 7;
+  // Skip verifying peer's certificate.
+  bool insecureSkipVerify = 8;
 }
 
 message Alpn {
diff --git a/pkg/test/echo/server/endpoint/hbone.go b/pkg/test/echo/server/endpoint/hbone.go
new file mode 100644
index 000000000000..88eb887e2721
--- /dev/null
+++ b/pkg/test/echo/server/endpoint/hbone.go
@@ -0,0 +1,99 @@
+// Copyright Istio Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package endpoint
+
+import (
+	"crypto/tls"
+	"fmt"
+	"net"
+	"net/http"
+
+	"istio.io/istio/pkg/hbone"
+)
+
+var _ Instance = &connectInstance{}
+
+type connectInstance struct {
+	Config
+	server *http.Server
+}
+
+func newHBONE(config Config) Instance {
+	return &connectInstance{
+		Config: config,
+	}
+}
+
+func (c connectInstance) Close() error {
+	if c.server != nil {
+		return c.server.Close()
+	}
+	return nil
+}
+
+func (c connectInstance) Start(onReady OnReadyFunc) error {
+	defer onReady()
+	c.server = hbone.NewServer()
+
+	var listener net.Listener
+	var port int
+	var err error
+	if c.Port.TLS {
+		cert, cerr := tls.LoadX509KeyPair(c.TLSCert, c.TLSKey)
+		if cerr != nil {
+			return fmt.Errorf("could not load TLS keys: %v", cerr)
+		}
+		config := &tls.Config{
+			Certificates: []tls.Certificate{cert},
+			NextProtos:   []string{"h2"},
+			GetConfigForClient: func(info *tls.ClientHelloInfo) (*tls.Config, error) {
+				// There isn't a way to pass through all ALPNs presented by the client down to the
+				// HTTP server to return in the response. However, for debugging, we can at least log
+				// them at this level.
+				epLog.Infof("TLS connection with alpn: %v", info.SupportedProtos)
+				return nil, nil
+			},
+		}
+		// Listen on the given port and update the port if it changed from what was passed in.
+		listener, port, err = listenOnAddressTLS(c.ListenerIP, c.Port.Port, config)
+		// Store the actual listening port back to the argument.
+		c.Port.Port = port
+	} else {
+		// Listen on the given port and update the port if it changed from what was passed in.
+		listener, port, err = listenOnAddress(c.ListenerIP, c.Port.Port)
+		// Store the actual listening port back to the argument.
+		c.Port.Port = port
+	}
+	if err != nil {
+		return err
+	}
+
+	if c.Port.TLS {
+		c.server.Addr = fmt.Sprintf(":%d", port)
+		fmt.Printf("Listening HBONE on %v\n", port)
+	} else {
+		c.server.Addr = fmt.Sprintf(":%d", port)
+		fmt.Printf("Listening HBONE (plaintext) on %v\n", port)
+	}
+	go func() {
+		err := c.server.Serve(listener)
+		epLog.Warnf("Port %d listener terminated with error: %v", port, err)
+	}()
+	return nil
+}
+
+func (c connectInstance) GetConfig() Config {
+	return c.Config
+}
diff --git a/pkg/test/echo/server/endpoint/http.go b/pkg/test/echo/server/endpoint/http.go
index e4547348c648..40e2b1620366 100644
--- a/pkg/test/echo/server/endpoint/http.go
+++ b/pkg/test/echo/server/endpoint/http.go
@@ -211,13 +211,7 @@ type codeAndSlices struct {
 
 func (h *httpHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	id := uuid.New()
-	remoteAddr, _, err := net.SplitHostPort(r.RemoteAddr)
-	if err != nil {
-		epLog.Warnf("failed to get host from remote address: %s", err)
-	}
-	epLog.WithLabels(
-		"remoteAddr", remoteAddr, "method", r.Method, "url", r.URL, "host", r.Host, "headers", r.Header, "id", id,
-	).Infof("HTTP Request")
+	epLog.WithLabels("method", r.Method, "url", r.URL, "host", r.Host, "headers", r.Header, "id", id).Infof("HTTP Request")
 	if h.Port == nil {
 		defer common.Metrics.HTTPRequests.With(common.PortLabel.Value("uds")).Increment()
 	} else {
diff --git a/pkg/test/echo/server/endpoint/instance.go b/pkg/test/echo/server/endpoint/instance.go
index f6a0c7de1679..80876e3d9ae2 100644
--- a/pkg/test/echo/server/endpoint/instance.go
+++ b/pkg/test/echo/server/endpoint/instance.go
@@ -54,6 +54,8 @@ type Instance interface {
 func New(cfg Config) (Instance, error) {
 	if cfg.Port != nil {
 		switch cfg.Port.Protocol {
+		case protocol.HBONE:
+			return newHBONE(cfg), nil
 		case protocol.HTTP, protocol.HTTPS:
 			return newHTTP(cfg), nil
 		case protocol.HTTP2, protocol.GRPC:
diff --git a/pkg/test/echo/server/forwarder/config.go b/pkg/test/echo/server/forwarder/config.go
index 33fc746fa939..f55ddd483c33 100644
--- a/pkg/test/echo/server/forwarder/config.go
+++ b/pkg/test/echo/server/forwarder/config.go
@@ -55,13 +55,24 @@ type Config struct {
 	urlPath                 string
 	method                  string
 	secure                  bool
+
+	// completed whether the Config is complete or still in progress
+	completed bool
+
+	hboneTLSConfig    *tls.Config
+	hboneClientConfig func(info *tls.CertificateRequestInfo) (*tls.Certificate, error)
+	hboneHeaders      http.Header
 }
 
 func (c *Config) fillDefaults() error {
+	if c.completed {
+		return nil
+	}
 	c.checkRedirect = checkRedirectFunc(c.Request)
 	c.timeout = common.GetTimeout(c.Request)
 	c.count = common.GetCount(c.Request)
 	c.headers = common.GetHeaders(c.Request)
+	c.hboneHeaders = common.ProtoToHTTPHeaders(c.Request.Hbone.GetHeaders())
 
 	// Extract the host from the headers and then remove it.
 	c.hostHeader = c.headers.Get(hostHeader)
@@ -91,6 +102,15 @@ func (c *Config) fillDefaults() error {
 	if err != nil {
 		return err
 	}
+	c.hboneClientConfig, err = getHBONEClientConfig(c.Request.Hbone)
+	if err != nil {
+		return err
+	}
+
+	c.hboneTLSConfig, err = newMtpTLSConfig(c)
+	if err != nil {
+		return err
+	}
 
 	// Parse the proxy if specified.
 	if len(c.Proxy) > 0 {
@@ -115,6 +135,8 @@ func (c *Config) fillDefaults() error {
 		c.forceDNSLookup = c.newConnectionPerRequest && c.Request.ForceDNSLookup
 	}
 
+	c.completed = true
+
 	return nil
 }
 
@@ -183,6 +205,60 @@ func getClientCertificateFunc(r *proto.ForwardEchoRequest) (func(info *tls.Certi
 	return nil, nil
 }
 
+func getHBONEClientConfig(r *proto.HBONE) (func(info *tls.CertificateRequestInfo) (*tls.Certificate, error), error) {
+	if r == nil {
+		return nil, nil
+	}
+	if r.KeyFile != "" && r.CertFile != "" {
+		certData, err := os.ReadFile(r.CertFile)
+		if err != nil {
+			return nil, fmt.Errorf("failed to load client certificate: %v", err)
+		}
+		r.Cert = string(certData)
+		keyData, err := os.ReadFile(r.KeyFile)
+		if err != nil {
+			return nil, fmt.Errorf("failed to load client certificate key: %v", err)
+		}
+		r.Key = string(keyData)
+	}
+
+	if r.Cert != "" && r.Key != "" {
+		cert, err := tls.X509KeyPair([]byte(r.Cert), []byte(r.Key))
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse x509 key pair: %v", err)
+		}
+
+		for _, c := range cert.Certificate {
+			cert, err := x509.ParseCertificate(c)
+			if err != nil {
+				fwLog.Errorf("Failed to parse client certificate: %v", err)
+			}
+			fwLog.Debugf("Using client certificate [%s] issued by %s", cert.SerialNumber, cert.Issuer)
+			for _, uri := range cert.URIs {
+				fwLog.Debugf("  URI SAN: %s", uri)
+			}
+		}
+		// nolint: unparam
+		return func(info *tls.CertificateRequestInfo) (*tls.Certificate, error) {
+			fwLog.Debugf("Peer asking for client certificate")
+			for i, ca := range info.AcceptableCAs {
+				x := &pkix.RDNSequence{}
+				if _, err := asn1.Unmarshal(ca, x); err != nil {
+					fwLog.Errorf("Failed to decode AcceptableCA[%d]: %v", i, err)
+				} else {
+					name := &pkix.Name{}
+					name.FillFromRDNSequence(x)
+					fwLog.Debugf("  AcceptableCA[%d]: %s", i, name)
+				}
+			}
+
+			return &cert, nil
+		}, nil
+	}
+
+	return nil, nil
+}
+
 func newTLSConfig(c *Config) (*tls.Config, error) {
 	r := c.Request
 	tlsConfig := &tls.Config{
@@ -238,6 +314,34 @@ func newTLSConfig(c *Config) (*tls.Config, error) {
 	return tlsConfig, nil
 }
 
+func newMtpTLSConfig(c *Config) (*tls.Config, error) {
+	r := c.Request.Hbone
+	if r == nil {
+		return nil, nil
+	}
+	tlsConfig := &tls.Config{
+		GetClientCertificate: c.hboneClientConfig,
+	}
+	if r.CaCertFile != "" {
+		certData, err := os.ReadFile(r.CaCertFile)
+		if err != nil {
+			return nil, fmt.Errorf("failed to load client certificate: %v", err)
+		}
+		r.CaCert = string(certData)
+	}
+	if r.InsecureSkipVerify || r.CaCert == "" {
+		tlsConfig.InsecureSkipVerify = true
+	} else if r.CaCert != "" {
+		certPool := x509.NewCertPool()
+		if !certPool.AppendCertsFromPEM([]byte(r.CaCert)) {
+			return nil, fmt.Errorf("failed to create cert pool")
+		}
+		tlsConfig.RootCAs = certPool
+	}
+
+	return tlsConfig, nil
+}
+
 func checkRedirectFunc(req *proto.ForwardEchoRequest) func(req *http.Request, via []*http.Request) error {
 	if req.FollowRedirects {
 		return nil
diff --git a/pkg/test/echo/server/forwarder/grpc.go b/pkg/test/echo/server/forwarder/grpc.go
index c8bb23ccf438..164abe3b9723 100644
--- a/pkg/test/echo/server/forwarder/grpc.go
+++ b/pkg/test/echo/server/forwarder/grpc.go
@@ -142,11 +142,10 @@ func newGRPCConnection(cfg *Config) (*grpc.ClientConn, error) {
 		security = grpc.WithTransportCredentials(insecure.NewCredentials())
 	}
 
-	forceDNSLookup := cfg.forceDNSLookup
 	opts := []grpc.DialOption{
 		grpc.WithAuthority(cfg.hostHeader),
 		grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) {
-			return newDialer(forceDNSLookup).DialContext(ctx, "tcp", addr)
+			return newDialer(cfg).DialContext(ctx, "tcp", addr)
 		}),
 		security,
 	}
diff --git a/pkg/test/echo/server/forwarder/http.go b/pkg/test/echo/server/forwarder/http.go
index fd95dc5abbd7..4122ff5e9c78 100644
--- a/pkg/test/echo/server/forwarder/http.go
+++ b/pkg/test/echo/server/forwarder/http.go
@@ -31,6 +31,7 @@ import (
 	"github.com/lucas-clemente/quic-go/http3"
 	"golang.org/x/net/http2"
 
+	"istio.io/istio/pkg/hbone"
 	"istio.io/istio/pkg/test/echo"
 	"istio.io/istio/pkg/test/echo/common/scheme"
 	"istio.io/istio/pkg/test/echo/proto"
@@ -107,7 +108,7 @@ func newHTTP2TransportGetter(cfg *Config) (httpTransportGetter, func()) {
 			return &http2.Transport{
 				TLSClientConfig: cfg.tlsConfig,
 				DialTLS: func(network, addr string, tlsConfig *tls.Config) (net.Conn, error) {
-					return tls.DialWithDialer(newDialer(cfg.forceDNSLookup), network, addr, tlsConfig)
+					return hbone.TLSDialWithDialer(newDialer(cfg), network, addr, tlsConfig)
 				},
 			}
 		}
@@ -119,7 +120,7 @@ func newHTTP2TransportGetter(cfg *Config) (httpTransportGetter, func()) {
 			AllowHTTP: true,
 			// Pretend we are dialing a TLS endpoint. (Note, we ignore the passed tls.Config)
 			DialTLS: func(network, addr string, _ *tls.Config) (net.Conn, error) {
-				return newDialer(cfg.forceDNSLookup).Dial(network, addr)
+				return newDialer(cfg).Dial(network, addr)
 			},
 		}
 	}
@@ -147,11 +148,11 @@ func newHTTP2TransportGetter(cfg *Config) (httpTransportGetter, func()) {
 func newHTTPTransportGetter(cfg *Config) (httpTransportGetter, func()) {
 	newConn := func() *http.Transport {
 		dialContext := func(ctx context.Context, network, addr string) (net.Conn, error) {
-			return newDialer(cfg.forceDNSLookup).DialContext(ctx, network, addr)
+			return newDialer(cfg).DialContext(ctx, network, addr)
 		}
 		if len(cfg.UDS) > 0 {
 			dialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
-				return newDialer(cfg.forceDNSLookup).DialContext(ctx, "unix", cfg.UDS)
+				return newDialer(cfg).DialContext(ctx, "unix", cfg.UDS)
 			}
 		}
 		out := &http.Transport{
diff --git a/pkg/test/echo/server/forwarder/tcp.go b/pkg/test/echo/server/forwarder/tcp.go
index 71e3955ee6b6..019183560c18 100644
--- a/pkg/test/echo/server/forwarder/tcp.go
+++ b/pkg/test/echo/server/forwarder/tcp.go
@@ -18,13 +18,13 @@ import (
 	"bufio"
 	"bytes"
 	"context"
-	"crypto/tls"
 	"fmt"
 	"io"
 	"net"
 	"net/http"
 	"strings"
 
+	"istio.io/istio/pkg/hbone"
 	"istio.io/istio/pkg/test/echo"
 	"istio.io/istio/pkg/test/echo/common"
 	"istio.io/istio/pkg/test/echo/proto"
@@ -134,10 +134,10 @@ func newTCPConnection(cfg *Config) (net.Conn, error) {
 	address := cfg.Request.Url[len(cfg.scheme+"://"):]
 
 	if cfg.secure {
-		return tls.DialWithDialer(newDialer(cfg.forceDNSLookup), "tcp", address, cfg.tlsConfig)
+		return hbone.TLSDialWithDialer(newDialer(cfg), "tcp", address, cfg.tlsConfig)
 	}
 
 	ctx, cancel := context.WithTimeout(context.Background(), common.ConnectionTimeout)
 	defer cancel()
-	return newDialer(cfg.forceDNSLookup).DialContext(ctx, "tcp", address)
+	return newDialer(cfg).DialContext(ctx, "tcp", address)
 }
diff --git a/pkg/test/echo/server/forwarder/tls.go b/pkg/test/echo/server/forwarder/tls.go
index 6015ddf40e8d..a9defdea2f41 100644
--- a/pkg/test/echo/server/forwarder/tls.go
+++ b/pkg/test/echo/server/forwarder/tls.go
@@ -22,6 +22,7 @@ import (
 	"strings"
 	"time"
 
+	"istio.io/istio/pkg/hbone"
 	"istio.io/istio/pkg/test/echo"
 	"istio.io/istio/pkg/test/echo/proto"
 )
@@ -117,7 +118,7 @@ func (c *tlsProtocol) Close() error {
 func newTLSConnection(cfg *Config) (*tls.Conn, error) {
 	address := cfg.Request.Url[len(cfg.scheme+"://"):]
 
-	con, err := tls.DialWithDialer(newDialer(cfg.forceDNSLookup), "tcp", address, cfg.tlsConfig)
+	con, err := hbone.TLSDialWithDialer(newDialer(cfg), "tcp", address, cfg.tlsConfig)
 	if err != nil {
 		return nil, err
 	}
diff --git a/pkg/test/echo/server/forwarder/util.go b/pkg/test/echo/server/forwarder/util.go
index 95afda4cba0f..0ac5d9c76231 100644
--- a/pkg/test/echo/server/forwarder/util.go
+++ b/pkg/test/echo/server/forwarder/util.go
@@ -25,6 +25,7 @@ import (
 
 	"github.com/hashicorp/go-multierror"
 
+	"istio.io/istio/pkg/hbone"
 	"istio.io/istio/pkg/test/echo"
 	"istio.io/istio/pkg/test/echo/common"
 	"istio.io/istio/pkg/test/echo/proto"
@@ -45,11 +46,19 @@ func writeForwardedHeaders(out *bytes.Buffer, requestID int, header http.Header)
 	}
 }
 
-func newDialer(forceDNSLookup bool) *net.Dialer {
+func newDialer(cfg *Config) hbone.Dialer {
+	if cfg.Request.Hbone.GetAddress() != "" {
+		out := hbone.NewDialer(hbone.Config{
+			ProxyAddress: cfg.Request.Hbone.GetAddress(),
+			Headers:      cfg.hboneHeaders,
+			TLS:          cfg.hboneTLSConfig,
+		})
+		return out
+	}
 	out := &net.Dialer{
 		Timeout: common.ConnectionTimeout,
 	}
-	if forceDNSLookup {
+	if cfg.forceDNSLookup {
 		out.Resolver = newResolver(common.ConnectionTimeout, "", "")
 	}
 	return out
diff --git a/pkg/test/echo/server/forwarder/websocket.go b/pkg/test/echo/server/forwarder/websocket.go
index 1ccff2023d41..939322656908 100644
--- a/pkg/test/echo/server/forwarder/websocket.go
+++ b/pkg/test/echo/server/forwarder/websocket.go
@@ -65,11 +65,11 @@ func (c *websocketProtocol) makeRequest(ctx context.Context, cfg *Config, reques
 	}
 
 	dialContext := func(network, addr string) (net.Conn, error) {
-		return newDialer(cfg.forceDNSLookup).Dial(network, addr)
+		return newDialer(cfg).Dial(network, addr)
 	}
 	if len(cfg.UDS) > 0 {
 		dialContext = func(network, addr string) (net.Conn, error) {
-			return newDialer(cfg.forceDNSLookup).Dial("unix", cfg.UDS)
+			return newDialer(cfg).Dial("unix", cfg.UDS)
 		}
 	}
 
diff --git a/pkg/test/echo/server/instance.go b/pkg/test/echo/server/instance.go
index fe60e431c979..67cb33de553a 100644
--- a/pkg/test/echo/server/instance.go
+++ b/pkg/test/echo/server/instance.go
@@ -255,6 +255,7 @@ func (s *Instance) validate() error {
 		case protocol.HTTPS:
 		case protocol.HTTP2:
 		case protocol.GRPC:
+		case protocol.HBONE:
 		default:
 			return fmt.Errorf("protocol %v not currently supported", port.Protocol)
 		}

From 75e2efd022be095c26e73fe1bed3275d4ed1dfe6 Mon Sep 17 00:00:00 2001
From: John Howard <howardjohn@google.com>
Date: Tue, 28 Jun 2022 11:33:56 -0700
Subject: [PATCH 2/4] some fixes

---
 pkg/config/protocol/instance.go          |  1 +
 pkg/config/protocol/instance_test.go     |  1 +
 pkg/hbone/dialer.go                      | 85 +++++++++++++-----------
 pkg/hbone/dialer_test.go                 | 53 ++++++++++-----
 pkg/hbone/server.go                      | 60 +++++++++--------
 pkg/test/echo/server/forwarder/config.go |  4 +-
 6 files changed, 121 insertions(+), 83 deletions(-)

diff --git a/pkg/config/protocol/instance.go b/pkg/config/protocol/instance.go
index e6811db19a47..d8973f9ff993 100644
--- a/pkg/config/protocol/instance.go
+++ b/pkg/config/protocol/instance.go
@@ -55,6 +55,7 @@ const (
 	// MySQL declares that the port carries MySQL traffic.
 	MySQL Instance = "MySQL"
 	// HBONE declares that the port carries HBONE traffic.
+	// This cannot be declared by Services, but is used for some internal code that uses Protocol
 	HBONE Instance = "HBONE"
 	// Unsupported - value to signify that the protocol is unsupported.
 	Unsupported Instance = "UnsupportedProtocol"
diff --git a/pkg/config/protocol/instance_test.go b/pkg/config/protocol/instance_test.go
index c63b10edca75..1a26e7d4e03f 100644
--- a/pkg/config/protocol/instance_test.go
+++ b/pkg/config/protocol/instance_test.go
@@ -60,6 +60,7 @@ func TestParse(t *testing.T) {
 		{"MySQL", protocol.MySQL},
 		{"", protocol.Unsupported},
 		{"SMTP", protocol.Unsupported},
+		{"HBONE", protocol.Unsupported},
 	}
 
 	for _, testPair := range testPairs {
diff --git a/pkg/hbone/dialer.go b/pkg/hbone/dialer.go
index 8fe7c7205cf7..7c1470d08192 100644
--- a/pkg/hbone/dialer.go
+++ b/pkg/hbone/dialer.go
@@ -34,7 +34,9 @@ import (
 
 var log = istiolog.RegisterScope("hbone", "", 0)
 
+// Config defines the configuration for a given dialer. All fields other than ProxyAddress are optional
 type Config struct {
+	// ProxyAddress defines the address of the HBONE proxy we are connecting to
 	ProxyAddress string
 	Headers      http.Header
 	TLS          *tls.Config
@@ -45,24 +47,44 @@ type Dialer interface {
 	proxy.ContextDialer
 }
 
+// NewDialer creates a Dialer that proxies connections over HBONE to the configured proxy.
 func NewDialer(cfg Config) Dialer {
-	return &dialer{cfg: cfg}
+	var transport *http2.Transport
+	if cfg.TLS != nil {
+		transport = &http2.Transport{
+			TLSClientConfig: cfg.TLS,
+		}
+	} else {
+		transport = &http2.Transport{
+			// For h2c
+			AllowHTTP: true,
+			DialTLS: func(network, addr string, cfg *tls.Config) (net.Conn, error) {
+				return net.Dial(network, addr)
+			},
+		}
+	}
+	return &dialer{
+		cfg:       cfg,
+		transport: transport,
+	}
 }
 
 type dialer struct {
-	cfg Config
+	cfg       Config
+	transport *http2.Transport
 }
 
-func (d dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
+// DialContext connects to `address` via the HBONE proxy.
+func (d *dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
 	if network != "tcp" {
 		return net.Dial(network, address)
 	}
 	// TODO: use context
 	c, s := net.Pipe()
-	go func() {
-		err := proxyTo(s, d.cfg, address)
-		log.Infof("Tunnel complete: %v", err)
-	}()
+	err := d.proxyTo(s, d.cfg, address)
+	if err != nil {
+		return nil, err
+	}
 	return c, nil
 }
 
@@ -70,8 +92,7 @@ func (d dialer) Dial(network, address string) (c net.Conn, err error) {
 	return d.DialContext(context.Background(), network, address)
 }
 
-func proxyTo(conn io.ReadWriteCloser, req Config, address string) error {
-	defer conn.Close()
+func (d *dialer) proxyTo(conn io.ReadWriteCloser, req Config, address string) error {
 	t0 := time.Now()
 
 	url := "http://" + req.ProxyAddress
@@ -82,35 +103,19 @@ func proxyTo(conn io.ReadWriteCloser, req Config, address string) error {
 	// * Less visibility into i/o
 	// * http will call conn.Close, which will close before we want to (finished writing response).
 	pr, pw := io.Pipe()
-	r, err := http.NewRequest("CONNECT", url, pr)
+	r, err := http.NewRequest(http.MethodConnect, url, pr)
 	if err != nil {
 		return fmt.Errorf("new request: %v", err)
 	}
 	r.Host = address
 
-	var transport *http2.Transport
-	if req.TLS != nil {
-		transport = &http2.Transport{
-			TLSClientConfig: req.TLS,
-		}
-	} else {
-		transport = &http2.Transport{
-			// For h2c
-			AllowHTTP: true,
-			DialTLS: func(network, addr string, cfg *tls.Config) (net.Conn, error) {
-				return net.Dial(network, addr)
-			},
-		}
-	}
-
 	// Initiate CONNECT.
 	log.Infof("initiate CONNECT to %v via %v", r.Host, url)
 
-	resp, err := transport.RoundTrip(r)
+	resp, err := d.transport.RoundTrip(r)
 	if err != nil {
 		return fmt.Errorf("round trip: %v", err)
 	}
-	defer resp.Body.Close()
 	var remoteID string
 	if resp.TLS != nil && len(resp.TLS.PeerCertificates) > 0 {
 		ids, _ := util.ExtractIDs(resp.TLS.PeerCertificates[0].Extensions)
@@ -119,19 +124,25 @@ func proxyTo(conn io.ReadWriteCloser, req Config, address string) error {
 		}
 	}
 	log.WithLabels("host", r.Host, "remote", remoteID).Info("CONNECT established")
-	wg := sync.WaitGroup{}
-	wg.Add(1)
 	go func() {
-		// handle upstream (mtp server) --> downstream (app)
-		copyBuffered(conn, resp.Body, log.WithLabels("name", "body to conn"))
-		wg.Done()
+		defer conn.Close()
+		defer resp.Body.Close()
+
+		wg := sync.WaitGroup{}
+		wg.Add(1)
+		go func() {
+			// handle upstream (hbone server) --> downstream (app)
+			copyBuffered(conn, resp.Body, log.WithLabels("name", "body to conn"))
+			wg.Done()
+		}()
+		// Copy from conn into the pipe, which will then be sent as part of the request
+		// handle upstream (hbone server) <-- downstream (app)
+		copyBuffered(pw, conn, log.WithLabels("name", "conn to pipe"))
+
+		wg.Wait()
+		log.Info("stream closed in ", time.Since(t0))
 	}()
-	// Copy from conn into the pipe, which will then be sent as part of the request
-	// handle upstream (mtp server) <-- downstream (app)
-	copyBuffered(pw, conn, log.WithLabels("name", "conn to pipe"))
 
-	wg.Wait()
-	log.Info("stream closed in ", time.Since(t0))
 	return nil
 }
 
diff --git a/pkg/hbone/dialer_test.go b/pkg/hbone/dialer_test.go
index 39c9434b8be4..615196b8a7c7 100644
--- a/pkg/hbone/dialer_test.go
+++ b/pkg/hbone/dialer_test.go
@@ -43,6 +43,20 @@ func newTCPServer(t testing.TB, data string) string {
 	return n.Addr().String()
 }
 
+func TestDialerError(t *testing.T) {
+	d := NewDialer(Config{
+		ProxyAddress: "127.0.0.10:1", // Random address that should fail to dial
+		Headers: map[string][]string{
+			"some-addition-metadata": {"test-value"},
+		},
+		TLS: nil, // No TLS for simplification
+	})
+	_, err := d.Dial("tcp", "fake")
+	if err == nil {
+		t.Fatal("expected error, got none.")
+	}
+}
+
 func TestDialer(t *testing.T) {
 	testAddr := newTCPServer(t, "hello")
 	proxy := newHBONEServer(t)
@@ -53,26 +67,31 @@ func TestDialer(t *testing.T) {
 		},
 		TLS: nil, // No TLS for simplification
 	})
-	client, err := d.Dial("tcp", testAddr)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer client.Close()
+	send := func() {
+		client, err := d.Dial("tcp", testAddr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer client.Close()
 
-	go func() {
-		n, err := client.Write([]byte("hello world"))
-		t.Logf("wrote %v/%v", n, err)
-	}()
+		go func() {
+			n, err := client.Write([]byte("hello world"))
+			t.Logf("wrote %v/%v", n, err)
+		}()
 
-	buf := make([]byte, 8)
-	n, err := client.Read(buf)
-	if err != nil {
-		t.Fatalf("err with %v: %v", n, err)
-	}
-	if string(buf[:n]) != "hello" {
-		t.Fatalf("got unexpected buffer: %v", string(buf[:n]))
+		buf := make([]byte, 8)
+		n, err := client.Read(buf)
+		if err != nil {
+			t.Fatalf("err with %v: %v", n, err)
+		}
+		if string(buf[:n]) != "hello" {
+			t.Fatalf("got unexpected buffer: %v", string(buf[:n]))
+		}
+		t.Logf("Read %v", string(buf[:n]))
 	}
-	t.Logf("Read %v", string(buf[:n]))
+	// Make sure we can create multiple connections
+	send()
+	send()
 }
 
 func newHBONEServer(t *testing.T) string {
diff --git a/pkg/hbone/server.go b/pkg/hbone/server.go
index 24f04f1df545..8155d079d951 100644
--- a/pkg/hbone/server.go
+++ b/pkg/hbone/server.go
@@ -33,38 +33,12 @@ func NewServer() *http.Server {
 	h2, _ := http2.ConfigureTransports(h1)
 	h2.ReadIdleTimeout = 10 * time.Minute // TODO: much larger to support long-lived connections
 	h2.AllowHTTP = true
-	h2.StrictMaxConcurrentStreams = false
 	h2Server := &http2.Server{}
 	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		t0 := time.Now()
 		if r.Method == http.MethodConnect {
-			log.WithLabels("host", r.Host).Info("Received CONNECT")
-			// Send headers back immediately so we can start getting the body
-			w.(http.Flusher).Flush()
-			ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-			defer cancel()
-
-			dst, err := (&net.Dialer{}).DialContext(ctx, "tcp", r.Host)
-			if err != nil {
-				w.WriteHeader(http.StatusServiceUnavailable)
-				log.Errorf("failed to dial upstream: %v", err)
+			if handleConnect(w, r) {
 				return
 			}
-			log.Infof("Connected to %v", r.Host)
-			w.WriteHeader(http.StatusOK)
-
-			wg := sync.WaitGroup{}
-			wg.Add(1)
-			go func() {
-				// downstream (hbone client) <-- upstream (app)
-				copyBuffered(w, dst, log.WithLabels("name", "dst to w"))
-				r.Body.Close()
-				wg.Done()
-			}()
-			// downstream (hbone client) --> upstream (app)
-			copyBuffered(dst, r.Body, log.WithLabels("name", "body to dst"))
-			wg.Wait()
-			log.Infof("connection closed in %v", time.Since(t0))
 		} else {
 			log.Errorf("non-CONNECT: %v", r.Method)
 			w.WriteHeader(http.StatusMethodNotAllowed)
@@ -75,3 +49,35 @@ func NewServer() *http.Server {
 	}
 	return hs
 }
+
+func handleConnect(w http.ResponseWriter, r *http.Request) bool {
+	t0 := time.Now()
+	log.WithLabels("host", r.Host, "source", r.RemoteAddr).Info("Received CONNECT")
+	// Send headers back immediately so we can start getting the body
+	w.(http.Flusher).Flush()
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	dst, err := (&net.Dialer{}).DialContext(ctx, "tcp", r.Host)
+	if err != nil {
+		w.WriteHeader(http.StatusServiceUnavailable)
+		log.Errorf("failed to dial upstream: %v", err)
+		return true
+	}
+	log.Infof("Connected to %v", r.Host)
+	w.WriteHeader(http.StatusOK)
+
+	wg := sync.WaitGroup{}
+	wg.Add(1)
+	go func() {
+		// downstream (hbone client) <-- upstream (app)
+		copyBuffered(w, dst, log.WithLabels("name", "dst to w"))
+		r.Body.Close()
+		wg.Done()
+	}()
+	// downstream (hbone client) --> upstream (app)
+	copyBuffered(dst, r.Body, log.WithLabels("name", "body to dst"))
+	wg.Wait()
+	log.Infof("connection closed in %v", time.Since(t0))
+	return false
+}
diff --git a/pkg/test/echo/server/forwarder/config.go b/pkg/test/echo/server/forwarder/config.go
index f55ddd483c33..f8e1bfff617c 100644
--- a/pkg/test/echo/server/forwarder/config.go
+++ b/pkg/test/echo/server/forwarder/config.go
@@ -107,7 +107,7 @@ func (c *Config) fillDefaults() error {
 		return err
 	}
 
-	c.hboneTLSConfig, err = newMtpTLSConfig(c)
+	c.hboneTLSConfig, err = newHBONETLSConfig(c)
 	if err != nil {
 		return err
 	}
@@ -314,7 +314,7 @@ func newTLSConfig(c *Config) (*tls.Config, error) {
 	return tlsConfig, nil
 }
 
-func newMtpTLSConfig(c *Config) (*tls.Config, error) {
+func newHBONETLSConfig(c *Config) (*tls.Config, error) {
 	r := c.Request.Hbone
 	if r == nil {
 		return nil, nil

From d6b55584219033536291f81a40bf658c3e78282d Mon Sep 17 00:00:00 2001
From: John Howard <howardjohn@google.com>
Date: Fri, 1 Jul 2022 08:15:15 -0700
Subject: [PATCH 3/4] cleanup

---
 pkg/test/echo/server/endpoint/hbone.go   | 4 ++--
 pkg/test/echo/server/endpoint/http.go    | 6 +++---
 pkg/test/echo/server/endpoint/tcp.go     | 4 ++--
 pkg/test/echo/server/forwarder/config.go | 8 --------
 4 files changed, 7 insertions(+), 15 deletions(-)

diff --git a/pkg/test/echo/server/endpoint/hbone.go b/pkg/test/echo/server/endpoint/hbone.go
index 88eb887e2721..da93d9b9603d 100644
--- a/pkg/test/echo/server/endpoint/hbone.go
+++ b/pkg/test/echo/server/endpoint/hbone.go
@@ -82,10 +82,10 @@ func (c connectInstance) Start(onReady OnReadyFunc) error {
 
 	if c.Port.TLS {
 		c.server.Addr = fmt.Sprintf(":%d", port)
-		fmt.Printf("Listening HBONE on %v\n", port)
+		epLog.Infof("Listening HBONE on %v\n", port)
 	} else {
 		c.server.Addr = fmt.Sprintf(":%d", port)
-		fmt.Printf("Listening HBONE (plaintext) on %v\n", port)
+		epLog.Infof("Listening HBONE (plaintext) on %v\n", port)
 	}
 	go func() {
 		err := c.server.Serve(listener)
diff --git a/pkg/test/echo/server/endpoint/http.go b/pkg/test/echo/server/endpoint/http.go
index 40e2b1620366..6d81a944ea4f 100644
--- a/pkg/test/echo/server/endpoint/http.go
+++ b/pkg/test/echo/server/endpoint/http.go
@@ -121,13 +121,13 @@ func (s *httpInstance) Start(onReady OnReadyFunc) error {
 	}
 
 	if s.isUDS() {
-		fmt.Printf("Listening HTTP/1.1 on %v\n", s.UDSServer)
+		epLog.Infof("Listening HTTP/1.1 on %v\n", s.UDSServer)
 	} else if s.Port.TLS {
 		s.server.Addr = fmt.Sprintf(":%d", port)
-		fmt.Printf("Listening HTTPS/1.1 on %v\n", port)
+		epLog.Infof("Listening HTTPS/1.1 on %v\n", port)
 	} else {
 		s.server.Addr = fmt.Sprintf(":%d", port)
-		fmt.Printf("Listening HTTP/1.1 on %v\n", port)
+		epLog.Infof("Listening HTTP/1.1 on %v\n", port)
 	}
 
 	// Start serving HTTP traffic.
diff --git a/pkg/test/echo/server/endpoint/tcp.go b/pkg/test/echo/server/endpoint/tcp.go
index abaab0d49c13..53030a14d0d7 100644
--- a/pkg/test/echo/server/endpoint/tcp.go
+++ b/pkg/test/echo/server/endpoint/tcp.go
@@ -75,9 +75,9 @@ func (s *tcpInstance) Start(onReady OnReadyFunc) error {
 
 	s.l = listener
 	if s.Port.TLS {
-		fmt.Printf("Listening TCP (over TLS) on %v\n", port)
+		epLog.Infof("Listening TCP (over TLS) on %v\n", port)
 	} else {
-		fmt.Printf("Listening TCP on %v\n", port)
+		epLog.Infof("Listening TCP on %v\n", port)
 	}
 
 	// Start serving TCP traffic.
diff --git a/pkg/test/echo/server/forwarder/config.go b/pkg/test/echo/server/forwarder/config.go
index f8e1bfff617c..2673a613e3e9 100644
--- a/pkg/test/echo/server/forwarder/config.go
+++ b/pkg/test/echo/server/forwarder/config.go
@@ -56,18 +56,12 @@ type Config struct {
 	method                  string
 	secure                  bool
 
-	// completed whether the Config is complete or still in progress
-	completed bool
-
 	hboneTLSConfig    *tls.Config
 	hboneClientConfig func(info *tls.CertificateRequestInfo) (*tls.Certificate, error)
 	hboneHeaders      http.Header
 }
 
 func (c *Config) fillDefaults() error {
-	if c.completed {
-		return nil
-	}
 	c.checkRedirect = checkRedirectFunc(c.Request)
 	c.timeout = common.GetTimeout(c.Request)
 	c.count = common.GetCount(c.Request)
@@ -135,8 +129,6 @@ func (c *Config) fillDefaults() error {
 		c.forceDNSLookup = c.newConnectionPerRequest && c.Request.ForceDNSLookup
 	}
 
-	c.completed = true
-
 	return nil
 }
 

From 49f78f0c36f55f09999a2abca3596ba4feba2602 Mon Sep 17 00:00:00 2001
From: John Howard <howardjohn@google.com>
Date: Fri, 1 Jul 2022 08:10:46 -0700
Subject: [PATCH 4/4] echo: avoid filling defaults twice

We already do this at a higher level in `(i *Instance) ForwardEcho`

(cherry picked from commit d99cb3c9101028c934788e07826d332441950e23)
---
 pkg/test/echo/server/forwarder/util.go | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/pkg/test/echo/server/forwarder/util.go b/pkg/test/echo/server/forwarder/util.go
index 0ac5d9c76231..c9cc564bdcd4 100644
--- a/pkg/test/echo/server/forwarder/util.go
+++ b/pkg/test/echo/server/forwarder/util.go
@@ -86,10 +86,6 @@ func newResolver(timeout time.Duration, protocol, dnsServer string) *net.Resolve
 
 // doForward sends the requests and collect the responses.
 func doForward(ctx context.Context, cfg *Config, e *executor, doReq func(context.Context, *Config, int) (string, error)) (*proto.ForwardEchoResponse, error) {
-	if err := cfg.fillDefaults(); err != nil {
-		return nil, err
-	}
-
 	// make the timeout apply to the entire set of requests
 	ctx, cancel := context.WithTimeout(ctx, cfg.timeout)
 	defer cancel()