Configuring Envoy as an edge proxy

Envoy is a production-ready edge proxy, however, the default settings are tailored for the service mesh use case, and some values need to be adjusted when using Envoy as an edge proxy.

TCP proxies should configure:

HTTP proxies should additionally configure:

The following is a YAML example of the above recommendation (taken from the Google VRP edge server configuration):

  1. overload_manager:
  2.   refresh_interval: 0.25s
  3.   resource_monitors:
  4.   - name: "envoy.resource_monitors.fixed_heap"
  5.     typed_config:
  6.       "@type": type.googleapis.com/envoy.config.resource_monitor.fixed_heap.v2alpha.FixedHeapConfig
  7.       # TODO: Tune for your system.
  8.       max_heap_size_bytes: 2147483648 # 2 GiB
  9.   actions:
  10.   - name: "envoy.overload_actions.shrink_heap"
  11.     triggers:
  12.     - name: "envoy.resource_monitors.fixed_heap"
  13.       threshold:
  14.         value: 0.95
  15.   - name: "envoy.overload_actions.stop_accepting_requests"
  16.     triggers:
  17.     - name: "envoy.resource_monitors.fixed_heap"
  18.       threshold:
  19.         value: 0.98
  21. admin:
  22.   access_log_path: "/var/log/envoy_admin.log"
  23.   address:
  24.     socket_address:
  25.       address: 127.0.0.1
  26.       port_value: 9090
  28. static_resources:
  29.   listeners:
  30.   - address:
  31.       socket_address:
  32.         address: 0.0.0.0
  33.         port_value: 443
  34.     listener_filters:
  35.     - name: "envoy.filters.listener.tls_inspector"
  36.       typed_config: {}
  37.     per_connection_buffer_limit_bytes: 32768 # 32 KiB
  38.     filter_chains:
  39.     - filter_chain_match:
  40.         server_names: ["example.com", "www.example.com"]
  41.       transport_socket:
  42.         name: envoy.transport_sockets.tls
  43.         typed_config:
  44.           "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext
  45.           common_tls_context:
  46.             tls_certificates:
  47.             - certificate_chain: { filename: "example_com_cert.pem" }
  48.               private_key: { filename: "example_com_key.pem" }
  49.       # Uncomment if Envoy is behind a load balancer that exposes client IP address using the PROXY protocol.
  50.       # use_proxy_proto: true
  51.       filters:
  52.       - name: envoy.filters.network.http_connection_manager
  53.         typed_config:
  54.           "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
  55.           stat_prefix: ingress_http
  56.           use_remote_address: true
  57.           common_http_protocol_options:
  58.             idle_timeout: 3600s # 1 hour
  59.             headers_with_underscores_action: REJECT_REQUEST
  60.           http2_protocol_options:
  61.             max_concurrent_streams: 100
  62.             initial_stream_window_size: 65536 # 64 KiB
  63.             initial_connection_window_size: 1048576 # 1 MiB
  64.           stream_idle_timeout: 300s # 5 mins, must be disabled for long-lived and streaming requests
  65.           request_timeout: 300s # 5 mins, must be disabled for long-lived and streaming requests
  66.           route_config:
  67.             virtual_hosts:
  68.             - name: default
  69.               domains: "*"
  70.               routes:
  71.               - match: { prefix: "/" }
  72.                 route:
  73.                   cluster: service_foo
  74.                   idle_timeout: 15s # must be disabled for long-lived and streaming requests
  75.   clusters:
  76.     name: service_foo
  77.     connect_timeout: 15s
  78.     per_connection_buffer_limit_bytes: 32768 # 32 KiB
  79.     load_assignment:
  80.       cluster_name: some_service
  81.       endpoints:
  82.         - lb_endpoints:
  83.           - endpoint:
  84.               address:
  85.                 socket_address:
  86.                   address: 127.0.0.1
  87.                   port_value: 8080
  88.     http2_protocol_options:
  89.       initial_stream_window_size: 65536 # 64 KiB
  90.       initial_connection_window_size: 1048576 # 1 MiB
  92. layered_runtime:
  93.   layers:
  94.     - name: static_layer_0
  95.       static_layer:
  96.         envoy:
  97.           resource_limits:
  98.             listener:
  99.               example_listener_name:
  100.                 connection_limit: 10000
  101.         overload:
  102.           global_downstream_max_connections: 50000