本博客是深入研究Envoy Proxy和Istio.io 以及它如何实现更优雅的方式来连接和管理微服务系列文章的一部分。
这是接下来几个部分的想法(将在发布时更新链接):
在envoy1.9版本中,已经直接暴露了prometheus 格式的metrics,也就是prometheus可以直接去采集,无需再通过配置Statsd来收集metrics了,使用statsd的流程大致上是:首先Envoy推送指标到statsd,然后我们用prometheus(一个时序数据库)从statsd拉取指标。
设置envoy 的配置文件中,有如下的设置:
admin: access_log_path: "/dev/null" address: socket_address: address: 0.0.0.0 port_value: 9000
envoy 启动成功以后,访问localhost:9000/stats/prometheus,可以返回类似下面的参数:
# TYPE envoy_listener_admin_http_downstream_rq_completed counter envoy_listener_admin_http_downstream_rq_completed{envoy_http_conn_manager_prefix="admin"} 3154 # TYPE envoy_listener_admin_http_downstream_rq_xx counter envoy_listener_admin_http_downstream_rq_xx{envoy_response_code_class="1",envoy_http_conn_manager_prefix="admin"} 0 # TYPE envoy_listener_admin_downstream_cx_total counter envoy_listener_admin_downstream_cx_total{} 146346 envoy_listener_admin_http_downstream_rq_xx{envoy_response_code_class="5",envoy_http_conn_manager_prefix="admin"} 0 envoy_listener_admin_http_downstream_rq_xx{envoy_response_code_class="3",envoy_http_conn_manager_prefix="admin"} 0 # TYPE envoy_listener_admin_downstream_pre_cx_timeout counter envoy_listener_admin_downstream_pre_cx_timeout{} 0 # TYPE envoy_listener_admin_no_filter_chain_match counter envoy_listener_admin_no_filter_chain_match{} 0 # TYPE envoy_listener_admin_downstream_cx_destroy counter envoy_listener_admin_downstream_cx_destroy{} 146344 envoy_listener_admin_http_downstream_rq_xx{envoy_response_code_class="2",envoy_http_conn_manager_prefix="admin"} 3154 envoy_listener_admin_http_downstream_rq_xx{envoy_response_code_class="4",envoy_http_conn_manager_prefix="admin"} 0 # TYPE envoy_cluster_upstream_flow_control_paused_reading_total counter envoy_cluster_upstream_flow_control_paused_reading_total{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_membership_change counter envoy_cluster_membership_change{envoy_cluster_name="hawkeye"} 1 # TYPE envoy_cluster_ext_authz_denied counter envoy_cluster_ext_authz_denied{envoy_cluster_name="hawkeye"} 15 # TYPE envoy_cluster_upstream_rq_completed counter envoy_cluster_upstream_rq_completed{envoy_cluster_name="hawkeye"} 255 # TYPE envoy_cluster_upstream_rq_pending_failure_eject counter envoy_cluster_upstream_rq_pending_failure_eject{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_upstream_cx_connect_attempts_exceeded counter envoy_cluster_upstream_cx_connect_attempts_exceeded{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_upstream_rq_timeout counter envoy_cluster_upstream_rq_timeout{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_upstream_rq counter envoy_cluster_upstream_rq{envoy_response_code="204",envoy_cluster_name="hawkeye"} 2 # TYPE envoy_cluster_upstream_cx_connect_timeout counter envoy_cluster_upstream_cx_connect_timeout{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_upstream_cx_none_healthy counter envoy_cluster_upstream_cx_none_healthy{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_upstream_rq_pending_total counter envoy_cluster_upstream_rq_pending_total{envoy_cluster_name="hawkeye"} 6 # TYPE envoy_cluster_internal_upstream_rq_completed counter envoy_cluster_internal_upstream_rq_completed{envoy_cluster_name="hawkeye"} 15 envoy_cluster_upstream_rq{envoy_response_code="200",envoy_cluster_name="hawkeye"} 209 envoy_cluster_upstream_rq{envoy_response_code="503",envoy_cluster_name="hawkeye"} 29 # TYPE envoy_cluster_lb_local_cluster_not_ok counter envoy_cluster_lb_local_cluster_not_ok{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_lb_zone_routing_sampled counter envoy_cluster_lb_zone_routing_sampled{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_upstream_cx_connect_fail counter envoy_cluster_upstream_cx_connect_fail{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_upstream_rq_retry_success counter envoy_cluster_upstream_rq_retry_success{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_bind_errors counter envoy_cluster_bind_errors{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_upstream_cx_total counter envoy_cluster_upstream_cx_total{envoy_cluster_name="hawkeye"} 6 # TYPE envoy_cluster_lb_zone_number_differs counter envoy_cluster_lb_zone_number_differs{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_original_dst_host_invalid counter envoy_cluster_original_dst_host_invalid{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_lb_zone_no_capacity_left counter envoy_cluster_lb_zone_no_capacity_left{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_upstream_cx_max_requests counter envoy_cluster_upstream_cx_max_requests{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_upstream_rq_per_try_timeout counter envoy_cluster_upstream_rq_per_try_timeout{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_upstream_rq_retry_overflow counter envoy_cluster_upstream_rq_retry_overflow{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_external_upstream_rq counter envoy_cluster_external_upstream_rq{envoy_response_code="204",envoy_cluster_name="hawkeye"} 2 envoy_cluster_external_upstream_rq{envoy_response_code="503",envoy_cluster_name="hawkeye"} 29 # TYPE envoy_cluster_upstream_cx_rx_bytes_total counter envoy_cluster_upstream_cx_rx_bytes_total{envoy_cluster_name="hawkeye"} 122326 # TYPE envoy_cluster_upstream_cx_http1_total counter envoy_cluster_upstream_cx_http1_total{envoy_cluster_name="hawkeye"} 6 # TYPE envoy_cluster_upstream_rq_pending_overflow counter envoy_cluster_upstream_rq_pending_overflow{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_lb_zone_routing_cross_zone counter envoy_cluster_lb_zone_routing_cross_zone{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_lb_subsets_created counter envoy_cluster_lb_subsets_created{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_upstream_flow_control_backed_up_total counter envoy_cluster_upstream_flow_control_backed_up_total{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_upstream_flow_control_drained_total counter envoy_cluster_upstream_flow_control_drained_total{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_external_upstream_rq_xx counter envoy_cluster_external_upstream_rq_xx{envoy_response_code_class="2",envoy_cluster_name="hawkeye"} 211 # TYPE envoy_cluster_upstream_rq_cancelled counter envoy_cluster_upstream_rq_cancelled{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_lb_subsets_removed counter envoy_cluster_lb_subsets_removed{envoy_cluster_name="hawkeye"} 0 envoy_cluster_upstream_rq{envoy_response_code="401",envoy_cluster_name="hawkeye"} 15 # TYPE envoy_cluster_retry_or_shadow_abandoned counter envoy_cluster_retry_or_shadow_abandoned{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_upstream_cx_tx_bytes_total counter envoy_cluster_upstream_cx_tx_bytes_total{envoy_cluster_name="hawkeye"} 179523 # TYPE envoy_cluster_lb_zone_cluster_too_small counter envoy_cluster_lb_zone_cluster_too_small{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_external_upstream_rq_completed counter envoy_cluster_external_upstream_rq_completed{envoy_cluster_name="hawkeye"} 240 # TYPE envoy_cluster_internal_upstream_rq_xx counter envoy_cluster_internal_upstream_rq_xx{envoy_response_code_class="4",envoy_cluster_name="hawkeye"} 15 # TYPE envoy_cluster_lb_recalculate_zone_structures counter envoy_cluster_lb_recalculate_zone_structures{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_upstream_cx_destroy_with_active_rq counter envoy_cluster_upstream_cx_destroy_with_active_rq{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_lb_subsets_fallback counter envoy_cluster_lb_subsets_fallback{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_upstream_rq_xx counter envoy_cluster_upstream_rq_xx{envoy_response_code_class="5",envoy_cluster_name="hawkeye"} 29 # TYPE envoy_cluster_upstream_rq_total counter envoy_cluster_upstream_rq_total{envoy_cluster_name="hawkeye"} 240 # TYPE envoy_cluster_update_no_rebuild counter envoy_cluster_update_no_rebuild{envoy_cluster_name="hawkeye"} 18890 # TYPE envoy_cluster_upstream_cx_destroy counter envoy_cluster_upstream_cx_destroy{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_upstream_cx_destroy_local counter envoy_cluster_upstream_cx_destroy_local{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_upstream_rq_rx_reset counter envoy_cluster_upstream_rq_rx_reset{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_upstream_cx_destroy_remote_with_active_rq counter envoy_cluster_upstream_cx_destroy_remote_with_active_rq{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_upstream_flow_control_resumed_reading_total counter envoy_cluster_upstream_flow_control_resumed_reading_total{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_update_attempt counter envoy_cluster_update_attempt{envoy_cluster_name="hawkeye"} 18891 # TYPE envoy_cluster_upstream_rq_maintenance_mode counter envoy_cluster_upstream_rq_maintenance_mode{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_update_failure counter envoy_cluster_update_failure{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_ext_authz_ok counter envoy_cluster_ext_authz_ok{envoy_cluster_name="hawkeye"} 460 envoy_cluster_external_upstream_rq_xx{envoy_response_code_class="5",envoy_cluster_name="hawkeye"} 29 # TYPE envoy_cluster_internal_upstream_rq counter envoy_cluster_internal_upstream_rq{envoy_response_code="401",envoy_cluster_name="hawkeye"} 15 # TYPE envoy_cluster_update_empty counter envoy_cluster_update_empty{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_lb_zone_routing_all_directly counter envoy_cluster_lb_zone_routing_all_directly{envoy_cluster_name="hawkeye"} 0 envoy_cluster_upstream_rq_xx{envoy_response_code_class="2",envoy_cluster_name="hawkeye"} 211 # TYPE envoy_cluster_upstream_cx_http2_total counter envoy_cluster_upstream_cx_http2_total{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_upstream_cx_destroy_remote counter envoy_cluster_upstream_cx_destroy_remote{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_upstream_rq_retry counter envoy_cluster_upstream_rq_retry{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_upstream_cx_destroy_local_with_active_rq counter envoy_cluster_upstream_cx_destroy_local_with_active_rq{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_upstream_cx_overflow counter envoy_cluster_upstream_cx_overflow{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_upstream_cx_protocol_error counter envoy_cluster_upstream_cx_protocol_error{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_upstream_rq_tx_reset counter envoy_cluster_upstream_rq_tx_reset{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_lb_healthy_panic counter envoy_cluster_lb_healthy_panic{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_upstream_cx_idle_timeout counter envoy_cluster_upstream_cx_idle_timeout{envoy_cluster_name="hawkeye"} 0 envoy_cluster_upstream_rq_xx{envoy_response_code_class="4",envoy_cluster_name="hawkeye"} 15 envoy_cluster_external_upstream_rq{envoy_response_code="200",envoy_cluster_name="hawkeye"} 209 # TYPE envoy_cluster_lb_subsets_selected counter envoy_cluster_lb_subsets_selected{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_update_success counter envoy_cluster_update_success{envoy_cluster_name="hawkeye"} 18891 # TYPE envoy_cluster_upstream_cx_close_notify counter envoy_cluster_upstream_cx_close_notify{envoy_cluster_name="hawkeye"} 0 envoy_cluster_upstream_cx_close_notify{envoy_cluster_name="ext-authz"} 0 envoy_cluster_upstream_rq_tx_reset{envoy_cluster_name="ext-authz"} 0 envoy_cluster_lb_local_cluster_not_ok{envoy_cluster_name="ext-authz"} 0 envoy_cluster_update_no_rebuild{envoy_cluster_name="ext-authz"} 18890 envoy_cluster_upstream_rq{envoy_response_code="401",envoy_cluster_name="ext-authz"} 15 envoy_cluster_upstream_cx_overflow{envoy_cluster_name="ext-authz"} 0 envoy_cluster_internal_upstream_rq{envoy_response_code="200",envoy_cluster_name="ext-authz"} 460 envoy_cluster_upstream_rq_xx{envoy_response_code_class="4",envoy_cluster_name="ext-authz"} 15 envoy_cluster_upstream_flow_control_paused_reading_total{envoy_cluster_name="ext-authz"} 0 envoy_cluster_internal_upstream_rq_xx{envoy_response_code_class="2",envoy_cluster_name="ext-authz"} 460 envoy_cluster_upstream_cx_tx_bytes_total{envoy_cluster_name="ext-authz"} 142288 envoy_cluster_update_empty{envoy_cluster_name="ext-authz"} 0 envoy_cluster_upstream_rq_cancelled{envoy_cluster_name="ext-authz"} 0 envoy_cluster_upstream_cx_destroy_remote{envoy_cluster_name="ext-authz"} 0 envoy_cluster_upstream_rq_timeout{envoy_cluster_name="ext-authz"} 0 envoy_cluster_upstream_rq_retry{envoy_cluster_name="ext-authz"} 0 envoy_cluster_upstream_cx_protocol_error{envoy_cluster_name="ext-authz"} 0 envoy_cluster_upstream_cx_destroy_local{envoy_cluster_name="ext-authz"} 0 envoy_cluster_update_attempt{envoy_cluster_name="ext-authz"} 18891 envoy_cluster_upstream_rq_retry_overflow{envoy_cluster_name="ext-authz"} 0 envoy_cluster_upstream_cx_idle_timeout{envoy_cluster_name="ext-authz"} 0 envoy_cluster_lb_zone_routing_sampled{envoy_cluster_name="ext-authz"} 0 envoy_cluster_original_dst_host_invalid{envoy_cluster_name="ext-authz"} 0 envoy_cluster_upstream_flow_control_resumed_reading_total{envoy_cluster_name="ext-authz"} 0 envoy_cluster_upstream_rq_pending_failure_eject{envoy_cluster_name="ext-authz"} 0 envoy_cluster_upstream_rq_per_try_timeout{envoy_cluster_name="ext-authz"} 0 envoy_cluster_upstream_cx_max_requests{envoy_cluster_name="ext-authz"} 0 envoy_cluster_upstream_cx_http2_total{envoy_cluster_name="ext-authz"} 0 envoy_cluster_lb_zone_no_capacity_left{envoy_cluster_name="ext-authz"} 0 envoy_cluster_upstream_cx_http1_total{envoy_cluster_name="ext-authz"} 4 envoy_cluster_upstream_rq_xx{envoy_response_code_class="2",envoy_cluster_name="ext-authz"} 460 envoy_cluster_upstream_rq{envoy_response_code="200",envoy_cluster_name="ext-authz"} 460 envoy_cluster_update_success{envoy_cluster_name="ext-authz"} 18891 envoy_cluster_lb_subsets_fallback{envoy_cluster_name="ext-authz"} 0 envoy_cluster_lb_healthy_panic{envoy_cluster_name="ext-authz"} 0 envoy_cluster_upstream_cx_destroy{envoy_cluster_name="ext-authz"} 0 envoy_cluster_upstream_cx_connect_fail{envoy_cluster_name="ext-authz"} 0 envoy_cluster_upstream_cx_connect_attempts_exceeded{envoy_cluster_name="ext-authz"} 0 envoy_cluster_upstream_cx_total{envoy_cluster_name="ext-authz"} 4 envoy_cluster_upstream_cx_rx_bytes_total{envoy_cluster_name="ext-authz"} 80016 envoy_cluster_internal_upstream_rq_completed{envoy_cluster_name="ext-authz"} 475 envoy_cluster_lb_zone_number_differs{envoy_cluster_name="ext-authz"} 0 envoy_cluster_lb_zone_routing_all_directly{envoy_cluster_name="ext-authz"} 0 envoy_cluster_upstream_rq_pending_total{envoy_cluster_name="ext-authz"} 4 envoy_cluster_lb_subsets_created{envoy_cluster_name="ext-authz"} 0 envoy_cluster_upstream_rq_retry_success{envoy_cluster_name="ext-authz"} 0 envoy_cluster_internal_upstream_rq_xx{envoy_response_code_class="4",envoy_cluster_name="ext-authz"} 15 envoy_cluster_upstream_rq_maintenance_mode{envoy_cluster_name="ext-authz"} 0 envoy_cluster_lb_zone_routing_cross_zone{envoy_cluster_name="ext-authz"} 0 envoy_cluster_upstream_rq_completed{envoy_cluster_name="ext-authz"} 475 envoy_cluster_upstream_flow_control_drained_total{envoy_cluster_name="ext-authz"} 0 envoy_cluster_membership_change{envoy_cluster_name="ext-authz"} 1 envoy_cluster_upstream_cx_none_healthy{envoy_cluster_name="ext-authz"} 0 envoy_cluster_bind_errors{envoy_cluster_name="ext-authz"} 0 envoy_cluster_upstream_rq_rx_reset{envoy_cluster_name="ext-authz"} 0 envoy_cluster_upstream_cx_destroy_remote_with_active_rq{envoy_cluster_name="ext-authz"} 0 envoy_cluster_upstream_flow_control_backed_up_total{envoy_cluster_name="ext-authz"} 0 envoy_cluster_upstream_cx_connect_timeout{envoy_cluster_name="ext-authz"} 0 envoy_cluster_update_failure{envoy_cluster_name="ext-authz"} 0 envoy_cluster_lb_subsets_removed{envoy_cluster_name="ext-authz"} 0 envoy_cluster_upstream_cx_destroy_with_active_rq{envoy_cluster_name="ext-authz"} 0 envoy_cluster_lb_subsets_selected{envoy_cluster_name="ext-authz"} 0 envoy_cluster_upstream_rq_total{envoy_cluster_name="ext-authz"} 475 envoy_cluster_internal_upstream_rq{envoy_response_code="401",envoy_cluster_name="ext-authz"} 15 envoy_cluster_lb_recalculate_zone_structures{envoy_cluster_name="ext-authz"} 0 envoy_cluster_lb_zone_cluster_too_small{envoy_cluster_name="ext-authz"} 0 envoy_cluster_upstream_rq_pending_overflow{envoy_cluster_name="ext-authz"} 0 envoy_cluster_retry_or_shadow_abandoned{envoy_cluster_name="ext-authz"} 0 envoy_cluster_upstream_cx_destroy_local_with_active_rq{envoy_cluster_name="ext-authz"} 0 envoy_cluster_upstream_cx_connect_timeout{envoy_cluster_name="sso"} 0 envoy_cluster_upstream_rq_pending_failure_eject{envoy_cluster_name="sso"} 0 envoy_cluster_upstream_flow_control_paused_reading_total{envoy_cluster_name="sso"} 0 envoy_cluster_upstream_flow_control_drained_total{envoy_cluster_name="sso"} 0 envoy_cluster_lb_local_cluster_not_ok{envoy_cluster_name="sso"} 0 envoy_cluster_lb_subsets_created{envoy_cluster_name="sso"} 0 envoy_cluster_lb_subsets_selected{envoy_cluster_name="sso"} 0 envoy_cluster_lb_zone_routing_cross_zone{envoy_cluster_name="sso"} 0 envoy_cluster_external_upstream_rq{envoy_response_code="405",envoy_cluster_name="sso"} 1 envoy_cluster_upstream_rq_timeout{envoy_cluster_name="sso"} 0 envoy_cluster_external_upstream_rq_xx{envoy_response_code_class="4",envoy_cluster_name="sso"} 8 envoy_cluster_external_upstream_rq_xx{envoy_response_code_class="2",envoy_cluster_name="sso"} 13 envoy_cluster_upstream_cx_overflow{envoy_cluster_name="sso"} 0 envoy_cluster_upstream_rq_per_try_timeout{envoy_cluster_name="sso"} 0 envoy_cluster_upstream_cx_max_requests{envoy_cluster_name="sso"} 0 envoy_cluster_upstream_cx_destroy_remote_with_active_rq{envoy_cluster_name="sso"} 0 envoy_cluster_retry_or_shadow_abandoned{envoy_cluster_name="sso"} 0 envoy_cluster_upstream_cx_close_notify{envoy_cluster_name="sso"} 0 envoy_cluster_upstream_cx_idle_timeout{envoy_cluster_name="sso"} 0 envoy_cluster_upstream_rq{envoy_response_code="415",envoy_cluster_name="sso"} 1 envoy_cluster_upstream_cx_destroy_with_active_rq{envoy_cluster_name="sso"} 0 envoy_cluster_lb_zone_cluster_too_small{envoy_cluster_name="sso"} 0 envoy_cluster_upstream_rq_xx{envoy_response_code_class="4",envoy_cluster_name="sso"} 8 envoy_cluster_upstream_rq{envoy_response_code="401",envoy_cluster_name="sso"} 6 envoy_cluster_upstream_rq{envoy_response_code="405",envoy_cluster_name="sso"} 1 envoy_cluster_upstream_flow_control_backed_up_total{envoy_cluster_name="sso"} 0 envoy_cluster_external_upstream_rq{envoy_response_code="401",envoy_cluster_name="sso"} 6 envoy_cluster_external_upstream_rq{envoy_response_code="415",envoy_cluster_name="sso"} 1 envoy_cluster_upstream_rq_retry{envoy_cluster_name="sso"} 0 envoy_cluster_lb_zone_no_capacity_left{envoy_cluster_name="sso"} 0 envoy_cluster_upstream_cx_http1_total{envoy_cluster_name="sso"} 7 envoy_cluster_upstream_rq_total{envoy_cluster_name="sso"} 21 envoy_cluster_upstream_cx_protocol_error{envoy_cluster_name="sso"} 0 envoy_cluster_upstream_rq_tx_reset{envoy_cluster_name="sso"} 0 envoy_cluster_update_success{envoy_cluster_name="sso"} 18891 envoy_cluster_upstream_rq_completed{envoy_cluster_name="sso"} 21 envoy_cluster_external_upstream_rq{envoy_response_code="200",envoy_cluster_name="sso"} 13 envoy_cluster_upstream_cx_destroy{envoy_cluster_name="sso"} 0 envoy_cluster_upstream_cx_destroy_local{envoy_cluster_name="sso"} 0 envoy_cluster_lb_recalculate_zone_structures{envoy_cluster_name="sso"} 0 envoy_cluster_upstream_cx_tx_bytes_total{envoy_cluster_name="sso"} 13072 envoy_cluster_upstream_rq_pending_overflow{envoy_cluster_name="sso"} 0 envoy_cluster_upstream_cx_total{envoy_cluster_name="sso"} 7 envoy_cluster_lb_subsets_fallback{envoy_cluster_name="sso"} 0 envoy_cluster_original_dst_host_invalid{envoy_cluster_name="sso"} 0 envoy_cluster_upstream_rq_pending_total{envoy_cluster_name="sso"} 7 envoy_cluster_upstream_rq_cancelled{envoy_cluster_name="sso"} 0 envoy_cluster_upstream_rq_maintenance_mode{envoy_cluster_name="sso"} 0 envoy_cluster_upstream_rq_retry_success{envoy_cluster_name="sso"} 0 envoy_cluster_lb_healthy_panic{envoy_cluster_name="sso"} 0 envoy_cluster_upstream_cx_http2_total{envoy_cluster_name="sso"} 0 envoy_cluster_update_failure{envoy_cluster_name="sso"} 0 envoy_cluster_upstream_cx_connect_fail{envoy_cluster_name="sso"} 0 envoy_cluster_update_attempt{envoy_cluster_name="sso"} 18891 envoy_cluster_upstream_flow_control_resumed_reading_total{envoy_cluster_name="sso"} 0 envoy_cluster_update_empty{envoy_cluster_name="sso"} 0 envoy_cluster_upstream_cx_destroy_remote{envoy_cluster_name="sso"} 0 envoy_cluster_update_no_rebuild{envoy_cluster_name="sso"} 18890 envoy_cluster_bind_errors{envoy_cluster_name="sso"} 0 envoy_cluster_lb_subsets_removed{envoy_cluster_name="sso"} 0 envoy_cluster_upstream_cx_connect_attempts_exceeded{envoy_cluster_name="sso"} 0 envoy_cluster_upstream_rq_retry_overflow{envoy_cluster_name="sso"} 0 envoy_cluster_upstream_rq_xx{envoy_response_code_class="2",envoy_cluster_name="sso"} 13 envoy_cluster_lb_zone_number_differs{envoy_cluster_name="sso"} 0 envoy_cluster_upstream_cx_rx_bytes_total{envoy_cluster_name="sso"} 12934 envoy_cluster_lb_zone_routing_sampled{envoy_cluster_name="sso"} 0 envoy_cluster_lb_zone_routing_all_directly{envoy_cluster_name="sso"} 0 envoy_cluster_upstream_cx_destroy_local_with_active_rq{envoy_cluster_name="sso"} 0 envoy_cluster_upstream_rq_rx_reset{envoy_cluster_name="sso"} 0 envoy_cluster_upstream_cx_none_healthy{envoy_cluster_name="sso"} 0 envoy_cluster_membership_change{envoy_cluster_name="sso"} 1 envoy_cluster_external_upstream_rq_completed{envoy_cluster_name="sso"} 21 envoy_cluster_upstream_rq{envoy_response_code="200",envoy_cluster_name="sso"} 13 # TYPE envoy_http_downstream_rq_too_large counter envoy_http_downstream_rq_too_large{envoy_http_conn_manager_prefix="admin"} 0 # TYPE envoy_http_downstream_cx_protocol_error counter envoy_http_downstream_cx_protocol_error{envoy_http_conn_manager_prefix="admin"} 0 # TYPE envoy_http_downstream_rq_timeout counter envoy_http_downstream_rq_timeout{envoy_http_conn_manager_prefix="admin"} 0 # TYPE envoy_runtime_load_error counter envoy_runtime_load_error{} 0 # TYPE envoy_runtime_override_dir_not_exists counter envoy_runtime_override_dir_not_exists{} 0 # TYPE envoy_http_no_cluster counter envoy_http_no_cluster{envoy_http_conn_manager_prefix="async-client"} 0 # TYPE envoy_server_watchdog_miss counter envoy_server_watchdog_miss{} 0 # TYPE envoy_http_downstream_rq_xx counter envoy_http_downstream_rq_xx{envoy_response_code_class="3",envoy_http_conn_manager_prefix="admin"} 0 # TYPE envoy_cluster_manager_cluster_removed counter envoy_cluster_manager_cluster_removed{} 0 # TYPE envoy_http_downstream_cx_upgrades_total counter envoy_http_downstream_cx_upgrades_total{envoy_http_conn_manager_prefix="admin"} 0 # TYPE envoy_cluster_manager_update_merge_cancelled counter envoy_cluster_manager_update_merge_cancelled{} 0 # TYPE envoy_http_downstream_cx_overload_disable_keepalive counter envoy_http_downstream_cx_overload_disable_keepalive{envoy_http_conn_manager_prefix="admin"} 0 envoy_http_downstream_rq_xx{envoy_response_code_class="1",envoy_http_conn_manager_prefix="admin"} 0 # TYPE envoy_http_downstream_cx_destroy counter envoy_http_downstream_cx_destroy{envoy_http_conn_manager_prefix="admin"} 146344 envoy_http_downstream_rq_xx{envoy_response_code_class="5",envoy_http_conn_manager_prefix="admin"} 0 # TYPE envoy_http_downstream_cx_tx_bytes_total counter envoy_http_downstream_cx_tx_bytes_total{envoy_http_conn_manager_prefix="admin"} 155797820 # TYPE envoy_http_downstream_cx_drain_close counter envoy_http_downstream_cx_drain_close{envoy_http_conn_manager_prefix="admin"} 0 # TYPE envoy_http_downstream_cx_destroy_local_active_rq counter envoy_http_downstream_cx_destroy_local_active_rq{envoy_http_conn_manager_prefix="admin"} 0 # TYPE envoy_http_downstream_cx_destroy_active_rq counter envoy_http_downstream_cx_destroy_active_rq{envoy_http_conn_manager_prefix="admin"} 0 # TYPE envoy_http_downstream_cx_destroy_local counter envoy_http_downstream_cx_destroy_local{envoy_http_conn_manager_prefix="admin"} 0 envoy_http_downstream_rq_xx{envoy_response_code_class="2",envoy_http_conn_manager_prefix="admin"} 3154 # TYPE envoy_http_downstream_rq_http1_total counter envoy_http_downstream_rq_http1_total{envoy_http_conn_manager_prefix="admin"} 3155 # TYPE envoy_http_rq_total counter envoy_http_rq_total{envoy_http_conn_manager_prefix="async-client"} 475 # TYPE envoy_http_downstream_cx_total counter envoy_http_downstream_cx_total{envoy_http_conn_manager_prefix="admin"} 146346 # TYPE envoy_runtime_load_success counter envoy_runtime_load_success{} 0 # TYPE envoy_filesystem_reopen_failed counter envoy_filesystem_reopen_failed{} 0 # TYPE envoy_http_downstream_rq_total counter envoy_http_downstream_rq_total{envoy_http_conn_manager_prefix="admin"} 3155 # TYPE envoy_http_downstream_rq_rx_reset counter envoy_http_downstream_rq_rx_reset{envoy_http_conn_manager_prefix="admin"} 0 # TYPE envoy_cluster_manager_cluster_modified counter envoy_cluster_manager_cluster_modified{} 0 # TYPE envoy_http_downstream_rq_idle_timeout counter envoy_http_downstream_rq_idle_timeout{envoy_http_conn_manager_prefix="admin"} 0 # TYPE envoy_http_downstream_cx_destroy_remote_active_rq counter envoy_http_downstream_cx_destroy_remote_active_rq{envoy_http_conn_manager_prefix="admin"} 0 # TYPE envoy_http_downstream_cx_ssl_total counter envoy_http_downstream_cx_ssl_total{envoy_http_conn_manager_prefix="admin"} 0 # TYPE envoy_http_downstream_cx_delayed_close_timeout counter envoy_http_downstream_cx_delayed_close_timeout{envoy_http_conn_manager_prefix="admin"} 0 # TYPE envoy_cluster_manager_update_out_of_merge_window counter envoy_cluster_manager_update_out_of_merge_window{} 0 # TYPE envoy_http_downstream_flow_control_resumed_reading_total counter envoy_http_downstream_flow_control_resumed_reading_total{envoy_http_conn_manager_prefix="admin"} 0 # TYPE envoy_listener_manager_listener_added counter envoy_listener_manager_listener_added{} 1 # TYPE envoy_http_downstream_rq_overload_close counter envoy_http_downstream_rq_overload_close{envoy_http_conn_manager_prefix="admin"} 0 # TYPE envoy_http_downstream_cx_http2_total counter envoy_http_downstream_cx_http2_total{envoy_http_conn_manager_prefix="admin"} 0 # TYPE envoy_listener_manager_listener_create_failure counter envoy_listener_manager_listener_create_failure{} 0 # TYPE envoy_stats_overflow counter envoy_stats_overflow{} 0 # TYPE envoy_listener_manager_listener_create_success counter envoy_listener_manager_listener_create_success{} 16 # TYPE envoy_http_downstream_cx_http1_total counter envoy_http_downstream_cx_http1_total{envoy_http_conn_manager_prefix="admin"} 2 # TYPE envoy_http_no_route counter envoy_http_no_route{envoy_http_conn_manager_prefix="async-client"} 0 # TYPE envoy_cluster_manager_cluster_updated counter envoy_cluster_manager_cluster_updated{} 0 # TYPE envoy_http_downstream_cx_destroy_remote counter envoy_http_downstream_cx_destroy_remote{envoy_http_conn_manager_prefix="admin"} 146344 # TYPE envoy_http_downstream_rq_non_relative_path counter envoy_http_downstream_rq_non_relative_path{envoy_http_conn_manager_prefix="admin"} 0 # TYPE envoy_http_rs_too_large counter envoy_http_rs_too_large{envoy_http_conn_manager_prefix="admin"} 0 # TYPE envoy_http_downstream_rq_tx_reset counter envoy_http_downstream_rq_tx_reset{envoy_http_conn_manager_prefix="admin"} 0 # TYPE envoy_server_watchdog_mega_miss counter envoy_server_watchdog_mega_miss{} 0 # TYPE envoy_http_downstream_flow_control_paused_reading_total counter envoy_http_downstream_flow_control_paused_reading_total{envoy_http_conn_manager_prefix="admin"} 0 # TYPE envoy_http_downstream_cx_rx_bytes_total counter envoy_http_downstream_cx_rx_bytes_total{envoy_http_conn_manager_prefix="admin"} 795620 # TYPE envoy_http_downstream_rq_completed counter envoy_http_downstream_rq_completed{envoy_http_conn_manager_prefix="admin"} 3154 # TYPE envoy_listener_manager_listener_removed counter envoy_listener_manager_listener_removed{} 0 envoy_http_downstream_rq_xx{envoy_response_code_class="4",envoy_http_conn_manager_prefix="admin"} 0 # TYPE envoy_http_rq_direct_response counter envoy_http_rq_direct_response{envoy_http_conn_manager_prefix="async-client"} 0 # TYPE envoy_filesystem_write_completed counter envoy_filesystem_write_completed{} 3366 # TYPE envoy_http_downstream_rq_ws_on_non_ws_route counter envoy_http_downstream_rq_ws_on_non_ws_route{envoy_http_conn_manager_prefix="admin"} 0 # TYPE envoy_cluster_manager_cluster_updated_via_merge counter envoy_cluster_manager_cluster_updated_via_merge{} 0 # TYPE envoy_filesystem_flushed_by_timer counter envoy_filesystem_flushed_by_timer{} 18495 # TYPE envoy_filesystem_write_buffered counter envoy_filesystem_write_buffered{} 3669 # TYPE envoy_runtime_override_dir_exists counter envoy_runtime_override_dir_exists{} 0 # TYPE envoy_cluster_manager_cluster_added counter envoy_cluster_manager_cluster_added{} 4 # TYPE envoy_listener_manager_listener_modified counter envoy_listener_manager_listener_modified{} 0 # TYPE envoy_http_downstream_rq_http2_total counter envoy_http_downstream_rq_http2_total{envoy_http_conn_manager_prefix="admin"} 0 # TYPE envoy_http_downstream_rq_response_before_rq_complete counter envoy_http_downstream_rq_response_before_rq_complete{envoy_http_conn_manager_prefix="admin"} 0 # TYPE envoy_http_downstream_cx_idle_timeout counter envoy_http_downstream_cx_idle_timeout{envoy_http_conn_manager_prefix="admin"} 0 # TYPE envoy_http_rq_redirect counter envoy_http_rq_redirect{envoy_http_conn_manager_prefix="async-client"} 0 # TYPE envoy_listener_http_downstream_rq_xx counter envoy_listener_http_downstream_rq_xx{envoy_response_code_class="1",envoy_http_conn_manager_prefix="ingress_http",envoy_listener_address="0.0.0.0_8000"} 0 envoy_listener_http_downstream_rq_xx{envoy_response_code_class="2",envoy_http_conn_manager_prefix="ingress_http",envoy_listener_address="0.0.0.0_8000"} 456 envoy_listener_http_downstream_rq_xx{envoy_response_code_class="4",envoy_http_conn_manager_prefix="ingress_http",envoy_listener_address="0.0.0.0_8000"} 30 # TYPE envoy_listener_downstream_pre_cx_timeout counter envoy_listener_downstream_pre_cx_timeout{envoy_listener_address="0.0.0.0_8000"} 0 # TYPE envoy_listener_no_filter_chain_match counter envoy_listener_no_filter_chain_match{envoy_listener_address="0.0.0.0_8000"} 0 # TYPE envoy_listener_downstream_cx_total counter envoy_listener_downstream_cx_total{envoy_listener_address="0.0.0.0_8000"} 147370 envoy_listener_http_downstream_rq_xx{envoy_response_code_class="5",envoy_http_conn_manager_prefix="ingress_http",envoy_listener_address="0.0.0.0_8000"} 29 envoy_listener_http_downstream_rq_xx{envoy_response_code_class="3",envoy_http_conn_manager_prefix="ingress_http",envoy_listener_address="0.0.0.0_8000"} 0 # TYPE envoy_listener_downstream_cx_destroy counter envoy_listener_downstream_cx_destroy{envoy_listener_address="0.0.0.0_8000"} 147369 # TYPE envoy_listener_http_downstream_rq_completed counter envoy_listener_http_downstream_rq_completed{envoy_http_conn_manager_prefix="ingress_http",envoy_listener_address="0.0.0.0_8000"} 515 envoy_http_downstream_cx_destroy{envoy_http_conn_manager_prefix="ingress_http"} 147369 envoy_http_downstream_rq_http1_total{envoy_http_conn_manager_prefix="ingress_http"} 515 envoy_http_downstream_cx_destroy_local_active_rq{envoy_http_conn_manager_prefix="ingress_http"} 0 envoy_http_downstream_flow_control_resumed_reading_total{envoy_http_conn_manager_prefix="ingress_http"} 0 # TYPE envoy_http_tracing_health_check counter envoy_http_tracing_health_check{envoy_http_conn_manager_prefix="ingress_http"} 0 envoy_http_downstream_rq_response_before_rq_complete{envoy_http_conn_manager_prefix="ingress_http"} 0 envoy_http_downstream_cx_protocol_error{envoy_http_conn_manager_prefix="ingress_http"} 2 envoy_http_downstream_cx_destroy_remote{envoy_http_conn_manager_prefix="ingress_http"} 147367 envoy_http_no_route{envoy_http_conn_manager_prefix="ingress_http"} 7 envoy_http_rq_redirect{envoy_http_conn_manager_prefix="ingress_http"} 0 envoy_http_downstream_cx_http1_total{envoy_http_conn_manager_prefix="ingress_http"} 41 # TYPE envoy_http_tracing_random_sampling counter envoy_http_tracing_random_sampling{envoy_http_conn_manager_prefix="ingress_http"} 0 envoy_http_downstream_cx_total{envoy_http_conn_manager_prefix="ingress_http"} 147370 envoy_http_downstream_cx_destroy_remote_active_rq{envoy_http_conn_manager_prefix="ingress_http"} 0 envoy_http_rs_too_large{envoy_http_conn_manager_prefix="ingress_http"} 0 envoy_http_downstream_rq_idle_timeout{envoy_http_conn_manager_prefix="ingress_http"} 0 envoy_http_downstream_cx_idle_timeout{envoy_http_conn_manager_prefix="ingress_http"} 0 envoy_http_downstream_cx_rx_bytes_total{envoy_http_conn_manager_prefix="ingress_http"} 306987 envoy_http_downstream_rq_xx{envoy_response_code_class="1",envoy_http_conn_manager_prefix="ingress_http"} 0 envoy_http_downstream_rq_ws_on_non_ws_route{envoy_http_conn_manager_prefix="ingress_http"} 0 envoy_http_downstream_rq_xx{envoy_response_code_class="3",envoy_http_conn_manager_prefix="ingress_http"} 0 envoy_http_downstream_cx_overload_disable_keepalive{envoy_http_conn_manager_prefix="ingress_http"} 0 # TYPE envoy_http_tracing_service_forced counter envoy_http_tracing_service_forced{envoy_http_conn_manager_prefix="ingress_http"} 0 envoy_http_downstream_cx_drain_close{envoy_http_conn_manager_prefix="ingress_http"} 0 envoy_http_downstream_cx_tx_bytes_total{envoy_http_conn_manager_prefix="ingress_http"} 227287 envoy_http_downstream_cx_delayed_close_timeout{envoy_http_conn_manager_prefix="ingress_http"} 2 envoy_http_downstream_flow_control_paused_reading_total{envoy_http_conn_manager_prefix="ingress_http"} 0 envoy_http_downstream_rq_timeout{envoy_http_conn_manager_prefix="ingress_http"} 0 envoy_http_downstream_rq_rx_reset{envoy_http_conn_manager_prefix="ingress_http"} 0 envoy_http_downstream_cx_destroy_local{envoy_http_conn_manager_prefix="ingress_http"} 2 # TYPE envoy_http_tracing_not_traceable counter envoy_http_tracing_not_traceable{envoy_http_conn_manager_prefix="ingress_http"} 0 # TYPE envoy_http_cors_origin_invalid counter envoy_http_cors_origin_invalid{envoy_http_conn_manager_prefix="ingress_http"} 0 envoy_http_no_cluster{envoy_http_conn_manager_prefix="ingress_http"} 0 # TYPE envoy_http_cors_origin_valid counter envoy_http_cors_origin_valid{envoy_http_conn_manager_prefix="ingress_http"} 485 envoy_http_downstream_rq_completed{envoy_http_conn_manager_prefix="ingress_http"} 515 envoy_http_rq_total{envoy_http_conn_manager_prefix="ingress_http"} 268 envoy_http_downstream_rq_tx_reset{envoy_http_conn_manager_prefix="ingress_http"} 0 envoy_http_downstream_rq_total{envoy_http_conn_manager_prefix="ingress_http"} 515 envoy_http_downstream_rq_overload_close{envoy_http_conn_manager_prefix="ingress_http"} 0 envoy_http_downstream_cx_http2_total{envoy_http_conn_manager_prefix="ingress_http"} 0 envoy_http_downstream_cx_destroy_active_rq{envoy_http_conn_manager_prefix="ingress_http"} 0 envoy_http_downstream_rq_http2_total{envoy_http_conn_manager_prefix="ingress_http"} 0 envoy_http_downstream_cx_upgrades_total{envoy_http_conn_manager_prefix="ingress_http"} 0 envoy_http_downstream_rq_non_relative_path{envoy_http_conn_manager_prefix="ingress_http"} 0 envoy_http_downstream_rq_xx{envoy_response_code_class="4",envoy_http_conn_manager_prefix="ingress_http"} 30 envoy_http_downstream_rq_xx{envoy_response_code_class="5",envoy_http_conn_manager_prefix="ingress_http"} 29 envoy_http_rq_direct_response{envoy_http_conn_manager_prefix="ingress_http"} 0 envoy_http_downstream_cx_ssl_total{envoy_http_conn_manager_prefix="ingress_http"} 0 envoy_http_downstream_rq_too_large{envoy_http_conn_manager_prefix="ingress_http"} 0 envoy_http_downstream_rq_xx{envoy_response_code_class="2",envoy_http_conn_manager_prefix="ingress_http"} 456 # TYPE envoy_http_tracing_client_enabled counter envoy_http_tracing_client_enabled{envoy_http_conn_manager_prefix="ingress_http"} 0 envoy_cluster_upstream_cx_protocol_error{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_flow_control_paused_reading_total{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_cx_http2_total{envoy_cluster_name="cost"} 0 envoy_cluster_lb_subsets_removed{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_cx_http1_total{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_cx_close_notify{envoy_cluster_name="cost"} 0 envoy_cluster_lb_zone_cluster_too_small{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_cx_max_requests{envoy_cluster_name="cost"} 0 envoy_cluster_membership_change{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_cx_destroy_local{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_rq_cancelled{envoy_cluster_name="cost"} 0 envoy_cluster_update_no_rebuild{envoy_cluster_name="cost"} 18872 envoy_cluster_upstream_rq_timeout{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_cx_rx_bytes_total{envoy_cluster_name="cost"} 0 envoy_cluster_lb_zone_no_capacity_left{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_cx_destroy_local_with_active_rq{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_cx_total{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_rq_per_try_timeout{envoy_cluster_name="cost"} 0 envoy_cluster_bind_errors{envoy_cluster_name="cost"} 0 envoy_cluster_update_success{envoy_cluster_name="cost"} 18872 envoy_cluster_upstream_rq_retry{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_rq_tx_reset{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_rq_retry_success{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_cx_connect_attempts_exceeded{envoy_cluster_name="cost"} 0 envoy_cluster_update_empty{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_cx_destroy_remote{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_cx_tx_bytes_total{envoy_cluster_name="cost"} 0 envoy_cluster_original_dst_host_invalid{envoy_cluster_name="cost"} 0 envoy_cluster_update_failure{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_rq_pending_overflow{envoy_cluster_name="cost"} 0 envoy_cluster_update_attempt{envoy_cluster_name="cost"} 18872 envoy_cluster_upstream_cx_connect_timeout{envoy_cluster_name="cost"} 0 envoy_cluster_lb_zone_routing_all_directly{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_cx_none_healthy{envoy_cluster_name="cost"} 0 envoy_cluster_retry_or_shadow_abandoned{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_flow_control_resumed_reading_total{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_cx_connect_fail{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_rq_retry_overflow{envoy_cluster_name="cost"} 0 envoy_cluster_lb_zone_routing_cross_zone{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_rq_pending_total{envoy_cluster_name="cost"} 0 envoy_cluster_lb_subsets_created{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_rq_maintenance_mode{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_rq_completed{envoy_cluster_name="cost"} 0 envoy_cluster_lb_healthy_panic{envoy_cluster_name="cost"} 0 envoy_cluster_lb_zone_routing_sampled{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_flow_control_drained_total{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_cx_destroy_remote_with_active_rq{envoy_cluster_name="cost"} 0 envoy_cluster_lb_local_cluster_not_ok{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_flow_control_backed_up_total{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_rq_rx_reset{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_cx_destroy_with_active_rq{envoy_cluster_name="cost"} 0 envoy_cluster_lb_subsets_fallback{envoy_cluster_name="cost"} 0 envoy_cluster_lb_zone_number_differs{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_cx_idle_timeout{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_cx_overflow{envoy_cluster_name="cost"} 0 envoy_cluster_lb_subsets_selected{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_rq_total{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_cx_destroy{envoy_cluster_name="cost"} 0 envoy_cluster_lb_recalculate_zone_structures{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_rq_pending_failure_eject{envoy_cluster_name="cost"} 0 # TYPE envoy_listener_admin_downstream_pre_cx_active gauge envoy_listener_admin_downstream_pre_cx_active{} 0 # TYPE envoy_listener_admin_downstream_cx_active gauge envoy_listener_admin_downstream_cx_active{} 2 # TYPE envoy_cluster_upstream_cx_rx_bytes_buffered gauge envoy_cluster_upstream_cx_rx_bytes_buffered{envoy_cluster_name="hawkeye"} 3402 # TYPE envoy_cluster_membership_healthy gauge envoy_cluster_membership_healthy{envoy_cluster_name="hawkeye"} 1 # TYPE envoy_cluster_circuit_breakers_high_cx_open gauge envoy_cluster_circuit_breakers_high_cx_open{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_circuit_breakers_default_rq_pending_open gauge envoy_cluster_circuit_breakers_default_rq_pending_open{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_upstream_cx_tx_bytes_buffered gauge envoy_cluster_upstream_cx_tx_bytes_buffered{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_circuit_breakers_high_rq_retry_open gauge envoy_cluster_circuit_breakers_high_rq_retry_open{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_lb_subsets_active gauge envoy_cluster_lb_subsets_active{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_circuit_breakers_default_cx_open gauge envoy_cluster_circuit_breakers_default_cx_open{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_max_host_weight gauge envoy_cluster_max_host_weight{envoy_cluster_name="hawkeye"} 1 # TYPE envoy_cluster_circuit_breakers_default_rq_retry_open gauge envoy_cluster_circuit_breakers_default_rq_retry_open{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_version gauge envoy_cluster_version{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_membership_total gauge envoy_cluster_membership_total{envoy_cluster_name="hawkeye"} 1 # TYPE envoy_cluster_circuit_breakers_high_rq_pending_open gauge envoy_cluster_circuit_breakers_high_rq_pending_open{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_circuit_breakers_default_rq_open gauge envoy_cluster_circuit_breakers_default_rq_open{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_circuit_breakers_high_rq_open gauge envoy_cluster_circuit_breakers_high_rq_open{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_upstream_cx_active gauge envoy_cluster_upstream_cx_active{envoy_cluster_name="hawkeye"} 3 # TYPE envoy_cluster_upstream_rq_pending_active gauge envoy_cluster_upstream_rq_pending_active{envoy_cluster_name="hawkeye"} 0 # TYPE envoy_cluster_upstream_rq_active gauge envoy_cluster_upstream_rq_active{envoy_cluster_name="hawkeye"} 0 envoy_cluster_membership_healthy{envoy_cluster_name="ext-authz"} 1 envoy_cluster_circuit_breakers_default_rq_pending_open{envoy_cluster_name="ext-authz"} 0 envoy_cluster_circuit_breakers_default_rq_open{envoy_cluster_name="ext-authz"} 0 envoy_cluster_membership_total{envoy_cluster_name="ext-authz"} 1 envoy_cluster_circuit_breakers_default_rq_retry_open{envoy_cluster_name="ext-authz"} 0 envoy_cluster_upstream_cx_active{envoy_cluster_name="ext-authz"} 4 envoy_cluster_version{envoy_cluster_name="ext-authz"} 0 envoy_cluster_circuit_breakers_high_rq_pending_open{envoy_cluster_name="ext-authz"} 0 envoy_cluster_upstream_cx_rx_bytes_buffered{envoy_cluster_name="ext-authz"} 714 envoy_cluster_upstream_rq_pending_active{envoy_cluster_name="ext-authz"} 0 envoy_cluster_circuit_breakers_high_cx_open{envoy_cluster_name="ext-authz"} 0 envoy_cluster_circuit_breakers_high_rq_retry_open{envoy_cluster_name="ext-authz"} 0 envoy_cluster_max_host_weight{envoy_cluster_name="ext-authz"} 1 envoy_cluster_upstream_rq_active{envoy_cluster_name="ext-authz"} 0 envoy_cluster_circuit_breakers_high_rq_open{envoy_cluster_name="ext-authz"} 0 envoy_cluster_upstream_cx_tx_bytes_buffered{envoy_cluster_name="ext-authz"} 0 envoy_cluster_circuit_breakers_default_cx_open{envoy_cluster_name="ext-authz"} 0 envoy_cluster_lb_subsets_active{envoy_cluster_name="ext-authz"} 0 envoy_cluster_circuit_breakers_default_rq_open{envoy_cluster_name="sso"} 0 envoy_cluster_upstream_rq_pending_active{envoy_cluster_name="sso"} 0 envoy_cluster_membership_healthy{envoy_cluster_name="sso"} 1 envoy_cluster_circuit_breakers_high_rq_pending_open{envoy_cluster_name="sso"} 0 envoy_cluster_circuit_breakers_high_rq_open{envoy_cluster_name="sso"} 0 envoy_cluster_circuit_breakers_high_rq_retry_open{envoy_cluster_name="sso"} 0 envoy_cluster_version{envoy_cluster_name="sso"} 0 envoy_cluster_circuit_breakers_high_cx_open{envoy_cluster_name="sso"} 0 envoy_cluster_membership_total{envoy_cluster_name="sso"} 1 envoy_cluster_upstream_cx_active{envoy_cluster_name="sso"} 0 envoy_cluster_upstream_cx_tx_bytes_buffered{envoy_cluster_name="sso"} 0 envoy_cluster_circuit_breakers_default_rq_pending_open{envoy_cluster_name="sso"} 0 envoy_cluster_lb_subsets_active{envoy_cluster_name="sso"} 0 envoy_cluster_upstream_cx_rx_bytes_buffered{envoy_cluster_name="sso"} 0 envoy_cluster_circuit_breakers_default_cx_open{envoy_cluster_name="sso"} 0 envoy_cluster_upstream_rq_active{envoy_cluster_name="sso"} 0 envoy_cluster_max_host_weight{envoy_cluster_name="sso"} 1 envoy_cluster_circuit_breakers_default_rq_retry_open{envoy_cluster_name="sso"} 0 # TYPE envoy_filesystem_write_total_buffered gauge envoy_filesystem_write_total_buffered{} 0 # TYPE envoy_http_downstream_cx_http1_active gauge envoy_http_downstream_cx_http1_active{envoy_http_conn_manager_prefix="admin"} 2 # TYPE envoy_listener_manager_total_listeners_warming gauge envoy_listener_manager_total_listeners_warming{} 0 # TYPE envoy_runtime_num_keys gauge envoy_runtime_num_keys{} 0 # TYPE envoy_http_downstream_cx_rx_bytes_buffered gauge envoy_http_downstream_cx_rx_bytes_buffered{envoy_http_conn_manager_prefix="admin"} 845 # TYPE envoy_runtime_admin_overrides_active gauge envoy_runtime_admin_overrides_active{} 0 # TYPE envoy_server_version gauge envoy_server_version{} 3653592 # TYPE envoy_cluster_manager_active_clusters gauge envoy_cluster_manager_active_clusters{} 4 # TYPE envoy_server_live gauge envoy_server_live{} 1 # TYPE envoy_http_downstream_cx_ssl_active gauge envoy_http_downstream_cx_ssl_active{envoy_http_conn_manager_prefix="admin"} 0 # TYPE envoy_http_downstream_cx_http2_active gauge envoy_http_downstream_cx_http2_active{envoy_http_conn_manager_prefix="admin"} 0 # TYPE envoy_http_downstream_cx_tx_bytes_buffered gauge envoy_http_downstream_cx_tx_bytes_buffered{envoy_http_conn_manager_prefix="admin"} 0 # TYPE envoy_http_downstream_cx_upgrades_active gauge envoy_http_downstream_cx_upgrades_active{envoy_http_conn_manager_prefix="admin"} 0 # TYPE envoy_http_downstream_rq_active gauge envoy_http_downstream_rq_active{envoy_http_conn_manager_prefix="admin"} 1 # TYPE envoy_listener_manager_total_listeners_active gauge envoy_listener_manager_total_listeners_active{} 1 # TYPE envoy_server_parent_connections gauge envoy_server_parent_connections{} 0 # TYPE envoy_cluster_manager_warming_clusters gauge envoy_cluster_manager_warming_clusters{} 0 # TYPE envoy_http_downstream_cx_active gauge envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="admin"} 2 # TYPE envoy_listener_manager_total_listeners_draining gauge envoy_listener_manager_total_listeners_draining{} 0 # TYPE envoy_server_total_connections gauge envoy_server_total_connections{} 1 # TYPE envoy_server_concurrency gauge envoy_server_concurrency{} 16 # TYPE envoy_server_memory_allocated gauge envoy_server_memory_allocated{} 4216000 # TYPE envoy_server_hot_restart_epoch gauge envoy_server_hot_restart_epoch{} 0 # TYPE envoy_server_uptime gauge envoy_server_uptime{} 94576 # TYPE envoy_server_memory_heap_size gauge envoy_server_memory_heap_size{} 6291456 # TYPE envoy_server_days_until_first_cert_expiring gauge envoy_server_days_until_first_cert_expiring{} 2147483647 # TYPE envoy_listener_downstream_cx_active gauge envoy_listener_downstream_cx_active{envoy_listener_address="0.0.0.0_8000"} 1 # TYPE envoy_listener_downstream_pre_cx_active gauge envoy_listener_downstream_pre_cx_active{envoy_listener_address="0.0.0.0_8000"} 0 envoy_http_downstream_cx_tx_bytes_buffered{envoy_http_conn_manager_prefix="ingress_http"} 0 envoy_http_downstream_cx_upgrades_active{envoy_http_conn_manager_prefix="ingress_http"} 0 envoy_http_downstream_cx_ssl_active{envoy_http_conn_manager_prefix="ingress_http"} 0 envoy_http_downstream_cx_active{envoy_http_conn_manager_prefix="ingress_http"} 1 envoy_http_downstream_cx_http1_active{envoy_http_conn_manager_prefix="ingress_http"} 1 envoy_http_downstream_cx_http2_active{envoy_http_conn_manager_prefix="ingress_http"} 0 envoy_http_downstream_cx_rx_bytes_buffered{envoy_http_conn_manager_prefix="ingress_http"} 403 envoy_http_downstream_rq_active{envoy_http_conn_manager_prefix="ingress_http"} 0 envoy_cluster_upstream_rq_pending_active{envoy_cluster_name="cost"} 0 envoy_cluster_circuit_breakers_default_cx_open{envoy_cluster_name="cost"} 0 envoy_cluster_circuit_breakers_default_rq_pending_open{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_rq_active{envoy_cluster_name="cost"} 0 envoy_cluster_circuit_breakers_high_rq_pending_open{envoy_cluster_name="cost"} 0 envoy_cluster_circuit_breakers_default_rq_open{envoy_cluster_name="cost"} 0 envoy_cluster_max_host_weight{envoy_cluster_name="cost"} 1 envoy_cluster_membership_total{envoy_cluster_name="cost"} 0 envoy_cluster_circuit_breakers_high_cx_open{envoy_cluster_name="cost"} 0 envoy_cluster_circuit_breakers_default_rq_retry_open{envoy_cluster_name="cost"} 0 envoy_cluster_membership_healthy{envoy_cluster_name="cost"} 0 envoy_cluster_version{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_cx_active{envoy_cluster_name="cost"} 0 envoy_cluster_circuit_breakers_high_rq_retry_open{envoy_cluster_name="cost"} 0 envoy_cluster_circuit_breakers_high_rq_open{envoy_cluster_name="cost"} 0 envoy_cluster_lb_subsets_active{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_cx_tx_bytes_buffered{envoy_cluster_name="cost"} 0 envoy_cluster_upstream_cx_rx_bytes_buffered{envoy_cluster_name="cost"} 0
由于我实际测试过程中,envoy是部署在k8s当中,所以,利用prometheus对pod 自动发现的能力,没有在prometheus做静态配置。自动发现的好处,就是当pod hpa的时候,不需要去更改配置文件。
要采集envoy metrics,在 envoy deployment当中要加入如下注解:
kind: Deployment apiVersion: apps/v1 metadata: name: gateway labels: app: gateway spec: replicas: 2 selector: matchLabels: app: gateway template: metadata: labels: app: gateway annotations: prometheus.io/scrape: 'true' prometheus.io/path: '/stats/prometheus' prometheus.io/port: '9000' spec: volumes: - name: config configMap: name: gateway-cm containers: - name: gateway image: 'envoyproxy/envoy:v1.9.0' env: - name: 'CPUS' value: '1' - name: 'SERVICE_NAME' value: '-gateway' resources: limits: cpu: '1' memory: 1Gi requests: cpu: '1' memory: 1Gi volumeMounts: - name: config mountPath: /etc/envoy/ imagePullPolicy: Always
注意如下的注解:
annotations: prometheus.io/scrape: 'true' prometheus.io/path: '/stats/prometheus' prometheus.io/port: '9000'
然后查看prometheus的管理界面,查询一下metrics:
prometheus已经成功采集metrics。
grafana dashbord中,已经有两个做好的模板。大家引入就可以了。
具体是envoy global看板(7253)和 Envoy Service Dashboard(7250)。
最后效果如下: