Monitoring the image renderer

Rendering images requires a lot of memory, mainly because Grafana creates browser instances in the background for the actual rendering. Monitoring your service can help you allocate the right amount of resources to your rendering service and set the right rendering mode.

Enable Prometheus metrics endpoint

Configure this service to expose a Prometheus metrics endpoint. For information on how to configure and monitor this service using Prometheus as a data source, refer to Grafana Image Rendering Service dashboard.

Metrics endpoint output example:

  1. # HELP process_cpu_user_seconds_total Total user CPU time spent in seconds.
  2. # TYPE process_cpu_user_seconds_total counter
  3. process_cpu_user_seconds_total 0.536 1579444523566
  5. # HELP process_cpu_system_seconds_total Total system CPU time spent in seconds.
  6. # TYPE process_cpu_system_seconds_total counter
  7. process_cpu_system_seconds_total 0.064 1579444523566
  9. # HELP process_cpu_seconds_total Total user and system CPU time spent in seconds.
  10. # TYPE process_cpu_seconds_total counter
  11. process_cpu_seconds_total 0.6000000000000001 1579444523566
  13. # HELP process_start_time_seconds Start time of the process since unix epoch in seconds.
  14. # TYPE process_start_time_seconds gauge
  15. process_start_time_seconds 1579444433
  17. # HELP process_resident_memory_bytes Resident memory size in bytes.
  18. # TYPE process_resident_memory_bytes gauge
  19. process_resident_memory_bytes 52686848 1579444523568
  21. # HELP process_virtual_memory_bytes Virtual memory size in bytes.
  22. # TYPE process_virtual_memory_bytes gauge
  23. process_virtual_memory_bytes 2055344128 1579444523568
  25. # HELP process_heap_bytes Process heap size in bytes.
  26. # TYPE process_heap_bytes gauge
  27. process_heap_bytes 1996390400 1579444523568
  29. # HELP process_open_fds Number of open file descriptors.
  30. # TYPE process_open_fds gauge
  31. process_open_fds 31 1579444523567
  33. # HELP process_max_fds Maximum number of open file descriptors.
  34. # TYPE process_max_fds gauge
  35. process_max_fds 1573877
  37. # HELP nodejs_eventloop_lag_seconds Lag of event loop in seconds.
  38. # TYPE nodejs_eventloop_lag_seconds gauge
  39. nodejs_eventloop_lag_seconds 0.000915922 1579444523567
  41. # HELP nodejs_active_handles Number of active libuv handles grouped by handle type. Every handle type is C++ class name.
  42. # TYPE nodejs_active_handles gauge
  43. nodejs_active_handles{type="WriteStream"} 2 1579444523566
  44. nodejs_active_handles{type="Server"} 1 1579444523566
  45. nodejs_active_handles{type="Socket"} 9 1579444523566
  46. nodejs_active_handles{type="ChildProcess"} 2 1579444523566
  48. # HELP nodejs_active_handles_total Total number of active handles.
  49. # TYPE nodejs_active_handles_total gauge
  50. nodejs_active_handles_total 14 1579444523567
  52. # HELP nodejs_active_requests Number of active libuv requests grouped by request type. Every request type is C++ class name.
  53. # TYPE nodejs_active_requests gauge
  54. nodejs_active_requests{type="FSReqCallback"} 2
  56. # HELP nodejs_active_requests_total Total number of active requests.
  57. # TYPE nodejs_active_requests_total gauge
  58. nodejs_active_requests_total 2 1579444523567
  60. # HELP nodejs_heap_size_total_bytes Process heap size from node.js in bytes.
  61. # TYPE nodejs_heap_size_total_bytes gauge
  62. nodejs_heap_size_total_bytes 13725696 1579444523567
  64. # HELP nodejs_heap_size_used_bytes Process heap size used from node.js in bytes.
  65. # TYPE nodejs_heap_size_used_bytes gauge
  66. nodejs_heap_size_used_bytes 12068008 1579444523567
  68. # HELP nodejs_external_memory_bytes Nodejs external memory size in bytes.
  69. # TYPE nodejs_external_memory_bytes gauge
  70. nodejs_external_memory_bytes 1728962 1579444523567
  72. # HELP nodejs_heap_space_size_total_bytes Process heap space size total from node.js in bytes.
  73. # TYPE nodejs_heap_space_size_total_bytes gauge
  74. nodejs_heap_space_size_total_bytes{space="read_only"} 262144 1579444523567
  75. nodejs_heap_space_size_total_bytes{space="new"} 1048576 1579444523567
  76. nodejs_heap_space_size_total_bytes{space="old"} 9809920 1579444523567
  77. nodejs_heap_space_size_total_bytes{space="code"} 425984 1579444523567
  78. nodejs_heap_space_size_total_bytes{space="map"} 1052672 1579444523567
  79. nodejs_heap_space_size_total_bytes{space="large_object"} 1077248 1579444523567
  80. nodejs_heap_space_size_total_bytes{space="code_large_object"} 49152 1579444523567
  81. nodejs_heap_space_size_total_bytes{space="new_large_object"} 0 1579444523567
  83. # HELP nodejs_heap_space_size_used_bytes Process heap space size used from node.js in bytes.
  84. # TYPE nodejs_heap_space_size_used_bytes gauge
  85. nodejs_heap_space_size_used_bytes{space="read_only"} 32296 1579444523567
  86. nodejs_heap_space_size_used_bytes{space="new"} 601696 1579444523567
  87. nodejs_heap_space_size_used_bytes{space="old"} 9376600 1579444523567
  88. nodejs_heap_space_size_used_bytes{space="code"} 286688 1579444523567
  89. nodejs_heap_space_size_used_bytes{space="map"} 704320 1579444523567
  90. nodejs_heap_space_size_used_bytes{space="large_object"} 1064872 1579444523567
  91. nodejs_heap_space_size_used_bytes{space="code_large_object"} 3552 1579444523567
  92. nodejs_heap_space_size_used_bytes{space="new_large_object"} 0 1579444523567
  94. # HELP nodejs_heap_space_size_available_bytes Process heap space size available from node.js in bytes.
  95. # TYPE nodejs_heap_space_size_available_bytes gauge
  96. nodejs_heap_space_size_available_bytes{space="read_only"} 229576 1579444523567
  97. nodejs_heap_space_size_available_bytes{space="new"} 445792 1579444523567
  98. nodejs_heap_space_size_available_bytes{space="old"} 417712 1579444523567
  99. nodejs_heap_space_size_available_bytes{space="code"} 20576 1579444523567
  100. nodejs_heap_space_size_available_bytes{space="map"} 343632 1579444523567
  101. nodejs_heap_space_size_available_bytes{space="large_object"} 0 1579444523567
  102. nodejs_heap_space_size_available_bytes{space="code_large_object"} 0 1579444523567
  103. nodejs_heap_space_size_available_bytes{space="new_large_object"} 1047488 1579444523567
  105. # HELP nodejs_version_info Node.js version info.
  106. # TYPE nodejs_version_info gauge
  107. nodejs_version_info{version="v14.16.1",major="14",minor="16",patch="1"} 1
  109. # HELP grafana_image_renderer_service_http_request_duration_seconds duration histogram of http responses labeled with: status_code
  110. # TYPE grafana_image_renderer_service_http_request_duration_seconds histogram
  111. grafana_image_renderer_service_http_request_duration_seconds_bucket{le="1",status_code="200"} 0
  112. grafana_image_renderer_service_http_request_duration_seconds_bucket{le="5",status_code="200"} 4
  113. grafana_image_renderer_service_http_request_duration_seconds_bucket{le="7",status_code="200"} 4
  114. grafana_image_renderer_service_http_request_duration_seconds_bucket{le="9",status_code="200"} 4
  115. grafana_image_renderer_service_http_request_duration_seconds_bucket{le="11",status_code="200"} 4
  116. grafana_image_renderer_service_http_request_duration_seconds_bucket{le="13",status_code="200"} 4
  117. grafana_image_renderer_service_http_request_duration_seconds_bucket{le="15",status_code="200"} 4
  118. grafana_image_renderer_service_http_request_duration_seconds_bucket{le="20",status_code="200"} 4
  119. grafana_image_renderer_service_http_request_duration_seconds_bucket{le="30",status_code="200"} 4
  120. grafana_image_renderer_service_http_request_duration_seconds_bucket{le="+Inf",status_code="200"} 4
  121. grafana_image_renderer_service_http_request_duration_seconds_sum{status_code="200"} 10.492873834
  122. grafana_image_renderer_service_http_request_duration_seconds_count{status_code="200"} 4
  124. # HELP up 1 = up, 0 = not up
  125. # TYPE up gauge
  126. up 1
  128. # HELP grafana_image_renderer_http_request_in_flight A gauge of requests currently being served by the image renderer.
  129. # TYPE grafana_image_renderer_http_request_in_flight gauge
  130. grafana_image_renderer_http_request_in_flight 1
  132. # HELP grafana_image_renderer_step_duration_seconds duration histogram of browser steps for rendering an image labeled with: step
  133. # TYPE grafana_image_renderer_step_duration_seconds histogram
  134. grafana_image_renderer_step_duration_seconds_bucket{le="0.3",step="launch"} 0
  135. grafana_image_renderer_step_duration_seconds_bucket{le="0.5",step="launch"} 0
  136. grafana_image_renderer_step_duration_seconds_bucket{le="1",step="launch"} 1
  137. grafana_image_renderer_step_duration_seconds_bucket{le="2",step="launch"} 1
  138. grafana_image_renderer_step_duration_seconds_bucket{le="3",step="launch"} 1
  139. grafana_image_renderer_step_duration_seconds_bucket{le="5",step="launch"} 1
  140. grafana_image_renderer_step_duration_seconds_bucket{le="+Inf",step="launch"} 1
  141. grafana_image_renderer_step_duration_seconds_sum{step="launch"} 0.7914972
  142. grafana_image_renderer_step_duration_seconds_count{step="launch"} 1
  143. grafana_image_renderer_step_duration_seconds_bucket{le="0.3",step="newPage"} 1
  144. grafana_image_renderer_step_duration_seconds_bucket{le="0.5",step="newPage"} 1
  145. grafana_image_renderer_step_duration_seconds_bucket{le="1",step="newPage"} 1
  146. grafana_image_renderer_step_duration_seconds_bucket{le="2",step="newPage"} 1
  147. grafana_image_renderer_step_duration_seconds_bucket{le="3",step="newPage"} 1
  148. grafana_image_renderer_step_duration_seconds_bucket{le="5",step="newPage"} 1
  149. grafana_image_renderer_step_duration_seconds_bucket{le="+Inf",step="newPage"} 1
  150. grafana_image_renderer_step_duration_seconds_sum{step="newPage"} 0.2217868
  151. grafana_image_renderer_step_duration_seconds_count{step="newPage"} 1
  152. grafana_image_renderer_step_duration_seconds_bucket{le="0.3",step="prepare"} 1
  153. grafana_image_renderer_step_duration_seconds_bucket{le="0.5",step="prepare"} 1
  154. grafana_image_renderer_step_duration_seconds_bucket{le="1",step="prepare"} 1
  155. grafana_image_renderer_step_duration_seconds_bucket{le="2",step="prepare"} 1
  156. grafana_image_renderer_step_duration_seconds_bucket{le="3",step="prepare"} 1
  157. grafana_image_renderer_step_duration_seconds_bucket{le="5",step="prepare"} 1
  158. grafana_image_renderer_step_duration_seconds_bucket{le="+Inf",step="prepare"} 1
  159. grafana_image_renderer_step_duration_seconds_sum{step="prepare"} 0.0819274
  160. grafana_image_renderer_step_duration_seconds_count{step="prepare"} 1
  161. grafana_image_renderer_step_duration_seconds_bucket{le="0.3",step="navigate"} 0
  162. grafana_image_renderer_step_duration_seconds_bucket{le="0.5",step="navigate"} 0
  163. grafana_image_renderer_step_duration_seconds_bucket{le="1",step="navigate"} 0
  164. grafana_image_renderer_step_duration_seconds_bucket{le="2",step="navigate"} 0
  165. grafana_image_renderer_step_duration_seconds_bucket{le="3",step="navigate"} 0
  166. grafana_image_renderer_step_duration_seconds_bucket{le="5",step="navigate"} 0
  167. grafana_image_renderer_step_duration_seconds_bucket{le="+Inf",step="navigate"} 1
  168. grafana_image_renderer_step_duration_seconds_sum{step="navigate"} 15.3311258
  169. grafana_image_renderer_step_duration_seconds_count{step="navigate"} 1
  170. grafana_image_renderer_step_duration_seconds_bucket{le="0.3",step="panelsRendered"} 1
  171. grafana_image_renderer_step_duration_seconds_bucket{le="0.5",step="panelsRendered"} 1
  172. grafana_image_renderer_step_duration_seconds_bucket{le="1",step="panelsRendered"} 1
  173. grafana_image_renderer_step_duration_seconds_bucket{le="2",step="panelsRendered"} 1
  174. grafana_image_renderer_step_duration_seconds_bucket{le="3",step="panelsRendered"} 1
  175. grafana_image_renderer_step_duration_seconds_bucket{le="5",step="panelsRendered"} 1
  176. grafana_image_renderer_step_duration_seconds_bucket{le="+Inf",step="panelsRendered"} 1
  177. grafana_image_renderer_step_duration_seconds_sum{step="panelsRendered"} 0.0205577
  178. grafana_image_renderer_step_duration_seconds_count{step="panelsRendered"} 1
  179. grafana_image_renderer_step_duration_seconds_bucket{le="0.3",step="screenshot"} 1
  180. grafana_image_renderer_step_duration_seconds_bucket{le="0.5",step="screenshot"} 1
  181. grafana_image_renderer_step_duration_seconds_bucket{le="1",step="screenshot"} 1
  182. grafana_image_renderer_step_duration_seconds_bucket{le="2",step="screenshot"} 1
  183. grafana_image_renderer_step_duration_seconds_bucket{le="3",step="screenshot"} 1
  184. grafana_image_renderer_step_duration_seconds_bucket{le="5",step="screenshot"} 1
  185. grafana_image_renderer_step_duration_seconds_bucket{le="+Inf",step="screenshot"} 1
  186. grafana_image_renderer_step_duration_seconds_sum{step="screenshot"} 0.2866623
  187. grafana_image_renderer_step_duration_seconds_count{step="screenshot"} 1
  189. # HELP grafana_image_renderer_browser_info A metric with a constant '1 value labeled by version of the browser in use
  190. # TYPE grafana_image_renderer_browser_info gauge
  191. grafana_image_renderer_browser_info{version="HeadlessChrome/79.0.3945.0"} 1