diff --git a/communicate/dashboards/overview.mdx b/communicate/dashboards/overview.mdx index 3de998dbb..035676c4c 100644 --- a/communicate/dashboards/overview.mdx +++ b/communicate/dashboards/overview.mdx @@ -30,6 +30,8 @@ Dashboards allow you to do the following: - Show the availability and p95 / p99 metrics over the last 24 hours, 7 days and 30 days. - Communicate custom incident messages and maintenance messages. +To build dashboards in Grafana or another tool, scrape your check metrics from the [Prometheus endpoint](/integrations/observability/prometheus). + You can create multiple, distinct dashboards based on your plan. Edit your dashboard by clicking on the **Dashboards** button on the Checkly dashboard page. diff --git a/detect/uptime-monitoring/heartbeat-monitors/overview.mdx b/detect/uptime-monitoring/heartbeat-monitors/overview.mdx index e4609549c..4076fe968 100644 --- a/detect/uptime-monitoring/heartbeat-monitors/overview.mdx +++ b/detect/uptime-monitoring/heartbeat-monitors/overview.mdx @@ -81,7 +81,7 @@ Heartbeat monitors provide different metrics and insights than other types of ch - **Alert Timeline**: When alerts were triggered and resolved - **Source Tracking**: Which systems or processes sent pings -Heartbeat metrics are also available via the [Prometheus V2 integration](/integrations/observability/prometheus-v2#heartbeat-metrics), including dead man's switch gauges for alerting in Grafana when a ping is overdue. +Heartbeat metrics are also available via the [Prometheus integration](/integrations/observability/prometheus#heartbeat-metrics), including dead man's switch gauges for alerting in Grafana when a ping is overdue. Remember: Heartbeat monitors detect when jobs fail to complete, but they can't tell you why a job failed. Combine heartbeat monitoring with application logging and error tracking for complete observability. diff --git a/docs.json b/docs.json index e1e419f0b..07468aaa7 100644 --- a/docs.json +++ b/docs.json @@ -344,10 +344,7 @@ { "group": "Exporting", "pages": [ - "resolve/traces/export/grafana-tempo", - "resolve/traces/export/honeycomb", - "resolve/traces/export/new-relic", - "resolve/traces/export/dash0" + "resolve/traces/export/overview" ] }, { @@ -463,9 +460,28 @@ { "group": "Observability", "pages": [ - "integrations/observability/prometheus", - "integrations/observability/prometheus-v2", - "integrations/observability/coralogix" + "integrations/observability/overview", + { + "group": "Metrics", + "pages": [ + "integrations/observability/prometheus" + ] + }, + { + "group": "Traces", + "pages": [ + "integrations/observability/traces/otlp", + "integrations/observability/coralogix", + "integrations/observability/traces/chronosphere", + "integrations/observability/traces/clickstack", + "integrations/observability/traces/cribl", + "integrations/observability/traces/dash0", + "integrations/observability/traces/datadog", + "integrations/observability/traces/grafana-tempo", + "integrations/observability/traces/honeycomb", + "integrations/observability/traces/new-relic" + ] + } ] }, { @@ -2441,6 +2457,30 @@ "source": "/detect/synthetic-monitoring/playwright-checks/private-packages-and-internal-registries", "destination": "/detect/synthetic-monitoring/playwright-checks/internal-registries" }, + { + "source": "/resolve/traces/export/grafana-tempo", + "destination": "/integrations/observability/traces/grafana-tempo" + }, + { + "source": "/resolve/traces/export/honeycomb", + "destination": "/integrations/observability/traces/honeycomb" + }, + { + "source": "/resolve/traces/export/new-relic", + "destination": "/integrations/observability/traces/new-relic" + }, + { + "source": "/resolve/traces/export/dash0", + "destination": "/integrations/observability/traces/dash0" + }, + { + "source": "/resolve/traces/export/to-chronosphere", + "destination": "/integrations/observability/traces/chronosphere" + }, + { + "source": "/integrations/observability/prometheus-v2", + "destination": "/integrations/observability/prometheus" + }, { "source": "/platform/rocky-ai", "destination": "/ai/rocky-ai" diff --git a/images/docs/images/integrations/coralogix/cx_otel_int1.png b/images/docs/images/integrations/coralogix/cx_otel_int1.png index a5016d284..38a54a6e5 100644 Binary files a/images/docs/images/integrations/coralogix/cx_otel_int1.png and b/images/docs/images/integrations/coralogix/cx_otel_int1.png differ diff --git a/images/docs/images/integrations/prometheus_integration_card.png b/images/docs/images/integrations/prometheus_integration_card.png new file mode 100644 index 000000000..5502fa00c Binary files /dev/null and b/images/docs/images/integrations/prometheus_integration_card.png differ diff --git a/images/docs/images/otel/export-traces/dash0-token.png b/images/docs/images/otel/export-traces/dash0-token.png index 9dd00f32b..16ca13216 100644 Binary files a/images/docs/images/otel/export-traces/dash0-token.png and b/images/docs/images/otel/export-traces/dash0-token.png differ diff --git a/images/docs/images/otel/otel_export_traces_settings.png b/images/docs/images/otel/otel_export_traces_settings.png index 927d1541e..98acff044 100644 Binary files a/images/docs/images/otel/otel_export_traces_settings.png and b/images/docs/images/otel/otel_export_traces_settings.png differ diff --git a/integrations/observability/coralogix.mdx b/integrations/observability/coralogix.mdx index 4b8c1c995..bc45c5e74 100644 --- a/integrations/observability/coralogix.mdx +++ b/integrations/observability/coralogix.mdx @@ -1,121 +1,84 @@ --- -title: 'Export Metrics & Data via Coralogix' -description: 'Deep integration with Coralogix for OpenTelemetry tracing, RUM monitoring, and alert notifications' +title: 'Send traces, RUM and alerts to Coralogix' +description: 'Send check runs to Coralogix as OpenTelemetry traces, add Coralogix RUM to Browser checks and Multistep checks, and send alerts to Coralogix Logs.' sidebarTitle: 'Coralogix' canonical: 'https://www.checklyhq.com/docs/integrations/observability/coralogix/' --- -Checkly deeply integrates with [Coralogix](https://coralogix.com/) in three ways: +Checkly connects to [Coralogix](https://coralogix.com/) in three ways. Each one works on its own. -1. Checkly can automatically instrument all check executions with **OpenTelemetry tracing** and send the data to Coralogix -as traces. -2. Checkly can automatically instrument your Browser checks with **Coralogix RUM (Real User Monitoring)**. -3. Checkly can send **alert notifications** when checks pass and/or fail to **Coralogix Logs**. +- **OpenTelemetry tracing**: every API check, URL monitor, Browser check and Multistep check run arrives in Coralogix as a trace, linked from the check result page. +- **RUM**: Browser checks and Multistep checks send Coralogix RUM data labeled `Checkly: true`, so you can compare synthetic and real users. +- **Alert channel**: your Checkly alert notifications arrive in Coralogix Logs. -## OpenTelemetry tracing integration +## OpenTelemetry tracing -To set up the OpenTelemetry tracing integration, first enable the Coralogix integration in your Checkly account in the -[integrations section](https://app.checklyhq.com/settings/account/integrations). + +- A [plan](https://www.checklyhq.com/pricing/) that includes the Coralogix integration. +- A Coralogix account where you can create Send-Your-Data API keys. + -![coralogix otel integration](/images/docs/images/integrations/coralogix/cx_otel_int1.png) +1. In Coralogix, create a Send-Your-Data API key under **Settings > Users and Teams > API Keys**. See the [Coralogix API keys documentation](https://coralogix.com/docs/user-guides/account-management/api-keys/api-keys/). +2. In Checkly, open [Settings > Integrations](https://app.checklyhq.com/settings/account/integrations) and click **Add Coralogix integration**. +3. Pick the region of your Coralogix tenant and enter your team name. The region selects the ingest endpoint. The team name only builds the permalink shown on the check result page. +4. Select **Enable sending traces to Coralogix**, paste the API key and save the integration. -1. Pick the region of your tenant and enter the team name. We use this to determine the correct trace ingest endpoint for Coralogix. -2. Create a "Send Your Data" API key in your Coralogix account, you can find these in the **Data Flow > API Keys** section -of your Coralogix dashboard. Copy and paste the API key. +![Coralogix integration form in Checkly](/images/docs/images/integrations/coralogix/cx_otel_int1.png) -The integration is now active and Checkly will start doing two things +Export to Coralogix depends only on the Coralogix integration being enabled. Import Traces and Export Traces on [Settings > Traces](https://app.checklyhq.com/settings/account/traces) do not need to be on. -1. Send a trace for every check run to Coralogix. -2. Instrument all HTTP requests generated by the check with the following headers: +Checkly adds `traceparent` and `tracestate` headers to the requests your checks make, so spans from your instrumented backend join the same trace. The [OTLP page](/integrations/observability/traces/otlp#trace-headers) documents both headers. -| Header | Value | Description | -|---------------|------------------------------------------------|-----------------------------------------------------------------------------------------------------------------------| -| `traceparent` | `{version}-{trace_id}-{span_id}-{trace_flags}` | A trace parent ID in [the canonical w3c format](https://www.w3.org/TR/trace-context/#traceparent-header-field-values) | -| `tracestate` | `checkly=true` | A vendor specific header so you can recognize traces originating from Checkly | +![Checkly traces in the Coralogix Tracing view](/images/docs/images/integrations/coralogix/cx_otel_int2.png) -![coralogix checkly traces in coralogix](/images/docs/images/integrations/coralogix/cx_otel_int2.png) +The check result page shows the trace ID and a permalink to the trace in Coralogix. -Moreover, it will show a **trace ID** and a **permalink** to the recorded trace in the check result page, so you can easily -correlate the check run with the trace in Coralogix. +![Coralogix trace link on a Checkly check result](/images/docs/images/integrations/coralogix/cx_otel_int3.png) -![coralogix link in checkly result](/images/docs/images/integrations/coralogix/cx_otel_int3.png) +### What is sent -Each trace will have the following metadata added to it as **tags**: +Exported spans carry the `checkly.*` attributes documented on the [OTLP page](/integrations/observability/traces/otlp#what-is-sent): `checkly.account.id`, `checkly.check.id`, `checkly.check.result.id`, `checkly.check.name`, `checkly.check.type`, `checkly.check.runtime`, `checkly.check.session.id`, `checkly.check.location`, `checkly.check.tags` and `checkly.check.status`. It also carries two Coralogix resource attributes: -| Tag | Description | -|----------------------------|---------------------------------------------------------------| -| `checkly.account.id` | The ID of the Checkly account the check belongs to. | -| `checkly.check.id` | The ID of the Checkly check that generated the trace. | -| `checkly.check.name` | The name of the Checkly check that generated the trace. | -| `checkly.check.type` | The type of the Checkly check that generated the trace. | -| `checkly.check.runtime` | The runtime ID of the Checkly check that generated the trace. | -| `checkly.check.session.id` | The unique ID of the check session the check belongs to. | -| `checkly.check.result.id` | The unique result ID of the check run. | -| `checkly.check.location` | The location ID of where the check was run, i.e. `us-east-1`. | +| Attribute | Value | +|---|---| +| `cx.application.name` | `checkly--check` | +| `cx.subsystem.name` | `-check` | -Using these tags, you can create filters, dashboards and slice & dice your Checkly data as your runs are executed and -ingested into Coralogix as traces. +When the Coralogix integration and the generic export on Settings > Traces are both on, these `cx.*` resource attributes may also reach your generic endpoint from API checks, URL monitors, Browser checks and Multistep checks. -If you are using a different trace header than the standard `traceparent`, you can configure this header in the *custom -trace header* field in the integration settings. +### Custom trace header -### What's included - -- Traces for every check execution -- HTTP request instrumentation with W3C headers -- Trace correlation metadata +If your backend reads a header other than `traceparent`, set its name in the **Custom trace header** field. When it is set, API checks and URL monitors send the custom header instead of `traceparent`, even when the generic OTLP export is also on, so backend spans no longer join those check runs by `traceparent`. Browser checks and Multistep checks send both headers unless every enabled export has a custom header. -Check **metrics** (response times, availability, etc.) are not sent through this integration. To send these metrics -to Coralogix, scrape them via the [Prometheus integration](https://www.checklyhq.com/docs/integrations/observability/prometheus-v2/) -and forward to Coralogix using an OpenTelemetry Collector. +Check metrics, such as response times and availability, are not sent through this integration. To get them into Coralogix, scrape the [Prometheus endpoint](/integrations/observability/prometheus) and forward the metrics with an OpenTelemetry Collector. -## RUM integration - -In addition to the OpenTelemetry tracing, Checkly can also send RUM data to Coralogix for all Browser checks. This enables -tracking your Playwright-powered checks to be recorded as (semi) real users, so you can compare and contrast the performance -between synthetic and real users. - -![coralogix rum integration](/images/docs/images/integrations/coralogix/cx_rum_int1.png) - -1. Click **Enable Coralogix RUM** and pick the region of your tenant. -2. Set the global name of the application you will be targeting with your checks, e.g. "Acme webshop". -3. Create a new RUM integration in your Coralogix account via **Data Flow > Integrations > Real User Monitoring** - -![rum api key area](/images/docs/images/integrations/coralogix/cx_rum_api_creation.png) - -4. You will need to create an **Integration Key** and a **SourceMap Upload Key**. -Copy and paste the integration key into Checkly. - -> Only the integration key is needed for the Checkly integration. +## RUM -5. As Checkly automatically injects the Coralogix RUM SDK to each Browser check run, uploading and setting up the SourceMap is not required. You can **skip** the **Install & setup Browser SDK Package** step and finish the set up there. +With RUM enabled, Checkly injects the Coralogix RUM SDK into your Browser checks and Multistep checks. Their RUM data reaches Coralogix with the label `Checkly: true`. -If you are targeting multiple applications with your checks, you can override the global application name by adding the -environment variable `CORALOGIX_RUM_APP_NAME` to your Browser check. You can find the Browser check specific environment -variables under **Settings** > **Environment variables** in the Browser check editor. +![Coralogix RUM settings in Checkly](/images/docs/images/integrations/coralogix/cx_rum_int1.png) -After enabling the RUM integration, Checkly will start sending RUM data to Coralogix for all Browser checks. +1. In Coralogix, create a Send-Your-Data API key with the **SendRUMData** role preset under **Settings > Users and Teams > API Keys**. The [Coralogix RUM documentation](https://coralogix.com/docs/user-guides/rum/sdk-installation/javascript/npm-browser/) covers the Coralogix side. + ![RUM integration keys in Coralogix](/images/docs/images/integrations/coralogix/cx_rum_api_creation.png) +2. In the Coralogix integration in Checkly, select **Enable Coralogix RUM** and pick the **RUM Region** of your tenant. +3. Set the **RUM application name** to the application your checks target, for example `my-website`. +4. Paste the key into **RUM API key** and save the integration. -> Note that you do not need to install the RUM SDK in your application, Checkly will automatically inject the necessary -> JavaScript snippet into your Browser checks. However, if you have installed the SDK we will use the existing installation and not inject the snippet. +You do not need to install the RUM SDK in your application or upload source maps. If your application already loads the Coralogix RUM SDK, the injected copy skips its own initialization. -## Alert channel → Coralogix Logs integration +If your checks target several applications, set the environment variable `CORALOGIX_RUM_APP_NAME` on a check to override the application name for that check. -You can set up Coralogix as an alert channel destination, so Coralogix receives any alert notifications you have set up as logs. -Note that you can use this integration separately from the other two integrations mentioned above. +## Alert channel to Coralogix Logs -1. In Checkly, navigate to [alert settings](https://app.checklyhq.com/alerts/settings/). Click the **Add more channels** -button, find Coralogix on the list, and click **Add channel**. +Coralogix can receive your alert notifications as logs. This alert channel works without the tracing and RUM integrations. -![coralogix alert channels](/images/docs/images/integrations/coralogix/coralogix_alert_channels.png) +1. In Checkly, go to [Alert channels](https://app.checklyhq.com/alerts/settings), click **+ Add more channels** (**New alert channel** if you have none yet), find Coralogix and click **Add channel**. -2. Create a "Send Your Data" API key in your Coralogix account, you can find these in the **Data Flow > API Keys** section -of your Coralogix dashboard. Copy and paste the API key. +![Coralogix in the Checkly alert channel list](/images/docs/images/integrations/coralogix/coralogix_alert_channels.png) -2. Give your channel a name, and provide the correct Coralogix Logs API endpoint for the appropriate region, e.g. -`https://ingress.eu2.coralogix.com/logs/v1/singles` Make sure that the right checks are subscribing to the channel and -that the **Send when** rules are correctly set, then hit **Save Coralogix Webhook**. +2. In Coralogix, create a Send-Your-Data API key under **Settings > Users and Teams > API Keys**. Paste it into the channel form. +3. Name the channel and enter the Coralogix Logs API endpoint for your region, for example `https://ingress.eu2.coralogix.com/logs/v1/singles`. Make sure the right checks subscribe to the channel and the **Send when** rules fit, then click **Save Coralogix webhook**. -![coralogix integration step 4](/images/docs/images/integrations/coralogix/coralogix_step4.png) +![Coralogix alert channel settings in Checkly](/images/docs/images/integrations/coralogix/coralogix_step4.png) diff --git a/integrations/observability/overview.mdx b/integrations/observability/overview.mdx new file mode 100644 index 000000000..12be77490 --- /dev/null +++ b/integrations/observability/overview.mdx @@ -0,0 +1,47 @@ +--- +title: 'Observability integrations' +description: 'Send Checkly metrics, traces and alerts to the tools you already use, and bring your application traces into Checkly.' +sidebarTitle: 'Overview' +canonical: 'https://www.checklyhq.com/docs/integrations/observability/overview/' +--- + +Turn on trace export and every API check, URL monitor, Browser check and Multistep check run is sent to your tracing backend as one OpenTelemetry span with the check name, location, tags and verdict, joined to your application's spans when your app honors the `traceparent` header. Point any scraper that sends a bearer token at one URL and it reads the status of every check type, plus timings and Browser check web vitals. + +## Choose an integration + +| I want | How Checkly does it | Go to | +| --- | --- | --- | +| Metrics in my dashboards | Your scraper reads one Prometheus endpoint with a bearer token. | [Prometheus](/integrations/observability/prometheus) | +| Check runs as traces in my backend | Checkly sends one span per run over OTLP/HTTP to any OpenTelemetry backend or collector. | [Export traces over OTLP](/integrations/observability/traces/otlp) | +| Alerts in my tools | Alert notifications go out as webhooks. Add `OTEL_TRACE_ID` to the payload to link the alert to the trace in your own tool, or `OTEL_TRACE_LINK` to open the trace in Checkly. Both are empty when the run has no trace. | [Webhooks](/integrations/alerts/webhooks) | +| Traces and metrics through my OpenTelemetry Collector | The Collector receives the spans on its OTLP/HTTP receiver and scrapes the Prometheus endpoint with its `prometheus` receiver. | [Collector receiver](/integrations/observability/traces/otlp#receive-spans-with-an-opentelemetry-collector), [Collector scraper](/integrations/observability/prometheus#scraping-with-other-tools) | +| My application's traces in Checkly | Your app sends its spans to Checkly over OTLP, so a failed check result shows what happened in your backend. | [Import traces](/resolve/traces/import/overview) | + +## Coverage and limits + +- Metrics cover every check type, including Heartbeat monitors, plus queue and agent metrics for your private locations. +- Traces are sent for API check, URL monitor, Browser check and Multistep check runs. +- The span's OpenTelemetry status is always OK. Filter on `checkly.check.status` to find failed and degraded runs in your tool. +- Traces carry no metrics, the Prometheus endpoint carries no traces, and neither carries logs. +- Export is account-wide to one endpoint. Failed exports are not shown in Checkly. +- Timings are averages only. Histograms expose only the `+Inf` bucket, and counters reset every hour. + +## Integration pages + + + +Scrape check, heartbeat and private location metrics from one endpoint. + + + +Send one span per check run to any backend or collector that accepts OTLP/HTTP. + + + +Send traces, RUM and alerts to Coralogix. + + + +Backend not listed? Use [Export traces over OTLP](/integrations/observability/traces/otlp) for traces and [Prometheus](/integrations/observability/prometheus) for metrics. Pages for specific backends: [Chronosphere](/integrations/observability/traces/chronosphere), [ClickStack](/integrations/observability/traces/clickstack), [Cribl Stream](/integrations/observability/traces/cribl), [Dash0](/integrations/observability/traces/dash0), [Datadog](/integrations/observability/traces/datadog), [Grafana Tempo](/integrations/observability/traces/grafana-tempo), [Honeycomb](/integrations/observability/traces/honeycomb), [New Relic](/integrations/observability/traces/new-relic). + +New to traces and spans? [OpenTelemetry concepts](/learn/opentelemetry/overview) covers the basics. diff --git a/integrations/observability/prometheus-v2.mdx b/integrations/observability/prometheus-v2.mdx deleted file mode 100644 index 74f05c34f..000000000 --- a/integrations/observability/prometheus-v2.mdx +++ /dev/null @@ -1,285 +0,0 @@ ---- -title: 'Exporting Metrics & Data via Prometheus V2' -description: 'Export Checkly monitoring data to Prometheus for advanced metrics analysis and Grafana dashboards' -sidebarTitle: 'Prometheus V2' -canonical: 'https://www.checklyhq.com/docs/integrations/observability/prometheus-v2/' ---- - -This page describes a new V2 version of the Prometheus exporter. For information about the old Prometheus exporter, see the [Prometheus V1 docs](/integrations/observability/prometheus). - - -If you are using [Prometheus](https://prometheus.io/) for monitoring and the popular [Grafana](https://grafana.com/) stack -for dashboarding, you can expose Checkly's core metrics on a dedicated, secured endpoint. - -## Activation - -Activating this integration is simple. - -1. Open **Account settings** from the left sidebar, then select **Integrations** (or go directly to the Integrations page) and click the **Create Prometheus endpoint** button. -![Prometheus integration step 1](/images/docs/images/integrations/prometheus_step1.png) - -2. We directly create an endpoint for you and provide its URL and the required Bearer token. -![Prometheus integration step 2](/images/docs/images/integrations/prometheus_v2_step2.png) - -3. Create a new job in your Prometheus `prometheus.yml` config and set up a scraping interval. Use a 60-second scrape interval and enable compression. Add the URL (divided into `metrics_path`, `scheme` and `target`) and `bearer_token`. -Here is an example - -```yaml -# prometheus.yml -- job_name: 'checkly' - scrape_interval: 60s - scrape_timeout: 30s - enable_compression: true - metrics_path: '/accounts//v2/prometheus/metrics' - bearer_token: '' - scheme: https - static_configs: - - targets: ['api.checklyhq.com'] -``` - -Now restart Prometheus and you should see metrics coming in. - - -The Prometheus metrics endpoint has a rate limit of 50 requests per minute. -The responses from this endpoint are cached for 60 seconds. -Any request made to this endpoint within 60 seconds of the initial request will receive the cached response. -Use a scrape interval of 60 seconds. Shorter intervals fetch the same cached data. -If you run multiple Prometheus instances, each instance scrapes independently. Align their scrape intervals where possible. - - - -Keep `enable_compression` set to `true` to reduce response size and make scrapes faster. Prometheus supports this setting in version 2.49 and later. If your account has many checks, use a longer `scrape_timeout`, such as 30 seconds, to give Prometheus enough time to scrape all metrics. - - -## Check Metrics - -The Prometheus exporter exposes several metrics you can use to monitor the status of your checks, as well as to inspect detailed information such as [Web Vitals](/detect/synthetic-monitoring/browser-checks/performance-metrics). - -The following metrics are available to monitor checks: - -| Metric | Type | Description | -|--------|------|-------------| -| `checkly_check_status` | Gauge | Indicates whether a given check is currently passing, degraded, or failing. | -| `checkly_check_result_total` | Counter | The number of passing, degraded, and failing check results. | -| `checkly_browser_check_web_vitals_seconds` | Histogram | The [Web Vitals](/detect/synthetic-monitoring/browser-checks/performance-metrics) timings. | -| `checkly_browser_check_duration_seconds` | Histogram | The total check duration. This includes all pages visited and any waits. | -| `checkly_browser_check_errors` | Histogram | The errors encountered during a full browser session. | -| `checkly_api_check_timing_seconds` | Histogram | The response time for the API request, as well as the duration of the different phases. | -| `checkly_url_monitor_timing_seconds` | Histogram | The response time for the HTTP request, as well as the duration of the different phases. | -| `checkly_check_http_status_code` | Gauge | The HTTP status code returned on the latest run of an API check or URL monitor. | -| `checkly_tcp_monitor_timing_seconds` | Histogram | The response time for the TCP request, as well as the duration of the different phases. | -| `checkly_tcp_check_timing_seconds` | Histogram | **Deprecated:** use `checkly_tcp_monitor_timing_seconds` instead. Emits identical values. | -| `checkly_icmp_monitor_timing_seconds` | Histogram | The timing phases of the ICMP monitor, including DNS resolution and latency measurements (avg, min, max). | -| `checkly_icmp_monitor_packet_loss_ratio` | Gauge | The average packet loss ratio (0-1) for the ICMP monitor. | -| `checkly_dns_monitor_timing_seconds` | Histogram | The total duration of the DNS monitor. | -| `checkly_grpc_monitor_timing_seconds` | Histogram | The timing phases of the gRPC monitor, including DNS resolution, connection, and total RPC duration. | -| `checkly_grpc_monitor_status_code` | Gauge | The gRPC status code returned on the latest run (`0` = OK). | -| `checkly_traceroute_monitor_timing_seconds` | Histogram | The timing phases of the Traceroute monitor, including DNS resolution and final hop latency. | -| `checkly_traceroute_monitor_hops` | Gauge | The total number of hops for the Traceroute monitor. | -| `checkly_traceroute_monitor_destination_reached` | Gauge | Whether the traceroute reached its destination on the latest run (`1`) or not (`0`). | -| `checkly_ssl_monitor_days_until_expiry` | Gauge | The average days remaining until the certificate checked by the SSL monitor expires, over the current scrape window. | -| `checkly_ssl_monitor_handshake_seconds` | Histogram | The TLS handshake duration of the SSL monitor. | -| `checkly_ssl_monitor_chain_trusted` | Gauge | Whether the certificate chain was trusted on the latest run (`1`) or not (`0`). | -| `checkly_ssl_monitor_hostname_verified` | Gauge | Whether the certificate hostname was verified on the latest run (`1`) or not (`0`). | -| `checkly_ssl_monitor_ocsp_stapled` | Gauge | Whether an OCSP staple was present on the latest run (`1`) or not (`0`). | -| `checkly_ssl_monitor_key_size_bits` | Gauge | The size in bits of the leaf certificate key seen on the latest run. | -| `checkly_ssl_monitor_info` | Gauge | Always `1`. Carries the latest negotiated TLS attributes (version, cipher suite) and the baseline assessment (verdict, grade) as labels. | -| `checkly_multistep_check_duration_seconds` | Histogram | The total check duration. This includes all requests done and any waits. | -| `checkly_playwright_check_duration_seconds` | Histogram | The total duration of the Playwright Check Suite. | -| `checkly_heartbeat_last_success_timestamp_seconds` | Gauge | The Unix timestamp of the last successful heartbeat ping. A value of `0` means no successful ping has ever been received. | -| `checkly_heartbeat_expected_interval_seconds` | Gauge | The configured maximum interval between heartbeat pings, in seconds. Derived from the Heartbeat monitor's period and period unit settings. | -| `checkly_time_to_ssl_expiry_seconds` | Gauge | The amount of time remaining before the SSL certificate of the monitored domain expires. See the [SSL certificate expiration docs](/communicate/alerts/ssl-expiration) for more information on monitoring SSL certificates with checks. | - -The `checkly_check_status` and `checkly_check_result_total` metrics contain a `status` label with values `passing`, `failing`, and `degraded`. -The `checkly_check_status` gauge is `1` when the check has the status indicated by the `status` label and is `0` otherwise. - -For example, if a check is passing the result will be: -```bash -checkly_check_status{name="Passing Browser Check",status="passing"} 1 -checkly_check_status{name="Passing Browser Check",status="failing"} 0 -checkly_check_status{name="Passing Browser Check",status="degraded"} 0 -``` - -`checkly_check_status` can be useful for viewing the current status of a check, whereas `checkly_check_result_total` can be useful for calculating overall statistics. - -The metrics `checkly_browser_check_web_vitals_seconds`, `checkly_browser_check_errors`, `checkly_api_check_timing_seconds`, `checkly_tcp_monitor_timing_seconds`, `checkly_icmp_monitor_timing_seconds`, and `checkly_dns_monitor_timing_seconds` contain a `type` label. -This label indicates the different Web Vitals, error types, and timing phases being measured. - -For `checkly_tcp_monitor_timing_seconds`, the `type` label has the following values: `dns`, `connection`, `data`, and `total`. -For `checkly_icmp_monitor_timing_seconds`, the `type` label has the following values: `dns` (DNS resolution time), `latency_avg`, `latency_min`, and `latency_max`. -For `checkly_dns_monitor_timing_seconds`, the `type` label has the value `total`. - -`checkly_time_to_ssl_expiry_seconds` contains a `domain` label giving the domain of the monitored SSL certificate. - -In addition, the check metrics all contain the following labels: - -| Label | Description | -|-------|-------------| -| `name` | The name of the check. | -| `check_id` | The unique UUID of the check. | -| `check_type` | The type of check: `api`, `browser`, `multi_step`, `playwright`, `heartbeat`, `tcp`, `icmp`, `dns`, `url`, `grpc`, `traceroute`, `ssl`. | -| `muted` | Whether the check is muted, configured to not send alerts. | -| `activated` | Whether the check is activated. Deactivated checks aren't be run. | -| `group` | The name of the check group. | -| `tags` | The tags of the check. | - - -> You can set `key:value` tags in your checks and groups and they will be exported as custom labels in Prometheus. For instance the tag `env:production` will be exposed as a custom label `env="production"`. You can disable this by adding the query param `disableTagParsing=true`. Please note that Prometheus label names may only contain ASCII letters, numbers, as well as underscores (see the official [docs](https://prometheus.io/docs/concepts/data_model/)). Tags containing other characters in the label name will be sanitized. - -> The counter and histogram metrics are reset every hour. These resets can be handled in Prometheus by using the [rate](https://prometheus.io/docs/prometheus/latest/querying/functions/#rate) or [increase](https://prometheus.io/docs/prometheus/latest/querying/functions/#increase) functions. - -### Check Run Location Label - -To avoid creating a high volume of metrics, by default the metrics don't include a label for the check run location. It is possible to enable this by adding the query param `locationLabelEnabled=true` to your API request. This will add a `location` label giving the location where the checks ran. - -Since check status and SSL days remaining is only tracked on a per-check basis rather than by location, `checkly_check_status` and `checkly_time_to_ssl_expiry_seconds` do not have the `location` label included. Heartbeat metrics (`checkly_heartbeat_last_success_timestamp_seconds`, `checkly_heartbeat_expected_interval_seconds`, and heartbeat entries in `checkly_check_result_total`) also do not have a `location` label, since Heartbeat monitors receive pings rather than running from specific locations. All other [check metrics](#check-metrics) will have the `location` label added. - -Here is an example for how to set this in your `prometheus.yml` config: - -```yaml -# prometheus.yml -- job_name: 'checkly' - scrape_interval: 60s - scrape_timeout: 30s - enable_compression: true - metrics_path: '/accounts//v2/prometheus/metrics' - bearer_token: '' - scheme: https - static_configs: - - targets: ['api.checklyhq.com'] - params: - locationLabelEnabled: ['true'] -``` - -### Check Retry Attempts - -By default, [retry attempts](/communicate/alerts/retries/) are not included in the reported metrics. For example, if a check is failing on the initial attempt and passing on the retry, `checkly_check_result_total` and other metrics will only report the final passing check result. - -It is possible to include retry attempts by adding the query param `includeRetryAttempts=true` to your API request. Metrics reporting information on check results will then include retry attempts. - -An exception to this behaviour is `checkly_check_status`, which reports whether the check is currently in a passing, degraded, or failing state. Since retry attempts don't affect the checks state and only the final check result determines the state, this metric ignores retry attempts regardless of the `includeRetryAttempts` setting. For example, if a check is failing on retry attempts and passing on the retry, `checkly_check_status` will report that the check is passing. - -Here is an example for how to configure `includeRetryAttempts` in your `prometheus.yml` config: - -```yaml -# prometheus.yml -- job_name: 'checkly' - scrape_interval: 60s - scrape_timeout: 30s - enable_compression: true - metrics_path: '/accounts//v2/prometheus/metrics' - bearer_token: '' - scheme: https - static_configs: - - targets: ['api.checklyhq.com'] - params: - includeRetryAttempts: ['true'] -``` - -### PromQL Examples - -This section contains a few PromQL queries that you can use to start working with the Prometheus data. - -#### Currently failing checks - -To graph whether checks are passing or failing, use the query: -```bash -checkly_check_status{status="passing"} -``` - -Passing checks will have the value `1` while failing and degraded checks will have the value `0`. -This can be used to build a [Grafana table](https://grafana.com/docs/grafana/latest/panels-visualizations/visualizations/table/) of currently failing checks. - -#### Failure percentage - -To calculate the percentage of check runs that failed in the last 24 hours, use: -```bash -increase(checkly_check_result_total{status="failing"}[24h]) / ignoring(status) sum without (status) (increase(checkly_check_result_total[24h])) -``` - -#### Histogram averages - -The different histogram metrics can all be used to compute averages. For example, query the average web vitals times for a check using: - -```bash -sum by(type) (rate(checkly_browser_check_web_vitals_seconds_sum{name="Check Name"}[30m])) / sum by(type) (rate(checkly_browser_check_web_vitals_seconds_count{name="Check Name"}[30m])) -``` - -## Heartbeat Metrics - -[Heartbeat monitors](/detect/uptime-monitoring/heartbeat-monitors/overview) are passive checks that wait for your scheduled jobs and automated processes to send a ping. The Prometheus exporter includes heartbeat checks in the standard `checkly_check_status` and `checkly_check_result_total` metrics, plus two heartbeat-specific gauges designed for [dead man's switch](https://en.wikipedia.org/wiki/Dead_man%27s_switch) alerting. - -- **`checkly_heartbeat_last_success_timestamp_seconds`** — The Unix timestamp of the last successful ping, searched across the entire event history (no time window). A value of `0` means no successful ping has ever been received — this immediately triggers a dead man's switch alert since `time() - 0` produces a very large value. -- **`checkly_heartbeat_expected_interval_seconds`** — The maximum expected time between pings, derived from the Heartbeat monitor's configured period and period unit (e.g., `5 minutes` = `300` seconds). Use this as a threshold for alerting. - -### How heartbeats map to `checkly_check_result_total` - -Heartbeat checks contribute to `checkly_check_result_total` using a 1-hour count window, the same as other check types. The `status` label values map to heartbeat event states as follows: - -| `status` label | Heartbeat states counted | -|----------------|--------------------------| -| `passing` | `RECEIVED`, `EARLY`, `GRACE`, `LATE` — any ping that arrived | -| `failing` | `FAILING` — no ping received before the grace period expired | -| `degraded` | Always `0` — Heartbeat monitors do not have a degraded state | - -### Dead Man's Switch PromQL Examples - -#### Alert when a heartbeat ping is overdue - -Compare the current time against the last success timestamp. If the difference exceeds the expected interval, the job has missed its window: - -```bash -time() - checkly_heartbeat_last_success_timestamp_seconds > checkly_heartbeat_expected_interval_seconds -``` - -#### Alert with a safety margin - -Add a grace period multiplier (e.g., 1.5x) to avoid false alarms from minor delays: - -```bash -time() - checkly_heartbeat_last_success_timestamp_seconds > 1.5 * checkly_heartbeat_expected_interval_seconds -``` - -#### Grafana table of heartbeat health - -Show how long ago each heartbeat was last seen: - -```bash -time() - checkly_heartbeat_last_success_timestamp_seconds -``` - -## Private location metrics - -The Prometheus exporter also contains metrics for monitoring [private locations](/platform/private-locations/overview). These metrics can be used to ensure that your private locations have enough Checkly Agent instances running to execute all of your checks, and to drive [agent autoscaling](/platform/private-locations/autoscaling). - -The following metrics are available to monitor private locations: - -| Metric | Type | Description | -|--------|------|-------------| -| `checkly_private_location_queue_size` | Gauge | The number of check runs scheduled to the private location and waiting to be executed. A high value indicates that checks are becoming backlogged and that you may need to [scale your Checkly Agents](/platform/private-locations/scaling-redundancy). | -| `checkly_private_location_oldest_scheduled_check_run` | Gauge | The age in seconds of the oldest check run job scheduled to the private location queue. A high value indicates that checks are becoming backlogged. | -| `checkly_private_location_queue_duration_ms` | Gauge | The scheduling delay in milliseconds for check runs routed through a private location, by queue and `stat`. Derived from the last 2 minutes of check run data. Used to evaluate whether sufficient agent capacity is provisioned for the private location. | -| `checkly_private_location_agent_count` | Gauge | The number of agents connected for the private location. | -| `checkly_private_location_check_runs` | Gauge | The count of check runs in the private location, broken down by lifecycle state via the `state` label and by routing queue via the `queue` label. Used as the signal for [autoscaling Checkly Agents](/platform/private-locations/autoscaling). | - -The private location metrics all contain the following labels: - -| Label | Description | -| ------|-------------| -| `private_location_name` | the name of the private location. | -| `private_location_slug_name` | the private location's human readable unique identifier. | -| `private_location_id` | the private location's UUID. | - -`checkly_private_location_check_runs` additionally carries a `queue` label (`uptime` or `main`) and a `state` label. The `state` values relevant for autoscaling are: - -- `queued` — the check run has been scheduled but not yet picked up by an agent. -- `inflight` — the check run is currently being executed by an agent. - -`checkly_private_location_queue_duration_ms` additionally carries two labels: - -- `stat` — the statistical aggregation over the 2-minute window. Values: `avg`, `p90`, `p99`. -- `queue` — the internal routing queue the check run was scheduled to. Values: `uptime`, `main`. - - -This gauge is aggregated on a ~1 minute interval, so check runs that start and finish within that window may be excluded. - diff --git a/integrations/observability/prometheus.mdx b/integrations/observability/prometheus.mdx index 3fe94b4d9..0665e3614 100644 --- a/integrations/observability/prometheus.mdx +++ b/integrations/observability/prometheus.mdx @@ -1,108 +1,339 @@ --- -title: 'Send Alerts via Prometheus' -description: 'Configure Prometheus integration to receive real-time alerts from Checkly monitors' +title: 'Export metrics to Prometheus' +description: 'Scrape check, heartbeat and private location metrics from Checkly with Prometheus or any compatible scraper.' sidebarTitle: 'Prometheus' canonical: 'https://www.checklyhq.com/docs/integrations/observability/prometheus/' --- - -This page describes the v1 version of the Prometheus exporter. If you are just starting with our Prometheus integration, we recommend using our [v2 version](/integrations/observability/prometheus-v2). - +If you use [Prometheus](https://prometheus.io/) for monitoring and the [Grafana](https://grafana.com/) stack for dashboards, you can scrape Checkly's metrics for every check type from a dedicated, secured endpoint. Any other scraper that sends a bearer token, such as an OpenTelemetry Collector, Grafana Alloy or the Datadog Agent, reads the same endpoint. + +A [plan](https://www.checklyhq.com/pricing/) that includes the Prometheus integration, otherwise the endpoint returns `403`. + -Checkly exposes the following metrics in a Prometheus-compatible format. +Looking for the v1 endpoint? See [Legacy v1 endpoint](#legacy-v1-endpoint). -| Metric | Description | -|--------|-------------| -| `checkly_check_status` | Whether a check is passing or failing. `1` means the check is passing, `0` means the check is failing. | -| `checkly_check_degraded_status` | Whether a check is degraded. `1` means the check is not-degraded, `0` means the check is degraded. | -| `checkly_check_result` | The last collected response time for a check in a specific region. This means you get one `checkly_check_result` stanza for each region the check is configured to run in. | -| `checkly_private_location_queue_size` | The number of check runs scheduled to a private location and waiting to be executed. In particular, this metric reports the maximum count of scheduled check runs over the past 10 minutes. | -| `checkly_private_location_oldest_scheduled_check_run` | The age in seconds of the oldest scheduled job in the private location's queue. In particular, this metric reports the maximum age from the past 10 minutes. | -| `checkly_private_location_agent_count` | The number of connected Checkly Agents connected for a private location. An Agent is considered as connected if it's communicated with the Checkly infrastructure in the past 10 minutes. | +## Set up + +1. Go to **Account settings** > [**Integrations**](https://app.checklyhq.com/settings/account/integrations) and find the **Prometheus** card. It shows your endpoint URL, your bearer tokens and a scrape config sample. If the card shows a **Create Prometheus endpoint** button instead, click it first; the URL and tokens appear once the endpoint exists. + ![Prometheus integration card with the Create Prometheus endpoint button](/images/docs/images/integrations/prometheus_step1.png) + ![Prometheus integration card in account settings](/images/docs/images/integrations/prometheus_integration_card.png) +2. Under **Bearer tokens**, click **Add token** and copy the value. The full value is shown only once, right after you create it. An account can have at most two tokens, so you can rotate: add a token, point your scrape config at it, then revoke the old one. +3. Add the `checkly` job under `scrape_configs` in your `prometheus.yml` and replace the placeholders with your account ID and token. It matches the sample on the card, plus `scrape_timeout: 30s`. + +```yaml +scrape_configs: + - job_name: checkly + scrape_interval: 60s + scrape_timeout: 30s + scheme: https + metrics_path: /accounts//v2/prometheus/metrics + bearer_token: + params: + includeRetryAttempts: ['true'] + locationLabelEnabled: ['true'] + static_configs: + - targets: ['api.checklyhq.com'] +``` + +4. Restart Prometheus. Metrics appear after the first scrape. + +## Options + +Add these query parameters to the endpoint URL, or under `params` in the scrape config. The sample above turns on `includeRetryAttempts` and `locationLabelEnabled`, like the sample on the card. + +| Parameter | Default | Effect | +|-----------|---------|--------| +| `disableTagParsing` | `false` | Stops turning `key:value` tags into labels. See tag parsing under [Behavior](#behavior). | +| `locationLabelEnabled` | `false` | Adds a `location` label to metrics built from check results. Off by default to keep the number of series down. | +| `includeRetryAttempts` | `false` | Counts [retry attempts](/communicate/alerts/retries/) in metrics built from check results. Without it, a run that fails and then passes on retry counts once, as passing. | + +The endpoint rejects any other query parameter, and values other than `true` or `false`, with `400`. Each combination of parameters is cached separately. + +## Check metrics + +Together these metrics cover every check type. Each one also carries the labels listed under [Labels](#labels). The Labels column below lists the extra ones. + +| Metric | Type | Labels | Description | +|--------|------|--------|-------------| +| `checkly_check_status` | Gauge | `status`: `passing`, `failing`, `degraded` | Whether a given check is currently passing, degraded, or failing. `1` for the series whose `status` matches, `0` for the other two. Absent for a check that has never produced a result. | +| `checkly_check_result_total` | Counter | `status`: `passing`, `failing`, `degraded` | The number of passing, degraded, and failing check results in the [hourly window](#behavior). | +| `checkly_browser_check_web_vitals_seconds` | Histogram | `type`: `CLS`, `FCP`, `LCP`, `TBT`, `TTFB`, `loaded`, `DOM content loaded` | The [web vitals](/detect/synthetic-monitoring/browser-checks/performance-metrics) timings of the first page visited. | +| `checkly_browser_check_duration_seconds` | Histogram | | The total check duration. This includes all pages visited and any waits. | +| `checkly_browser_check_errors` | Histogram | `type`: `console`, `network`, `user`, `document` | The errors encountered during a full browser session. | +| `checkly_api_check_timing_seconds` | Histogram | `type`: `total`, `wait`, `dns`, `tcp`, `first byte` | The response time for the API request, as well as the duration of the different phases. | +| `checkly_url_monitor_timing_seconds` | Histogram | `type`: `total`, `wait`, `dns`, `tcp`, `first byte` | The response time for the HTTP request, as well as the duration of the different phases. | +| `checkly_check_http_status_code` | Gauge | | The HTTP status code returned on the latest run of an API check or URL monitor. Reads `0` when the latest run got no HTTP response. Absent when there was no run in the window. | +| `checkly_tcp_monitor_timing_seconds` | Histogram | `type`: `dns`, `connection`, `data`, `total` | The response time for the TCP request, as well as the duration of the different phases. | +| `checkly_tcp_check_timing_seconds` | Histogram | `type`: `dns`, `connection`, `data`, `total` | **Deprecated:** use `checkly_tcp_monitor_timing_seconds` instead. Emits identical values. | +| `checkly_icmp_monitor_timing_seconds` | Histogram | `type`: `dns`, `latency_avg`, `latency_min`, `latency_max` | The timing phases of the ICMP monitor, including DNS resolution and latency measurements. | +| `checkly_icmp_monitor_packet_loss_ratio` | Gauge | | The average packet loss ratio (0-1) for the ICMP monitor over the window. Reads `0` when there was no run in the window. | +| `checkly_dns_monitor_timing_seconds` | Histogram | `type`: `total` | The total duration of the DNS monitor. | +| `checkly_grpc_monitor_timing_seconds` | Histogram | `type`: `dns`, `connect`, `total` | The timing phases of the gRPC monitor, including DNS resolution, connection, and total RPC duration. | +| `checkly_grpc_monitor_status_code` | Gauge | | The gRPC status code returned on the latest run (`0` = OK). Absent when there was no run in the window. | +| `checkly_traceroute_monitor_timing_seconds` | Histogram | `type`: `dns`, `latency_avg`, `latency_min`, `latency_max` | The timing phases of the Traceroute monitor, including DNS resolution and final hop latency. | +| `checkly_traceroute_monitor_hops` | Gauge | | The total number of hops for the Traceroute monitor, averaged over the window. Reads `0` when there was no run in the window. | +| `checkly_traceroute_monitor_destination_reached` | Gauge | | Whether the traceroute reached its destination on the latest run (`1`) or not (`0`). Absent when there was no run in the window. | +| `checkly_ssl_monitor_days_until_expiry` | Gauge | | The average days remaining until the certificate checked by the SSL monitor expires, over the window. Reads `0` when there was no run in the window. | +| `checkly_ssl_monitor_handshake_seconds` | Histogram | | The TLS handshake duration of the SSL monitor. | +| `checkly_ssl_monitor_chain_trusted` | Gauge | | Whether the certificate chain was trusted on the latest run (`1`) or not (`0`). Absent when there was no run in the window. | +| `checkly_ssl_monitor_hostname_verified` | Gauge | | Whether the certificate hostname was verified on the latest run (`1`) or not (`0`). Absent when there was no run in the window. | +| `checkly_ssl_monitor_ocsp_stapled` | Gauge | | Whether an OCSP staple was present on the latest run (`1`) or not (`0`). Absent when there was no run in the window. | +| `checkly_ssl_monitor_key_size_bits` | Gauge | | The size in bits of the leaf certificate key seen on the latest run. Absent when there was no run in the window. | +| `checkly_ssl_monitor_info` | Gauge | `tls_version`, `cipher_suite`, `baseline_verdict`, `baseline_grade` | Always `1`. Carries the latest negotiated TLS attributes (version, cipher suite) and the baseline assessment (verdict, grade) as labels. Absent when there was no run in the window. | +| `checkly_multistep_check_duration_seconds` | Histogram | | The total check duration. This includes all requests done and any waits. | +| `checkly_playwright_check_duration_seconds` | Histogram | | The total duration of the Playwright Check Suite. | +| `checkly_time_to_ssl_expiry_seconds` | Gauge | `domain`: the domain of the monitored certificate | The time in seconds until the SSL certificate of the monitored domain expires, rounded to whole days. Only for checks with an SSL check domain. See the [SSL certificate expiration docs](/communicate/alerts/ssl-expiration). | + +For example, a passing check reads: + +```bash +checkly_check_status{name="Passing Browser Check",status="passing"} 1 +checkly_check_status{name="Passing Browser Check",status="failing"} 0 +checkly_check_status{name="Passing Browser Check",status="degraded"} 0 +``` + +`checkly_check_status` can be useful for viewing the current status of a check, whereas `checkly_check_result_total` can be useful for calculating overall statistics. + +### Labels + +| Label | Present on | Value | +|-------|------------|-------| +| `name` | All check metrics | The check name. | +| `check_id` | All check metrics | The check UUID. | +| `check_type` | All check metrics | `api`, `browser`, `multi_step`, `playwright`, `heartbeat`, `tcp`, `icmp`, `dns`, `url`, `grpc`, `traceroute`, `ssl` | +| `muted` | All check metrics | `true` when the check or its check group is muted, so it sends no alerts. | +| `activated` | All check metrics | `true` when both the check and its check group are activated. Deactivated checks don't run. | +| `group` | Checks in a check group only | The check group name. | +| `tags` | All check metrics | All tags of the check and its check group, comma separated, including `key:value` tags. | +| Tag labels | All check metrics, unless `disableTagParsing=true` | One label per `key:value` tag, for example `env:production` becomes `env="production"`. | +| `location` | Metrics built from check results, only with `locationLabelEnabled=true` | The region code of the run, for example `eu-west-1`, or the private location's slug name. Never on `checkly_check_status`, `checkly_time_to_ssl_expiry_seconds` or heartbeat metrics. | + +## Heartbeat metrics + +[Heartbeat monitors](/detect/uptime-monitoring/heartbeat-monitors/overview) are passive checks that wait for your scheduled jobs and automated processes to send a ping. The Prometheus exporter includes Heartbeat monitors in the standard `checkly_check_status` and `checkly_check_result_total` metrics, plus two heartbeat-specific gauges designed for [dead man's switch](https://en.wikipedia.org/wiki/Dead_man%27s_switch) alerting. + +| Metric | Type | Description | +|--------|------|-------------| +| `checkly_heartbeat_last_success_timestamp_seconds` | Gauge | The Unix timestamp of the last successful ping, searched across the entire event history with no time window. `0` means no successful ping has ever been received, which fires a dead man's switch alert right away because `time() - 0` is very large. | +| `checkly_heartbeat_expected_interval_seconds` | Gauge | The maximum expected time between pings, from the monitor's period and period unit. For example, 5 minutes reads `300`. Use it as the alert threshold. | + +### How heartbeats map to `checkly_check_result_total` + +Heartbeat monitors count in `checkly_check_result_total` over the same hourly window as other check types. The `status` label values map to heartbeat event states as follows: + +| `status` label | Heartbeat states counted | +|----------------|--------------------------| +| `passing` | `RECEIVED`, `EARLY`, `GRACE`, `LATE`: any ping that arrived | +| `failing` | `FAILING`: no ping received before the grace period expired | +| `degraded` | Always `0`. Heartbeat monitors have no degraded state. | + +### Dead man's switch PromQL examples + +#### Alert when a heartbeat ping is overdue + +Compare the current time against the last success timestamp. If the difference exceeds the expected interval, the job has missed its window: + +```bash +time() - checkly_heartbeat_last_success_timestamp_seconds > checkly_heartbeat_expected_interval_seconds +``` + +#### Alert with a safety margin + +Multiply the interval by a margin such as 1.5 to avoid false alarms from minor delays: + +```bash +time() - checkly_heartbeat_last_success_timestamp_seconds > 1.5 * checkly_heartbeat_expected_interval_seconds +``` + +#### Grafana table of heartbeat health + +Show how long ago each heartbeat was last seen: + +```bash +time() - checkly_heartbeat_last_success_timestamp_seconds +``` + +## Private location metrics + +The Prometheus exporter also contains metrics for monitoring [private locations](/platform/private-locations/overview). Use them to make sure your private locations have enough Checkly Agent instances running to execute all of your checks, and to drive [agent autoscaling](/platform/private-locations/autoscaling). + +Every private location metric carries the labels `private_location_name`, `private_location_slug_name` and `private_location_id`. Check runs flow through two queues: `main` carries Browser checks, Multistep checks and Playwright Check Suites, and `uptime` carries API checks, URL monitors and the TCP, DNS, ICMP, SSL, gRPC and Traceroute monitors. + +| Metric | Type | Labels | Description | +|--------|------|--------|-------------| +| `checkly_private_location_queue_size` | Gauge | | The number of check runs scheduled to the private location and waiting to be executed, on the `main` queue. The highest value in the latest 1-minute period, looking back 2 minutes. A high value indicates that checks are becoming backlogged and that you may need to [scale your Checkly Agents](/platform/private-locations/scaling-redundancy). Absent when the queue reports no data. | +| `checkly_private_location_oldest_scheduled_check_run` | Gauge | | The age in seconds of the oldest check run job scheduled to the `main` queue. Same sampling as `checkly_private_location_queue_size`. A high value indicates that checks are becoming backlogged. | +| `checkly_private_location_queue_duration_ms` | Gauge | `queue`, `stat` | The scheduling delay in milliseconds for check runs routed through the private location. Derived from the last 2 minutes of check run data. Use it to judge whether the private location has enough agent capacity. Absent for a queue with no check runs in the last 2 minutes. | +| `checkly_private_location_agent_count` | Gauge | | The number of agents connected for the private location. An agent counts as connected if it reported in the last 250 seconds. Always present. | +| `checkly_private_location_check_runs` | Gauge | `queue`, `state` | The number of check runs waiting in or picked up from the private location's queues. The `main` `queued` series is the same sample as `checkly_private_location_queue_size`, the other three are live queue reads, and a series is absent when its queue reports no data. Used as the signal for [autoscaling Checkly Agents](/platform/private-locations/autoscaling). | + +| Label | Values | +|-------|--------| +| `queue` | `main` for Browser checks, Multistep checks and Playwright Check Suites. `uptime` for API checks, URL monitors and TCP, DNS, ICMP, SSL, gRPC and Traceroute monitors. | +| `state` | `queued`: scheduled but not yet picked up by an agent. `inflight`: being executed by an agent. | +| `stat` | `avg`, `p90`, `p99` over the 2-minute window. | + +For metrics scraped from the agent container itself, including an opt-in per-check set with the `checkly_agent_` prefix, see [Agent metrics](/platform/private-locations/agent-metrics). + +## Behavior + +- **Caching and limits.** Responses are cached for 60 seconds, so a scrape interval below 60 seconds fetches the same data. Responses are gzip compressed when the scraper asks for it, which Prometheus does by default. The endpoint allows 50 requests per minute per account. Above that it returns `429` with `Retry-After`, `X-RateLimit-Limit`, `X-RateLimit-Remaining` and `X-RateLimit-Reset` headers. If you run multiple Prometheus instances, each instance scrapes independently. Align their scrape intervals where possible. +- **Errors.** An unknown token or a lapsed account returns `401`. A revoked token can keep working for up to 10 minutes. Tokens do not expire. A plan without the Prometheus integration returns `403`. +- **Timeouts.** A scrape fails as a whole if any part of the backing query fails. The query times out after 25 seconds, and a scrape that arrives while another one is refreshing the cache waits for that refresh. For accounts with many checks, keep `scrape_timeout` at `30s` or higher. +- **Hourly window.** Counters and histograms reset every hour at HH:01:30, for example 14:01:30. Handle the resets with the [rate](https://prometheus.io/docs/prometheus/latest/querying/functions/#rate) or [increase](https://prometheus.io/docs/prometheus/latest/querying/functions/#increase) functions. Gauges built from runs reset too. After the reset and until a check's next run: + - `checkly_ssl_monitor_days_until_expiry`, `checkly_icmp_monitor_packet_loss_ratio` and `checkly_traceroute_monitor_hops` read `0`. + - `checkly_check_http_status_code`, `checkly_grpc_monitor_status_code`, `checkly_traceroute_monitor_destination_reached`, `checkly_ssl_monitor_info`, `checkly_ssl_monitor_hostname_verified`, `checkly_ssl_monitor_chain_trusted`, `checkly_ssl_monitor_ocsp_stapled` and `checkly_ssl_monitor_key_size_bits` disappear. + - With `locationLabelEnabled`, each location reads `0` or is absent until it runs, so checks that run round-robin read `0` on most locations. + - A monitor that runs every hour reads `0` from the reset until its run. One that runs less often reads `0` in every hour without a run. + + An alert on the raw `checkly_ssl_monitor_days_until_expiry` gauge below a threshold therefore fires after every reset until the monitor runs again. Alert on its highest value over a range longer than the monitor's interval instead, for example with `max_over_time`. `checkly_time_to_ssl_expiry_seconds` and `checkly_check_status` read the current status and do not reset. +- **Histograms.** Histograms expose only the `+Inf` bucket. Use `_sum / _count` averages as in [Histogram averages](#histogram-averages), not `histogram_quantile()`. A missing web vital counts as `0` and pulls averages down. +- **Tag parsing.** Checkly does not clean up tag keys. The label name is the run of letters, digits and underscores right before the first colon, and the value is everything after that colon: `my-env:prod` becomes `env="prod"`, and `a:b:c` becomes `a="b:c"`. A key that starts with a digit produces an invalid label name, and a tag whose key matches a built-in label name is ignored. See the Prometheus [data model](https://prometheus.io/docs/concepts/data_model/) for valid label names. +- **Retries.** Retry attempts are left out unless you set `includeRetryAttempts=true`. `checkly_check_status`, `checkly_time_to_ssl_expiry_seconds` and all heartbeat metrics ignore the parameter. + +## PromQL examples + +### Currently failing checks + +To graph whether checks are passing or failing, use the query: +```bash +checkly_check_status{status="passing"} +``` + +Passing checks will have the value `1` while failing and degraded checks will have the value `0`. +This can be used to build a [Grafana table](https://grafana.com/docs/grafana/latest/panels-visualizations/visualizations/table/) of currently failing checks. + +### Failure percentage + +To calculate the share of check runs that failed in the last 24 hours, as a ratio from 0 to 1, use: +```bash +increase(checkly_check_result_total{status="failing"}[24h]) / ignoring(status) sum without (status) (increase(checkly_check_result_total[24h])) +``` + +With `locationLabelEnabled`, a location with no runs in the window returns `NaN` for this query. + +### Histogram averages -Each `checkly_check` metric has the following labels: +The different histogram metrics can all be used to compute averages. For example, query the average web vitals times for a check using: -- `check_name`, the name of your check. -- `check_type`, either `api` or `browser`. -- `tags`, this check's tags. +```bash +sum by(type) (rate(checkly_browser_check_web_vitals_seconds_sum{name="Check Name"}[30m])) / sum by(type) (rate(checkly_browser_check_web_vitals_seconds_count{name="Check Name"}[30m])) +``` + +## Scraping with other tools + +The `prometheus` receiver of the OpenTelemetry Collector, included in the `otelcol-contrib` distribution, scrapes the endpoint with the same settings. Replace `debug` with the exporter for your backend. + +```yaml +receivers: + prometheus: + config: + scrape_configs: + - job_name: checkly + scrape_interval: 60s + scrape_timeout: 30s + scheme: https + metrics_path: /accounts//v2/prometheus/metrics + authorization: + type: Bearer + credentials_file: /etc/otelcol-contrib/checkly.token + params: + includeRetryAttempts: ['true'] + locationLabelEnabled: ['true'] + static_configs: + - targets: ['api.checklyhq.com'] + +exporters: + debug: + verbosity: basic + +service: + pipelines: + metrics: + receivers: [prometheus] + exporters: [debug] +``` + +The token file holds only the bearer token. Exporters for every common backend are listed in the [Collector configuration docs](https://opentelemetry.io/docs/collector/configuration/). If you scrape with the [Datadog Agent OpenMetrics check](https://docs.datadoghq.com/integrations/openmetrics/) or Grafana Alloy [`prometheus.scrape`](https://grafana.com/docs/alloy/latest/reference/components/prometheus/prometheus.scrape/), use the same URL, bearer token and interval. + +## Legacy v1 endpoint -> You can set `key:value` tags in your checks/groups and they will be exported as custom labels in Prometheus. For instance the tag `env:production` will be exposed as a custom label `env="production"`. You can disable this by adding the query param `disableTagParsing=true`. + +The v1 endpoint is deprecated. Use the v2 endpoint described above for new setups. The v1 endpoint keeps working for existing scrape jobs. + + +The v1 endpoint is `/accounts//prometheus/metrics`. It takes the same bearer tokens, allows 300 requests per minute, caches responses for 30 seconds and accepts only the `disableTagParsing` parameter. -The `checkly_private_location` metrics contain the labels: +### Legacy v1 metrics and labels + +| Metric | Description | +|--------|-------------| +| `checkly_check_status` | The check's current status: `1` is passing, `0` is failing. It is not a per-region result. | +| `checkly_check_degraded_status` | Whether the check is degraded: `1` is not degraded, `0` is degraded. | +| `checkly_check_result` | The response time in milliseconds of the latest run in each region. One row for each region that ran the check in the last hour. | +| `checkly_private_location_queue_size` | Same as the v2 metric of the same name. | +| `checkly_private_location_oldest_scheduled_check_run` | Same as the v2 metric of the same name. | +| `checkly_private_location_agent_count` | The number of agents that reported in the last 250 seconds. Always present. | -- `private_location_name`, the name of the private location. -- `private_location_slug_name`, the private location's human readable unique identifier. -- `private_location_id`, the private location's UUID. +Checks with no run in the last hour are absent from the three check metrics. Labels: -> If a private location has no check runs for six hours, it will be considered inactive and `checkly_private_location` metrics won't be reported for it. +- `checkly_check_status` and `checkly_check_degraded_status`: `check_name`, `check_type`, `muted`, `activated`, `degraded`, `group` for checks in a check group, `tags` and tag labels. +- `checkly_check_result`: `check_name`, `check_type`, `region`, `group` for checks in a check group, `http_status_code` on API checks only, `tags` and tag labels. +- `check_type` is any check type in lowercase. `muted` and `activated` are the check's own settings, without its check group. +- `tags` holds only the tags that are not `key:value`. Each `key:value` tag becomes a label: the key is the text before the first colon, with any characters, so `team-x:y` produces the invalid label name `team-x`. The value stops at the second colon. Label values are not escaped. +- The private location metrics carry `private_location_name`, `private_location_slug_name` and `private_location_id`. -Here is an example: +### Legacy v1 example output ```bash -# HELP checkly_check_status The status of the last check. 1 is passing, 0 is failing +# HELP checkly_check_status The passing/failing status of the last check. 1 is passing, 0 is failing # TYPE checkly_check_status gauge -checkly_check_status{check_name="Customer API",check_type="api",muted="false",activated="true",tags="alerts,public"} 1 -checkly_check_status{check_name="Email login",check_type="browser",muted="false",activated="true",tags="auth,browser-checks,public"} 0 +checkly_check_status{check_name="Customer API",check_type="api",muted="false",activated="true",degraded="false",group="Public APIs",tags="alerts,public"} 1 +checkly_check_status{check_name="Homepage",check_type="browser",muted="false",activated="true",degraded="false",tags="public"} 1 +checkly_check_status{check_name="Internal API",check_type="api",muted="false",activated="true",degraded="false",tags="public"} 1 # HELP checkly_check_degraded_status The degraded status of the last check. 1 is not-degraded, 0 is degraded # TYPE checkly_check_degraded_status gauge -checkly_check_degraded_status{check_name="Customer API",check_type="api",muted="false",activated="true",tags="alerts,public"} 0 -checkly_check_degraded_status{check_name="Email login",check_type="browser",muted="false",activated="true",tags="auth,browser-checks,public"} 1 +checkly_check_degraded_status{check_name="Customer API",check_type="api",muted="false",activated="true",degraded="false",group="Public APIs",tags="alerts,public"} 1 +checkly_check_degraded_status{check_name="Homepage",check_type="browser",muted="false",activated="true",degraded="false",tags="public"} 1 +checkly_check_degraded_status{check_name="Internal API",check_type="api",muted="false",activated="true",degraded="false",tags="public"} 1 # HELP checkly_check_result The response time of the last check per region. # TYPE checkly_check_result gauge -checkly_check_result{check_name="Customer API",check_type="api",region="ap-northeast-2",tags="alerts,public"} 1168 -checkly_check_result{check_name="Customer API",check_type="api",region="ap-southeast-1",tags="alerts,public"} 932 -checkly_check_result{check_name="Customer API",check_type="api",region="ca-central-1",tags="alerts,public"} 424 -checkly_check_result{check_name="Customer API",check_type="api",region="eu-west-2",tags="alerts,public"} 138 -checkly_check_result{check_name="Customer API",check_type="api",region="us-east-2",tags="alerts,public"} 432 -checkly_check_result{check_name="Email login",check_type="browser",region="ap-south-1",tags="auth,browser-checks,public"} 10174 +checkly_check_result{check_name="Customer API",check_type="api",region="eu-west-1",group="Public APIs",http_status_code="200",tags="alerts,public"} 5 +checkly_check_result{check_name="Homepage",check_type="browser",region="us-east-1",tags="public"} 4376 +checkly_check_result{check_name="Internal API",check_type="api",region="acme-dc",http_status_code="200",tags="public"} 3 # HELP checkly_private_location_queue_size The number of check runs scheduled to the private location and waiting to be executed. # TYPE checkly_private_location_queue_size gauge -checkly_private_location_queue_size{private_location_name="Internal CI",private_location_slug_name="internal-ci",private_location_id="cac52f2d-8b8c-4ca5-9711-1836be02eda4"} 0 +checkly_private_location_queue_size{private_location_name="ACME data center",private_location_slug_name="acme-dc",private_location_id=""} 0 # HELP checkly_private_location_oldest_scheduled_check_run The age in seconds of the oldest check run job scheduled to the private location queue. # TYPE checkly_private_location_oldest_scheduled_check_run gauge -checkly_private_location_oldest_scheduled_check_run{private_location_name="Internal CI",private_location_slug_name="internal-ci",private_location_id="cac52f2d-8b8c-4ca5-9711-1836be02eda4"} 0 +checkly_private_location_oldest_scheduled_check_run{private_location_name="ACME data center",private_location_slug_name="acme-dc",private_location_id=""} 0 # HELP checkly_private_location_agent_count The number of agents connected for the private location. # TYPE checkly_private_location_agent_count gauge -checkly_private_location_agent_count{private_location_name="Internal CI",private_location_slug_name="internal-ci",private_location_id="cac52f2d-8b8c-4ca5-9711-1836be02eda4"} 1 +checkly_private_location_agent_count{private_location_name="ACME data center",private_location_slug_name="acme-dc",private_location_id=""} 1 ``` Notice that: -- The check for "Email login" is failing. -- The check for "Customer API" is degraded, but not failing. -- The `checkly_check_status` metric has `muted` and `activated` labels, reflecting if a check is sending out alerts or is actually -running. -- The `checkly_check_result` metric has a `region` label. -- The private location "Internal CI" has one Checkly Agent connected. From `checkly_private_location_queue_size` and `checkly_private_location_oldest_scheduled_check_run`, we see that there's no backlog of check run jobs. - - - -Activating this integration is simple. - -1. Open **Account settings** from the left sidebar, then select **Integrations** (or go directly to the Integrations page) and click the **Create Prometheus endpoint** button. -![Prometheus integration step 1](/images/docs/images/integrations/prometheus_step1.png) - -2. We directly create an endpoint for you and provide its URL and the required Bearer token. -![Prometheus integration step 2](/images/docs/images/integrations/prometheus_step2.png) +- `checkly_check_status` and `checkly_check_degraded_status` carry `muted`, `activated` and `degraded`, reflecting whether a check sends alerts, is running and is degraded. "Customer API" is in a check group, so it also carries `group`. +- `checkly_check_result` has one row per region that ran the check in the last hour, with a `region` label. A run from a private location carries the private location's slug as its region. Rows for API checks also carry `http_status_code`. +- The private location "ACME data center" has one Checkly Agent connected. `checkly_private_location_queue_size` and `checkly_private_location_oldest_scheduled_check_run` show no backlog of check run jobs. -3. Create a new job in your Prometheus `prometheus.yml` config and set up a scraping interval. We recommend an interval -between 30 seconds and 60 seconds. Add the URL (divided into `metrics_path`, `scheme` and `target`) and `bearer_token`. -Here is an example +### Legacy v1 scrape config ```yaml -# prometheus.yml -- job_name: 'checkly' - scrape_interval: 30s - metrics_path: '/accounts/993adb-8ac6-3432-9e80-cb43437bf263/prometheus/metrics' - bearer_token: 'lSAYpOoLtdAa7ajasoNNS234' - scheme: https - static_configs: - - targets: ['api.checklyhq.com'] +scrape_configs: + - job_name: checkly + scrape_interval: 60s + scrape_timeout: 30s + scheme: https + metrics_path: /accounts//prometheus/metrics + bearer_token: + fallback_scrape_protocol: PrometheusText0.0.4 + static_configs: + - targets: ['api.checklyhq.com'] ``` -Now restart Prometheus and you should see metrics coming in. +The v1 endpoint answers with the content type `text/html`. Prometheus 3 and later reject that unless the job sets `fallback_scrape_protocol`, as above. Prometheus 2 does not know that setting, so remove the line there. - -💡 Check out our [blog post on using this integration with Prometheus and Grafana](https://blog.checklyhq.com/monitoring-website-performance-with-checkly-prometheus-grafana/) for some cool real-life applications. - \ No newline at end of file +For a worked example with Grafana, see the blog post on [monitoring website performance with Checkly, Prometheus and Grafana](https://blog.checklyhq.com/monitoring-website-performance-with-checkly-prometheus-grafana/). diff --git a/integrations/observability/traces/chronosphere.mdx b/integrations/observability/traces/chronosphere.mdx new file mode 100644 index 000000000..58cad7f68 --- /dev/null +++ b/integrations/observability/traces/chronosphere.mdx @@ -0,0 +1,25 @@ +--- +title: 'Export traces to Chronosphere' +description: 'Send Checkly traces to Chronosphere with a restricted service account token.' +sidebarTitle: 'Chronosphere' +canonical: 'https://www.checklyhq.com/docs/integrations/observability/traces/chronosphere/' +--- + +import OtlpVendorIntro from '/snippets/otlp-vendor-intro.mdx'; +import OtlpVendorOutro from '/snippets/otlp-vendor-outro.mdx'; + + + +Chronosphere is now Palo Alto Networks Cortex XCOR. The endpoint host is still `.chronosphere.io` and its documentation lives at docs-xcor.paloaltonetworks.com. + +1. Get the endpoint. It is `https://.chronosphere.io/data/opentelemetry/v1/traces`. +2. Get the key. In Chronosphere, go to the admin view, then **Platform** > **Service Accounts**, and add a service account. Make it restricted and write-only. Copy the token right away, because Chronosphere shows it once. +3. Paste the values into Checkly. On the [Traces settings page](https://app.checklyhq.com/settings/account/traces), turn on **Enable exporting traces**, paste the URL, and add the header from the table below. +4. Find the spans. In Chronosphere, the spans appear under service `checkly`. + +| Header name | Value | +|---|---| +| `API-Token` | `` | + +Chronosphere documents the endpoint in [OTLP endpoints](https://docs-xcor.paloaltonetworks.com/ingest/metrics-traces/otel/otlp-endpoints). + diff --git a/integrations/observability/traces/clickstack.mdx b/integrations/observability/traces/clickstack.mdx new file mode 100644 index 000000000..6813ea786 --- /dev/null +++ b/integrations/observability/traces/clickstack.mdx @@ -0,0 +1,27 @@ +--- +title: 'Export traces to ClickStack' +description: 'Send Checkly traces to a ClickStack collector, self-hosted or with Managed ClickStack.' +sidebarTitle: 'ClickStack' +canonical: 'https://www.checklyhq.com/docs/integrations/observability/traces/clickstack/' +--- + +import OtlpVendorIntro from '/snippets/otlp-vendor-intro.mdx'; +import OtlpVendorOutro from '/snippets/otlp-vendor-outro.mdx'; + + + +Self-hosted ClickStack and Managed ClickStack in ClickHouse Cloud end the same way. Checkly sends the spans to your collector on port 4318 with an `authorization` header. + +1. Get the endpoint. It is your collector at `https://:4318/v1/traces`. The collector must be reachable from where your checks run. For public locations that means a public address with TLS in front. For private locations, an address reachable from the agent's network is enough. +2. Get the key. It depends on your setup. + - Self-hosted ClickStack: in the HyperDX UI, open **Team Settings** > **API Keys** and copy the ingestion key. + - Managed ClickStack in ClickHouse Cloud: there is no hosted OTLP endpoint. Run the ClickStack collector pointed at ClickHouse Cloud, and use the `OTLP_AUTH_TOKEN` you set on it as the key. +3. Paste the values into Checkly. On the [Traces settings page](https://app.checklyhq.com/settings/account/traces), turn on **Enable exporting traces**, paste the URL, and add the header from the table below. Use the bare key, without a `Bearer` prefix. +4. Find the spans. In ClickStack, the spans appear under service `checkly`. + +| Header name | Value | +|---|---| +| `authorization` | `` | + +ClickStack documents the collector for [self-hosted ClickStack](https://clickhouse.com/docs/use-cases/observability/clickstack/ingesting-data/opentelemetry) and for [Managed ClickStack](https://clickhouse.com/docs/clickstack/managed-onboarding/setting-up-your-opentelemetry-collector). + diff --git a/integrations/observability/traces/cribl.mdx b/integrations/observability/traces/cribl.mdx new file mode 100644 index 000000000..5d60a9204 --- /dev/null +++ b/integrations/observability/traces/cribl.mdx @@ -0,0 +1,27 @@ +--- +title: 'Export traces to Cribl Stream' +description: 'Send Checkly traces to Cribl Stream with an OpenTelemetry Source.' +sidebarTitle: 'Cribl Stream' +canonical: 'https://www.checklyhq.com/docs/integrations/observability/traces/cribl/' +--- + +import OtlpVendorIntro from '/snippets/otlp-vendor-intro.mdx'; +import OtlpVendorOutro from '/snippets/otlp-vendor-outro.mdx'; + + + +Export traces to Cribl Stream once, then use Routes to send them to one or more destinations. + +1. Get the endpoint. In Cribl Stream, create an **OpenTelemetry** Source with **Protocol** set to `HTTP` on a Cribl.Cloud port between `20000` and `20010`. Turn on TLS with the private key path `$CRIBL_CLOUD_KEY` and the certificate path `$CRIBL_CLOUD_CRT`, which Cribl.Cloud requires on those ports, and add a **Token** authentication method. The URL is `https://default.main..cribl.cloud:/v1/traces`, built from your Organization ID and the Source's port. +2. Get the key. Copy the token you added to the Source. +3. Paste the values into Checkly. On the [Traces settings page](https://app.checklyhq.com/settings/account/traces), turn on **Enable exporting traces**, paste the URL, and add the header from the table below. +4. Find the spans. Add a Route that sends the Source's data to your destination. There, the spans appear under service `checkly`. + +| Header name | Value | +|---|---| +| `authorization` | `Bearer ` | + +This page covers traces only. Checkly's [Prometheus endpoint](/integrations/observability/prometheus) accepts only a bearer token and answers `401` to Basic auth, which is what Cribl's Prometheus Scraper Source sends. + +Cribl documents the Source in [OpenTelemetry Source](https://docs.cribl.io/stream/sources-otel/) and the certificates in [TLS in Cribl.Cloud](https://docs.cribl.io/stream/securing-tls-cloud). + diff --git a/integrations/observability/traces/dash0.mdx b/integrations/observability/traces/dash0.mdx new file mode 100644 index 000000000..6b74ce676 --- /dev/null +++ b/integrations/observability/traces/dash0.mdx @@ -0,0 +1,28 @@ +--- +title: 'Export traces to Dash0' +description: 'Send Checkly traces to Dash0 with a static auth token.' +sidebarTitle: 'Dash0' +canonical: 'https://www.checklyhq.com/docs/integrations/observability/traces/dash0/' +--- + +import OtlpVendorIntro from '/snippets/otlp-vendor-intro.mdx'; +import OtlpVendorOutro from '/snippets/otlp-vendor-outro.mdx'; + + + +1. Get the endpoint. In Dash0, open **Settings** > **Endpoints** and copy the URL for your organization. It ends in `/v1/traces` with no trailing slash, for example `https://ingress.eu-west-1.aws.dash0.com/v1/traces`. +2. Get the key. In Dash0, open **Settings** > **Auth Tokens** and create a token. Use a static token that starts with `auth_`. + ![Auth Tokens page in Dash0](/images/docs/images/otel/export-traces/dash0-token.png) +3. Paste the values into Checkly. On the [Traces settings page](https://app.checklyhq.com/settings/account/traces), turn on **Enable exporting traces**, paste the URL, and add the header from the table below. + ![Export Traces settings with an endpoint URL and an authorization header](/images/docs/images/otel/otel_export_traces_settings.png) +4. Find the spans. In Dash0, the spans appear under service `checkly`. + ![Checkly spans in the Dash0 Tracing view](/images/docs/images/otel/export-traces/dash0-traces.png) + +| Header name | Value | +|---|---| +| `Authorization` | `Bearer auth_...` | + +Add a `Dash0-Dataset` header row to write to a specific dataset. A token restricted to a dataset gets a 403 without it. + +Dash0 documents the endpoints in [Setup without Kubernetes](https://www.dash0.com/docs/dash0/telemetry/signal-control/edge/setup-without-kubernetes). + diff --git a/integrations/observability/traces/datadog.mdx b/integrations/observability/traces/datadog.mdx new file mode 100644 index 000000000..15148774f --- /dev/null +++ b/integrations/observability/traces/datadog.mdx @@ -0,0 +1,39 @@ +--- +title: 'Export traces to Datadog' +description: 'Send Checkly traces to Datadog through direct OTLP intake.' +sidebarTitle: 'Datadog' +canonical: 'https://www.checklyhq.com/docs/integrations/observability/traces/datadog/' +--- + +import OtlpVendorIntro from '/snippets/otlp-vendor-intro.mdx'; +import OtlpVendorOutro from '/snippets/otlp-vendor-outro.mdx'; + + + +1. Get the endpoint. Use the URL for your Datadog site. + + | Site | URL | + |---|---| + | US1 (`datadoghq.com`) | `https://otlp.datadoghq.com/v1/traces` | + | US3 (`us3.datadoghq.com`) | `https://otlp.us3.datadoghq.com/v1/traces` | + | US5 (`us5.datadoghq.com`) | `https://otlp.us5.datadoghq.com/v1/traces` | + | EU (`datadoghq.eu`) | `https://otlp.datadoghq.eu/v1/traces` | + | AP1 (`ap1.datadoghq.com`) | `https://otlp.ap1.datadoghq.com/v1/traces` | + | AP2 (`ap2.datadoghq.com`) | `https://otlp.ap2.datadoghq.com/v1/traces` | + | UK1 (`uk1.datadoghq.com`) | `https://otlp.uk1.datadoghq.com/v1/traces` | + | US1-FED (`ddog-gov.com`) | `https://otlp.ddog-gov.com/v1/traces` | + | US2-FED (`us2.ddog-gov.com`) | `https://otlp.us2.ddog-gov.com/v1/traces` | + +2. Get the key. In Datadog, go to **Organization Settings** > **API Keys** and create a key. +3. Paste the values into Checkly. On the [Traces settings page](https://app.checklyhq.com/settings/account/traces), turn on **Enable exporting traces**, paste the URL, and add both headers from the table below. Without `compute_stats`, Datadog does not compute trace metrics. +4. Find the spans. In Datadog APM, the spans appear under service `checkly`. + +| Header name | Value | +|---|---| +| `dd-api-key` | `` | +| `compute_stats` | `true` | + +If no spans arrive, check the site first. Datadog answers 403 when the site does not match your organization or the key is wrong. + +Datadog documents the intake in [OTLP traces ingest](https://docs.datadoghq.com/opentelemetry/setup/otlp_ingest/traces/). + diff --git a/integrations/observability/traces/grafana-tempo.mdx b/integrations/observability/traces/grafana-tempo.mdx new file mode 100644 index 000000000..be01281e0 --- /dev/null +++ b/integrations/observability/traces/grafana-tempo.mdx @@ -0,0 +1,36 @@ +--- +title: 'Export traces to Grafana Tempo' +description: 'Send Checkly traces to Grafana Cloud Tempo through the Grafana OTLP gateway.' +sidebarTitle: 'Grafana Tempo' +canonical: 'https://www.checklyhq.com/docs/integrations/observability/traces/grafana-tempo/' +--- + +import OtlpVendorIntro from '/snippets/otlp-vendor-intro.mdx'; +import OtlpVendorOutro from '/snippets/otlp-vendor-outro.mdx'; + + + +1. Get the endpoint. In the Grafana Cloud portal, open your stack, select the **OpenTelemetry** tile, then **Configure**. Copy the host from the tile instead of building it yourself, because it depends on your stack's region. The Checkly URL ends in `/otlp/v1/traces`, for example `https://otlp-gateway-prod-eu-west-2.grafana.net/otlp/v1/traces`. Grafana rejects traces sent to `.grafana.net/tempo`. + ![OpenTelemetry tile on the Grafana Cloud stack page](/images/docs/images/otel/export-traces/grafana-cloud-opentelemetry.png) + ![OTLP endpoint and instance ID on the Grafana Cloud OpenTelemetry page](/images/docs/images/otel/export-traces/grafana-otlp-endpoint-config.png) +2. Get the key. On the same page, create a token with the `traces:write` scope and copy your instance ID. Then base64 encode both on your own machine. The output must be one line without a trailing newline. + ![Create an API token dialog in Grafana Cloud with the traces:write scope](/images/docs/images/otel/export-traces/create-otlp-token-grafana.png) + + ```bash + printf '%s' ':' | base64 | tr -d '\n' + ``` + +3. Paste the values into Checkly. On the [Traces settings page](https://app.checklyhq.com/settings/account/traces), turn on **Enable exporting traces**, paste the URL, and add the header from the table below. + ![Export Traces settings with an endpoint URL and an authorization header](/images/docs/images/otel/otel_export_traces_settings.png) +4. Find the spans. In Grafana, open **Explore** and select your stack's traces data source, named `grafanacloud--traces`. The spans appear under service `checkly`. + ![Traces data source selected in Grafana Explore](/images/docs/images/otel/export-traces/grafana-cloud-tempo-source.png) + ![Checkly traces in Grafana Explore](/images/docs/images/otel/export-traces/grafana-explore-checkly-traces.png) + +| Header name | Value | +|---|---| +| `Authorization` | `Basic ` | + +For self-hosted Tempo, use the [OTLP export page](/integrations/observability/traces/otlp) with your distributor's `/v1/traces` URL. + +Grafana documents the endpoint and token in [Send data using OpenTelemetry Protocol](https://grafana.com/docs/grafana-cloud/send-data/otlp/send-data-otlp/). + diff --git a/integrations/observability/traces/honeycomb.mdx b/integrations/observability/traces/honeycomb.mdx new file mode 100644 index 000000000..eef36e487 --- /dev/null +++ b/integrations/observability/traces/honeycomb.mdx @@ -0,0 +1,25 @@ +--- +title: 'Export traces to Honeycomb' +description: 'Send Checkly traces to Honeycomb with an Ingest API key.' +sidebarTitle: 'Honeycomb' +canonical: 'https://www.checklyhq.com/docs/integrations/observability/traces/honeycomb/' +--- + +import OtlpVendorIntro from '/snippets/otlp-vendor-intro.mdx'; +import OtlpVendorOutro from '/snippets/otlp-vendor-outro.mdx'; + + + +1. Get the endpoint for your Honeycomb region. US is `https://api.honeycomb.io/v1/traces` and EU is `https://api.eu1.honeycomb.io/v1/traces`. +2. Get the key. In Honeycomb, select **Environments** > **Manage Environments**, open your environment, then go to **API Keys** > **Ingest** and select **Create Ingest API Key**. Copy the key. +3. Paste the values into Checkly. On the [Traces settings page](https://app.checklyhq.com/settings/account/traces), turn on **Enable exporting traces**, paste the URL, and add the header from the table below. +4. Find the spans. In Honeycomb, open the environment you used. The spans appear under service `checkly`. + +| Header name | Value | +|---|---| +| `x-honeycomb-team` | `` | + +Only Honeycomb Classic also needs an `x-honeycomb-dataset` header row. Environment keys do not. + +Honeycomb documents the endpoints in [Send data with OpenTelemetry](https://docs.honeycomb.io/send-data/opentelemetry/). + diff --git a/integrations/observability/traces/new-relic.mdx b/integrations/observability/traces/new-relic.mdx new file mode 100644 index 000000000..a3f29bb6a --- /dev/null +++ b/integrations/observability/traces/new-relic.mdx @@ -0,0 +1,25 @@ +--- +title: 'Export traces to New Relic' +description: 'Send Checkly traces to New Relic with an Ingest - License key.' +sidebarTitle: 'New Relic' +canonical: 'https://www.checklyhq.com/docs/integrations/observability/traces/new-relic/' +--- + +import OtlpVendorIntro from '/snippets/otlp-vendor-intro.mdx'; +import OtlpVendorOutro from '/snippets/otlp-vendor-outro.mdx'; + + + +1. Get the endpoint for your New Relic region. US is `https://otlp.nr-data.net/v1/traces`, EU is `https://otlp.eu01.nr-data.net/v1/traces`, JP is `https://otlp.jp.nr-data.net/v1/traces` and FedRAMP is `https://gov-otlp.nr-data.net/v1/traces`. +2. Get the key. In New Relic, go to **Administration** > **API Keys**, create a key, and set the key type to **Ingest - License**. Copy the key. +3. Paste the values into Checkly. On the [Traces settings page](https://app.checklyhq.com/settings/account/traces), turn on **Enable exporting traces**, paste the URL, and add the header from the table below. +4. Find the spans. In New Relic, the spans appear under service `checkly`. + +| Header name | Value | +|---|---| +| `api-key` | `` | + +New Relic limits each request to 1 MB. + +New Relic documents the endpoints in its [OTLP best practices](https://docs.newrelic.com/docs/opentelemetry/best-practices/opentelemetry-otlp/). + diff --git a/integrations/observability/traces/otlp.mdx b/integrations/observability/traces/otlp.mdx new file mode 100644 index 000000000..fd20ce32b --- /dev/null +++ b/integrations/observability/traces/otlp.mdx @@ -0,0 +1,150 @@ +--- +title: 'Export traces to any OpenTelemetry backend' +description: 'Send every API check, URL monitor, Browser check and Multistep check run as an OpenTelemetry span to any OTLP/HTTP endpoint.' +sidebarTitle: 'OTLP (any backend)' +canonical: 'https://www.checklyhq.com/docs/integrations/observability/traces/otlp/' +--- + +When trace export is on, Checkly sends each API check, URL monitor, Browser check and Multistep check run as one OpenTelemetry span to an OTLP/HTTP endpoint you choose. Any OpenTelemetry backend or Collector that accepts OTLP/HTTP works. If your application honors the `traceparent` header, its spans join the same trace as the check. Export is account-wide to one endpoint. To reach several backends, send to a Collector and route from there. + + +- A [plan](https://www.checklyhq.com/pricing/) that includes trace export, otherwise the export settings cannot be saved. +- Export is controlled by **Enable exporting traces** under **Export Traces** on [Settings > Traces](https://app.checklyhq.com/settings/account/traces). Import Traces does not need to be on. +- You enable export for the whole account. No per-check or per-group setting exists. +- An endpoint reachable from Checkly's public locations for checks that run there, and from your private location's network for checks that run there. Runs from public locations connect from Checkly's [static IP ranges](/platform/ip-information#static-ip-ranges), which include IPv6 addresses. + + +## Set up + + + +Open [Settings > Traces](https://app.checklyhq.com/settings/account/traces) and, under **Export Traces**, select **Enable exporting traces**. + + +Enter your backend's OTLP/HTTP traces URL, for example `https://otel.acme.com/v1/traces`. Use an `https` URL that ends in `/v1/traces`. Checkly only checks that the URL is well formed when you save, not the scheme, the path or that the endpoint answers. + + +Click **Add header** and enter each header your endpoint expects as a key and a value, for example `Authorization` with `Bearer `. Put credentials in header rows, not in the URL. API checks and URL monitors drop the URL's query string when they export. Click the lock next to a value to store it as an account secret, which the row then references as `{{NAME}}`. Any value written as `{{NAME}}` is replaced with the matching environment variable or secret when the check runs. Rows with an empty key or value are discarded. + + + +![Export Traces settings with an endpoint URL and an authorization header](/images/docs/images/otel/otel_export_traces_settings.png) + +## What is sent + +Checkly sends one root span per check run, with no child spans. These check types take part: + +| Check type | Trace headers on its requests | Span exported | +|---|---|---| +| API check | Yes | Yes | +| URL monitor | Yes | Yes | +| Browser check | Yes for `@playwright/test` scripts, unless the check [sets its own `extraHTTPHeaders`](/resolve/traces/overview#add-trace-headers-to-custom-http-headers). Other scripts only get the `CHECKLY_TRACEPARENT` variable | Yes | +| Multistep check | Yes, unless the check [sets its own `extraHTTPHeaders`](/resolve/traces/overview#add-trace-headers-to-custom-http-headers) | Yes | + +The span is named after the check and has kind `CLIENT`. Its OpenTelemetry status is always `OK`, even when the check fails. The check's verdict is in `checkly.check.status`. + +### Span attributes + +| Attribute | Type | Example | Present when | +|---|---|---|---| +| `checkly.account.id` | string | `97bbd3a5-1f0c-4e2a-9b7d-5c8e2a6f4d10` | Always | +| `checkly.check.id` | string | `438481ea-0eab-43d6-8932-ab51bd0d49d6` | Always. Empty on `checkly test` runs | +| `checkly.check.result.id` | string | `6030240b-7a4e-4c1f-8d2b-9e3f1a5c7b82` | Always | +| `checkly.check.name` | string | `ACME homepage` | Always | +| `checkly.check.type` | string | The check type | Always | +| `checkly.check.runtime` | string | `2026.04` | Always | +| `checkly.check.session.id` | string | `b2c4e6f8-3a5d-4b7e-9c1f-2d4a6b8c0e13` | Scheduled runs. Empty or absent on `checkly test` and `checkly trigger` runs | +| `checkly.check.location` | string | Region code of the run, for example `eu-west-1` | Always | +| `checkly.check.tags` | string array | `["production", "checkout"]` | The check has at least one tag | +| `checkly.check.status` | string | `PASSED`, `FAILED` or `DEGRADED` | Always | + +Each span also carries the resource attributes `service.name`, set to `checkly`, and `telemetry.sdk.language`. Its instrumentation scope is `checkly-tracer`, or `checkly-browser-tracer` for Browser checks on the current runtime. When the [Coralogix integration](/integrations/observability/coralogix) is on as well, spans can include the resource attributes `cx.application.name` and `cx.subsystem.name`. + +`checkly.check.tags` carries the check's tags as a string array, the check's own tags first, then tags inherited from its group, deduplicated. Use it to filter or group traces by the same labels you organize your checks by, such as a team or workflow tag. + +- The attribute is only present when the check has tags. +- At most 20 tags are exported per span. A tag longer than 128 characters is omitted entirely rather than truncated. +- Attributes are set on Checkly's root span only. They do not propagate to your application's own spans within the trace. To find a whole trace by tag, filter on the root span, then use its trace ID. + +### Trace headers + +Checkly adds these headers to the requests a check makes: + +| Header | Value | Description | +|---|---|---| +| `traceparent` | `{version}-{trace_id}-{span_id}-{trace_flags}` | A trace parent ID in [the canonical W3C format](https://www.w3.org/TR/trace-context/#traceparent-header-field-values) | +| `tracestate` | `checkly=true`, `check_id=`, `check_result_id=`, `checkly_timestamp=`, and on ad hoc and test session runs of Browser checks and Multistep checks `checkly_debug=true` | Marks the request as coming from a Checkly check. Members may appear in any order and some may be absent | + +## Correlate with your application's traces + +The exported span's trace ID is the one in the `traceparent` header the check sent to your application, and it equals the [runtime variable](/detect/synthetic-monitoring/browser-checks/mac-structure#built-in-runtime-variables) `CHECKLY_TRACE_ID`. If your application honors `traceparent`, its spans and Checkly's root span share one trace. The check result page shows the same ID under **Traces**, and a [webhook](/integrations/alerts/webhooks) template can include it as `{{OTEL_TRACE_ID}}` to build a link into your tool. + +Two settings change the header your application receives: + +- On API checks and URL monitors, a request header you define named `traceparent` replaces Checkly's. +- On API checks and URL monitors, a [Coralogix](/integrations/observability/coralogix) custom trace header replaces `traceparent`. Browser checks and Multistep checks send both headers. + +To send your application's spans into Checkly instead, see [How it works](/resolve/traces/how-it-works) and [Import traces](/resolve/traces/import/overview). + +## Receive spans with an OpenTelemetry Collector + +This configuration runs on the `otelcol-contrib` distribution of the OpenTelemetry Collector. It accepts OTLP/HTTP on port 4318 and requires a bearer token. It listens on plain HTTP, so put TLS in front of it, with a reverse proxy or the receiver's `tls` settings, so that Checkly reaches an `https` URL. Replace `debug` with the exporter for your backend. + +```yaml +extensions: + bearertokenauth: + token: ${env:CHECKLY_EXPORT_TOKEN} + +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + auth: + authenticator: bearertokenauth + +exporters: + debug: + verbosity: detailed + +service: + extensions: [bearertokenauth] + pipelines: + traces: + receivers: [otlp] + exporters: [debug] +``` + +In Checkly, add a header row with the key `Authorization` and the value `Bearer `. Set the URL to the https address of this receiver, ending in `/v1/traces`. + +For receivers, exporters, TLS and authentication, see the [Collector configuration docs](https://opentelemetry.io/docs/collector/configuration/) and the [bearer token auth extension](https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/extension/bearertokenauthextension). + +## Delivery + +- Checkly sends OTLP/HTTP with protobuf encoding. There is no gRPC, JSON or gzip option. +- Each export request times out after 5 seconds. +- Checkly may retry briefly on 429, 502, 503 and 504. +- Failures are logged on Checkly's side and never shown in the UI. + +A slow or unavailable endpoint can leave gaps in your traces that Checkly does not report. + +## Troubleshooting + +- **Nothing arrives.** Check that **Enable exporting traces** is on, the URL ends in `/v1/traces`, and the header rows match what your endpoint expects. Confirm the endpoint accepts OTLP/HTTP protobuf, since a gRPC-only receiver on port 4317 receives nothing, allows Checkly's [static IP ranges](/platform/ip-information#static-ip-ranges) or your private location's network, and answers within 5 seconds. Checkly shows no export errors, so read your endpoint's logs and search your backend for the trace ID shown under **Traces** on a recent check result. +- **Only Browser check and Multistep check spans arrive.** API checks and URL monitors drop the URL's query string. Move credentials from the URL into a header row. +- **Every span has status `OK`.** This is expected. Filter on `checkly.check.status` to find failed or degraded runs. +- **A check type you expected is missing.** Only the types in [What is sent](#what-is-sent) export spans. + +For the wire format, see the [OTLP/HTTP specification](https://opentelemetry.io/docs/specs/otlp/#otlphttp). + +## Backends with their own page + +- [Coralogix](/integrations/observability/coralogix) +- [Chronosphere](/integrations/observability/traces/chronosphere) +- [ClickStack](/integrations/observability/traces/clickstack) +- [Cribl Stream](/integrations/observability/traces/cribl) +- [Dash0](/integrations/observability/traces/dash0) +- [Datadog](/integrations/observability/traces/datadog) +- [Grafana Tempo](/integrations/observability/traces/grafana-tempo) +- [Honeycomb](/integrations/observability/traces/honeycomb) +- [New Relic](/integrations/observability/traces/new-relic) diff --git a/platform/private-locations/agent-metrics.mdx b/platform/private-locations/agent-metrics.mdx index 10a26c7c1..8918663b6 100644 --- a/platform/private-locations/agent-metrics.mdx +++ b/platform/private-locations/agent-metrics.mdx @@ -48,7 +48,7 @@ Prefix|Component|Source Starting with agent v8.13.0, set `AGENT_METRICS_PER_CHECK=true` to expose one set of series per check. These series carry the labels `check_id`, `name`, `check_type` and `tags`. Timings are in seconds. Histograms have five buckets. -The metrics follow the [Prometheus V2 integration](/integrations/observability/prometheus-v2/), with these differences: +The metrics follow the [Prometheus v2 integration](/integrations/observability/prometheus), with these differences: - Names use the `checkly_agent_` prefix. - `check_type` values are upper-case, for example `API` or `BROWSER`. diff --git a/platform/private-locations/autoscaling.mdx b/platform/private-locations/autoscaling.mdx index 3a8e68f10..384b97b23 100644 --- a/platform/private-locations/autoscaling.mdx +++ b/platform/private-locations/autoscaling.mdx @@ -8,7 +8,7 @@ Scale Checkly Agent pods automatically in relation to live load. This page cover -- Prometheus V2 metrics are being ingested for your account — the only source for this gauge. See [Exporting Metrics & Data via Prometheus V2](/integrations/observability/prometheus-v2/). +- Prometheus v2 metrics are being ingested for your account. They are the only source for this gauge. See [Export metrics to Prometheus](/integrations/observability/prometheus). - Checkly Agents are deployed via the [Checkly agent Helm chart](https://github.com/checkly/helm-charts/tree/main/charts/agent) (or an equivalent `Deployment`). See [Kubernetes Deployment](/platform/private-locations/kubernetes-deployment). - [KEDA](https://keda.sh) is installed in the cluster. @@ -16,7 +16,7 @@ Scale Checkly Agent pods automatically in relation to live load. This page cover ## The signal -Checkly exposes the `checkly_private_location_check_runs` gauge through the Prometheus V2 exporter. Filtered by `state` and a `private_location_slug_name`, it provides the count of pending and currently-executing check runs in a single private location — the signal you drive replica count from. +Checkly exposes the `checkly_private_location_check_runs` gauge through the [Prometheus v2 endpoint](/integrations/observability/prometheus). Filtered by `state` and a `private_location_slug_name`, it provides the count of pending and currently-executing check runs in a single private location — the signal you drive replica count from. The relevant `state` values are: @@ -136,5 +136,5 @@ Maximum runtime by check type: ## See also - [Scaling and Redundancy](/platform/private-locations/scaling-redundancy) -- [Exporting Metrics & Data via Prometheus V2](/integrations/observability/prometheus-v2/) +- [Export metrics to Prometheus](/integrations/observability/prometheus) - [Kubernetes Deployment](/platform/private-locations/kubernetes-deployment) diff --git a/resolve/overview.mdx b/resolve/overview.mdx index cbc6e77bc..937ce5592 100644 --- a/resolve/overview.mdx +++ b/resolve/overview.mdx @@ -40,7 +40,7 @@ Experience AI-powered root cause analysis and insights Add OpenTelemetry instrumentation to your codebase - + Connect traces with your existing observability platform diff --git a/resolve/traces/export/dash0.mdx b/resolve/traces/export/dash0.mdx deleted file mode 100644 index ae86dd365..000000000 --- a/resolve/traces/export/dash0.mdx +++ /dev/null @@ -1,19 +0,0 @@ ---- -title: 'Exporting Traces to Dash0' -description: 'Learn how to export traces from Checkly to Dash0.' -sidebarTitle: 'Dash0' -canonical: 'https://www.checklyhq.com/docs/resolve/traces/export/dash0/' ---- - - -1. Create an API key in the **Settings** > **Auth Tokens** section. - ![Generate an auth token](/images/docs/images/otel/export-traces/dash0-token.png) -2. Grab the right endpoint URL for your Dash0 organization from the **Settings** > **Endpoints** section. It should look like `https://ingress.eu-west-1.aws.dash0.com/v1/traces`. - Make sure it ends with `/v1/traces/` -3. Add the endpoint and the auth token to the Checkly integration settings. The auth token is added as an HTTP header named - `Authorization` with the value of `Bearer ` and your auth token. - ![Fill in the values into Checkly](/images/docs/images/otel/export-traces/dash0-checkly.png) -4. Observe Spans appearing in Dash0 on the `Tracing` tab. - ![Traces displayed in Dash0](/images/docs/images/otel/export-traces/dash0-traces.png) - -Find all the details in the [Dash0 documentation](https://www.dash0.com/documentation). diff --git a/resolve/traces/export/grafana-tempo.mdx b/resolve/traces/export/grafana-tempo.mdx deleted file mode 100644 index bc5b27fca..000000000 --- a/resolve/traces/export/grafana-tempo.mdx +++ /dev/null @@ -1,37 +0,0 @@ ---- -title: 'Exporting Traces to Grafana Tempo' -description: 'Learn how to export traces from Checkly to Grafana Tempo.' -sidebarTitle: 'Grafana Tempo' -canonical: 'https://www.checklyhq.com/docs/resolve/traces/export/grafana-tempo/' ---- - -You can connect the Traces export to Grafana using the OpenTelemetry Integration and Grafana Tempo. - -1. Make sure that you have Tempo running on your Grafana Cloud Instance at `yourOrganization.grafana.net`. Find it at **Connections > Search for Tempo > Check if Tempo Data source is marked as installed**. - -1. In Grafana Cloud (`grafana.com/orgs/yourOrganization`), head over to **Connections** > **Add new connection** > **OpenTelemetry (OTLP)**. - ![Add an OTLP connection in Grafana Cloud](/images/docs/images/otel/export-traces/grafana-cloud-opentelemetry.png) - -1. Copy the endpoint URL and append `v1/traces` to it. - ![Grafana OTLP endpoint](/images/docs/images/otel/export-traces/grafana-otlp-endpoint-config.png) - It should be similar to:`https://otlp-gateway-prod-eu-west-2.grafana.net/otlp/v1/traces`. - -1. Generate an OTLP Token, and copy both the Instance ID and the OTLP Token as well. - ![Generate an OTLP token](/images/docs/images/otel/export-traces/create-otlp-token-grafana.png) - -1. Head back to the [Traces Settings page](https://app.checklyhq.com/settings/account/traces) on Checkly. - - * Enable exporting traces and add the endpoint URL from step 3. - * and in the **Headers** section, specify the HTTP Header: `Authorization` and `Basic instance:token`, where `instance:token` is base64 encoded. - You can use an online tool like [base64encode.net](https://www.base64encode.net/) - to encode your instance and token. - ![Export Traces configuration](/images/docs/images/otel/export-traces/export-traces-to-grafana-config.png) - -2. Back in your Grafana Cloud Instance (for example danube.grafana.net), head over to "Explore", select the *Tempo* source that is named `grafanacloud-yourOrganization-traces`: - ![Select Tempo traces data source](/images/docs/images/otel/export-traces/grafana-cloud-tempo-source.png) - - Now, click **Search** to see the table of Traces received. The ones exported by Checkly, have `checkly` in the service column. - ![See Checkly Traces in Grafana](/images/docs/images/otel/export-traces/grafana-explore-checkly-traces.png) - - -Find more the details in the [Grafana OpenTelemetry documentation](https://grafana.com/docs/grafana-cloud/send-data/otlp/send-data-otlp/?pg=traces&plcmt=hero-btn-2#before-you-begin). diff --git a/resolve/traces/export/honeycomb.mdx b/resolve/traces/export/honeycomb.mdx deleted file mode 100644 index 0ed70bc42..000000000 --- a/resolve/traces/export/honeycomb.mdx +++ /dev/null @@ -1,14 +0,0 @@ ---- -title: 'Exporting Traces to Honeycomb' -description: 'Learn how to export traces from Checkly to Honeycomb.' -sidebarTitle: 'Honeycomb' -canonical: 'https://www.checklyhq.com/docs/resolve/traces/export/honeycomb/' ---- - - -1. Grab the relevant API endpoint from [the Honeycomb documentation](https://docs.honeycomb.io/send-data/opentelemetry/#using-the-honeycomb-opentelemetry-endpoint). It should look like `https://api.honeycomb.io/v1/traces`. -2. Grab your Honeycomb Ingest API key from the **Account** > **Team settings** > **Environments and API keys** section. -3. Add the endpoint the Checkly integration settings and provide the API key as an HTTP header `x-honeycomb-team` with - the value of your API key. - - diff --git a/resolve/traces/export/image.png b/resolve/traces/export/image.png deleted file mode 100644 index c50626652..000000000 Binary files a/resolve/traces/export/image.png and /dev/null differ diff --git a/resolve/traces/export/new-relic.mdx b/resolve/traces/export/new-relic.mdx deleted file mode 100644 index 2b46db7de..000000000 --- a/resolve/traces/export/new-relic.mdx +++ /dev/null @@ -1,15 +0,0 @@ ---- -title: 'Exporting Traces to New Relic' -description: 'Learn how to export traces from Checkly to New Relic.' -sidebarTitle: 'New Relic' -canonical: 'https://www.checklyhq.com/docs/resolve/traces/export/new-relic/' ---- - -1. Make sure you have the **Traces** feature enabled in your New Relic account. -2. Create an *Ingest - License* type API key in the **Administration** > **API keys** section. -3. Grab the right endpoint URL for your New Relic account. It should look like `https://otlp.nr-data.net/v1/traces`. - Check [the full list of all New Relic OTel endpoints](https://docs.newrelic.com/docs/more-integrations/open-source-telemetry-integrations/opentelemetry/get-started/opentelemetry-set-up-your-app/#ports-and-endpoints). -4. Add the endpoint and the API key to the Checkly integration settings. The API key is added as an HTTP header named - `api-key` with the value of your API key. - -Find all the details in the [New Relic OpenTelemetry documentation](https://docs.newrelic.com/docs/more-integrations/open-source-telemetry-integrations/opentelemetry/get-started/opentelemetry-set-up-your-app/). diff --git a/resolve/traces/export/overview.mdx b/resolve/traces/export/overview.mdx new file mode 100644 index 000000000..59b238789 --- /dev/null +++ b/resolve/traces/export/overview.mdx @@ -0,0 +1,19 @@ +--- +title: 'Export traces to your observability backend' +description: 'Send every API check, URL monitor, Browser check and Multistep check run as a span to the OpenTelemetry backend you already use.' +sidebarTitle: 'Exporting' +canonical: 'https://www.checklyhq.com/docs/resolve/traces/export/overview/' +--- + +Every API check, URL monitor, Browser check and Multistep check run can be sent as one span to a single OTLP endpoint that you configure on [Settings > Traces](https://app.checklyhq.com/settings/account/traces). Setup, the span schema and a collector recipe live on the OTLP page, the first link below. + +- [OTLP (any backend)](/integrations/observability/traces/otlp) +- [Coralogix](/integrations/observability/coralogix) +- [Chronosphere](/integrations/observability/traces/chronosphere) +- [ClickStack](/integrations/observability/traces/clickstack) +- [Cribl Stream](/integrations/observability/traces/cribl) +- [Dash0](/integrations/observability/traces/dash0) +- [Datadog](/integrations/observability/traces/datadog) +- [Grafana Tempo](/integrations/observability/traces/grafana-tempo) +- [Honeycomb](/integrations/observability/traces/honeycomb) +- [New Relic](/integrations/observability/traces/new-relic) diff --git a/resolve/traces/export/to-chronosphere.mdx b/resolve/traces/export/to-chronosphere.mdx deleted file mode 100644 index 507e6da65..000000000 --- a/resolve/traces/export/to-chronosphere.mdx +++ /dev/null @@ -1,15 +0,0 @@ ---- -title: 'Exporting Traces to Chronosphere' -description: 'Learn how to export traces from Checkly to Chronosphere.' -sidebarTitle: 'To Chronosphere' -canonical: 'https://www.checklyhq.com/docs/resolve/traces/export/to-chronosphere/' ---- - - - -1. Create a service account in your Chronosphere account with write permissions for traces. Chronosphere recommends creating a restricted service account with a write-only scope. -2. Generate an API token for the service account from the **Account** > **Service Accounts** section. -3. Determine your Chronosphere OTLP endpoint. It should follow the format: `https://YOUR_COMPANY.chronosphere.io/data/opentelemetry/v1/traces` where `YOUR_COMPANY` is your organization name. -4. Add the endpoint to the Checkly integration settings and provide the API token as an HTTP header `API-Token` with the value of your API token. - -Find all the details in the [Chronosphere OpenTelemetry documentation](https://docs.chronosphere.io/ingest/metrics-traces/otel/otlp-endpoints). diff --git a/resolve/traces/how-it-works.mdx b/resolve/traces/how-it-works.mdx index acacf903c..b063e965c 100644 --- a/resolve/traces/how-it-works.mdx +++ b/resolve/traces/how-it-works.mdx @@ -8,43 +8,36 @@ Checkly Traces use OpenTelemetry to correlate synthetic check results with your ![Checkly OpenTelemetry diagram](/images/docs/images/otel/checkly_otel_diagram.png) -## The Flow +{/* +Vendor-neutral Mermaid version of the diagram above, kept for reference. To swap it in, delete the PNG line and this comment wrapper. +```mermaid +flowchart LR + run["Check run"] -->|"Step 1: trace headers"| app["Your app"] + run -->|"Step 2: check run span"| backend["Your OpenTelemetry backend"] + app -->|"Your app spans"| backend + app -->|"Step 3: spans with checkly=true"| checkly["Checkly"] + checkly -->|"Step 4: alert"| result["Check result with the full trace"] +``` +*/} + +## The flow -When enabling the integration, Checkly automatically instruments all HTTP requests made by your checks with `traceparent` and `tracestate` headers. These HTTP requests hit your web application and/or API when running checks. +When trace export or the Coralogix integration is enabled, Checkly automatically instruments the HTTP requests made by your checks with `traceparent` and `tracestate` headers. These HTTP requests hit your web application or API when running checks. -All synthetic check types, including API, Browser and Multistep checks, include these headers: +API checks, URL monitors, Browser checks and Multistep checks include these headers: -- **traceparent**: W3C compliant trace ID generated for each HTTP request that is part of a check. For Browser checks and Multistep checks, there can be multiple requests instrumented. -- **tracestate**: Set to `checkly=true` to identify Checkly as the vendor that generated the trace +- **traceparent**: W3C compliant trace context. Each check run is one trace, and every request in that run carries the same `traceparent`. +- **tracestate**: Carries `checkly=true` to identify Checkly as the vendor that generated the trace, plus the members documented on the [OTLP page](/integrations/observability/traces/otlp#trace-headers). Together, these headers propagate the trace context along the request chain. This is the most basic way of tying a synthetic check to your backend traces, though it won't provide much context about the check run itself, like the check name or location. -You can configure your 3rd party backend in the Checkly UI so we can export every check result as a span to your backend. This correlates all backend spans with the check run that triggered them, together with full context of the check run. - -Check details are stored in the `checkly` namespace as attributes on the span: - -```yaml -checkly.check.name: "ACME homepage" -checkly.check.id: "438481ea-0eab-43d6-8932-ab51bd0d49d6" -checkly.check.type: "browser" -checkly.check.location: "eu-west-1" -checkly.check.tags: ["production", "checkout"] -``` - -This provides rich context about the run location, check name, check type, and more. - -`checkly.check.tags` carries the check's tags as a string array — the check's own tags first, then tags inherited from its group, deduplicated. Use it to filter or group traces by the same labels you organize your checks by, such as a team or workflow tag. - - -Tag caveats: +You can configure your OpenTelemetry backend on [Settings > Traces](https://app.checklyhq.com/settings/account/traces) so Checkly exports every API check, URL monitor, Browser check and Multistep check run as a span to your backend. This correlates all backend spans with the check run that triggered them when your application honors `traceparent`, together with full context of the check run. -- The attribute is only present when the check has tags. At most 20 tags are exported per span, and a tag longer than 128 characters is omitted entirely rather than truncated. -- Attributes are set on Checkly's root span only — they do not propagate to your application's own spans within the trace. To find a whole trace by tag, filter on the root span, then use its trace ID. - +Check details are stored in the `checkly` namespace as attributes on the span, such as the check name, check type, run location and tags. See [What is sent](/integrations/observability/traces/otlp#what-is-sent) for the span schema and the tag rules. diff --git a/resolve/traces/overview.mdx b/resolve/traces/overview.mdx index 1b6e8d0f6..12d66ce38 100644 --- a/resolve/traces/overview.mdx +++ b/resolve/traces/overview.mdx @@ -38,7 +38,7 @@ To get started with Checkly Traces using OpenTelemetry, pick the scenario that b Send your infrastructure traces to Checkly to get contextualized check failure analysis. - + Export check results as traces to your 3rd party OTel tooling @@ -63,6 +63,8 @@ test.use({ }) ``` +`CHECKLY_TRACEPARENT` is empty when the run carries no trace context. Check that it is set before relying on it. + See the built-in variables for [Browser checks](/detect/synthetic-monitoring/browser-checks/mac-structure#built-in-runtime-variables) and [Playwright Check Suites](/detect/synthetic-monitoring/playwright-checks/environment-variables#built-in-variables-in-playwright-check-suites). ## Works hand-in-hand with Rocky AI diff --git a/sitemap-crawl.md b/sitemap-crawl.md index b0cdf2067..1baf70d85 100644 --- a/sitemap-crawl.md +++ b/sitemap-crawl.md @@ -168,10 +168,7 @@ This file contains all valid URLs extracted from docs.json organized by director - /resolve/traces/import/https-grpc #### Resolve - Traces - Exporting -- /resolve/traces/export/grafana-tempo -- /resolve/traces/export/honeycomb -- /resolve/traces/export/new-relic -- /resolve/traces/export/dash0 +- /resolve/traces/export/overview #### Resolve - Traces - Instrumentation - /resolve/traces/instrumentation/nextjs @@ -232,9 +229,18 @@ This file contains all valid URLs extracted from docs.json organized by director - /integrations/ci-cd/vercel/managing-plan ### Integrations - Observability +- /integrations/observability/overview - /integrations/observability/prometheus -- /integrations/observability/prometheus-v2 +- /integrations/observability/traces/otlp - /integrations/observability/coralogix +- /integrations/observability/traces/chronosphere +- /integrations/observability/traces/clickstack +- /integrations/observability/traces/cribl +- /integrations/observability/traces/dash0 +- /integrations/observability/traces/datadog +- /integrations/observability/traces/grafana-tempo +- /integrations/observability/traces/honeycomb +- /integrations/observability/traces/new-relic ### Integrations - Infrastructure as Code diff --git a/sitemap.xml b/sitemap.xml index 2c11945f3..94c1d1885 100644 --- a/sitemap.xml +++ b/sitemap.xml @@ -1479,11 +1479,38 @@ https://www.checklyhq.com/docs/integrations/observability/coralogix/ + + https://www.checklyhq.com/docs/integrations/observability/overview/ + https://www.checklyhq.com/docs/integrations/observability/prometheus/ - https://www.checklyhq.com/docs/integrations/observability/prometheus-v2/ + https://www.checklyhq.com/docs/integrations/observability/traces/chronosphere/ + + + https://www.checklyhq.com/docs/integrations/observability/traces/clickstack/ + + + https://www.checklyhq.com/docs/integrations/observability/traces/cribl/ + + + https://www.checklyhq.com/docs/integrations/observability/traces/dash0/ + + + https://www.checklyhq.com/docs/integrations/observability/traces/datadog/ + + + https://www.checklyhq.com/docs/integrations/observability/traces/grafana-tempo/ + + + https://www.checklyhq.com/docs/integrations/observability/traces/honeycomb/ + + + https://www.checklyhq.com/docs/integrations/observability/traces/new-relic/ + + + https://www.checklyhq.com/docs/integrations/observability/traces/otlp/ https://www.checklyhq.com/docs/learn/incidents/anatomy-of-a-status-page/ @@ -1876,16 +1903,7 @@ https://www.checklyhq.com/docs/resolve/overview/ - https://www.checklyhq.com/docs/resolve/traces/export/dash0/ - - - https://www.checklyhq.com/docs/resolve/traces/export/grafana-tempo/ - - - https://www.checklyhq.com/docs/resolve/traces/export/honeycomb/ - - - https://www.checklyhq.com/docs/resolve/traces/export/new-relic/ + https://www.checklyhq.com/docs/resolve/traces/export/overview/ https://www.checklyhq.com/docs/resolve/traces/how-it-works/ diff --git a/snippets/otlp-vendor-intro.mdx b/snippets/otlp-vendor-intro.mdx new file mode 100644 index 000000000..27699a92d --- /dev/null +++ b/snippets/otlp-vendor-intro.mdx @@ -0,0 +1,2 @@ +Before you start, read the prerequisites on the [OTLP export page](/integrations/observability/traces/otlp) and check [what is sent](/integrations/observability/traces/otlp#what-is-sent). +This page only covers the values {vendor} needs. diff --git a/snippets/otlp-vendor-outro.mdx b/snippets/otlp-vendor-outro.mdx new file mode 100644 index 000000000..711a919ab --- /dev/null +++ b/snippets/otlp-vendor-outro.mdx @@ -0,0 +1,2 @@ +To see failed runs, filter on the attribute `checkly.check.status` with the value `FAILED`. +The [OTLP export page](/integrations/observability/traces/otlp) has the span schema.