From c17bb5ea078de7854e42e0c8592c0304bd1da3c4 Mon Sep 17 00:00:00 2001 From: Brandon Morelli Date: Mon, 5 Nov 2018 19:54:28 -0800 Subject: [PATCH 01/12] docs: rework apm overview --- .../guide/agent-server-compatibility.asciidoc | 70 ++++ docs/guide/apm-data-model.asciidoc | 79 ++++ docs/guide/apm-release-notes.asciidoc | 84 ++++ docs/guide/index.asciidoc | 374 +----------------- docs/guide/install-and-run.asciidoc | 260 ++++++++++++ docs/guide/overview.asciidoc | 51 +++ 6 files changed, 555 insertions(+), 363 deletions(-) create mode 100644 docs/guide/agent-server-compatibility.asciidoc create mode 100644 docs/guide/apm-data-model.asciidoc create mode 100644 docs/guide/apm-release-notes.asciidoc create mode 100644 docs/guide/install-and-run.asciidoc create mode 100644 docs/guide/overview.asciidoc diff --git a/docs/guide/agent-server-compatibility.asciidoc b/docs/guide/agent-server-compatibility.asciidoc new file mode 100644 index 00000000000..0c0942b2da1 --- /dev/null +++ b/docs/guide/agent-server-compatibility.asciidoc @@ -0,0 +1,70 @@ +[[agent-server-compatibility]] +== Agent/Server Compatibility + +When instrumenting an APM Agent within your codebase, it's important to use a compatible version of APM Server. Below is a chart that outlines the compatibility between agents and server versions. + +[float] +[[nodejs-compatibility]] +==== Node.js Agent Compatibility + +[options="header"] +|======================================================================= +|Agent Version |APM Server Version +|1.x |6.2.0-6.5.0 +|2.x |6.5.0+ +|======================================================================= + +[float] +[[python-compatibility]] +==== Python Agent Compatibility + +[options="header"] +|======================================================================= +|Agent Version |APM Server Version +|1.x |6.2.0-6.5.0 +|2.x |6.5.0+ +|======================================================================= + +[float] +[[ruby-compatibility]] +==== Ruby Agent Compatibility + +[options="header"] +|======================================================================= +|Agent Version |APM Server Version +|1.x |6.2.0-6.5.0 +|2.x |6.5.0+ +|======================================================================= + +[float] +[[java-compatibility]] +==== Java Agent Compatibility + +[options="header"] +|======================================================================= +|Agent Version |APM Server Version +|1.x |6.2.0-6.5.0 +|2.x |6.5.0+ +|======================================================================= + +[float] +[[go-compatibility]] +==== Go Agent Compatibility + +[options="header"] +|======================================================================= +|Agent Version |APM Server Version +|1.x |6.2.0-6.5.0 +|2.x |6.5.0+ +|======================================================================= + +[float] +[[rum-compatibility]] +==== RUM Agent Compatibility + +[options="header"] +|======================================================================= +|Agent Version |APM Server Version +|1.x |6.2.0-6.5.0 +|2.x |6.5.0+ +|======================================================================= diff --git a/docs/guide/apm-data-model.asciidoc b/docs/guide/apm-data-model.asciidoc new file mode 100644 index 00000000000..ea5ac049cd2 --- /dev/null +++ b/docs/guide/apm-data-model.asciidoc @@ -0,0 +1,79 @@ +[[apm-data-model]] +== Data Model + +Elastic APM agents capture different types of information from within their instrumented applications - namely `transactions`, `spans`, and `errors`. + +* <> +* <> +* <> + +[[transactions]] +=== Transactions + +A transaction describes an event captured by an Elastic APM agent instrumenting a service. Some examples of a transaction might be: + +* An HTTP request +* An asynchronous background job + +A transaction contains: + +* The timestamp and duration of the event +* A unique id, type, and name +* A result (e.g. a response code) +* Some contextual data (see below for details) +* Other relevant information depending on the agent. Example: The JavaScript RUM captures transaction marks, +which are points in time relative to the start of the transaction with some label. + +[[transactions-context]] +include::../context.asciidoc[] + +Agents decide whether to sample transactions or not, +and provide settings to control sampling behavior. +If sampled, +the {apm-server-ref}/transaction-spans[spans] of a transaction are sent and stored as separate documents. Within one transaction there can be several, or no spans captured. + +Transactions are stored in {apm-server-ref}/transaction-indices[transaction indices]. + +[[transaction-spans]] +=== Spans +Transactions can have 0, 1, or many spans. Spans have a `transaction.id` attribute that refer to their parent transaction. + +Agents typically automatically instrument a variety of of libraries, +but also provide an API for ad hoc instrumentation of specific code paths. +A span contains information about a specific code path that has been executed as part of a transaction. +This information includes: + +* start time +* duration +* name +* type +* `stack trace` (optional). + +As an example, if a database query happens within a sampled transaction, +a span describing the database query will be created. +The name of this span will contain information about the query itself, +and the type of this span will contain information about the database type. + +Spans are stored in {apm-server-ref}/span-indices[span indices]. Note that these indices are separate from {apm-server-ref}/transaction-indices[transaction indices] by default. + +[[errors]] +=== Errors + +Errors are identified by a unique ID. +An error event contains at least +information about the original `exception` that occured +or about a `log` created when the exception occured. + +Both the captured `exception` and the captured `log` of an error can contain a `stack trace`, +helpful for debugging. + +The `culprit` of an error indicates where is originated. + +An error might relate to the <> during which it happened, +via the `transaction.id`. + +Errors also have some contextual data. + +include::../context.asciidoc[] + +Errors are stored in {apm-server-ref}/error-indices[error indices]. \ No newline at end of file diff --git a/docs/guide/apm-release-notes.asciidoc b/docs/guide/apm-release-notes.asciidoc new file mode 100644 index 00000000000..87d6f1684fb --- /dev/null +++ b/docs/guide/apm-release-notes.asciidoc @@ -0,0 +1,84 @@ +[[kibana]] +[[apm-release-notes]] +== Release notes + +[float] +==== APM version 6.4.1 + +[float] +===== Bug Fixes +Changes introduced in 6.4.0 potentially caused an empty APM Kibana UI. +This happened in case the APM Server was using an outdated configuration file, not configured to index events into separate indices. +To fix this, the APM Kibana UI now falls back to use `apm-*` as default indices to query. +Users can still leverage separate indices for queries by overriding the default values described in {kibana-ref}/apm-settings-kb.html[Kibana APM settings]. + + +[float] +==== APM version 6.4.0 + +[float] +===== Breaking changes + +We previously split APM data into separate indices (transaction, span, error, etc.). +In 6.4 APM Kibana UI starts to leverage those separate indices for queries. + +In case you only update Kibana but run an older version of APM Server you will not be able to see any APM data by default. +To fix this, use the {kibana-ref}/apm-settings-kb.html[Kibana APM settings] to specify the location of the APM index: +["source","sh"] +------------------------------------------------------------ +apm_oss.errorIndices: apm-* +apm_oss.spanIndices: apm-* +apm_oss.transactionIndices: apm-* +apm_oss.onboardingIndices: apm-* +------------------------------------------------------------ + +In case you are upgrading APM Server from an older version, you might need to refresh your APM index pattern for certain APM UI features to work. +Also ensure to add the new config options in `apm-server.yml` in case you keep your existing configuration file: +["source","sh"] +------------------------------------------------------------ +output.elasticsearch: + indices: + - index: "apm-%{[beat.version]}-sourcemap" + when.contains: + processor.event: "sourcemap" + - index: "apm-%{[beat.version]}-error-%{+yyyy.MM.dd}" + when.contains: + processor.event: "error" + - index: "apm-%{[beat.version]}-transaction-%{+yyyy.MM.dd}" + when.contains: + processor.event: "transaction" + - index: "apm-%{[beat.version]}-span-%{+yyyy.MM.dd}" + when.contains: + processor.event: "span" + - index: "apm-%{[beat.version]}-metric-%{+yyyy.MM.dd}" + when.contains: + processor.event: "metric" + - index: "apm-%{[beat.version]}-onboarding-%{+yyyy.MM.dd}" + when.contains: + processor.event: "onboarding" +------------------------------------------------------------ + +[float] +===== New features + +*APM Server* + +* Logstash output +* Kafka output + + +*APM UI* + +* Query bar +* Machine Learning integration: Anomaly detection on service response times +* Kibana objects (index pattern, dashboards, etc.) can now be imported via the Kibana setup instuctions + + +*APM agents* + +* RUM is now GA +* Ruby is now GA +* Java is now Beta +* Go is now Beta +* Python added instrumentation for Cassandra, PyODBC and PyMSSQL +* Node.js added instrumentation for Cassandra and broader MySQL support diff --git a/docs/guide/index.asciidoc b/docs/guide/index.asciidoc index 0845225e031..f780c2f593d 100644 --- a/docs/guide/index.asciidoc +++ b/docs/guide/index.asciidoc @@ -1,373 +1,21 @@ include::../version.asciidoc[] include::{asciidoc-dir}/../../shared/attributes.asciidoc[] -[[gettting-started]] -= Getting Started with APM - -[[overview]] -== Overview - -Elastic APM is an application performance monitoring system built on the Elastic Stack. -It allows you to monitor software services and applications in real time, -collecting detailed performance information on response time for incoming requests, -database queries, -calls to caches, -external HTTP requests, -etc. -This makes it easier to pinpoint and fix performance problems quickly. - -Elastic APM also collects automatically unhandled errors and exceptions. -Errors are grouped based primarily on the stacktrace, -so you can identify new errors as they appear and keep an eye on how many times specific errors happen. - -NOTE: this guide will indiscriminately use the word service for both services and applications. - -[[components]] -[float] -=== APM Components - -Elastic APM consists of four components: - -* {ref}/index.html[Elasticsearch] -* {apm-agents-ref}/index.html[APM agents] -* {apm-server-ref}/index.html[APM Server] -* {kibana-ref}/xpack-apm.html[Kibana APM UI] - -image::apm-architecture.png[Architecture of Elastic APM] - -*APM agents* are open source libraries written in the same language as your service. -You install them into your service as you would install any other library. -They instrument your code and collect performance data and errors at runtime. -This data is buffered for a short period and sent on to APM Server. - -Out of the box, -APM automatically instruments web frameworks, -database drivers, -calls to caching servers, -and HTTP libraries for requests to external services. -Additionally, -agents provide an API to manually instrument anything else you find interesting. - -*APM Server* is an open source application written in Go which typically runs on dedicated servers. -It listens on port 8200 by default and receives data from agents through a JSON HTTP API. -Then it creates documents from that data and stores them in Elasticsearch. - -To visualize the data after it's sent to Elasticsearch, -you can use the the dedicated APM UI bundled in X-Pack, -or the pre-built open source Kibana dashboards that can be loaded directly via the {kibana-ref}/apm-getting-started.html[APM Kibana UI]. - - -[[install-and-run]] -== Install and run Elastic APM - -To get started using Elastic APM, -you need to have: - -* an Elasticsearch cluster and Kibana (version 5.6 or above) -* APM Server -* APM agents installed in your services - - -For information about setting up an Elasticsearch cluster, -see the Elasticsearch {ref}/getting-started.html[Getting Started]. - -The following sections show how to get started quickly with Elastic APM on a local machine. - -[[apm-server]] -[float] -=== Install and run APM Server - -First, https://www.elastic.co/downloads/apm/apm-server[download APM Server] for your operating system and extract the package. - -In a production environment you would put APM Server on its own machines, -similar to how you run Elasticsearch. -You _can_ run it on the same machines as Elasticsearch, -but this is not recommended, -as the processes will be competing for resources. - -To start APM Server, run: - -[source,bash] ----------------------------------- -./apm-server -e ----------------------------------- - -It will try to connect to Elasticsearch on localhost port 9200 and expose an API to agents on port 8200. -You can change the defaults by supplying different addresses on the command line: - -[source,bash] ----------------------------------- -./apm-server -e -E output.elasticsearch.hosts=ElasticsearchAddress:9200 -E apm-server.host=localhost:8200 ----------------------------------- - -Or you can update the `apm-server.yml` configuration file: - -[source,yaml] ----------------------------------- -apm-server: - host: localhost:8200 - -output: - elasticsearch: - hosts: ElasticsearchAddress:9200 ----------------------------------- - -NOTE: If you are using an X-Pack secured version of Elastic Stack, -you need to specify credentials in the config file: - -[source,yaml] ----- -output.elasticsearch: - hosts: ["ElasticsearchAddress:9200"] - username: "elastic" - password: "elastic" ----- - - - -[[secure-api-access]] -[float] -==== Secure access to the API -If you change the listen address from `localhost` to something that is accessible from outside of the machine, -we recommend setting up firewall rules to ensure that only your own systems can access the API. -Alternatively, -you can use a {apm-server-ref}/securing-apm-server.html[secret token and TLS]. - -If you have APM Server running on the same host as your service, you can configure it to listen on a Unix domain socket. - -[[kibana-dashboards]] -[float] -==== Install the Kibana dashboards - -From APM Server and Kibana 6.4 on you can load dashboards directly via the {kibana-ref}/apm-getting-started.html[APM -Kibana UI]. - -See an example screenshot of a Kibana dashboard: - -image::kibana-dashboard.png[Screenshot of a Kibana Dashboard] - -[[more-information]] -[float] -=== More information -For detailed instructions on how to install and secure APM Server in your server environment, -including details on how to run APM Server in a highly available environment, -please see {apm-server-ref}/index.html[APM Server documentation]. - -Once APM Server is up and running, -you need to install an agent in your service. - -[[agents]] -[float] -=== Install and configure APM agents - -Agents are written in the same language as your service. -Currently Elastic APM has agents for Node.js, Python, Ruby, and JavaScript. - -Setting up a new service to be monitored requires installing the agent, -and configuring it with the address of your APM Server and the service name. - -[[choose-service-name]] -[float] -==== Choose a service name - -The service name is used by Elastic APM to differentiate between data coming from different services. - -Elastic APM includes the service name field on every document that it saves in Elasticsearch. -If you change the service name after using Elastic APM, -you will see the old service name and the new service name as two separate services. -Make sure you choose a good service name before you get started. - -The service name can only contain alphanumeric characters, -spaces, -underscores, -and dashes (must match `^[a-zA-Z0-9 _-]+$`). - -[[nodejs-agent]] -[float] -==== Install the Node.js agent - -Install the elastic-apm-node module from npm in your service: - -[source,bash] ----------------------------------- -npm install elastic-apm-node --save ----------------------------------- - -Then configure the elastic-apm-node module by adding the following lines to the top of your code: - -[source,javascript] ----------------------------------- -// Add this to the VERY top of the first file loaded in your app -var apm = require('elastic-apm-node').start({ - // Override service name from package.json - // Allowed characters: a-z, A-Z, 0-9, -, _, and space - serviceName: '', - - // Use if APM Server requires a token - secretToken: '', - - // Set custom APM Server URL (default: http://localhost:8200) - serverUrl: '' -}) ----------------------------------- - -The Node.js agent automatically instruments Express, -hapi, -Koa, -and Restify out of the box. -See the {apm-node-ref}/index.html[APM Node.js Agent documentation] for more details. - -[[python-agent]] -[float] -==== Install the Python agent - -Install the Elastic APM module from pypi: - -[source,bash] ----------------------------------- -pip install elastic-apm ----------------------------------- - -The Python agent automatically instruments Django and Flask out of the box. -See the {apm-py-ref}/index.html[APM Python Agent documentation] for more details. - -[[ruby-agent]] -[float] -==== Install the Ruby agent - -Add the gem to your Gemfile: - -[source,bash] ----------------------------------- -gem 'elastic-apm' ----------------------------------- - -Create a config file config/elastic_apm.yml: - -[source,bash] ----------------------------------- -server_url: http://localhost:8200 -secret_token: '' ----------------------------------- - -The Ruby agent automatically instruments Rails out of the box. -See the {apm-ruby-ref}/index.html[APM Ruby Agent documentation] for more details. - - -[[rum-agent]] -[float] -==== Install the RUM JavaScript agent - -Install the agent as a dependency to your application: - -[source,bash] ----------------------------------- -npm install elastic-apm-js-base --save ----------------------------------- - -Configure the agent: - -[source,bash] ----------------------------------- -import { init as initApm } from 'elastic-apm-js-base' -var apm = initApm({ - - // Set required service name (allowed characters: a-z, A-Z, 0-9, -, _, and space) - serviceName: '', - - // Set custom APM Server URL (default: http://localhost:8200) - serverUrl: 'http://localhost:8200', - - // Set service version (required for sourcemap feature) - serviceVersion: '' -}) ----------------------------------- - -See the {apm-rum-ref}/index.html[APM RUM JavaScript Agent documentation] for more details. - - -[[kibana]] -[[apm-release-notes]] -== Release notes - -[float] -==== APM version 6.4.1 - -[float] -===== Bug Fixes -Changes introduced in 6.4.0 potentially caused an empty APM Kibana UI. -This happened in case the APM Server was using an outdated configuration file, not configured to index events into separate indices. -To fix this, the APM Kibana UI now falls back to use `apm-*` as default indices to query. -Users can still leverage separate indices for queries by overriding the default values described in {kibana-ref}/apm-settings-kb.html[Kibana APM settings]. - - -[float] -==== APM version 6.4.0 - -[float] -===== Breaking changes - -We previously split APM data into separate indices (transaction, span, error, etc.). -In 6.4 APM Kibana UI starts to leverage those separate indices for queries. - -In case you only update Kibana but run an older version of APM Server you will not be able to see any APM data by default. -To fix this, use the {kibana-ref}/apm-settings-kb.html[Kibana APM settings] to specify the location of the APM index: -["source","sh"] ------------------------------------------------------------- -apm_oss.errorIndices: apm-* -apm_oss.spanIndices: apm-* -apm_oss.transactionIndices: apm-* -apm_oss.onboardingIndices: apm-* ------------------------------------------------------------- - -In case you are upgrading APM Server from an older version, you might need to refresh your APM index pattern for certain APM UI features to work. -Also ensure to add the new config options in `apm-server.yml` in case you keep your existing configuration file: -["source","sh"] ------------------------------------------------------------- -output.elasticsearch: - indices: - - index: "apm-%{[beat.version]}-sourcemap" - when.contains: - processor.event: "sourcemap" - - index: "apm-%{[beat.version]}-error-%{+yyyy.MM.dd}" - when.contains: - processor.event: "error" - - index: "apm-%{[beat.version]}-transaction-%{+yyyy.MM.dd}" - when.contains: - processor.event: "transaction" - - index: "apm-%{[beat.version]}-span-%{+yyyy.MM.dd}" - when.contains: - processor.event: "span" - - index: "apm-%{[beat.version]}-metric-%{+yyyy.MM.dd}" - when.contains: - processor.event: "metric" - - index: "apm-%{[beat.version]}-onboarding-%{+yyyy.MM.dd}" - when.contains: - processor.event: "onboarding" ------------------------------------------------------------- - -[float] -===== New features - -*APM Server* - -* Logstash output -* Kafka output +ifdef::env-github[] +NOTE: For the best reading experience, +please view this documentation at https://www.elastic.co/guide/en/apm/get-started[elastic.co] +endif::[] +[[gettting-started]] += APM Overview -*APM UI* -* Query bar -* Machine Learning integration: Anomaly detection on service response times -* Kibana objects (index pattern, dashboards, etc.) can now be imported via the Kibana setup instuctions +include::./overview.asciidoc[] +include::./apm-data-model.asciidoc[] -*APM agents* +include::./install-and-run.asciidoc[] -* RUM is now GA -* Ruby is now GA -* Java is now Beta -* Go is now Beta -* Python added instrumentation for Cassandra, PyODBC and PyMSSQL -* Node.js added instrumentation for Cassandra and broader MySQL support +include::./agent-server-compatibility.asciidoc[] +include::./apm-release-notes.asciidoc[] diff --git a/docs/guide/install-and-run.asciidoc b/docs/guide/install-and-run.asciidoc new file mode 100644 index 00000000000..822a4e3ed72 --- /dev/null +++ b/docs/guide/install-and-run.asciidoc @@ -0,0 +1,260 @@ +[[install-and-run]] +== Install and run Elastic APM + +To get started using Elastic APM, +you need to have: + +* an Elasticsearch cluster and Kibana (version 5.6 or above) +* APM Server +* APM agents installed in your services + + +For information about setting up an Elasticsearch cluster, +see the Elasticsearch {ref}/getting-started.html[Getting Started]. + +The following sections show how to get started quickly with Elastic APM on a local machine. + +[[apm-server]] +[float] +=== Install and run APM Server + +First, https://www.elastic.co/downloads/apm/apm-server[download APM Server] for your operating system and extract the package. + +In a production environment you would put APM Server on its own machines, +similar to how you run Elasticsearch. +You _can_ run it on the same machines as Elasticsearch, +but this is not recommended, +as the processes will be competing for resources. + +To start APM Server, run: + +[source,bash] +---------------------------------- +./apm-server -e +---------------------------------- + +It will try to connect to Elasticsearch on localhost port 9200 and expose an API to agents on port 8200. +You can change the defaults by supplying different addresses on the command line: + +[source,bash] +---------------------------------- +./apm-server -e -E output.elasticsearch.hosts=ElasticsearchAddress:9200 -E apm-server.host=localhost:8200 +---------------------------------- + +Or you can update the `apm-server.yml` configuration file: + +[source,yaml] +---------------------------------- +apm-server: + host: localhost:8200 + +output: + elasticsearch: + hosts: ElasticsearchAddress:9200 +---------------------------------- + +NOTE: If you are using an X-Pack secured version of Elastic Stack, +you need to specify credentials in the config file: + +[source,yaml] +---- +output.elasticsearch: + hosts: ["ElasticsearchAddress:9200"] + username: "elastic" + password: "elastic" +---- + + + +[[secure-api-access]] +[float] +==== Secure access to the API +If you change the listen address from `localhost` to something that is accessible from outside of the machine, +we recommend setting up firewall rules to ensure that only your own systems can access the API. +Alternatively, +you can use a {apm-server-ref}/securing-apm-server.html[secret token and TLS]. + +If you have APM Server running on the same host as your service, you can configure it to listen on a Unix domain socket. + +[[kibana-dashboards]] +[float] +==== Install the Kibana dashboards + +From APM Server and Kibana 6.4 on you can load dashboards directly via the {kibana-ref}/apm-getting-started.html[APM +Kibana UI]. + +See an example screenshot of a Kibana dashboard: + +image::kibana-dashboard.png[Screenshot of a Kibana Dashboard] + +[[more-information]] +[float] +=== More information +For detailed instructions on how to install and secure APM Server in your server environment, +including details on how to run APM Server in a highly available environment, +please see {apm-server-ref}/index.html[APM Server documentation]. + +Once APM Server is up and running, +you need to install an agent in your service. + +[[agents]] +[float] +=== Install and configure APM agents + +Agents are written in the same language as your service. +Currently Elastic APM has agents for Node.js, Python, Ruby, Java, Go, and JavaScript. + +Setting up a new service to be monitored requires installing the agent, +and configuring it with the address of your APM Server and the service name. + +[[choose-service-name]] +[float] +==== Choose a service name + +The service name is used by Elastic APM to differentiate between data coming from different services. + +Elastic APM includes the service name field on every document that it saves in Elasticsearch. +If you change the service name after using Elastic APM, +you will see the old service name and the new service name as two separate services. +Make sure you choose a good service name before you get started. + +The service name can only contain alphanumeric characters, +spaces, +underscores, +and dashes (must match `^[a-zA-Z0-9 _-]+$`). + +[[nodejs-agent]] +[float] +==== Install the Node.js agent + +Install the elastic-apm-node module from npm in your service: + +[source,bash] +---------------------------------- +npm install elastic-apm-node --save +---------------------------------- + +Then configure the elastic-apm-node module by adding the following lines to the top of your code: + +[source,javascript] +---------------------------------- +// Add this to the VERY top of the first file loaded in your app +var apm = require('elastic-apm-node').start({ + // Override service name from package.json + // Allowed characters: a-z, A-Z, 0-9, -, _, and space + serviceName: '', + + // Use if APM Server requires a token + secretToken: '', + + // Set custom APM Server URL (default: http://localhost:8200) + serverUrl: '' +}) +---------------------------------- + +The Node.js agent automatically instruments Express, +hapi, +Koa, +and Restify out of the box. +See the {apm-node-ref}/index.html[APM Node.js Agent documentation] for more details. + +[[python-agent]] +[float] +==== Install the Python agent + +Install the Elastic APM module from pypi: + +[source,bash] +---------------------------------- +pip install elastic-apm +---------------------------------- + +The Python agent automatically instruments Django and Flask out of the box. +See the {apm-py-ref}/index.html[APM Python Agent documentation] for more details. + +[[ruby-agent]] +[float] +==== Install the Ruby agent + +Add the gem to your Gemfile: + +[source,bash] +---------------------------------- +gem 'elastic-apm' +---------------------------------- + +Create a config file config/elastic_apm.yml: + +[source,bash] +---------------------------------- +server_url: http://localhost:8200 +secret_token: '' +---------------------------------- + +The Ruby agent automatically instruments Rails out of the box. +See the {apm-ruby-ref}/index.html[APM Ruby Agent documentation] for more details. + +[[java-agent]] +[float] +==== Install the Java Agent + +When starting your application, add the JVM flag: +`-javaagent:/path/to/elastic-apm-agent-.jar` + +For a Spring Boot application, start your application and add the `-javaagent` JVM flag: + +[source,bash] +---- +java -javaagent:/path/to/elastic-apm-agent-.jar -jar my-spring-boot-application.jar +---- + +For Apache Tomcat, add the `javaagent` flag. Create `$CATALINA_BASE/bin/setenv.sh` (or modify if the file already exists). Add the following line: + +[source,bash] +.setenv.sh +---- +export CATALINA_OPTS="$CATALINA_OPTS -javaagent:/path/to/elastic-apm-agent-.jar" +---- + +The Java agent automatically instruments Servlet API, Spring MVC, and Spring Boot out of the box. +See the {apm-java-ref}/index.html[APM Java Agent documentation] for more details. + +[[go-agent]] +[float] +==== Install the Go Agent + +At present, Go applications must be instrumented manually at the source code level. Where possible, you should use the {apm-go-ref}/instrumenting-source.html#builtin-modules[built-in instrumentation modules] to report transactions served by web and RPC frameworks in your application. If the built-in modules are not suitable, please refer to {apm-go-ref}/instrumenting-source.html#custom-instrumentation[Custom Instrumentation]. + +The Go agent automatically instruments Gorilla and Gin, as well as support for Go's built-in net/http and database/sql drivers. +See the {apm-go-ref}/index.html[APM Go Agent documentation] for more details. + +[[rum-agent]] +[float] +==== Install the RUM JavaScript agent + +Install the agent as a dependency to your application: + +[source,bash] +---------------------------------- +npm install elastic-apm-js-base --save +---------------------------------- + +Configure the agent: + +[source,bash] +---------------------------------- +import { init as initApm } from 'elastic-apm-js-base' +var apm = initApm({ + + // Set required service name (allowed characters: a-z, A-Z, 0-9, -, _, and space) + serviceName: '', + + // Set custom APM Server URL (default: http://localhost:8200) + serverUrl: 'http://localhost:8200', + + // Set service version (required for sourcemap feature) + serviceVersion: '' +}) +---------------------------------- + +See the {apm-rum-ref}/index.html[APM RUM JavaScript Agent documentation] for more details. diff --git a/docs/guide/overview.asciidoc b/docs/guide/overview.asciidoc new file mode 100644 index 00000000000..72033d2e803 --- /dev/null +++ b/docs/guide/overview.asciidoc @@ -0,0 +1,51 @@ +[[overview]] +== Overview + +Elastic APM is an application performance monitoring system built on the Elastic Stack. +It allows you to monitor software services and applications in real time, +collecting detailed performance information on response time for incoming requests, +database queries, +calls to caches, +external HTTP requests, +etc. +This makes it easier to pinpoint and fix performance problems quickly. + +Elastic APM also collects automatically unhandled errors and exceptions. +Errors are grouped based primarily on the stacktrace, +so you can identify new errors as they appear and keep an eye on how many times specific errors happen. + +NOTE: This guide will indiscriminately use the word service for both services and applications. + +[[components]] +[float] +=== APM Components + +Elastic APM consists of four components: + +* {ref}/index.html[Elasticsearch] +* {apm-agents-ref}/index.html[APM agents] +* {apm-server-ref}/index.html[APM Server] +* {kibana-ref}/xpack-apm.html[Kibana APM UI] + +image::apm-architecture.png[Architecture of Elastic APM] + +*APM agents* are open source libraries written in the same language as your service. +You install them into your service as you would install any other library. +They instrument your code and collect performance data and errors at runtime. +This data is buffered for a short period and sent on to APM Server. + +Out of the box, +APM automatically instruments web frameworks, +database drivers, +calls to caching servers, +and HTTP libraries for requests to external services. +Additionally, +agents provide an API to manually instrument anything else you find interesting. + +*APM Server* is an open source application written in Go which typically runs on dedicated servers. +It listens on port 8200 by default and receives data from agents through a JSON HTTP API. +Then it creates documents from that data and stores them in Elasticsearch. + +To visualize the data after it's sent to Elasticsearch, +you can use the the dedicated APM UI bundled in X-Pack, +or the pre-built open source Kibana dashboards that can be loaded directly via the {kibana-ref}/apm-getting-started.html[APM Kibana UI]. \ No newline at end of file From 39c7472c8631751ef3380484bfdb97c6b274eec0 Mon Sep 17 00:00:00 2001 From: Brandon Morelli Date: Mon, 5 Nov 2018 20:39:05 -0800 Subject: [PATCH 02/12] docs: move events and add distributed tracing --- docs/errors.asciidoc | 21 ------------- docs/event-types.asciidoc | 13 -------- docs/guide/apm-data-model.asciidoc | 6 ++-- docs/guide/distributed-tracing.asciidoc | 6 ++++ docs/guide/index.asciidoc | 2 ++ docs/index.asciidoc | 2 -- docs/transactions.asciidoc | 40 ------------------------- 7 files changed, 11 insertions(+), 79 deletions(-) delete mode 100644 docs/errors.asciidoc delete mode 100644 docs/event-types.asciidoc create mode 100644 docs/guide/distributed-tracing.asciidoc delete mode 100644 docs/transactions.asciidoc diff --git a/docs/errors.asciidoc b/docs/errors.asciidoc deleted file mode 100644 index 2c046199294..00000000000 --- a/docs/errors.asciidoc +++ /dev/null @@ -1,21 +0,0 @@ -[[errors]] -== Errors - -Errors are identified by a unique ID. -An error event contains at least -information about the original `exception` that occured -or about a `log` created when the exception occured. - -Both the captured `exception` and the captured `log` of an error can contain a `stack trace`, -helpful for debugging. - -The `culprit` of an error indicates where is originated. - -An error might relate to the <> during which it happened, -via the `transaction.id`. - -Errors also have some contextual data. - -include::./context.asciidoc[] - -Errors are stored in <>. diff --git a/docs/event-types.asciidoc b/docs/event-types.asciidoc deleted file mode 100644 index 070d92df1f6..00000000000 --- a/docs/event-types.asciidoc +++ /dev/null @@ -1,13 +0,0 @@ -[[event-types]] -= Event Types - -[partintro] --- -Agents capture different types of information, namely `transactions` and `errors`. - -* <> -* <> --- - -include::./transactions.asciidoc[] -include::./errors.asciidoc[] diff --git a/docs/guide/apm-data-model.asciidoc b/docs/guide/apm-data-model.asciidoc index ea5ac049cd2..5449846bfe9 100644 --- a/docs/guide/apm-data-model.asciidoc +++ b/docs/guide/apm-data-model.asciidoc @@ -61,13 +61,13 @@ Spans are stored in {apm-server-ref}/span-indices[span indices]. Note that these Errors are identified by a unique ID. An error event contains at least -information about the original `exception` that occured -or about a `log` created when the exception occured. +information about the original `exception` that occurred +or about a `log` created when the exception occurred. Both the captured `exception` and the captured `log` of an error can contain a `stack trace`, helpful for debugging. -The `culprit` of an error indicates where is originated. +The `culprit` of an error indicates where it originated. An error might relate to the <> during which it happened, via the `transaction.id`. diff --git a/docs/guide/distributed-tracing.asciidoc b/docs/guide/distributed-tracing.asciidoc new file mode 100644 index 00000000000..54f1cb412f9 --- /dev/null +++ b/docs/guide/distributed-tracing.asciidoc @@ -0,0 +1,6 @@ +[[distributed-tracing]] +== Distributed tracing + +Elastic APM supports distributed tracing to enable you to analyze performance through your microservices architecture by tracing all the requests from the initial web request to your front-end service, to queries made to your back-end services all in one view. This makes finding possible bottlenecks through-out your application much easier and faster. No additional configuration is needed to get distributed tracing, but please see the supported technologies for the agent in use to get the most out of the integration. + +The APM UI in Kibana also supports displaying traces and an updated Timeline visualization has been redesigned to show all the transactions from individual services that are connected in a trace. As we're still working on enhancing the distributed tracing experience in the UI, we're flagging the feature beta in the UI. Should you still encounter any bug or issues, please reach out in our https://discuss.elastic.co/c/apm[discussion forum]. \ No newline at end of file diff --git a/docs/guide/index.asciidoc b/docs/guide/index.asciidoc index f780c2f593d..f8e69e036cd 100644 --- a/docs/guide/index.asciidoc +++ b/docs/guide/index.asciidoc @@ -14,6 +14,8 @@ include::./overview.asciidoc[] include::./apm-data-model.asciidoc[] +include::./distributed-tracing.asciidoc[] + include::./install-and-run.asciidoc[] include::./agent-server-compatibility.asciidoc[] diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 2f8ac88c7e5..c3c30798cfb 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -46,8 +46,6 @@ include::./copied-from-beats/shared-securing-beat.asciidoc[] include::./copied-from-beats/monitoring/monitoring-beats.asciidoc[] -include::./event-types.asciidoc[] - include::./rum.asciidoc[] include::./data-ingestion.asciidoc[] diff --git a/docs/transactions.asciidoc b/docs/transactions.asciidoc deleted file mode 100644 index b73d74d7e3c..00000000000 --- a/docs/transactions.asciidoc +++ /dev/null @@ -1,40 +0,0 @@ -[[transactions]] -== Transactions - -A transaction describes an event captured by an Elastic APM agent instrumeting a service. -A transaction can for example be an HTTP request or an asynchrounous background job. - -It contains the timestamp and duration of the event, -a unique id, type, name, -a result (for example a response code), some contextual data, and other relevant information depending on the agent. -For instance, the JavaScript RUM captures transaction marks, -which are points in time relative to the start of the transaction with some label. - -[[transactions-context]] -include::./context.asciidoc[] - -Agents decide whether to sample transactions or not, and provide settings to control sampling behaviour. -If sampled, -the <> of a transaction are sent and stored seperate documents. - -Transactions are stored in <>. - -[[transaction-spans]] -[float] -=== Spans -Transactions can have 0, 1, or many spans. Spans have a `transaction.id` attribute that refer to their transaction. - -A span contains information about a specific code path, -executed as part of a transaction. Such information includes start time, duration, name, type, and optionally a `stack trace`. - -If for example a database query happens within a sampled transaction, -a span describing the database query will be created. -In such a case the name of the span will contain information about the query itself, -and the type about the database. - -Typically, agents instrument automatically a variety of libraries, -but also provide an API for ad hoc instrumentation of specific code paths. - -Transactions are stored in <> and spans are stored in separated indices by default. - -Spans are stored in <>. From c632c8c3df8e0326589618fb8dc27559b2ff4370 Mon Sep 17 00:00:00 2001 From: Brandon Morelli Date: Tue, 6 Nov 2018 16:45:23 -0800 Subject: [PATCH 03/12] docs: add more v2 and change layout --- docs/common-problems.asciidoc | 12 +- docs/configuration-process.asciidoc | 2 +- docs/configuration-rum.asciidoc | 20 +-- docs/configuring-ingest.asciidoc | 2 +- docs/configuring-output-after.asciidoc | 2 +- docs/error-api.asciidoc | 101 ++------------ docs/events-api.asciidoc | 91 +++++++++++++ docs/exploring-es-data.asciidoc | 7 +- .../guide/agent-server-compatibility.asciidoc | 17 +-- docs/guide/apm-data-model.asciidoc | 16 +++ docs/intake-api.asciidoc | 30 +++-- docs/metadata-api.asciidoc | 15 +++ docs/metricset-api.asciidoc | 7 + docs/metricset-indices.asciidoc | 13 ++ docs/overview.asciidoc | 5 +- docs/rum.asciidoc | 6 +- docs/security.asciidoc | 14 +- docs/server-info.asciidoc | 3 +- docs/storage-management.asciidoc | 12 +- docs/transaction-api.asciidoc | 105 +++------------ docs/upgrading-to-65.asciidoc | 124 ++++++++++++++++++ docs/upgrading.asciidoc | 22 ++-- 22 files changed, 382 insertions(+), 244 deletions(-) create mode 100644 docs/events-api.asciidoc create mode 100644 docs/metadata-api.asciidoc create mode 100644 docs/metricset-api.asciidoc create mode 100644 docs/metricset-indices.asciidoc create mode 100644 docs/upgrading-to-65.asciidoc diff --git a/docs/common-problems.asciidoc b/docs/common-problems.asciidoc index aa0e807980a..b769ee6a464 100644 --- a/docs/common-problems.asciidoc +++ b/docs/common-problems.asciidoc @@ -36,8 +36,8 @@ Particularly, if you are using Docker, ensure to bind to the right interface (fo -If you see requests coming trough the APM Server but they are not accepted (response code other than `202`), consider -the response code to narrow down the possible causes (see sections bellow). +If you see requests coming through the APM Server but they are not accepted (response code other than `202`), consider +the response code to narrow down the possible causes (see sections below). Another reason for data not showing up is that the agent is not auto-instrumenting something you were expecting, check the {apm-agents-ref}/index.html[agent documentation] for details on what is automatically instrumented. @@ -46,7 +46,7 @@ the {apm-agents-ref}/index.html[agent documentation] for details on what is auto [float] === HTTP 400: Data decoding error / Data validation error -The most likely cause for this is that you are using non compatible versions of agent and APM Server. +The most likely cause for this is that you are using incompatible versions of agent and APM Server. For instance, APM Server 6.2.0 changed the Intake API spec and requires a minimum version of each agent. [[unauthorized]] @@ -157,20 +157,20 @@ To resolve this problem, try one of these solutions: * Create a DNS entry for the hostname mapping it to the server's IP. * Create an entry in `/etc/hosts` for the hostname. Or on Windows add an entry to `C:\Windows\System32\drivers\etc\hosts`. -* Re-create the server certificate and add a SubjectAltName (SAN) for the IP address of the server. This make the +* Re-create the server certificate and add a SubjectAltName (SAN) for the IP address of the server. This makes the server's certificate valid for both the hostname and the IP address. [float] [[getsockopt-no-route-to-host]] ===== getsockopt: no route to host -This is not a SSL problem. It's a networking problem. Make sure the two hosts can communicate. +This is not an SSL problem. It's a networking problem. Make sure the two hosts can communicate. [float] [[getsockopt-connection-refused]] ===== getsockopt: connection refused -This is not a SSL problem. Make sure that Logstash is running and that there is no firewall blocking the traffic. +This is not an SSL problem. Make sure that Logstash is running and that there is no firewall blocking the traffic. [float] [[target-machine-refused-connection]] diff --git a/docs/configuration-process.asciidoc b/docs/configuration-process.asciidoc index d7d28db0d41..cb27fec7957 100644 --- a/docs/configuration-process.asciidoc +++ b/docs/configuration-process.asciidoc @@ -87,7 +87,7 @@ Disabled by default. Authorization token for sending data to the APM server. If a token is set, the agents must send it in the following format: Authorization: Bearer . -The token is not used for RUM endpoints. By default no authorization token is set. +The token is not used for RUM endpoints. By default, no authorization token is set. It is recommended to use an authorization token in combination with SSL enabled. Read more about <> and the <>. diff --git a/docs/configuration-rum.asciidoc b/docs/configuration-rum.asciidoc index c6dd09cc6ed..8a211e35655 100644 --- a/docs/configuration-rum.asciidoc +++ b/docs/configuration-rum.asciidoc @@ -34,25 +34,25 @@ Default value is set to 10. [float] ==== `allow_origins` -Comma separated list of permitted origins for RUM supprt. +Comma separated list of permitted origins for RUM support. User-agents send an Origin header that will be validated against this list. This is done automatically by modern browsers as part of the https://www.w3.org/TR/cors/[CORS specification]. -An origin is made of a protocol scheme, host and port, without the url path. +An origin is made of a protocol scheme, host and port, without the URL path. Default value is set to `['*']`, which allows everything. [float] ==== `library_pattern` -Regexp to be matched against a stacktrace frame's `file_name` and `abs_path` attributes. -If the regexp matches, the stacktrace frame is considered to be a library frame. -When source mapping is applied the `error.culprit` is set to reflect the _function_ and the _filename_ -of the first not library frame. +RegExp to be matched against a stacktrace frame's `file_name` and `abs_path` attributes. +If the RegExp matches, the stacktrace frame is considered to be a library frame. +When source mapping is applied, the `error.culprit` is set to reflect the _function_ and the _filename_ +of the first non library frame. This aims to provide an entry point for identifying issues. Default value is `"node_modules|bower_components|~"`. [float] ==== `exclude_from_grouping` -Regexp to be matched against a stacktrace frame's `file_name`. -If the regexp matches, the stacktrace frame is excluded from being used for calculating error groups. +RegExp to be matched against a stacktrace frame's `file_name`. +If the RegExp matches, the stacktrace frame is excluded from being used for calculating error groups. The default pattern excludes stacktrace frames that have a filename starting with `/webpack`. [[rum-sourcemap-cache]] @@ -69,10 +69,10 @@ Default value is 5 minutes. ==== `source_mapping.elasticsearch` Configure the Elasticsearch source map retrieval location, taking the same options as <>. This must be set when using an output other than Elasticsearch, and that output is writing to Elasticsearch. -Othewise leave this section empty. +Otherwise leave this section empty. [float] ==== `source_mapping.index_pattern` -Source maps are stored in a seperate index `apm-%{[beat.version]}-sourcemap` by default. +Source maps are stored in a separate index `apm-%{[beat.version]}-sourcemap` by default. If changed, a matching index pattern needs to be specified here. diff --git a/docs/configuring-ingest.asciidoc b/docs/configuring-ingest.asciidoc index 47aaa63ad9b..8dcfeacda99 100644 --- a/docs/configuring-ingest.asciidoc +++ b/docs/configuring-ingest.asciidoc @@ -26,7 +26,7 @@ Automatic pipeline registration requires `output.elasticsearch` to be enabled an as well as having the required Elasticsearch plugins installed. APM Server default pipelines require you to have the Ingest user agent plugin installed. -If pipelines are enabeld but required plugins are missing, +If pipelines are enabled but required plugins are missing, the APM Server will not be able to properly connect to the Elasticsearch output. Find the default pipeline configuration at `ingest/pipeline/definition.json` of your APM Server installation's home directory. diff --git a/docs/configuring-output-after.asciidoc b/docs/configuring-output-after.asciidoc index 04f3788243f..d2934d87aab 100644 --- a/docs/configuring-output-after.asciidoc +++ b/docs/configuring-output-after.asciidoc @@ -24,4 +24,4 @@ fields: {project: "myproject", instance-id: "574734885120952459"} ------------------------------------------------------------------------------ To store the custom fields as top-level fields, set the `fields_under_root` option to true. -This is not recommended as backwards compatibility can not be ensured as new fields are added to APM documents. +This is not recommended as when new fields are added to APM documents backward compatibility cannot be ensured. diff --git a/docs/error-api.asciidoc b/docs/error-api.asciidoc index 432b2b1c4d5..e65edb9fec3 100644 --- a/docs/error-api.asciidoc +++ b/docs/error-api.asciidoc @@ -1,115 +1,40 @@ [[error-api]] -== Error API +=== Error API The following section contains information about: -* <> * <> +* <> * <> -[[error-endpoint]] -[float] -=== Endpoint - -Send a `HTTP POST` request to the APM Server `errors` endpoint: -[source,bash] ------------------------------------------------------------- -http(s)://{hostname}:{port}/v1/errors ------------------------------------------------------------- - -For <> send a `HTTP POST` request to the `rum errors` endpoint instead: - -[source,bash] ------------------------------------------------------------- -http(s)://{hostname}:{port}/v1/rum/errors ------------------------------------------------------------- - [[error-schema-definition]] [float] -=== Schema Definition - -The APM Server uses JSON Schema for validating requests. The specification for errors is defined bellow: -* <> -* <> -* <> -* <> -* <> -* <> -* <> - - -[[error-payload-schema]] -[float] -==== Payload - -[source,json] ----- -include::./spec/errors/v1_error.json[] ----- +==== Schema Definition -[[error-service-schema]] +The APM Server uses JSON Schema for validating requests. The specification for errors is defined below: [float] -==== Service +[[error-schema]] +===== Error [source,json] ---- -include::./spec/service.json[] ----- - -[[error-system-schema]] -[float] -==== System - -[source,json] +include::./spec/errors/common_error.json[] ---- -include::./spec/system.json[] ----- - -[[error-context-schema]] -[float] -==== Context - -[source,json] ----- -include::./spec/context.json[] ----- - -[[error-stacktraceframe-schema]] -[float] -==== Stacktrace Frame - -[source,json] ----- -include::./spec/stacktrace_frame.json[] ----- - -[[error-request-schema]] -[float] -==== Request - -[source,json] ----- -include::./spec/request.json[] ----- - -[[error-user-schema]] -[float] -==== User [source,json] ---- -include::./spec/user.json[] +include::./spec/errors/v2_error.json[] ---- [[error-api-examples]] [float] -=== Examples +==== Examples Request example: ["source","sh",subs="attributes"] ------------------------------------------------------------ -curl http://localhost:8200/v1/errors \ +curl http://localhost:8200/intake/v2/events \ --header "Content-Type: application/json" \ --data @docs/data/intake-api/generated/error/payload.json ------------------------------------------------------------ @@ -122,7 +47,7 @@ Example error requests: [[payload-with-error]] [float] -==== Payload with an Error +===== Payload with an Error [source,json] ---- @@ -131,7 +56,7 @@ include::./data/intake-api/generated/error/payload.json[] [[payload-with-minimal-exception]] [float] -==== Payload with an Error with minimal Exception Information +===== Payload with an Error with minimal Exception Information [source,json] ---- @@ -140,7 +65,7 @@ include::./data/intake-api/generated/error/minimal_payload_exception.json[] [[payload-with-minimal-log]] [float] -==== Payload with an Error with minimal Log Information +===== Payload with an Error with minimal Log Information [source,json] ---- diff --git a/docs/events-api.asciidoc b/docs/events-api.asciidoc new file mode 100644 index 00000000000..22aafee579a --- /dev/null +++ b/docs/events-api.asciidoc @@ -0,0 +1,91 @@ +[[events-api]] +== Events API + +The following section contains information about: + +* <> +* <> +* <> +* <> + +Agents capture different types of information, known as events. These events are sent to a single endpoint which then sorts and processes the events. Events can be: + +* Transactions +* Spans +* Errors +* Metricsets + +You can learn more about events in the {apm-get-started-ref}/apm-data-model.html[APM Data Model] documentation. + +[[events-api-endpoint]] +[float] +=== Endpoint +Send an `HTTP POST` request to the APM Server `intake/v2/events` endpoint: + +[source,bash] +------------------------------------------------------------ +http(s)://{hostname}:{port}/intake/v2/events +------------------------------------------------------------ + +For <> send an `HTTP POST` request to the APM Server `intake/v2/rum/events` endpoint instead: + +[source,bash] +------------------------------------------------------------ +http(s)://{hostname}:{port}/intake/v2/rum/events +------------------------------------------------------------ + +[[events-api-response]] +[float] +=== Response + +On success, the server will respond with a 202 Accepted status code. + +[[events-api-errors]] +[float] +=== Errors + +Events are processed one after the either. If an error is encountered while processing an event, the error encountered, as well as the document causing the error are added to an internal array. The server will wait to encounter up to five errors before sending a response to the agent. If there are multiple errors, the highest status code is returned. + +As an example, here's the error response for invalid JSON: + +[source,json] +------------------------------------------------------------ +{ + "accepted": 1, + "errors": [ + { + "document": "{ \"invalid-json\" }", + "message": "data read error: invalid character '}' after object key" + } + ] +} +------------------------------------------------------------ + +Some errors, not relating to specific events, may be immediately returnable. +For example: Request limit reached, bad header, or internal queue full. +If at any point one of these errors is encountered, +it is added to the internal array and immediately returned. + +If you're developing an agent, these errors can be useful for debugging your agent while building it. + +// todo: check troubleshooting section +The queue is full error is interesting, and can indicate that you probably want to reduce your sampling rate, get a bigger APM Server or elasticsearch - depending on where your bottlenecks are. + +// todo: what errors are unrecoverable? +NOTE: If the APM Server runs into certain errors, it may not consume the whole body from the agent. The server just stops reading if the error is unrecoverable. Agents should be able to handle these situations + +[[events-api-schema-definition]] +[float] +=== Event Schema + +The APM Server uses a JSON Schema for validating requests: + +* <> +* <> +* <> +* <> + +include::./metadata-api.asciidoc[] +include::./transaction-api.asciidoc[] +include::./error-api.asciidoc[] +include::./metricset-api.asciidoc[] \ No newline at end of file diff --git a/docs/exploring-es-data.asciidoc b/docs/exploring-es-data.asciidoc index 65969df3510..a95ec9589dd 100644 --- a/docs/exploring-es-data.asciidoc +++ b/docs/exploring-es-data.asciidoc @@ -6,7 +6,9 @@ By default Elastic APM data is stored in separated indices following the format: `apm-%{[version]}-{type}-%{+yyyy.MM.dd}`. -Data types are described <>. +NOTE: If you are updating APM Server from version 6.3 or earlier, you may still be using an outdated `apm-server.yml` file. You must update your `apm-server.yml` file in order to take advantage of the new format of indices. + +Data types are described {apm-get-started-ref}/apm-data-model.html[here]. To get an overview of existing indices you can run: ["source","sh"] @@ -39,7 +41,7 @@ GET apm-*transactions-*/_search // CONSOLE If you are interested in the _settings_ and _mappings_ of the Elastic APM indices, -first run a query to find template names: +first, run a query to find template names: ["source","sh"] ------------------------------------------------------------ @@ -65,4 +67,5 @@ When clicking on a specific index you can view the _settings_ and _mapping_ for include::./transaction-indices.asciidoc[] include::./span-indices.asciidoc[] include::./error-indices.asciidoc[] +include::./metricset-indices.asciidoc[] include::./sourcemap-indices.asciidoc[] diff --git a/docs/guide/agent-server-compatibility.asciidoc b/docs/guide/agent-server-compatibility.asciidoc index 0c0942b2da1..7b76fa9f23a 100644 --- a/docs/guide/agent-server-compatibility.asciidoc +++ b/docs/guide/agent-server-compatibility.asciidoc @@ -21,8 +21,9 @@ When instrumenting an APM Agent within your codebase, it's important to use a co [options="header"] |======================================================================= |Agent Version |APM Server Version -|1.x |6.2.0-6.5.0 -|2.x |6.5.0+ +|1.x |6.1 +|2.x/3.x |6.2-6.x +|4.0+ |6.5.0+ |======================================================================= [float] @@ -32,8 +33,8 @@ When instrumenting an APM Agent within your codebase, it's important to use a co [options="header"] |======================================================================= |Agent Version |APM Server Version -|1.x |6.2.0-6.5.0 -|2.x |6.5.0+ +|1.x |<6.5.x +|2.0+ |6.5.0+ |======================================================================= [float] @@ -43,8 +44,7 @@ When instrumenting an APM Agent within your codebase, it's important to use a co [options="header"] |======================================================================= |Agent Version |APM Server Version -|1.x |6.2.0-6.5.0 -|2.x |6.5.0+ +|1.0+ |6.5.0+ |======================================================================= [float] @@ -65,6 +65,7 @@ When instrumenting an APM Agent within your codebase, it's important to use a co [options="header"] |======================================================================= |Agent Version |APM Server Version -|1.x |6.2.0-6.5.0 -|2.x |6.5.0+ +|0.x |6.3.x-6.4.0 +|1.x |6.4.x +|2.0+ |6.5.0+ |======================================================================= diff --git a/docs/guide/apm-data-model.asciidoc b/docs/guide/apm-data-model.asciidoc index 5449846bfe9..a250138f4ab 100644 --- a/docs/guide/apm-data-model.asciidoc +++ b/docs/guide/apm-data-model.asciidoc @@ -56,6 +56,22 @@ and the type of this span will contain information about the database type. Spans are stored in {apm-server-ref}/span-indices[span indices]. Note that these indices are separate from {apm-server-ref}/transaction-indices[transaction indices] by default. +[float] +[[dropped-spans]] +==== Dropped Spans + +For performance reasons, APM agents can choose to purposefully omit spans. One example of this might be for long running transactions with over 100 transactions. These edge cases can overload both the agent and the APM Server. To avoid this, Agents can drop spans. When they do this, they notify the server of exactly how many spans were dropped. This note is then passed on to the user in the UI. + +Settings affecting dropped spans, and more details on why they might occur, are available in the relevant {apm-agents-ref}/index.html[APM Agent] documentation. + +[float] +[[missing-spans]] +==== Missing Spans + +Similarly to dropped spans, transactions may be missing spans. An agent informs the server of how many spans it has "started", but sometimes, not all of the spans make it to the server. This could be because + +Because spans are sent to the APM Server separately from their transaction, it may happen that we lose a transaction, but have spans belonging that transaction stored in Elasticsearch. The reverse can also happen, a transaction is successfully sent and stored in Elasticsearch, but one or more of it's spans are lost. + [[errors]] === Errors diff --git a/docs/intake-api.asciidoc b/docs/intake-api.asciidoc index f2783a51b67..8f6d31ae6ec 100644 --- a/docs/intake-api.asciidoc +++ b/docs/intake-api.asciidoc @@ -2,17 +2,31 @@ = Intake API [partintro] -- -The Intake API is HTTP JSON based. The APM Server exposes endpoints for +NOTE: Most users will not need to interact directly with the intake API. Therefor, unless you are implementing an agent, you don't need to know about the specifics of this API. -* <> -* <> +The Intake API is what we call the internal protocol that APM agents use to talk to the APM Server. This API has been redesigned in 6.5. Read the <> for more information. + +APM Agents communicate with the APM server by sending events in an HTTP request. Each event is sent as its own line in the HTTP request body. This is known as http://ndjson.org[newline delimited JSON (NDJSON)]. The request body looks roughly like this: + +[source,bash] +------------------------------------------------------------ +{"metadata": {"service": {"name": "ecommerce-front"}}} +{"span": {"name": "SELECT FROM products", "duration": 323, "transaction_id": "A"}} +{"span": {"name": "SELECT FROM users", "duration": 202, "transaction_id": "A"}} +{"transaction": {"name": "GET /index", "id": "A"}} +------------------------------------------------------------ + +With NDJSON, agents can open an HTTP POST request and use chunked encoding to stream events to the APM Server as soon as they are recorded in the agent. This makes it simple for agents to serialize each event to a stream of newline delimited JSON. + +The APM Server also treats the HTTP body as a compressed stream and thus reads and handles each event independently. + +The APM Server exposes endpoints for + +* <> * <> * <> -Unless you are implementing an agent, you don't need to know about the specifics of this API. - -- -include::./transaction-api.asciidoc[] -include::./error-api.asciidoc[] +include::./events-api.asciidoc[] include::./sourcemap-api.asciidoc[] -include::./server-info.asciidoc[] +include::./server-info.asciidoc[] \ No newline at end of file diff --git a/docs/metadata-api.asciidoc b/docs/metadata-api.asciidoc new file mode 100644 index 00000000000..8c546ab66c9 --- /dev/null +++ b/docs/metadata-api.asciidoc @@ -0,0 +1,15 @@ +[[metadata-schema]] +=== Metadata + +Every new connection to the APM Server starts with a `metadata` stanza. This provides general metadata concerning the other objects in the stream. + +Rather then send this metadata stanza multiple times, the APM Server hangs on to this information and reuses it to apply to every document that goes into elasticsearch. + +The specification for metadata is outlined below: + +[source,json] +---- +include::./spec/metadata.json[] +---- + +NOTE: Metadata is stored under `context` when viewing documents in Elasticsearch. \ No newline at end of file diff --git a/docs/metricset-api.asciidoc b/docs/metricset-api.asciidoc new file mode 100644 index 00000000000..f4c42167688 --- /dev/null +++ b/docs/metricset-api.asciidoc @@ -0,0 +1,7 @@ +[[metric-schema]] +=== Metricsets + +[source,json] +---- +include::./spec/metricsets/v2_metricset.json[] +---- \ No newline at end of file diff --git a/docs/metricset-indices.asciidoc b/docs/metricset-indices.asciidoc new file mode 100644 index 00000000000..3ac705c1c6f --- /dev/null +++ b/docs/metricset-indices.asciidoc @@ -0,0 +1,13 @@ +[[metricset-indices]] +== Metricset Indices + +Metricsets are by default stored to indices of the format `apm-[version]-metric-[date]`. + +[[metricset-example]] +[float] +=== Example Document +See an example metricset indexed in Elasticsearch: +[source,json] +---- +include::./data/elasticsearch/metric.json[] +---- \ No newline at end of file diff --git a/docs/overview.asciidoc b/docs/overview.asciidoc index 056de8606cc..097639f810b 100644 --- a/docs/overview.asciidoc +++ b/docs/overview.asciidoc @@ -14,6 +14,8 @@ To get an overview of the whole Elastic APM system, * {apm-node-ref}/index.html[APM Node.js Agent] * {apm-py-ref}/index.html[APM Python Agent] * {apm-ruby-ref}/index.html[APM Ruby Agent] +* {apm-go-ref}/index.html[APM Go Agent] +* {apm-java-ref}/index.html[APM Java Agent] * {apm-rum-ref}/index.html[APM RUM JavaScript Agent] * {ref}/index.html[Elasticsearch] @@ -26,8 +28,7 @@ The APM Server is a separate component for the following reasons: * It helps to keep the agents as light as possible and since the APM Server is a stateless separate component, it can be scaled independently. * For Real User monitoring data is collected in browsers. - APM Server prevents browsers from interacting directly with Elasticsearch (which poses a security risk), - and controls the amount of data flowing into Elasticsearch. + APM Server prevents browsers from interacting directly with Elasticsearch (which poses a security risk) and controls the amount of data flowing into Elasticsearch. * In cases where Elasticsearch becomes unresponsive, APM Server can buffer data temporarily without adding overhead to the agents. * Acts as a middleware for source mapping for javascript in the browser. diff --git a/docs/rum.asciidoc b/docs/rum.asciidoc index da2a81e259f..8d183e97ecf 100644 --- a/docs/rum.asciidoc +++ b/docs/rum.asciidoc @@ -9,9 +9,9 @@ To use it you need to <> in the APM Server. [[sourcemaps]] == Source Maps -It is common practice to minify JavaScript code, for example to reduce network latency. -While improving performance, minified code is also hard to debug. -A source map library helps by mapping the minified files back the the original source code. +For a number of reasons, it is common practice to minify JavaScript code, for example to reduce network latency. +While improving performance, minified code can be hard to debug. +A source map library helps by mapping the minified files back to the the original source code. APM Server provides a <> for uploading source maps. diff --git a/docs/security.asciidoc b/docs/security.asciidoc index b839e3f89ee..15bf075d3e3 100644 --- a/docs/security.asciidoc +++ b/docs/security.asciidoc @@ -22,11 +22,9 @@ Secret tokens are not applicable for RUM, as they would be publicly exposed. ==== SSL/TLS setup To enable SSL/TLS you need a private key and a certificate issued by a certification authority (CA). -Then you can specify the path to those files in the configuration properties -`apm-server.ssl.key` and -`apm-server.ssl.certificate` -respectively. -This will make the APM Server to serve HTTPS requests instead of HTTP. -Hence, you also need to enable SSL in the agent. -For agent specific details, -please check the {apm-agents-ref}/index.html[agent documentation] for how to do it. +You can then specify the path to those files in your configuration properties: +`apm-server.ssl.key` and `apm-server.ssl.certificate` respectively. +This will make the APM Server serve HTTPS requests instead of HTTP. +Don't forget, you also need to enable SSL in the agent. +For agent specific details on enabling SSL/TLS, +please see the {apm-agents-ref}/index.html[agent documentation]. \ No newline at end of file diff --git a/docs/server-info.asciidoc b/docs/server-info.asciidoc index 11de3259dc3..d13e0228f65 100644 --- a/docs/server-info.asciidoc +++ b/docs/server-info.asciidoc @@ -1,8 +1,7 @@ [[server-info]] == Server Information -The APM Server exposes an API endpoint to query some server information. - +The APM Server exposes an API endpoint to query general server information. This lightweight endpoint is useful as a server up/down healthcheck. [[server-info-endpoint]] diff --git a/docs/storage-management.asciidoc b/docs/storage-management.asciidoc index 96f2adf30db..91aa81863c4 100644 --- a/docs/storage-management.asciidoc +++ b/docs/storage-management.asciidoc @@ -3,11 +3,11 @@ [partintro] -- -In the following section you can learn how to +In the following section you can learn how to: -* <> -* <> -* <> +* <> +* <> +* <> -- [[manage-indices-kibana]] @@ -15,7 +15,7 @@ In the following section you can learn how to The Kibana UI for managing indices allows you to view indices, index settings, mappings, docs count, used storage per index and much more. You can perform management operations, -for example deleting indices directly via the Kibana UI. +for example, deleting indices directly via the Kibana UI. The UI also supports applying bulk operations on several indices at once. Check out the {kibana-ref}/managing-indices.html[Kibana Managing Indices] documentation for more details. @@ -140,7 +140,7 @@ Follow the {kibana-ref}/managing-indices.html[Kibana Index Management] documenta You might want to update documents already indexed, for example if you your service name was set incorrectly. -For this you can use the {ref}/docs-update-by-query.html[Update By Query API]. +For this, you can use the {ref}/docs-update-by-query.html[Update By Query API]. [[update-data-rename-a-service]] [float] diff --git a/docs/transaction-api.asciidoc b/docs/transaction-api.asciidoc index 9718f5c6c6b..6d1f34338b7 100644 --- a/docs/transaction-api.asciidoc +++ b/docs/transaction-api.asciidoc @@ -1,125 +1,56 @@ [[transaction-api]] -== Transaction API +=== Transactions and Spans The following section contains information about: -* <> * <> +* <> +* <> * <> -[[transaction-endpoint]] -[float] -=== Endpoint -Send a `HTTP POST` request to the APM Server `transactions` endpoint: - -[source,bash] ------------------------------------------------------------- -http(s)://{hostname}:{port}/v1/transactions ------------------------------------------------------------- - -For <> send a `HTTP POST` request to the APM Server `rum transactions` endpoint instead: - -[source,bash] ------------------------------------------------------------- -http(s)://{hostname}:{port}/v1/rum/transactions ------------------------------------------------------------- - [[transaction-schema-definition]] [float] -=== Schema Definition +==== Schema Definition -The APM Server uses JSON Schema for validating requests. The specification for transactions is defined bellow: +The APM Server uses JSON Schema for validating requests. The specification for transactions and spans is defined below: -* <> -* <> -* <> -* <> -* <> -* <> -* <> -* <> - -[[transaction-payload-schema]] [float] -==== Payload +[[transaction-schema]] +===== Transaction Schema [source,json] ---- -include::./spec/transactions/v1_transaction.json[] +include::./spec/transactions/common_transaction.json[] ---- -[[transaction-span-schema]] -[float] -==== Span - -[source,json] ----- -include::./spec/spans/v1_span.json[] ----- - -[[transaction-service-schema]] -[float] -==== Service - [source,json] ---- -include::./spec/service.json[] +include::./spec/transactions/v2_transaction.json[] ---- -[[transaction-system-schema]] [float] -==== System - -[source,json] ----- -include::./spec/system.json[] ----- - -[[transaction-context-schema]] -[float] -==== Context - -[source,json] ----- -include::./spec/context.json[] ----- - -[[transaction-stacktraceframe-schema]] -[float] -==== Stacktrace Frame - -[source,json] ----- -include::./spec/stacktrace_frame.json[] ----- - -[[transaction-request-schema]] -[float] -==== Request +[[transaction-span-schema]] +===== Span Schema [source,json] ---- -include::./spec/request.json[] +include::./spec/spans/common_span.json[] ---- -[[transaction-user-schema]] -[float] -==== User - [source,json] ---- -include::./spec/user.json[] +include::./spec/spans/v2_span.json[] ---- [[transaction-api-examples]] [float] -=== Examples +==== Examples Request example: ["source","sh",subs="attributes"] ------------------------------------------------------------ -curl http://localhost:8200/v1/transactions \ +curl http://localhost:8200/intake/v2/events \ --header "Content-Type: application/json" \ --data @docs/data/intake-api/generated/transaction/payload.json ------------------------------------------------------------ @@ -132,7 +63,7 @@ Example transaction requests: [[payload-with-transactions]] [float] -==== Payload with several Transactions +===== Payload with several Transactions [source,json] ---- @@ -141,7 +72,7 @@ include::./data/intake-api/generated/transaction/payload.json[] [[payload-with-minimal-transaction]] [float] -==== Payload with a minimal Transaction +===== Payload with a minimal Transaction [source,json] ---- @@ -150,7 +81,7 @@ include::./data/intake-api/generated/transaction/minimal_payload.json[] [[payload-with-minimal-span]] [float] -==== Payload with a Transaction with a minimal Span +===== Payload with a Transaction with a minimal Span [source,json] ---- diff --git a/docs/upgrading-to-65.asciidoc b/docs/upgrading-to-65.asciidoc new file mode 100644 index 00000000000..32405c27a89 --- /dev/null +++ b/docs/upgrading-to-65.asciidoc @@ -0,0 +1,124 @@ +[[upgrading-to-65]] +=== Upgrading to APM Server v6.5 + +If you're using an Elastic APM Agent, upgrading to version 6.5 of the APM Server is easy - just ensure you're using the most recent version of your applicable APM agent. Check the {apm-get-started-ref}/agent-server-compatibility.html[agent/server compatibility matrix] for details. + +If you're an advanced user, are implementing a custom agent, or provide custom data to Elasticsearch, there are some changes in 6.5 that you should be aware of. + +NOTE: None of the changes outlined below are breaking changes. APM Server 6.5 should still work with your currently implemented agent. To take advantage of new features, it's recommended you read about the changes below and update accordingly. + +[float] +[[intake-api-changes-65]] +==== Intake API Changes + +The Intake API is what we call the internal protocol that APM agents use to talk to the APM Server. +This API has been redesigned in 6.5 to increase the memory efficiency and predictability of the APM Server and APM agents. + +There have been a number of changes to the Intake API for 6.5. Notably, the endpoint itself has changed to `intake/v2/events`. Full documentation of this change is in the <>. + +[float] +[[error-api-changes-65]] +===== Error + +The error schema adds three new properties, and removes two others. The full schema is available here. + +* _remove_ pattern of `error.id`, change format to hex encoded 64 random bits +* _remove_ `transaction.id` +* _add_ `transaction_id` referring to transaction +* _add_ `parent_id` +* _add_ `trace_id` + +[float] +[[transaction-api-changes-65]] +===== Transaction + +The transaction schema adds two new properties, and removes two others. The full schema is available here. + +* _remove_ pattern of `transaction.id`, change format to hex encoded 64 random bits +* _remove_ spans (don't forbid them, just don't process them) +* _add_ `parent_id` +* _add_ `trace_id` (required attribute) + +[float] +[[span-api-changes-65]] +===== Span + +The span schema adds three new properties, removes two, and changes one. The full schema is available here. + +* _change_ pattern of `id` from long to string, format to hex encoded 64 random bits +* _remove_ pattern of `transaction_id` +* _remove_ parent referring to long id +* _add_ `parent_id` referring to transaction or span +* _add_ timestamp +* _add_ `trace_id` (required attribute) + +[float] +[[healthcheck-api-changes-65]] +===== Healthcheck Endpoint + +The Healthcheck endpoint, previously `/healthcheck`, has been deprecated. The <> api has replaced it. + +[float] +[[assets-api-changes-65]] +===== Assets Endpoint + +The assets endpoint, `/assets/v1/sourcemaps`, remains unchanged. + +[float] +[[metrics-api-changes-65]] +===== Metricsets Endpoint + +The Metricsets endpoint, previously `/v1/metrics`, has also integrated with the new intake endpoint at `/intake/v2/events`. In terms of functionality, the only change relates to timestamps. More information is available here. + +[float] +[[server-config-changes-65]] +==== Server Configuration Changes + +On the server there are some configuration options that are worth mentioning: + +`max_unzipped_size`, `max_request_queue_time`, and `concurrent_requests` have all been deprecated with 6.4. + +New with 6.5 is `max_event_size` which has a default value of `307200` bytes. You'll notice this value is much smaller than the previous `max_unzipped_size` value. This is because in APM Server 6.5, an event is much smaller than an event in 6.4. Instead of getting the whole payload at once, and event is just a single line of data. + +`read_timeout` and `write_timeout` remain at 30 seconds, but if you have a load balancer in front of the APM server you may notice connections are a little bit longer lived than they used to be. + +NOTE: If you are updating APM Server from version 6.3 or earlier, you may not be using an updated `apm-server.yml` file. How APM Server stores data in Elasticsearch has changed overtime, and in order to take advantage of new features, you must be using an update `apm-server.yml` file. + +[float] +[[es-schema-changes-65]] +==== Elasticsearch Schema Changes + +//todo: ADD LINKS TO sample docs created by the APM server + +The Elasticsearch schema defines how APM data is stored in Elasticsearch. +There have been a number of changes to the Elasticsearch schema for 6.5. + +[float] +[[es-error-changes-65]] +===== Error + +The Elasticsearch error schema adds two new properties. The full schema is available here. + +* _add_ `error.parent_id` +* _add_ `error.trace_id` + +[float] +[[es-transaction-changes-65]] +===== Transaction + +The Elasticsearch transaction schema adds two new properties. The full schema is available here. + +* _add_ `transaction.parent_id` +* _add_ `transaction.trace_id` + +[float] +[[es-span-changes-65]] +===== Span + +The Elasticsearch span schema adds two new properties. The full schema is available here. + +* _add_ `span.parent_id` +* _add_ `span.trace_id` +* _add_ `span.hex_id` +* _deprecate_ `span.id` +* _deprecate_ `span.parent` \ No newline at end of file diff --git a/docs/upgrading.asciidoc b/docs/upgrading.asciidoc index 84ce51d1624..04187d761e6 100644 --- a/docs/upgrading.asciidoc +++ b/docs/upgrading.asciidoc @@ -1,7 +1,7 @@ [[upgrading]] == Upgrading APM Server -We do our best to keep APM Server backwards compatible between minor releases. +We do our best to keep APM Server backward compatible between minor releases. However, check out the <> section for exceptions. Before upgrading: @@ -31,25 +31,25 @@ and applies the correct template automatically. [[breaking-changes]] === Breaking Changes APM Server is built on top of {beats-ref}/index.html[libbeat]. -As such any breaking change in libbeat is also considered to be a breaking change in APM Server. +As such, any breaking change in libbeat is also considered to be a breaking change in APM Server. [float] ==== HEAD -no breaking changes in APM Server - -https://www.elastic.co/guide/en/beats/libbeat/master/breaking-changes.html[breaking changes in libbeat] +* There are no breaking changes in APM Server. +* Advanced users may find the <> useful. +* https://www.elastic.co/guide/en/beats/libbeat/master/breaking-changes.html[Breaking changes in libbeat] [float] ==== 6.4 -Indexing the `onboarding` document in it's own index by default. - +* Indexing the `onboarding` document in it's own index by default. [float] ==== 6.3 -Indexing events in separate indices by default. - -https://www.elastic.co/guide/en/beats/libbeat/current/breaking-changes-6.3.html[breaking changes in libbeat] +* Indexing events in separate indices by default. +* https://www.elastic.co/guide/en/beats/libbeat/current/breaking-changes-6.3.html[Breaking changes in libbeat] [float] ==== 6.2 -APM Server GA +* APM Server GA + +include::./upgrading-to-65.asciidoc[] \ No newline at end of file From d79b3921cbec6386658f63b1c96741d09578b43e Mon Sep 17 00:00:00 2001 From: Brandon Morelli Date: Wed, 7 Nov 2018 13:57:36 -0800 Subject: [PATCH 04/12] docs: more v2 additions --- docs/events-api.asciidoc | 62 +++++----- docs/exploring-es-data.asciidoc | 19 +++- .../guide/agent-server-compatibility.asciidoc | 6 +- docs/guide/apm-data-model.asciidoc | 23 ++-- docs/guide/distributed-tracing.asciidoc | 12 +- docs/guide/install-and-run.asciidoc | 8 +- docs/intake-api.asciidoc | 14 ++- docs/metadata-api.asciidoc | 56 ++++++++- docs/metricset-api.asciidoc | 15 ++- docs/upgrading-to-65.asciidoc | 106 +++++++++++++----- 10 files changed, 235 insertions(+), 86 deletions(-) diff --git a/docs/events-api.asciidoc b/docs/events-api.asciidoc index 22aafee579a..a9e811638d0 100644 --- a/docs/events-api.asciidoc +++ b/docs/events-api.asciidoc @@ -1,14 +1,9 @@ [[events-api]] == Events API -The following section contains information about: - -* <> -* <> -* <> -* <> - -Agents capture different types of information, known as events. These events are sent to a single endpoint which then sorts and processes the events. Events can be: +Agents capture different types of information, known as events. +These events are sent to a single endpoint which then sorts and processes the events. +Events can be: * Transactions * Spans @@ -38,26 +33,39 @@ http(s)://{hostname}:{port}/intake/v2/rum/events [float] === Response -On success, the server will respond with a 202 Accepted status code. +On success, the server will respond with a 202 Accepted status code and no body. [[events-api-errors]] [float] === Errors -Events are processed one after the either. If an error is encountered while processing an event, the error encountered, as well as the document causing the error are added to an internal array. The server will wait to encounter up to five errors before sending a response to the agent. If there are multiple errors, the highest status code is returned. +Events are processed one after the either. +If an error is encountered while processing an event, +the error encountered as well as the document causing the error are added to an internal array. +The server will wait to encounter up to five errors before sending a response to the agent. +If there are multiple errors, the highest status code is returned. -As an example, here's the error response for invalid JSON: +Here's an example response: [source,json] ------------------------------------------------------------ { - "accepted": 1, - "errors": [ - { - "document": "{ \"invalid-json\" }", - "message": "data read error: invalid character '}' after object key" - } - ] + "errors": [ + { + "message": "queue is full" + },{ + "message": "timeout while waiting to process" + },{ + "message": "", + "document": "" + },{ + "message": "", + "document": "" + },{ + "message": "invalid content-type: 'html/text'" + } + ], + "accepted": 2320 } ------------------------------------------------------------ @@ -68,22 +76,16 @@ it is added to the internal array and immediately returned. If you're developing an agent, these errors can be useful for debugging your agent while building it. -// todo: check troubleshooting section -The queue is full error is interesting, and can indicate that you probably want to reduce your sampling rate, get a bigger APM Server or elasticsearch - depending on where your bottlenecks are. - -// todo: what errors are unrecoverable? -NOTE: If the APM Server runs into certain errors, it may not consume the whole body from the agent. The server just stops reading if the error is unrecoverable. Agents should be able to handle these situations - [[events-api-schema-definition]] [float] -=== Event Schema +=== Event API Schemas -The APM Server uses a JSON Schema for validating requests: +The APM Server uses a collection of JSON Schemas for validating requests to the intake API: -* <> -* <> -* <> -* <> +* <> +* <> +* <> +* <> include::./metadata-api.asciidoc[] include::./transaction-api.asciidoc[] diff --git a/docs/exploring-es-data.asciidoc b/docs/exploring-es-data.asciidoc index a95ec9589dd..76376150333 100644 --- a/docs/exploring-es-data.asciidoc +++ b/docs/exploring-es-data.asciidoc @@ -3,12 +3,23 @@ [partintro] -- -By default Elastic APM data is stored in separated indices following the format: -`apm-%{[version]}-{type}-%{+yyyy.MM.dd}`. -NOTE: If you are updating APM Server from version 6.3 or earlier, you may still be using an outdated `apm-server.yml` file. You must update your `apm-server.yml` file in order to take advantage of the new format of indices. +NOTE: Templates describing how the APM Server stores data in Elasticsearch have changed in v6.5. +Read the <> for more information. -Data types are described {apm-get-started-ref}/apm-data-model.html[here]. +By default, Elastic APM data is stored in separated indices in the following formats: +------------------------------------------------------------ +apm-%{[version]}-sourcemap +apm-%{[version]}-error-%{+yyyy.MM.dd} +apm-%{[version]}-transaction-%{+yyyy.MM.dd} +apm-%{[version]}-span-%{+yyyy.MM.dd} +apm-%{[version]}-metric-%{+yyyy.MM.dd} +apm-%{[version]}-onboarding-%{+yyyy.MM.dd} +------------------------------------------------------------ + +If you're unfamiliar with the data types shown above, they are described in the {apm-get-started-ref}/apm-data-model.html[APM data model]. + +TIP: If your APM data is being stored in a different format, you may be using an outdated `apm-server.yml` file. You must update your `apm-server.yml` file in order to take advantage of the new format of indices. To get an overview of existing indices you can run: ["source","sh"] diff --git a/docs/guide/agent-server-compatibility.asciidoc b/docs/guide/agent-server-compatibility.asciidoc index 7b76fa9f23a..105bf695bdc 100644 --- a/docs/guide/agent-server-compatibility.asciidoc +++ b/docs/guide/agent-server-compatibility.asciidoc @@ -1,7 +1,11 @@ [[agent-server-compatibility]] == Agent/Server Compatibility -When instrumenting an APM Agent within your codebase, it's important to use a compatible version of APM Server. Below is a chart that outlines the compatibility between agents and server versions. +Below is a chart that outlines the compatibility between different versions of the APM agents and the APM Server. + +NOTE: APM Server version 6.5 introduced a new way for APM agents to communicate with the APM Server. +To take advantage of new features, +it is recommended that you update to the newest version of the applicable agent. [float] [[nodejs-compatibility]] diff --git a/docs/guide/apm-data-model.asciidoc b/docs/guide/apm-data-model.asciidoc index a250138f4ab..e383b380f18 100644 --- a/docs/guide/apm-data-model.asciidoc +++ b/docs/guide/apm-data-model.asciidoc @@ -29,8 +29,6 @@ include::../context.asciidoc[] Agents decide whether to sample transactions or not, and provide settings to control sampling behavior. -If sampled, -the {apm-server-ref}/transaction-spans[spans] of a transaction are sent and stored as separate documents. Within one transaction there can be several, or no spans captured. Transactions are stored in {apm-server-ref}/transaction-indices[transaction indices]. @@ -54,23 +52,32 @@ a span describing the database query will be created. The name of this span will contain information about the query itself, and the type of this span will contain information about the database type. -Spans are stored in {apm-server-ref}/span-indices[span indices]. Note that these indices are separate from {apm-server-ref}/transaction-indices[transaction indices] by default. +Spans are stored in {apm-server-ref}/span-indices[span indices]. +Note that these indices are separate from {apm-server-ref}/transaction-indices[transaction indices] by default. [float] [[dropped-spans]] ==== Dropped Spans -For performance reasons, APM agents can choose to purposefully omit spans. One example of this might be for long running transactions with over 100 transactions. These edge cases can overload both the agent and the APM Server. To avoid this, Agents can drop spans. When they do this, they notify the server of exactly how many spans were dropped. This note is then passed on to the user in the UI. +For performance reasons, APM agents can choose to purposefully sample or omit spans. +One example of this might be for long running transactions with over 100 spans. +These edge cases can overload both the agent and the APM Server. +To avoid this, Agents can drop spans. When they do this, +they notify the server of exactly how many spans were dropped, +this note is then passed on to the user in the UI. -Settings affecting dropped spans, and more details on why they might occur, are available in the relevant {apm-agents-ref}/index.html[APM Agent] documentation. +Settings affecting dropped spans, and more details on why they might occur, +are available in the relevant {apm-agents-ref}/index.html[APM Agent] documentation. [float] [[missing-spans]] ==== Missing Spans -Similarly to dropped spans, transactions may be missing spans. An agent informs the server of how many spans it has "started", but sometimes, not all of the spans make it to the server. This could be because - -Because spans are sent to the APM Server separately from their transaction, it may happen that we lose a transaction, but have spans belonging that transaction stored in Elasticsearch. The reverse can also happen, a transaction is successfully sent and stored in Elasticsearch, but one or more of it's spans are lost. +Similarly to dropped spans, transactions may have missing spans. +This can happen because spans are streamed from the APM Agent to the APM Server separately from their transaction. +Unforseen errors may cause spans to go missing. +Because the agent notifies the server about how many spans their should be, +we're able to easily determine if a span has gone missing. [[errors]] === Errors diff --git a/docs/guide/distributed-tracing.asciidoc b/docs/guide/distributed-tracing.asciidoc index 54f1cb412f9..8ee06d1c854 100644 --- a/docs/guide/distributed-tracing.asciidoc +++ b/docs/guide/distributed-tracing.asciidoc @@ -1,6 +1,14 @@ [[distributed-tracing]] == Distributed tracing -Elastic APM supports distributed tracing to enable you to analyze performance through your microservices architecture by tracing all the requests from the initial web request to your front-end service, to queries made to your back-end services all in one view. This makes finding possible bottlenecks through-out your application much easier and faster. No additional configuration is needed to get distributed tracing, but please see the supported technologies for the agent in use to get the most out of the integration. +Elastic APM supports distributed tracing to enable you to analyze performance through your microservices architecture by tracing all the requests from the initial web request to your front-end service, +to queries made to your back-end services all in one view. +This makes finding possible bottlenecks through-out your application much easier and faster. +There's no additional configuration needed for distributed tracing, +but please ensure you are using the latest version of the agent. -The APM UI in Kibana also supports displaying traces and an updated Timeline visualization has been redesigned to show all the transactions from individual services that are connected in a trace. As we're still working on enhancing the distributed tracing experience in the UI, we're flagging the feature beta in the UI. Should you still encounter any bug or issues, please reach out in our https://discuss.elastic.co/c/apm[discussion forum]. \ No newline at end of file +The APM UI in Kibana also supports displaying traces and an updated Timeline visualization has been redesigned to show all the transactions from individual services that are connected in a trace. +As we're still working on enhancing the distributed tracing experience in the UI, +we're flagging the feature beta in the UI. +Should you still encounter any bug or issues, +please reach out in our https://discuss.elastic.co/c/apm[discussion forum]. \ No newline at end of file diff --git a/docs/guide/install-and-run.asciidoc b/docs/guide/install-and-run.asciidoc index 822a4e3ed72..40ed0f789e0 100644 --- a/docs/guide/install-and-run.asciidoc +++ b/docs/guide/install-and-run.asciidoc @@ -223,9 +223,13 @@ See the {apm-java-ref}/index.html[APM Java Agent documentation] for more details [float] ==== Install the Go Agent -At present, Go applications must be instrumented manually at the source code level. Where possible, you should use the {apm-go-ref}/instrumenting-source.html#builtin-modules[built-in instrumentation modules] to report transactions served by web and RPC frameworks in your application. If the built-in modules are not suitable, please refer to {apm-go-ref}/instrumenting-source.html#custom-instrumentation[Custom Instrumentation]. +At present, Go applications must be instrumented manually at the source code level. +Where possible, you should use the {apm-go-ref}/instrumenting-source.html#builtin-modules[built-in instrumentation modules] to report transactions served by web and RPC frameworks in your application. +If the built-in modules are not suitable, +please refer to {apm-go-ref}/instrumenting-source.html#custom-instrumentation[Custom Instrumentation]. -The Go agent automatically instruments Gorilla and Gin, as well as support for Go's built-in net/http and database/sql drivers. +The Go agent automatically instruments Gorilla and Gin, +as well as support for Go's built-in net/http and database/sql drivers. See the {apm-go-ref}/index.html[APM Go Agent documentation] for more details. [[rum-agent]] diff --git a/docs/intake-api.asciidoc b/docs/intake-api.asciidoc index 8f6d31ae6ec..a4ac879514a 100644 --- a/docs/intake-api.asciidoc +++ b/docs/intake-api.asciidoc @@ -2,11 +2,16 @@ = Intake API [partintro] -- -NOTE: Most users will not need to interact directly with the intake API. Therefor, unless you are implementing an agent, you don't need to know about the specifics of this API. +NOTE: Most users will not need to interact directly with the intake API unless they are implementing an agent. -The Intake API is what we call the internal protocol that APM agents use to talk to the APM Server. This API has been redesigned in 6.5. Read the <> for more information. +The Intake API is what we call the internal protocol that APM agents use to talk to the APM Server. +This API has been redesigned in 6.5. +Read the <> for more information. -APM Agents communicate with the APM server by sending events in an HTTP request. Each event is sent as its own line in the HTTP request body. This is known as http://ndjson.org[newline delimited JSON (NDJSON)]. The request body looks roughly like this: +APM Agents communicate with the APM server by sending events in an HTTP request. +Each event is sent as its own line in the HTTP request body. +This is known as http://ndjson.org[newline delimited JSON (NDJSON)]. +The request body looks roughly like this: [source,bash] ------------------------------------------------------------ @@ -16,7 +21,8 @@ APM Agents communicate with the APM server by sending events in an HTTP request. {"transaction": {"name": "GET /index", "id": "A"}} ------------------------------------------------------------ -With NDJSON, agents can open an HTTP POST request and use chunked encoding to stream events to the APM Server as soon as they are recorded in the agent. This makes it simple for agents to serialize each event to a stream of newline delimited JSON. +With NDJSON, agents can open an HTTP POST request and use chunked encoding to stream events to the APM Server as soon as they are recorded in the agent. +This makes it simple for agents to serialize each event to a stream of newline delimited JSON. The APM Server also treats the HTTP body as a compressed stream and thus reads and handles each event independently. diff --git a/docs/metadata-api.asciidoc b/docs/metadata-api.asciidoc index 8c546ab66c9..c5412e71cfe 100644 --- a/docs/metadata-api.asciidoc +++ b/docs/metadata-api.asciidoc @@ -1,15 +1,63 @@ -[[metadata-schema]] +[[metadata-api]] === Metadata -Every new connection to the APM Server starts with a `metadata` stanza. This provides general metadata concerning the other objects in the stream. +Every new connection to the APM Server starts with a `metadata` stanza. +This provides general metadata concerning the other objects in the stream. + +Rather then send this metadata information from the agent multiple times, +the APM Server hangs on to this information and applies it to other objects in the stream as necessary. -Rather then send this metadata stanza multiple times, the APM Server hangs on to this information and reuses it to apply to every document that goes into elasticsearch. +[[metadata-schema-definition]] +[float] +==== Schema Definition The specification for metadata is outlined below: +* <> +* <> +* <> +* <> +* <> + +[[metadata-schema]] +[float] +===== Metadata [source,json] ---- include::./spec/metadata.json[] ---- -NOTE: Metadata is stored under `context` when viewing documents in Elasticsearch. \ No newline at end of file +[[metadata-service-schema]] +[float] +===== Service +[source,json] +---- +include::./spec/service.json[] +---- + +[[metadata-process-schema]] +[float] +===== Process +[source,json] +---- +include::./spec/process.json[] +---- + +[[metadata-system-schema]] +[float] +===== System +[source,json] +---- +include::./spec/system.json[] +---- + +[[metadata-user-schema]] +[float] +===== User +[source,json] +---- +include::./spec/user.json[] +---- + + +TIP: Metadata is stored under `context` when viewing documents in Elasticsearch. \ No newline at end of file diff --git a/docs/metricset-api.asciidoc b/docs/metricset-api.asciidoc index f4c42167688..17828e7872a 100644 --- a/docs/metricset-api.asciidoc +++ b/docs/metricset-api.asciidoc @@ -1,6 +1,19 @@ -[[metric-schema]] +[[metricset-api]] === Metricsets +Metricsets contain application metric data captured by an APM agent. + +[[metricset-schema-definition]] +[float] +==== Schema Definition + +The APM Server uses JSON Schema for validating requests. The specification for metricsets is defined below: + +[source,json] +---- +include::./spec/metricsets/common_metricset.json[] +---- + [source,json] ---- include::./spec/metricsets/v2_metricset.json[] diff --git a/docs/upgrading-to-65.asciidoc b/docs/upgrading-to-65.asciidoc index 32405c27a89..7d8b00aaeb1 100644 --- a/docs/upgrading-to-65.asciidoc +++ b/docs/upgrading-to-65.asciidoc @@ -1,11 +1,20 @@ [[upgrading-to-65]] === Upgrading to APM Server v6.5 -If you're using an Elastic APM Agent, upgrading to version 6.5 of the APM Server is easy - just ensure you're using the most recent version of your applicable APM agent. Check the {apm-get-started-ref}/agent-server-compatibility.html[agent/server compatibility matrix] for details. +If you're using an Elastic APM Agent, upgrading to version 6.5 of the APM Server is easy, +just ensure you're using the most recent version of your applicable APM agent. +Check the {apm-get-started-ref}/agent-server-compatibility.html[agent/server compatibility matrix] for details. -If you're an advanced user, are implementing a custom agent, or provide custom data to Elasticsearch, there are some changes in 6.5 that you should be aware of. +If you're an advanced user, are implementing a custom agent, +or provide custom data to Elasticsearch, +there are some changes in 6.5 that you should be aware of. -NOTE: None of the changes outlined below are breaking changes. APM Server 6.5 should still work with your currently implemented agent. To take advantage of new features, it's recommended you read about the changes below and update accordingly. +NOTE: None of the changes outlined below are breaking changes. +APM Server 6.5 should still work with your currently implemented agent. +To take advantage of new features, +it's recommended you read about the changes below and update accordingly. +If you'd like to continue using the deprecated Intake API and Elasticsearch Schema, +you can view https://www.elastic.co/guide/en/apm/server/6.4/overview.html[previous documentation]. [float] [[intake-api-changes-65]] @@ -14,49 +23,72 @@ NOTE: None of the changes outlined below are breaking changes. APM Server 6.5 sh The Intake API is what we call the internal protocol that APM agents use to talk to the APM Server. This API has been redesigned in 6.5 to increase the memory efficiency and predictability of the APM Server and APM agents. -There have been a number of changes to the Intake API for 6.5. Notably, the endpoint itself has changed to `intake/v2/events`. Full documentation of this change is in the <>. +Notably, most of the endpoints associated with this API have been deprecated, +and replaced with a new endpoint: `intake/v2/events`. +Details about these changes, +and the information you can send to the new endpoint, +is available in the <> documentation. + +[float] +[[metadata-api-changes-65]] +===== Metadata + +Metadata is now sent only once from the agent to the server. APM Server will retain this information and apply it to the applicable documents when it writes to Elasticsearch. + +Metadata should be sent to the new `intake/v2/events` endpoint. +See the <> for more information. [float] [[error-api-changes-65]] ===== Error -The error schema adds three new properties, and removes two others. The full schema is available here. +The error schema adds three new properties, and deprecates two others. -* _remove_ pattern of `error.id`, change format to hex encoded 64 random bits -* _remove_ `transaction.id` -* _add_ `transaction_id` referring to transaction -* _add_ `parent_id` -* _add_ `trace_id` +* _deprecate_ pattern of `error.id`, change format to hex encoded 64 random bits +* _deprecate_ `transaction.id` +* _add_ `transaction_id` (required, hex encoded 64 random bits) +* _add_ `parent_id` (optional, hex encoded 64 random bits) +* _add_ `trace_id` (optional, hex encoded 128 random bits) + +Error events should be sent to the new `intake/v2/events` endpoint. +See the <> for more information. [float] [[transaction-api-changes-65]] ===== Transaction -The transaction schema adds two new properties, and removes two others. The full schema is available here. +The transaction schema adds two new properties, and deprecates two others. + +* _deprecate_ pattern of `transaction.id`, change format to hex encoded 64 random bits +* _deprecate_ spans (don't forbid them, just don't process them) +* _add_ `parent_id` (optional, hex encoded 64 random bits string) +* _add_ `trace_id` (required, hex encoded 128 random bits) -* _remove_ pattern of `transaction.id`, change format to hex encoded 64 random bits -* _remove_ spans (don't forbid them, just don't process them) -* _add_ `parent_id` -* _add_ `trace_id` (required attribute) +Transaction events should be sent to the new `intake/v2/events` endpoint. +See the <> for more information. [float] [[span-api-changes-65]] ===== Span -The span schema adds three new properties, removes two, and changes one. The full schema is available here. +The span schema adds three new properties, removes two, and changes one. + +* _change_ pattern of `id` to hex encoded 64 random bits +* _deprecate_ pattern of `transaction_id` +* _deprecate_ use of `parent` +* _add_ `parent_id` (required, hex encoded 64 random bits) +* _add_ `trace_id` (required, hex encoded 128 random bits) +* _add_ `timestamp` -* _change_ pattern of `id` from long to string, format to hex encoded 64 random bits -* _remove_ pattern of `transaction_id` -* _remove_ parent referring to long id -* _add_ `parent_id` referring to transaction or span -* _add_ timestamp -* _add_ `trace_id` (required attribute) +Span events should be sent to the new `intake/v2/events` endpoint. +See the <> for more information. [float] [[healthcheck-api-changes-65]] ===== Healthcheck Endpoint -The Healthcheck endpoint, previously `/healthcheck`, has been deprecated. The <> api has replaced it. +The Healthcheck endpoint, previously `/healthcheck`, has been deprecated. +The <> api has replaced it. [float] [[assets-api-changes-65]] @@ -68,7 +100,10 @@ The assets endpoint, `/assets/v1/sourcemaps`, remains unchanged. [[metrics-api-changes-65]] ===== Metricsets Endpoint -The Metricsets endpoint, previously `/v1/metrics`, has also integrated with the new intake endpoint at `/intake/v2/events`. In terms of functionality, the only change relates to timestamps. More information is available here. +The Metricsets endpoint, previously `/v1/metrics`, +has also integrated with the new intake endpoint at `/intake/v2/events`. +In terms of functionality, the only change relates to timestamps. +See the <> for more information. [float] [[server-config-changes-65]] @@ -78,11 +113,19 @@ On the server there are some configuration options that are worth mentioning: `max_unzipped_size`, `max_request_queue_time`, and `concurrent_requests` have all been deprecated with 6.4. -New with 6.5 is `max_event_size` which has a default value of `307200` bytes. You'll notice this value is much smaller than the previous `max_unzipped_size` value. This is because in APM Server 6.5, an event is much smaller than an event in 6.4. Instead of getting the whole payload at once, and event is just a single line of data. +New with 6.5 is `max_event_size` which has a default value of `307200` bytes. +You'll notice this value is much smaller than the previous `max_unzipped_size` value. +This is because in APM Server 6.5, an event is much smaller than an event in 6.4. +Instead of getting the whole payload at once, and event is just a single line of data. -`read_timeout` and `write_timeout` remain at 30 seconds, but if you have a load balancer in front of the APM server you may notice connections are a little bit longer lived than they used to be. +`read_timeout` and `write_timeout` remain at 30 seconds, +but if you have a load balancer in front of the APM server you may notice connections are a little bit longer lived than they used to be. -NOTE: If you are updating APM Server from version 6.3 or earlier, you may not be using an updated `apm-server.yml` file. How APM Server stores data in Elasticsearch has changed overtime, and in order to take advantage of new features, you must be using an update `apm-server.yml` file. +NOTE: If you are updating APM Server from version 6.3 or earlier, +you may not be using an updated `apm-server.yml` file. +How APM Server stores data in Elasticsearch has changed overtime, +and in order to take advantage of new features, +you must be using an updated `apm-server.yml` file. [float] [[es-schema-changes-65]] @@ -97,7 +140,8 @@ There have been a number of changes to the Elasticsearch schema for 6.5. [[es-error-changes-65]] ===== Error -The Elasticsearch error schema adds two new properties. The full schema is available here. +The Elasticsearch error schema adds two new properties. +The full schema is available here. * _add_ `error.parent_id` * _add_ `error.trace_id` @@ -106,7 +150,8 @@ The Elasticsearch error schema adds two new properties. The full schema is avail [[es-transaction-changes-65]] ===== Transaction -The Elasticsearch transaction schema adds two new properties. The full schema is available here. +The Elasticsearch transaction schema adds two new properties. +The full schema is available here. * _add_ `transaction.parent_id` * _add_ `transaction.trace_id` @@ -115,7 +160,8 @@ The Elasticsearch transaction schema adds two new properties. The full schema is [[es-span-changes-65]] ===== Span -The Elasticsearch span schema adds two new properties. The full schema is available here. +The Elasticsearch span schema adds two new properties. +The full schema is available here. * _add_ `span.parent_id` * _add_ `span.trace_id` From eb5a74bad2f1500c24f93467d5bf479254edae52 Mon Sep 17 00:00:00 2001 From: Brandon Morelli Date: Mon, 5 Nov 2018 19:54:28 -0800 Subject: [PATCH 05/12] docs: rework apm overview --- docs/guide/apm-data-model.asciidoc | 2 ++ docs/guide/index.asciidoc | 3 +++ 2 files changed, 5 insertions(+) diff --git a/docs/guide/apm-data-model.asciidoc b/docs/guide/apm-data-model.asciidoc index e383b380f18..db9b00b0c3d 100644 --- a/docs/guide/apm-data-model.asciidoc +++ b/docs/guide/apm-data-model.asciidoc @@ -29,6 +29,8 @@ include::../context.asciidoc[] Agents decide whether to sample transactions or not, and provide settings to control sampling behavior. +If sampled, +the {apm-server-ref}/transaction-spans[spans] of a transaction are sent and stored as separate documents. Within one transaction there can be several, or no spans captured. Transactions are stored in {apm-server-ref}/transaction-indices[transaction indices]. diff --git a/docs/guide/index.asciidoc b/docs/guide/index.asciidoc index 838200521ca..0c145ab5755 100644 --- a/docs/guide/index.asciidoc +++ b/docs/guide/index.asciidoc @@ -6,6 +6,9 @@ NOTE: For the best reading experience, please view this documentation at https://www.elastic.co/guide/en/apm/get-started[elastic.co] endif::[] +[[gettting-started]] += APM Overview + include::./overview.asciidoc[] include::./apm-data-model.asciidoc[] From 592f15e74f983df3a203257f577c3dfc43712cbc Mon Sep 17 00:00:00 2001 From: Brandon Morelli Date: Wed, 7 Nov 2018 15:30:27 -0800 Subject: [PATCH 06/12] docs: more v2 --- docs/guide/apm-data-model.asciidoc | 23 +++++++++-------- docs/transaction-indices.asciidoc | 2 ++ docs/upgrading-to-65.asciidoc | 40 +++++++++++++++++------------- 3 files changed, 38 insertions(+), 27 deletions(-) diff --git a/docs/guide/apm-data-model.asciidoc b/docs/guide/apm-data-model.asciidoc index db9b00b0c3d..f5d57df9085 100644 --- a/docs/guide/apm-data-model.asciidoc +++ b/docs/guide/apm-data-model.asciidoc @@ -30,9 +30,9 @@ include::../context.asciidoc[] Agents decide whether to sample transactions or not, and provide settings to control sampling behavior. If sampled, -the {apm-server-ref}/transaction-spans[spans] of a transaction are sent and stored as separate documents. Within one transaction there can be several, or no spans captured. +the {apm-server-ref}/transaction-spans.html[spans] of a transaction are sent and stored as separate documents. Within one transaction there can be several, or no spans captured. -Transactions are stored in {apm-server-ref}/transaction-indices[transaction indices]. +Transactions are stored in {apm-server-ref}/transaction-indices.html[transaction indices]. [[transaction-spans]] === Spans @@ -54,22 +54,25 @@ a span describing the database query will be created. The name of this span will contain information about the query itself, and the type of this span will contain information about the database type. -Spans are stored in {apm-server-ref}/span-indices[span indices]. -Note that these indices are separate from {apm-server-ref}/transaction-indices[transaction indices] by default. +Spans are stored in {apm-server-ref}/span-indices.html[span indices]. +Note that these indices are separate from {apm-server-ref}/transaction-indices.html[transaction indices] by default. [float] [[dropped-spans]] ==== Dropped Spans -For performance reasons, APM agents can choose to purposefully sample or omit spans. +For performance reasons, some APM agents can choose to purposefully sample or omit spans. One example of this might be for long running transactions with over 100 spans. These edge cases can overload both the agent and the APM Server. -To avoid this, Agents can drop spans. When they do this, -they notify the server of exactly how many spans were dropped, -this note is then passed on to the user in the UI. +To avoid this, Agents will drop spans. When they do this, +they notify the server of exactly how many spans were dropped. +This note is then passed on to the user in the UI. Settings affecting dropped spans, and more details on why they might occur, -are available in the relevant {apm-agents-ref}/index.html[APM Agent] documentation. +are available in the relevant agent documentation: + +* {apm-node-ref}/agent-api.html#transaction-max-spans[Node.js Agent max spans] +* {apm-agents-ref}/configuration.html#config-transaction-max-spans[Python Agent max spans] [float] [[missing-spans]] @@ -101,4 +104,4 @@ Errors also have some contextual data. include::../context.asciidoc[] -Errors are stored in {apm-server-ref}/error-indices[error indices]. \ No newline at end of file +Errors are stored in {apm-server-ref}/error-indices.html[error indices]. \ No newline at end of file diff --git a/docs/transaction-indices.asciidoc b/docs/transaction-indices.asciidoc index e63910fab1e..a81f1ba185c 100644 --- a/docs/transaction-indices.asciidoc +++ b/docs/transaction-indices.asciidoc @@ -6,7 +6,9 @@ Transactions are by default stored to indices of the format `apm-[version]-trans [[transaction-example]] [float] === Example Document + See an example transaction indexed in Elasticsearch: + [source,json] ---- include::./data/elasticsearch/transaction.json[] diff --git a/docs/upgrading-to-65.asciidoc b/docs/upgrading-to-65.asciidoc index 7d8b00aaeb1..ce7174c9b83 100644 --- a/docs/upgrading-to-65.asciidoc +++ b/docs/upgrading-to-65.asciidoc @@ -131,40 +131,46 @@ you must be using an updated `apm-server.yml` file. [[es-schema-changes-65]] ==== Elasticsearch Schema Changes -//todo: ADD LINKS TO sample docs created by the APM server - The Elasticsearch schema defines how APM data is stored in Elasticsearch. There have been a number of changes to the Elasticsearch schema for 6.5. +An important change to note is the addition of the `trace` and `parent` keys, +which have been added to errors, transactions, and spans. +Both only hold a field `id`. +These new keys are essential to taking advantage of APM's new {apm-get-started-ref}/distributed-tracing.html[distributed tracing] feature. + [float] [[es-error-changes-65]] ===== Error -The Elasticsearch error schema adds two new properties. -The full schema is available here. +The Elasticsearch error schema adds two new keys: + +* _add_ `trace.id` +* _add_ `parent.id` -* _add_ `error.parent_id` -* _add_ `error.trace_id` +View the sample Elasticsearch <> for more information. [float] [[es-transaction-changes-65]] ===== Transaction -The Elasticsearch transaction schema adds two new properties. -The full schema is available here. +The Elasticsearch transaction schema adds two new keys: -* _add_ `transaction.parent_id` -* _add_ `transaction.trace_id` +* _add_ `trace.id` +* _add_ `parent.id` + +View the sample Elasticsearch <> for more information. [float] [[es-span-changes-65]] ===== Span -The Elasticsearch span schema adds two new properties. -The full schema is available here. +The Elasticsearch span schema adds three new keys, and deprecates two: + +* _add_ `trace.id` +* _add_ `parent.id` +* _add_ `hex_id` +* _deprecate_ `parent` long +* _deprecate_ `id` long -* _add_ `span.parent_id` -* _add_ `span.trace_id` -* _add_ `span.hex_id` -* _deprecate_ `span.id` -* _deprecate_ `span.parent` \ No newline at end of file +View the sample Elasticsearch <> for more information. \ No newline at end of file From 8312f9a2482210f3211adf388fecf03f9e7b203d Mon Sep 17 00:00:00 2001 From: Brandon Morelli Date: Thu, 8 Nov 2018 07:25:56 -0800 Subject: [PATCH 07/12] docs: review improvements and fix compat --- docs/error-api.asciidoc | 1 + docs/events-api.asciidoc | 1 + .../guide/agent-server-compatibility.asciidoc | 23 ++++++++++--------- docs/guide/apm-data-model.asciidoc | 2 +- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/docs/error-api.asciidoc b/docs/error-api.asciidoc index e65edb9fec3..11bac108bd8 100644 --- a/docs/error-api.asciidoc +++ b/docs/error-api.asciidoc @@ -12,6 +12,7 @@ The following section contains information about: ==== Schema Definition The APM Server uses JSON Schema for validating requests. The specification for errors is defined below: + [float] [[error-schema]] ===== Error diff --git a/docs/events-api.asciidoc b/docs/events-api.asciidoc index a9e811638d0..f83c14dbbba 100644 --- a/docs/events-api.asciidoc +++ b/docs/events-api.asciidoc @@ -15,6 +15,7 @@ You can learn more about events in the {apm-get-started-ref}/apm-data-model.html [[events-api-endpoint]] [float] === Endpoint + Send an `HTTP POST` request to the APM Server `intake/v2/events` endpoint: [source,bash] diff --git a/docs/guide/agent-server-compatibility.asciidoc b/docs/guide/agent-server-compatibility.asciidoc index 105bf695bdc..8e0aa0044c4 100644 --- a/docs/guide/agent-server-compatibility.asciidoc +++ b/docs/guide/agent-server-compatibility.asciidoc @@ -14,8 +14,9 @@ it is recommended that you update to the newest version of the applicable agent. [options="header"] |======================================================================= |Agent Version |APM Server Version -|1.x |6.2.0-6.5.0 -|2.x |6.5.0+ +|0.x |<6.2 +|1.x |6.2-6.5 +|2.x |6.5+ |======================================================================= [float] @@ -27,7 +28,7 @@ it is recommended that you update to the newest version of the applicable agent. |Agent Version |APM Server Version |1.x |6.1 |2.x/3.x |6.2-6.x -|4.0+ |6.5.0+ +|4.x |6.5+ |======================================================================= [float] @@ -37,8 +38,8 @@ it is recommended that you update to the newest version of the applicable agent. [options="header"] |======================================================================= |Agent Version |APM Server Version -|1.x |<6.5.x -|2.0+ |6.5.0+ +|1.x |<6.5 +|2.x |6.5+ |======================================================================= [float] @@ -48,7 +49,7 @@ it is recommended that you update to the newest version of the applicable agent. [options="header"] |======================================================================= |Agent Version |APM Server Version -|1.0+ |6.5.0+ +|1.x |6.5.0+ |======================================================================= [float] @@ -58,8 +59,8 @@ it is recommended that you update to the newest version of the applicable agent. [options="header"] |======================================================================= |Agent Version |APM Server Version -|1.x |6.2.0-6.5.0 -|2.x |6.5.0+ +|0.x |<7.0 +|1.x |6.5+ |======================================================================= [float] @@ -69,7 +70,7 @@ it is recommended that you update to the newest version of the applicable agent. [options="header"] |======================================================================= |Agent Version |APM Server Version -|0.x |6.3.x-6.4.0 -|1.x |6.4.x -|2.0+ |6.5.0+ +|0.x |6.3-6.4 +|1.x |6.4 +|2.x |6.5+ |======================================================================= diff --git a/docs/guide/apm-data-model.asciidoc b/docs/guide/apm-data-model.asciidoc index f5d57df9085..98e867cc185 100644 --- a/docs/guide/apm-data-model.asciidoc +++ b/docs/guide/apm-data-model.asciidoc @@ -13,7 +13,7 @@ Elastic APM agents capture different types of information from within their inst A transaction describes an event captured by an Elastic APM agent instrumenting a service. Some examples of a transaction might be: * An HTTP request -* An asynchronous background job +* A background job A transaction contains: From 1fed7ab19fbecef9e85a48bd13e9ff9967113c43 Mon Sep 17 00:00:00 2001 From: Brandon Morelli Date: Thu, 8 Nov 2018 14:17:20 -0800 Subject: [PATCH 08/12] docs: v2 revisions and fixes --- docs/common-problems.asciidoc | 2 +- docs/configuration-process.asciidoc | 12 +- docs/data-ingestion.asciidoc | 2 +- docs/error-api.asciidoc | 51 +------- docs/events-api.asciidoc | 47 ++++--- docs/exploring-es-data.asciidoc | 1 - .../guide/agent-server-compatibility.asciidoc | 22 ++-- docs/guide/apm-data-model.asciidoc | 14 +-- docs/guide/distributed-tracing.asciidoc | 28 +++-- docs/guide/distributed-tracing.jpg | Bin 0 -> 111617 bytes docs/guide/install-and-run.asciidoc | 115 ++---------------- docs/guide/overview.asciidoc | 8 -- docs/metadata-api.asciidoc | 9 +- docs/span-api.asciidoc | 27 ++++ docs/transaction-api.asciidoc | 68 +---------- docs/upgrading-to-65.asciidoc | 65 ++++++---- 16 files changed, 167 insertions(+), 304 deletions(-) create mode 100644 docs/guide/distributed-tracing.jpg create mode 100644 docs/span-api.asciidoc diff --git a/docs/common-problems.asciidoc b/docs/common-problems.asciidoc index b769ee6a464..d5a9c574a39 100644 --- a/docs/common-problems.asciidoc +++ b/docs/common-problems.asciidoc @@ -103,7 +103,7 @@ You have a few options to solve this problem: === HTTP 503: Request timed out waiting to be processed This happens when APM Server exceeds the maximum number of requests that it can process concurrently. -This limit is determined by the `apm-server.concurrent_requests` configuration parameter. +This limit is determined by the `apm-server.concurrent_requests` configuration parameter deprecated[6.5]. To alleviate this problem, you can try to: diff --git a/docs/configuration-process.asciidoc b/docs/configuration-process.asciidoc index cb27fec7957..9aa82473eaf 100644 --- a/docs/configuration-process.asciidoc +++ b/docs/configuration-process.asciidoc @@ -28,10 +28,16 @@ Defaults to 'localhost:8200'. [[max_unzipped_size]] [float] -==== `max_unzipped_size` +==== `max_unzipped_size` deprecated[6.5] Maximum permitted size of an unzipped request accepted by the server to be processed (in Bytes). Defaults to 31457280 Bytes (30 MB). +[[max_event_size]] +[float] +==== `max_event_size added[6.5] +Maximum permitted size of an event accepted by the server to be processed (in Bytes). +Defaults to 307200 Bytes. + [[max_header_size]] [float] ==== `max_header_size` @@ -40,7 +46,7 @@ Defaults to 1048576 Bytes (1 MB). [[max_request_queue_time]] [float] -==== `max_request_queue_time` +==== `max_request_queue_time` deprecated[6.5] Maximum duration a request will be queued before being read. Defaults to 2 seconds. @@ -64,7 +70,7 @@ Defaults to 5 seconds. [[concurrent_requests]] [float] -==== `concurrent_request` +==== `concurrent_request` deprecated[6.5] Maximum number of requests the server can process concurrently. Read more about how to tune data ingestion by <>. Default value is 5. diff --git a/docs/data-ingestion.asciidoc b/docs/data-ingestion.asciidoc index ce877e75984..dab9cab4644 100644 --- a/docs/data-ingestion.asciidoc +++ b/docs/data-ingestion.asciidoc @@ -50,7 +50,7 @@ Increasing `queue.mem.events` can significantly affect APM Server memory usage. [float] === Adjust concurrent requests APM Server has a limit to how many requests can be processed concurrently. -This limit is determined by the `apm-server.concurrent_requests` setting. +This limit is determined by the `apm-server.concurrent_requests` setting deprecated[6.5]. Increasing this value will improve throughput, but it can significantly affect APM Server memory usage. [[add-apm-server-instances]] diff --git a/docs/error-api.asciidoc b/docs/error-api.asciidoc index 11bac108bd8..c14d85b7daa 100644 --- a/docs/error-api.asciidoc +++ b/docs/error-api.asciidoc @@ -5,7 +5,6 @@ The following section contains information about: * <> * <> -* <> [[error-schema-definition]] [float] @@ -17,58 +16,12 @@ The APM Server uses JSON Schema for validating requests. The specification for e [[error-schema]] ===== Error -[source,json] ----- -include::./spec/errors/common_error.json[] ----- - [source,json] ---- include::./spec/errors/v2_error.json[] ---- -[[error-api-examples]] -[float] -==== Examples - -Request example: - -["source","sh",subs="attributes"] ------------------------------------------------------------- -curl http://localhost:8200/intake/v2/events \ - --header "Content-Type: application/json" \ - --data @docs/data/intake-api/generated/error/payload.json ------------------------------------------------------------- - -Example error requests: - -* <> -* <> -* <> - -[[payload-with-error]] -[float] -===== Payload with an Error - [source,json] ---- -include::./data/intake-api/generated/error/payload.json[] ----- - -[[payload-with-minimal-exception]] -[float] -===== Payload with an Error with minimal Exception Information - -[source,json] ----- -include::./data/intake-api/generated/error/minimal_payload_exception.json[] ----- - -[[payload-with-minimal-log]] -[float] -===== Payload with an Error with minimal Log Information - -[source,json] ----- -include::./data/intake-api/generated/error/minimal_payload_log.json[] ----- +include::./spec/errors/common_error.json[] +---- \ No newline at end of file diff --git a/docs/events-api.asciidoc b/docs/events-api.asciidoc index f83c14dbbba..60f94f20567 100644 --- a/docs/events-api.asciidoc +++ b/docs/events-api.asciidoc @@ -36,26 +36,41 @@ http(s)://{hostname}:{port}/intake/v2/rum/events On success, the server will respond with a 202 Accepted status code and no body. +Keep in mind that events can succeed and fail independently of each other. Only if all events succeed does the server respond with a 202. + [[events-api-errors]] [float] === Errors -Events are processed one after the either. +There are two types of errors that the APM Server may return to an agent: + +* Event related errors (typically validation errors) +* Non-event related errors + +The APM Server process events one after the other. If an error is encountered while processing an event, the error encountered as well as the document causing the error are added to an internal array. -The server will wait to encounter up to five errors before sending a response to the agent. -If there are multiple errors, the highest status code is returned. +The APM Server will only process and save 5 event related errors. +If it encounters more than 5 event related errors, +the additional errors will not be returned to agent. +Once all events have been processed, +the error response is sent. + +Some errors, not relating to specific events, +may terminate the request immediately. +For example: queue is full, IP rate limit reached, wrong metadata, etc. +If at any point one of these errors is encountered, +it is added to the internal array and immediately returned. -Here's an example response: +An example error response might look something like this: [source,json] ------------------------------------------------------------ { "errors": [ { - "message": "queue is full" - },{ - "message": "timeout while waiting to process" + "message": "", <1> + "document": "" <2> },{ "message": "", "document": "" @@ -63,17 +78,17 @@ Here's an example response: "message": "", "document": "" },{ - "message": "invalid content-type: 'html/text'" - } + "message": "queue is full" <3> + }, ], - "accepted": 2320 + "accepted": 2320 <4> } ------------------------------------------------------------ -Some errors, not relating to specific events, may be immediately returnable. -For example: Request limit reached, bad header, or internal queue full. -If at any point one of these errors is encountered, -it is added to the internal array and immediately returned. +<1> An event related error +<2> The document causing the error +<3> An immediately returning non-event related error +<4> The number of accepted events If you're developing an agent, these errors can be useful for debugging your agent while building it. @@ -84,11 +99,13 @@ If you're developing an agent, these errors can be useful for debugging your age The APM Server uses a collection of JSON Schemas for validating requests to the intake API: * <> -* <> +* <> +* <> * <> * <> include::./metadata-api.asciidoc[] include::./transaction-api.asciidoc[] +include::./span-api.asciidoc[] include::./error-api.asciidoc[] include::./metricset-api.asciidoc[] \ No newline at end of file diff --git a/docs/exploring-es-data.asciidoc b/docs/exploring-es-data.asciidoc index 76376150333..a6bb79bdf03 100644 --- a/docs/exploring-es-data.asciidoc +++ b/docs/exploring-es-data.asciidoc @@ -14,7 +14,6 @@ apm-%{[version]}-error-%{+yyyy.MM.dd} apm-%{[version]}-transaction-%{+yyyy.MM.dd} apm-%{[version]}-span-%{+yyyy.MM.dd} apm-%{[version]}-metric-%{+yyyy.MM.dd} -apm-%{[version]}-onboarding-%{+yyyy.MM.dd} ------------------------------------------------------------ If you're unfamiliar with the data types shown above, they are described in the {apm-get-started-ref}/apm-data-model.html[APM data model]. diff --git a/docs/guide/agent-server-compatibility.asciidoc b/docs/guide/agent-server-compatibility.asciidoc index 8e0aa0044c4..e1022c328fd 100644 --- a/docs/guide/agent-server-compatibility.asciidoc +++ b/docs/guide/agent-server-compatibility.asciidoc @@ -14,9 +14,9 @@ it is recommended that you update to the newest version of the applicable agent. [options="header"] |======================================================================= |Agent Version |APM Server Version -|0.x |<6.2 -|1.x |6.2-6.5 -|2.x |6.5+ +|0.x |< 6.2 +|1.x |6.2-6.x +|2.x |>= 6.5 |======================================================================= [float] @@ -26,9 +26,9 @@ it is recommended that you update to the newest version of the applicable agent. [options="header"] |======================================================================= |Agent Version |APM Server Version -|1.x |6.1 +|1.x |6.1 beta[] |2.x/3.x |6.2-6.x -|4.x |6.5+ +|4.x |>= 6.5 |======================================================================= [float] @@ -38,8 +38,8 @@ it is recommended that you update to the newest version of the applicable agent. [options="header"] |======================================================================= |Agent Version |APM Server Version -|1.x |<6.5 -|2.x |6.5+ +|1.x |\<= 6.x +|2.x |>= 6.5 |======================================================================= [float] @@ -49,7 +49,7 @@ it is recommended that you update to the newest version of the applicable agent. [options="header"] |======================================================================= |Agent Version |APM Server Version -|1.x |6.5.0+ +|1.x |>= 6.5.0 |======================================================================= [float] @@ -59,8 +59,8 @@ it is recommended that you update to the newest version of the applicable agent. [options="header"] |======================================================================= |Agent Version |APM Server Version -|0.x |<7.0 -|1.x |6.5+ +|0.x |\<= 6.x +|1.x |>= 6.5 |======================================================================= [float] @@ -72,5 +72,5 @@ it is recommended that you update to the newest version of the applicable agent. |Agent Version |APM Server Version |0.x |6.3-6.4 |1.x |6.4 -|2.x |6.5+ +|2.x |>= 6.5 |======================================================================= diff --git a/docs/guide/apm-data-model.asciidoc b/docs/guide/apm-data-model.asciidoc index 98e867cc185..25479a895fd 100644 --- a/docs/guide/apm-data-model.asciidoc +++ b/docs/guide/apm-data-model.asciidoc @@ -36,7 +36,7 @@ Transactions are stored in {apm-server-ref}/transaction-indices.html[transaction [[transaction-spans]] === Spans -Transactions can have 0, 1, or many spans. Spans have a `transaction.id` attribute that refer to their parent transaction. +Transactions can have 0, 1, or many spans. Spans have a `transaction.id` attribute that refers to their parent transaction. They also have a `parent.id` attribute that refers to the parent span, or their transaction. Agents typically automatically instrument a variety of of libraries, but also provide an API for ad hoc instrumentation of specific code paths. @@ -47,12 +47,12 @@ This information includes: * duration * name * type -* `stack trace` (optional). +* `stack trace` (optional) As an example, if a database query happens within a sampled transaction, a span describing the database query will be created. The name of this span will contain information about the query itself, -and the type of this span will contain information about the database type. +and the type of this span will contain information about the database. Spans are stored in {apm-server-ref}/span-indices.html[span indices]. Note that these indices are separate from {apm-server-ref}/transaction-indices.html[transaction indices] by default. @@ -64,7 +64,7 @@ Note that these indices are separate from {apm-server-ref}/transaction-indices.h For performance reasons, some APM agents can choose to purposefully sample or omit spans. One example of this might be for long running transactions with over 100 spans. These edge cases can overload both the agent and the APM Server. -To avoid this, Agents will drop spans. When they do this, +To avoid this, agents will drop spans. When they do this, they notify the server of exactly how many spans were dropped. This note is then passed on to the user in the UI. @@ -81,13 +81,13 @@ are available in the relevant agent documentation: Similarly to dropped spans, transactions may have missing spans. This can happen because spans are streamed from the APM Agent to the APM Server separately from their transaction. Unforseen errors may cause spans to go missing. -Because the agent notifies the server about how many spans their should be, -we're able to easily determine if a span has gone missing. +Because the agent notifies the server about how many spans there should be, +the number of missing spans is able to be calculated and shown in the UI. [[errors]] === Errors -Errors are identified by a unique ID. +Errors are represented by a unique ID. An error event contains at least information about the original `exception` that occurred or about a `log` created when the exception occurred. diff --git a/docs/guide/distributed-tracing.asciidoc b/docs/guide/distributed-tracing.asciidoc index 8ee06d1c854..4bc69f5ed01 100644 --- a/docs/guide/distributed-tracing.asciidoc +++ b/docs/guide/distributed-tracing.asciidoc @@ -1,14 +1,22 @@ [[distributed-tracing]] == Distributed tracing -Elastic APM supports distributed tracing to enable you to analyze performance through your microservices architecture by tracing all the requests from the initial web request to your front-end service, -to queries made to your back-end services all in one view. -This makes finding possible bottlenecks through-out your application much easier and faster. -There's no additional configuration needed for distributed tracing, -but please ensure you are using the latest version of the agent. - -The APM UI in Kibana also supports displaying traces and an updated Timeline visualization has been redesigned to show all the transactions from individual services that are connected in a trace. -As we're still working on enhancing the distributed tracing experience in the UI, -we're flagging the feature beta in the UI. +Elastic APM supports distributed tracing. +Distributed tracing enables you to analyze performance throughout your microservices architecture all in one view. +This is accomplished by tracing all of the requests - from the initial web request to your front-end service - to queries made to your back-end services. +This makes finding possible bottlenecks throughout your application much easier and faster. +Best of all, there's no additional configuration needed for distributed tracing, just ensure you're using the latest version of the applicable {apm-agents-ref}/index.html[agent]. + +The APM UI in Kibana also supports distributed tracing. +The Timeline visualization has been redesigned to show all of the transactions from individual services that are connected in a trace: + +[role="screenshot"] +image::distributed-tracing.jpg[distribute tracing in the UI] + +NOTE: As we're still working on enhancing the distributed tracing experience in the UI, +we're flagging the feature as beta in the UI. + Should you still encounter any bug or issues, -please reach out in our https://discuss.elastic.co/c/apm[discussion forum]. \ No newline at end of file +please reach out in the https://github.com/elastic/apm[APM GitHub repository]. +For questions and feature requests, +visit our https://discuss.elastic.co/c/apm[discussion forum]. \ No newline at end of file diff --git a/docs/guide/distributed-tracing.jpg b/docs/guide/distributed-tracing.jpg new file mode 100644 index 0000000000000000000000000000000000000000..99f641043cb71d8c21d6818bd6b250ed8b0cb13c GIT binary patch literal 111617 zcmeFZcUY6nwkRG%iXtFgsY(%*rUK1^^y2iTne0 zVLJQwGKUytE&>==01qQSXHxzH=4ZgKLm_~}e}DEJf5Npr)yr?g?u9Z#t$swiHNWIX-c;fcD*E#1FA%lM=z`S(Wp z`1mOLTvCKRcT&1|_3Bln3(88$$_k7U3Ld_0o_25rHxH>lG2C+WuzwDD<_Uqho%)T@ z?lH{EQ&W;r^WT~TefAHu|CaXO1)r&FKX?4i?guc~Q%%W#W&hub7{VMNPke9Lc{*xJ z{_YNiix(9xTzv3ncStHRNK*PcO~1i^Fh}duU+FIn{KbL4IPez-{^G!29Qc2a1OFjj zIl3`qDj$aM1fUZESMEB)JYgQruxAW0?*ib;O#{QjzvV(k`U95v13W$;_GTmoPGAWRBS!^=PMj2$IU{@a+<6sMwaZtos%ziU(bdy8xPAY@LsK(z3rl+k$0ttC5El0-#Ze3SNf%o3uYD z`>zoe^nXR!KM4B|x<&!o3{~TII>dPYk(hs9nGZAnPKW;`vHX!(e%sc=ffabJ-x=t~~A1JkE3Lup5hEQRU!u^(1 z(==byVJV^&UN~ntEYsQf{q&`WE?$k3M2S#H* zUD8;O=4pLsY&%2;s33V95EsMR+dMw>%}x-DX?JL9XHhGWG|~Qj7CPW37KNOm?a%>{ zs7@uyIoXv_I^gimK};QwM6d?ON0(&man*02Men%fVGo_WQ76{DRY^KPIb8q~Mh6r< zjbA_4bA~VC!q(FH2h(RJr|aS|^jwjb7A zqXTk5;H7T$qjbQz7>Y~}H79h+_#hF+MF%XJ4DXdk2Vt+0wGZY(DM-`6_NnCkC?uH{ z)T<+~`*Q#8Jsp0)?|fI+IqQXnPh;kxKrsq;TTN&TLKEhV8KwiAIYf6ndR(7i$|KXz z<(OEUh)!+wA;^%--Bl*di?*#g{4x%4Bg{T-#@_t*bO3*5ftgGp+A%n2+BU{6i_!qK zJyFqR%IVU(8Fh2oIM$8$%2%<=ZE-Fv1(T&U%ggfY^7+_h&$}wPU!-v&e0|uU7&?7uyGMGv9+<8w0_OoqtX}Gxl;N;Hl{YCTDE^7E6oRx z6>;tl)-l+}w2}XY4se(pS#GzLO>5g54z8#|e zlVf%swf~E7l#5&D#bb}I0O>1P5sxN!S@`c=4~jVZ;SeKpf!i85Noy?&Up9+IY2}xIq!(IK>NPGj6-&{oFui z1+y{6<9DxSchA1qE3-OaAOa92H}icp^sad*cvI&%)sKb9(|IeXq z7EIMWcprEY@qlcIibUAKeC0{nFjsdN=M2dbJc_dpD47r$^>h~)xI0GM61;w0`qf)~ zlxJ3OO+aX1=nFV#+lg{-s)ut~OQ5e9HCr)k7u3B&9NxKXQk?j~5nUc+;oub1*K5;q zyzSCd!d$$XKQ^zi% zo@YfD@0jlLe>QN=%6J^fx%Bwh=laaJwJu|bLv)UAKHs{ZtXKIBPeiauZ2ah*=EL8Y0hO?Z& zS&reXK~VW~pLAwA?-3jw-+mAgMSKf%JhcJVfY1S@vW*|FROVv8#MBq^B$WD4xo_uZFlt377`K#NdINwEc*IM$o_!$VO58Z1B4n@6G-(LFTC`Sjx+=u=8f*WlxN`zKQti&IOk!5^ zz{{${iu<(~B|{*m*)-~tC!<2=7)^=|sksTsMipUui4frLHE-1H8s59p<1QaRVJOqG z{$2WYxkmZz)$uYo!0h8a9ZpPDTxFvUzsZT;liCbO0COx%D13>zGb{x;9kGtv6|l*c zbTu%6r=7pC>!5R`o9hC+Bq&0dc}~-5Tc2{RcnKF$?ZH))Y?jRJ3y~Mj9YcQn^pXB#kRfv%CMAzm}$Tpu-Oa@y0QF^DSfK{;?!;i;#Q1W`<54=lbYg z-0Ukh^ieJ4`OeT~56iJ;_w3sGgrRlasYZDPDeCu~GHnNU>*Uu!NA6eRtYpDt0+?&W zwp}r%xn&Nab3f2 zc}lMt^lkNVRcT_<%2M%_{X5O>LvD7pbu=xF?^C)b_Fq7)59wG^d`Yg2lbost{;ifg z1F{2D0}sx&Vr0Pt$X6w6#0EOeDJNEXJx;kTJ1gA@U_ZE!lMG(2!s2Ome~whfoh^g< zkm87sc@qd2Xb`6K5V0eE6_OKMp8M5BPvEZ#6Zl4JZhxM=SrgJ+aQX?cUy3Up%sxg@!ztWc;^;EzFt4i_K&`@*+!Q_MEB$qomMnyQtlXUr=>}K6zFyV% zbPnFJ?MMMnLwV<$_kvu!hjzY9;3lGgcdIHPM6_IilH%0cY$|EYP%c?Jwt{#9VD>ZN ze(D@Xt&*{iQ!CDBt;Y86Vdo1{vKIDWp%wl%%(`JWAofs!_eH0YjvPD7BP0(J`}Dyf zF2E4dt8>)=<1JmB^?L;L(0@`7AEXz#c>!?}UxlHmA*u82TH=^+gRe?wa$P59X^8L@# z0ao=+mS0G;WW|C{mA|HC0ReN5jRbUkANvJ*?F)td`rcP~L&tWJ=1lV^`77pIafT<1 znz~a{sMUdENc8Y)Gc5)3otwzx<0s|oF5^cqdaA_rXhdrph4|sH#_ILyqgqb{ zsJhg9l+5XvHxs}?Em?#fHrZszD>AR9z9#u)d{d5V-<8MK((b6EADF-k7wG^N^@CU% zfHp2J#6foXhNFCD7w6`n#NTO}Ty&2!8am100jZkPcfgM(I%*7)4EI5S-9R3sBQjDw zw8=P}M+paY*Npz?cEWN=(1dT+#wuy1jdD6+J^Cm{Dii#&5I<`;Vp%wGP*+#GON~h< z134mtoyRAFaN~49{rqTd-Ku{km3N+IqD_*b{lJ7GHw#=3EZT2?XBllfTulcgyrr_! z0cFMb$%FHjbbzV*PqQDCkF*VKDzH#%r%`D~m=2geV9Als4$?V?0+PCeKQ3Fc*tc%8=W4A=$V#I0l4(X_n`oi7pa@>e#xMg}i#kV6TeqWH;4`Jf4$#pzNs z;FVO6uKUuj@4zNB%_Pd+L-Px z`Z?IeByYC6zc;t^;9zMlDYwK$*(Ei4JL!J>dL_U2!|C9AUs2zBDDvDZ!}$n@gCte% zaBtIB`^c~Cd&JzK&TEkBL$t9<<-2)j9@6&Q2Zq-N874xrePd*=E_s;7t%Rg(nG9ed znYhaq{rP>WNC`^cQs0sgrn<6v2qqn2o^!&%6LP!e!j>yOn0a?+6Mt=2FU=agxrVll#16$VmaC{X)JP2N=#hOx1S=g7X|a=%_Z zP&e^hRpWzk*Y(Ps`bx`1WF0N~6RNET2cjL#Cj%LFqKny7J4z!-wnf4*V)OjRrnEylBck*;Ig<-a4iHrPdd!Ec48pusO zNqI>YoNUTu-vmZYNDgeKbx9cpwKw6$gp%1;Yl3H#H8QG1tCBO( z>m6uXyI(B0hK~1e8794{u)vKEOf9bMWPT3<>QjD2GDtN8M}>$uAoj|OlP=m07|P3a{T zoe{i$1#yGoTW})>9^bBR{6ZyD2uF!;#wd3ni?thurWP)`D3s{g%?A<;W-3kIT!x+t zxdeTTO{u8EB`aV_3PL{qProI^y~)`IrR=G8A#1@h)q7}zx9Y<)p|6os`5Bs&#%W{W zn30Vsa5NP>rNpvClO}c20la1pe{OcE?^}}UE(MfL40UhsY2E~R1TdSo`^<#lo@Q1Z zkx)eNGWt6v@F!3rkTtL)Q#4Rv!S)md;!XwC~7hlmEEuW&9ddFle}4{0b?-Y?3wb)S?zM8QHqu^RfonVpZF9dIgI= zkFKpNyCryWLhZvgpYf9g#g-e|ZYQo8p1q%td!p4hUyu(gx?{IgCQbsRWF zoER&suhnWN&I;@%?t38{4Ju&2nhD9xs32kFD^-aT@Ir!v-#~*zg>ZK)zT=RJ{>M0T zuH52roCSKT@x9;%!iY(idAxOD4y(YRS!A=$VK)uP(7{fQO)t~9>G5V>x zAX63X6|uL9Gjlw_I2tZis3eDd|Y%a1Bj*+l0w@v*sDqTjUqibj^?47 z*@<}h21!y>n);;=6}9P*;npBlbj|5(uFJ7O=N?3cURFn={1;u%Oitx4cgWdxdJ3W4 zqcp8kLES_d$`7F5Y79=<{d*Rc4GpBo!NRpp!`WJC>_ZGQ(9YCO81)8;xn)wQfoC~G zCJj_;^SI;A7l`EpE4j#0`%e}XKdnVT=<(SLkFSo}LXnIajwMa5ERviWwIs;-pp!ZZU_ze-_wYN;p$A zh#57z2kTGli%VRI|C!o3#!(X|D2d*CC(4E@#9Xea4mkYOFm%`p{GcEj%$M`j;>_!->}z~gQKT4jn84uH z`Gy|cx|S^+fXWB0xHJ5Ocefkb-+Oo;XhMrImHB1&EXtSFbt>~&Bi+4VVzUF!J6xUD z&vz+nV>+FuWj%`ghkftfTs_&Q^8_M(i4=|fY_s4eSdec!VQx?oaNL!)MfQ7dt7S#wU+%`gG`YBG;&3naI)}~~ zos;~43RBg|uuTq*eM7~)?LLvT-I>Z#Owt1jMVSb-Bz=O3kNzU>rLs@bXQkg9{q-&X zkzmC4VO1huv-$(vd>BgMseJL*OXCwizBHeBa+IU?ZDGv*Z#VzXSMJZn_P(IaKG z{v3_a^FJO{zc~kYz!`mipq!-J2zV3i^8TE3nx$soE!`X|h-QWAr<`xurv?NS_*3|+ zyZ&&gx~sQGsGjgTrxL?uE_T!3GaB^83FH-KSle;(*=R}---|0^##f5gDEhQh3?Fn2 zmU1za4v6YUFucPuA}!#p<~wviug74FIAh;8ZoX3dgq+y9-2q<{s}Aa3S}E6MR@14t z#Q#+)>8obB;h9ZcFV~yt)=LJ`HT68)RB7aK{YZGWtVc|O+N#GJX4@2Bq+@=T zL4eBLqPU`qDdwjjqWHUfgYQUeCVqDbLs5L%JVr&niv|WhM$4otz(2{!{2MvFj0IAn zMPg6rK4UAnKnIX(mM2?;`TfeAm2E8}N0d!Ee`GyrO2%KRqQ1Zwy;3br)oo8lGNe;B zTJQj4S$hMdy2bp)iTqjfca+0f<>!OCzA{UvHj{QDG&BhvX9f_ezOVqdxC&0y@^C)C z0DRRm{T>&)b@1_JbQ#UJ@j{xg>Ug}HuO&RqcCp5LXeQM`a8 zdu}>Fq@?-Q;MCAc;K z6d~@=_b#!yhPEc&psvhIz2Kjhn~}fzX-B5XY53Iw6#P;nxFRHFij%h+zr~FMn?p4< zt}=zXXl~-K!)V;&taoDTkhy#RDkf@kaeoHvFCdR_pX)buYyeBGq z93yqj$LZu8R-u>?-{EdWYX&mY0guVkkfT&#ve`?-od$-Pnh9+`RebXr`3QR7RzYVi zc{Z63H0$IoksDZk@8NAXzK2pZ?#L^(D2DmE@w|zfXbco7;6#xK0b)X_JlQ4;@xytI zAK~@0LHZZPmRk&=WE>nQ`(+Rzln>9^8#WA^$h>@gK&eDZU3!>xGcr$KM%mZp$Gt%@ z%@{_rgREL}tP!Ae_{bRb);qn+$`}b zLzYs2r%*Xa^EXM}DXz>^U}+WkVf}otDU<4I6hUu*rTfiZN@Yx^P6+HA=+hG}*UWMT z0^ucZiHiEQwlr`=W%iA<>DJ8lUn;%~x;juDg^lVzwuL6n_-6-rO$Ci~Y(;=f;1Er3>$t?L2X@>jJt}&I%W=K_#fl23z1oXvuPpr<*?5ylE#>g_I zeP0z}Lzc;G&Wv0MJOa;d6_+7L?KJt*0UVZw0W3Qah(**RH2HXq0lrjd13l(0qW6~f z?)Qq+^_Z7nk^w9Nd@N7|af+f%W}fDyFhZ0>$ZG9v8O$*>9(~VIbZ?P7T12H!=}F|> zShpugOsFfK_qf#P40SY=*vh}~z_ah}V#hq`TC+b@QzOkvEyea|wp)O2U4tqFN*zb6 z>=>Y$P>f}YqzmJts*+=xdstjEPpaB@OvN-wNOX~9@z^E|Y3qG4F}g^mHlt01=qu}s zT2c9qdgAXKT%`Cs%Rp2~s~kT`kx>!mZp4NnkHkdhy`!*g$0TnD*=X;}JeX9eDamK7 z{toXNh7gGoI4v{WtI~ssHMuz)5PbG?w&jCeUq1_Kf284#&l{hWq~`PoX|TQe;SNI5 z_-dddV_+H}?SK&ynB*7427WlQjmAyd2`+gGdy3G@f;WFv(^3Uts@cET{`&6Y|76ec zAZ|D0$n<2WBZBn?I30KbJ2jD_5#Uw4D9%Y~+H(sfz+O-AQ_M;X2w>L5i8Ey3QOeDZ z74LY+-uPhW^Tk+OYT4598(fR&ym2j~Ho)BT=g%25s*0Dabq}Ktu4ZHehQy_LV~sbA zH$Zz&sc$F@wd61=l%R{n-!{ObncK`{{!9*<6yiC_EgMNtus1%4LOdi%fk;m>b1N|F=+54Q61jJ_ zS2MI#)&MURc+bdN^PJI@+un{a-|gW!*_P9wh&`wD!+1@lqwv7;S%P4!yEkKGy-A*C zKbUw*3q(2@Q)HCduqP=J`jnUZ4-nwNx_bqB``0EAr%8fQK6rndlT;a3LX_Hs*v)|J zH=bLW4oaF6FivtVxsMrwdTU5#3~(xZZ^l-MglXw{K?m?qFMou`7*Qu2`g-)Ky81+ zwQsCq5qDk*yu>j}y<$0PWt7Jj?5Rtd4Dnp!s*CM>Di2-h{L&~^;r1x%Zdvnjo|{Ex zk}6Dh#R|U)V08qP5C zZ!w`{w(Oafmry#|Y7&xT3hVRUe;%>c7AUvr?vTG=ppq0jsyCaTUOA@|E@0I26DUYg zYSPl^6S+<C9 z4l9GGm0GsGRSPmzy{mthpYJDt?NI%U{{1Vx#xc>TlvZJlx%OB!9rr3bmZ+?%=8ld(|uQOP-Ln%<~Wf5Lh8OcPz6gWnp?< z-|W+pM-@f{lDChwNzO7|3KVh`7d)tK53N(3LWWLAlG}16!ZkE-_3e@Q<@NoU2ANGl zJD9hS^-&j*ReUlSlhOV4yT|_MyAQt79lXYs@X;o)P~chG*JVOXge5W>PiDmu4lcFV zU_w+z*37JXH%GjdUCZ-sh-J$R5if)8=1nN2b?G{=kikV&Sv4;1{JGx-Rnnj2du2m! zZ$qtg69mJp;+DEi8YGrBEz=!Mtjk^nm*5QAlW&a<+QO_bJx_3Npu2K}_acOeP&_y+ zL#r~9@Z_@>P{f=9zFr=Gci27>L@N3MVR-YQb`D$xYTs|Q;qmw3Ge3g#+f!_BOwj>M zv_bK!6!1}a;rNmE5}e=B9eu6j?%POBiVrDKu5%p zsjfS+{O<~~TXlUbZUPNlmj&x~=A8{kffxyAwtS zK#bYpYi$juLJcS}(~srLT$bu3W8GFhc7VA0?X<-wllkO__=KbJEHQi zk$^1opn5-ljoTv3(y)_u0e1 z8q${XxeY%axJhjn@;@%K)>ar-nXGQWvIVGaE~uviySnz?>d{1o-r#J73OrcL2H4@l~o%5}%IlO-^G z&#OQ!NT!&lahQ7UWXnmZfd)CsC}?N}Ez(g?0k4x!!o7k>TNuzHVbJ;Jr$fW+g;JLi zH%^M4I?&7d65=DByDIl%C&0@9_=8roNNbDwD0JDmV)*-yNw1Z99!kglEx00#$Au0M zr3uS6BYEJFEi~1Eit;HN+sT{pD|fAhep2hhdYn+u4WS9DPteE8t{c;O$pvai+M}!XNu1SI&0Fs^<*MCCayW&3kox z3ySqO_D>SJjia6H!ed883@E{5t}%FId-x~Fo=;@arU-vOy0#+#WDse1`6?&4Wb$wwk5V%u5XjFgbaf3)yuG|qIyPwwkK2RZq=5=zZSCdD zFFGFJuS6Yc%k&a>z+_we5@b!G&Y4HJ=MfU)>Whm}BZ@miv^7Q>GQ3g(h_)0R}khv*w0rC2UhgG1M1lb*N3%MAQLI-5cqx3&;cTa}mSc6H^))hm6 znsAH_DX$pnsaSEqZAyvtD#A^SzQlB3*Byjp>7DZWDu0AoHUCzet6}G& zN5SPY#r?(&d$pNt204f)wS5Fia4rN>sUY<4f{ZFap8@3d=+c6-J`KlskWp=XAS#zhA9`P+{Q9B{#D~B4^ zv*IV{fT14h+LDA*^4=oNROhpxYlCM7=OPHJ(*A4_b08;!jq)Ff)OzB{V^&Pj?%-$l z9-g$28L=W)WvmA<;+QxjGJP;~fS4lnM$tIMyT7F}&^xmrCn?%(jz{Hw)0bW4vCMLj zaD^eWS1Ui*I+^5M*4_}Q0V;b>0}s`b6Z0asuf<3%(G-m4$X>v!S5pL%eBS81A(!`lp41HX5k_E=*Kov)2AGC2rUSmz zXVOA#;EVddg9O>={WPcLH6TUSxZOAe%S_X95zG`;blY5GgYPsl0^gdI_*}htPL_B^ z0(q{NDrSqiXn)_n7Dhwf09!Ie2$WaiU%aB2Z&U{uri#g&32XN8 zN6IHZ`Jo9U#+=iaPH(G1eWe2^XG4jcQ%Ic0_7^f+OHA-v8!O7~zK+|Z%-5}?*3O#C zp-(^d7t6kTBoH@u1wklIx{EiT5vIKQ_6nIQ72~s0`GyAXGO&GO6aSfa{(0?iE(SeX z3&l7V6CJbk>fe*)TKgXiTs2K*95>-yNlY+_nDcWP^?_!>k&TMTS!v)Q;7TFfhYq-% zgdI%x6rkOgAb~P>4YPAlO7vL#~H|; zsc`e(FZV0fU=CR;a9DnNkhHD;Vg$F)D`Q(9H2 zzx@N+;8;Ii6vf}G*4I&Ul_!BH?eI(=wUW8*PGOLJIQ}3R2&DsFPV&*1DWP8*g!+0K zvAh-t-NC1_RqG+u>I-1U=#?K5PuDNh{VIGzM7(?n{thEvLoGDOALMGCrW7`tU7Uc_ zG%k)Pq448xDIy)}8yo>0I%$eqTFxYRj6X%M#MRTgn zAZ>|?3-dlqSh}jRYw0pp3fWoe-_XPUo4Nkp?>`^%();H_UMl~X?7ja7lU)?q))N+Z znqoF3(j&HDsnbwa^IRmd1F{zNiXAYUjVvEy3uxhdeYVYL861ZY0acZ=qScIUM z<_;~eG8U`-wZP$C__po)+U(;FC~Uu~_-(D$nD_pegco`El}b#QEf1DtTIXb?d3$Ws zTEuC6mc4}!{pA^a`IRjag*Z&o3#)tRJI%2(z%ousFp^4_M>h128bS-dOdp7S7iCD` zM2nu(o`i|a^8U2FQQDFDVQ?k55hTWfULHX^n{^s14h5*gOB^x&ru9};crR1e`_55H zX$u#lep#m7Jt%1qZWrk435BW>pL5vQw(GdQmu#F3UPVpFMA>^gPFzp15_Myn?Xkq5 z0vR6IV_UmKrQBqbkkuv*Iad(c)facAsMw+W;reQEiNYA@SZJpbpV(JrqGZ(l?S~|- zQi5EJpzTsBX~0IluW#w3f}QI{I}grrY=TsUPI5<*z>)3P3%BIM1k59FYIDP`Gzn6m z9&ls>GkxHfyoJI=;xj|h3iove<1M21?dqOUKYx zYipbP_1ErOg_YpuxkZJ9TZGXgZ*~uUzw)?hQdV`JmCxBCs{iwcF^eB2I6etscvompUP;05E<#|wsK z2=L#Pi+!KV6CG-#e3C^B-b`P+apgd6W{saV_}eq>`jL7I$}=N88LWgbszkv{*;(F| zYk__|exx0MN;!nT$yYHSQ#=zTb2AQbw&5VrdiALLM{V%sXYulzJi_(IpoaYky2I-7o)uvDjM$w4SNh8tz-(thUD+6Hl_W zblG<3*r-iNd1+w)*#51<*e}WZCyNrkyFM$`RkTYZR!sLogmr-JJZ^_8LdLXT78@VUu)tn1ZUn%L@G2wGBKh{P#TR1pN z$)3whUbbb@L0S&xo@{M$S={u?rtk*)_N5jW85%t8F9%=RQld#{GeQU?7{0I#Lpq?z zta*}JgbHrZKp-fE`yi@*H3dV?G>J4FUL``GmMA?cLx0f?=?0z`SeM5Kv79^6c$&$r z$N(jlRsG44;%73Kd;4NQpFXj5{z?sb24!htHS!(3qZYsjvHEu>3I2C}C7nP0O0?G` z&_NrPLI+&epN^RkDYf0*w??p#eTZ@- z%MCo+>pG`T6sdiq^`A?rNof#)=?O^bZSiHpL>YJr)Qi16_IWQzH1CX~nN7+SGH!;#=qjNYbj|3)Mu3Y)Aia-6H4`D8Cp5)Q$rfG z0N3pFc~RXc+2`?gTpdWAFMHHR`&_K;)#Sq(jR)O*0uTikcjKs?v9g z2Auz!;9ga4z*1?yno6{^ueuwHZnvn};#zlQdsq?+V6XgWY}e2fy>yj}Cl zbQq627Pn1$aZeL+VKEcVYYGtD@GHi@O`y!Q-8zUXsI(4GAw-X zM*MWl!-x&z7-RpAQLA5t#<)3vzAIDvBWt5;bgZ>qCz1*#as@}LYtNvlra!l30 z;hVS#h)TZIoW%*>XBV7s`U0Z}wu3SnE3G#}GP8-HWR+~)_f*LtbET>VoY?^xTInR+ zKkS#6y7R0BgY)lH3(t~%{^q)c)Dvnv$#_L)81Y6MLQtQAPS%_4{iEC~cIRne=Yc0hJjk65)t*f~YJI>;t@V$1G&nnlID}WF_+L zsr>3{{3D)bq{rF2K7k7w{ym-UxKY|LklT}Z8iueJ+j{@4W{}qFYUk=+5D?d_KOplZ zj<#4#fR$Sb#mLM6n5&KvA77;dLXl;~=#W?^{94%a8v*t?5UtassN!_6h8a!`<-^&j zq%~_6GIxStK5{p4>~heRB1742xhhh7Fp+u<4-|1VjcW)+;@`X8tE$A*Rt=~uFISh4 z?xgGZ@Khz5PPu)1ZgHedxv!(2N(>y8!|m6roX@iE9tH0mKo@E=1_Snu5rX|HiNmU5 z;O}TC$fU0mb$C*`=7EW^XZIoAQfYlL2PxN3!<;qijY-mQCR=yB;K!50(p72ql5g9b zQ4rtSlBtduCg$4qKHf;Q?BDq@19j-;x*^cv3M@5le(!2S=K82P_aRArCxk6js&9Io zS}5cUCC=8rPLv+&VR>I>P$yt_{M&BYH3@E@V<9*5_@eNe)Zp-{J}7YDHK*Nu zyI$iLH09KjiEkVr>{WU17eW!?UKpq*Y;SF2U2@I1eUGYT>!eBO8MVSQ=((vZwUC{gDL}lNkA8Fhj6&aKwFdwZ|Fz{lHmONRvAvH5RXGnHnRaFe6 zkFNO=cXb6{wtuxh$NsoJ9upqhlJO`OjTw4$$P&XP z{#px2z09z?z2Sa^5Yk-DSN=7+q-={Du~rSb;d97(G^uW#)?KWjD6FDTwSKYHow%=W z;hdpNZi^u^-_E?=ry-JS=`w9rn{vi=*)?48@g0k0F z`_pD`8bql&eVdubo&Dg9jix{?-U-Q-1-|dg+imq-F|b8po5S2xDB5^Zh>bfcuR1fB zapkfff&2!p()j~G%%90$1-^R}(Bn%k%T8Ui|H z9$y9vD>leY!t)G>CB?}gL%}seOp1c%kRjt}YBI~~Bm%L)H}~PjiQvP;yA`jACXPYaE^2xV0m)$6*GkKP*{v4SJa|P1Ou_9* zyxT3gIdxsp%w+jSJrmd2`1@@*Ft~=+uB{GN}s8~RKT=%RH3hA?useaV( zOjkm(Rpg||%aTJC&;W-h8y-{kvqZftoY1ht1qo3M6e+D+)Y~dPSr~BCY{i zJW&^8wy_npq{(KYp7BqZ^Z){;^ zPsfBJd8x)eex697;;6(R;g_}t2#QQ!337jA@2s@(YT)K35&t5mVb<5w*-Yxmi-fGA9BV3uD z(&keMcRD*e5@&+@$n7g@WU02A)o_$8d~Y`GdUb7GKT)y5^;+Z&50#>!Va_8C7K-xF zUpEfNR7E=wbQLVTGt2JaV9~SWh7^MDRHABtRg0xZl-gj))WDX%Q;=TVV!2FLk)f1< zSmBFw$x=CCeE*svLnmI|Ybf|G>sf~dQzW92_4*O|lwPtDzCEU*_~^!sUX)7lV;a0z z8a*(lp81PmSg5I~aWySJ9{f`3)XS?K8ZKPcW)bC8ri~y+T9m9VS@0dn z^`<`uRLQzVFkG$hS<-${S6Z2Bd$Q=~hgfeTE-h4_7Jr=t4<7**u4=GlqdVFBjN;b` zI_9p!3hu0{x@B}2Qzox_=h6ALPjzNJI})X~;r<8NMPn|NKbOt?LehL)eexG7O_S8f zi^-fGqTF^T)>q>oIZjoO9L1S)CTT3kO}&!}X<~@A?8QNEz5ugv3Qf4A{^T#J)Ish% zCA@&*VLZ6`t_V+74M8cU6j<1#)q)D&^Ui;*x@aj4sONi{tt~t{pIRaQ?uU%XrN5-jL}C+$mpv5m5!6wJ zb$%*+LSm4{vygdon(T&R+vaYrxBu#`9)}yPOG9hwfd&W-@OSSzaAS~rOx$C!ClkY_ zIrgYBdn1TFw+AK-4d7HP-otN_c2tLC!fD@T=@e06EaB`9S2Fpz>H7b%_vYbHhJD|- zN=Zoe5DKAe$yV7a*|UVmj4>6mj4_pcog!pQhzX&rS;mAhc9T6M*+yov46-jnmSL9O z>%O1oe&6T0pXd4A_j~+~_jeq>_jvwr%;CVy={nEzy3X%s`}Q?kw%aeGf@5Av)xhM< z?#;%T6f^`h#_7pEqJKg(_6H9{Aibs{!nFp+9EZ=3DWb{MV-+n z&qnQHgWsH_uWxOgYf%2zWW;q)iy?J|#uxrQxZ~QZ#%XTRtbS;xxDti!rC&5erg!9? zXO3unW!#1X?sr9o6I)faUD#!PnR}OBIkw0T7c(O14rPF02mnTsW}GU0JZ8~Gwxcc5 z>Mdz*-s*kX)?sRw+atC-2$de0As#)f*1C&s0m(U5XbY1Sw+Y#W^_3}BKsLG+N#GJir6$-ONA_pUuJ%- zKE7G-#G}C}*<&}vjN@_sTWb$*Zxip^sh<`Fn{v8$RliG8^(px2Xux466wKY84K^EH zhv%F|(Spqdq(hExu=EyOnmUa*arWHYbAEb#e+$^b4>p&T#`ILuT%XOi5BMgcHi@&G zUlR@6vaWSi^tbEC+RdSo$cO|aH=TV`^K+(5a$)H8Mu=c5I@Dx$;_1|#c*88a#vrx9 zVgtS(4n2tT1=Z&VUEwKjI3&@*5^;bAe}I?jy!(lgw7Q@iiFuVcgXXXs$=htIB@B`j zchi$iz0Or}?)j)hn{QdP$>nR^xt-AMhLT;4*?cs$TM0PbJih3j<0gP*mzO7-_od z8X%LHUvMQ%?{*99MaVsG@67;TrkTl;i|>;&q!R6kLenQpN~w+H+75{`f$6{RjcFe# z^z_#Rm?@W9tVsViw8v3&=SbysK`pzl;t#u7j&q#4#8TqZ6jblEGfdcp_)m?QTaRuL z?V3kxe}nNklW_-{#b2S7V|0Fz+1*lVGcfvega~x8lPI#X(6X-lP!q?@ z>6$x!VSZ1hS5Ua7Yf&-dd0@E>?Too<6-Nt_b4Q=(wYX^}ps;Ses;y5kx&!g~oXfNB zwIe3go=bTdXm1%J z86z+OV{%Z_$~GqX+8MmA(DU80wXDXT3zwNZp3EIS9%CV8-TkMp$koC zF}T|?@2L}LuvslUYZ1VwVo@4C zQOA8YeGE7IJ4=)JSHyLOA3{!c?#Lr~U?Mh4DP}SsI|h7;+UJF5hI7*eelrWgut|J-LwGjQ2l>EbF8iRcF=)`?=TEijegvo7 zSLsVZEf%)j@>Wr~&FhmiWQgnFT5Sa`!drW2c!DrNZ}`Ik@!xCGHvw>!6uL3$=1>=& zvfK;U?+X3j_U$H-*6iBdiL+KF+Vc9QP4`ylLwG{Au= zw4KvwdT`kASHb0FJjEPtxL}&OU`S0xFEP(;s-s?Fb7MhtkZVEWj3_t%gpps^8Nu}~ z#f$eiAIh`mUCs(LlP~X&5ppM)G4RXjTrfn=- zVN)O-h#|>ri6vt=0s|bIM|{C1I~+^O^xH1!0~YZX5wdB8V1ZZ5K^#$uj}kNv$<)JP z%oA=j*~JFk#2J}UbrM?7jxranoVjQDU0thAty!Sr*Qb;{+d%j`4+_btRa!_qoDiEF zuhjYIw5}pB1%Zpf-EB*uN6Axs5QhoNO)=fG6BC|xJf3!%EAZo0)k8I0I=`ajCp7wQ z#}SBf1*scg1Nmd{&-8%;V5?mUmkIP{gr5kM>ds7$rR*?yjDYhxWc1-Mwg=(&a>H*8 zdIyh-lb9&VQ1C~saSHL=+LW|tbygM;7^P`yDGF@kz3ml2EkjuUaWB$2$oT-^l==L8 z-u7O@@+#4ocBBJ9MVMLSK&jeDvl#G*lJ5BxakfZ{LAOkk^zIuqxmF{H4A+{(959$8 zz(3;D;#Sk0g5q!wJO_R{A6mTt(EBN&RwFhYEyDM-RX+q)%^GehB?h>CWJGqPibAfg zk+(=|dGl-L4WbXHzt|#q7r+4MyQ)K0K1v6Vw#Y$Yng=OP#j!%>Lq*YYZp zKF;_Ial43K;<(s8&xiFjsN9e6R_Li&>HtNntzoB=i6JMw0EKp2Z@!CYzn$bby!{&? z8-u;DO@qb9XO&qn<)ud7BQJ6_SaK(-AuI8q*cj0FDM%7?k^0x^!h&9Y9cVcu_krRpPJxv5; zWcM5EYK;+v!xbuL2CqBx+_$>+ZWN59%I&TG3M;(VQ^YL@SqV7!H5D_~N@zJV;<|ap z-e2SOve03O;sZ^U0)L@P5JW2%f2qxrFA}nJK)4a{FtB#Ut@Zb`C9%B|6!k4LU0iKg z&652GY5eSv9{IxPqk7HD&|9t|srufZ80i=CZ4)6w4^(I^i8BbQjNeoo^Eke>ae#+< z`ST=9b-Cdc`hBR*5VmB`1SWtLvb#xG)*Rl6*D#TZw28BkY|8Qvaxb~Vp&l?Ewl}s8 zr7&=rjs#1yzI9wK082#c*`XJ}Z+Xc_KKh``UE;!w{L6vXerdrAJ|+AKbYLT z4-D^Rn&MmiLKe~U2TB{2d+~vMOm?53hdJymfVm;^=1>ke9znh56ZrZkb#kax;X|_} zpQkPtO|CY+y^Vb~_Zy@V>zd~3529lkWq3CUer69|qDh}3+KKYdeAA#t>2ogkSkP$5 z$qG3Exd+vMmn=3>P}$3$Vw+Nbr5|}i@07MOcgNJYMC;U%7J2$3DkqbN0-qY!BNuEB z@M9;{8!6YyTeYu>M~XECk5ugoak2f7Opj8)QsFFS$u^hqk&rDoZ*%%oOSYoZoCCU3r+ zW#8mmYm3E?Z5r~vxU^Iwpw*q;V=!L)HS<-XtV%Ol3opBB7_#fEB zzxr*D{2BX;k*zRdGNgn+92Ivsvt~SNz25!(VuZ==VSEMmZ+fK5Lrb-b8%*Z0jR9eX z$LL}z;LNE-kViK|nf#kKJ(ROE%a(-t9{sp-rS4LYEbAYZ;5yOxka&|x;__9=6b+K} z(-I?_#oo`CbURMG?bpdDG_^^rPc@o={wIR0e+_l``@pfW_n0nit3ND3AD9xOxkFCu z$hxsVEE0=s|Bm9T`1tR=Wq-*M>kqZ3#`;laZ&l%kv4m0Y9kGXf2H0wcp&x2vRbF-; z5Qn=$thK$C?<FKK$Zh1}c5S2OYq(pfh91xX*B zG%O&Fr|MY&VgHObgToq57 z0p+fylC#z>tiqf0XTTw4D`6I6PNQ~U5AU6OI{~#JAFig4&L-qmbyQ~gCK!pVS{;>{ z!2YJE;YFu@JhBwzb(MSXr^?9%1VNtz6M&o;(7RicZnaoV5*#7)L+ofE~WN+ggMx8oqHMp|CB8<&6BX zR<4K>>jQw|FMk>@5$K_<5>(>33kY1xJUhB~J<@U=_&B0e9B(MF_3^3HzV32~dgkgi zm)=K{4HQ`(nl)@0@yo#04`eakxkTNSfy>jxP3-QzFmEus|LR6q*~Mt9L-MAM1pn0* zhVs_WJGB>E4yFK+qpJ1BZY5RO*;#C2bpB&ik9vnpJ59<;b1)0++mu#VL(?QmCc>p1 zI$k}$?ez#-*;NF+!`%WA>_vWEv2_yDn@j^vv+jPo9Aiwb0=qt&2Ab+uZv;& zhsA`lLh61}@g)je{}h%KC57aU=wfSvrGBM8W{8#Zt@ZtUDA|&4%5)J%Zr1 zu-Dke+EXGxmvOxqPj#mS}HGb4{#_`I2eQ zOPO@|a{G1D$3Cgcy4{DqPbWNs-AYTrLqbtI)TK{Zqa>KeN*EF|;<~pcLBSX|HGLYZ zz-*6uVTVhgVYy;!UuCtRD_xrQLn-J8X;yj zTh^jxG>$U+I9T$9+KAcy*rPR_ZS7J9oSl%iIZmG3DT*KUL*FvZe&?V zhye16UI(9(Y?VV)?x%Z~$PDiqzPxnz%NOYm-2u(ND7Bw6ypWn?Dza5Kg-v_#UsKBe z7U}%|^x7R32d+4zmlg`2Rwik+hRaknjcA)WE1~!`b~Ocao$He9gT8&x#!Gs3}aOLrhJgjRRRyV0Ydu{*l1Cv(3 zq9t(nhq@cmH$BoNlT8v|ICV!^+PocfS>}q{M-m=kK z7q-9H9er-&+t^hsin6|bR=#D`%wBOZeB8l`{U+N>r5&?!7%E_c&JG!I2z zcX=#)+AZz=ac!Q=%Dd&{%*i1+rSl$h#-mPdC(u%dBIe}o9!-*#V{vi|J0LE) z{?8te1*V0d65Co_BJdC^Y^Ep-DLg=Cp9|vS$qJG-YS7#Ma=Z^t`+-4fZ$h=qY1bA9 zW-|O%;4pAR&(3(0jd-jmILo*uHqX6(2vzIx#m@Qi=PfnefUj0b>1lv^CX=Dm9O4~a zQOg7XB`^|nWu8%{%K>r^4tQC^b#sd_Y&)U&4h8a?GeFz)Oa%6R|D|9=l$7wiFWL9J z@$7R<0PO}<63QA_+{F|DPnb@%!Tf9MaGAubZdH>q(d-L}k{QY0%hbGji%T#UY8Y|h z;v_TJ$FP=la9>DYS$ax{;$^ViE6QirCH2n#)S?R%@ByI?Cam-1!ffAM8EH-b$73wI6TPsd z$4^zEx&ZlC-4Y3(m7`mJgo%u<72@!IGa+Ip$ZLc)E{hF+K&R!0IP?2-ZGF_Ev{4S; zW7nk?ud8(OP^H2S7=1hV-Y!!H5)EBKWVZn!#{WB!CrAEH_4QQu-`7EPzVDCzRdoCv zD0}uKm|Voofv8YE00`5gd=3aI|^hvg2EJDi{^uqi?3=dfWUN^MHS7Kj+dq;INS)jk(+G$Av=Wnuu| zVKL=e_Y7n%qF72Y5@JzUo&LZ)1?(JvgyacT1?1u8-ty$gclaTw7uo3iQV^;Zpa$o*r#8--C7lpeDQd~t^H|Wy zOU#?tc%g5VB8IzhvpaH+k=Gy#8)u8J z#+I?0m8-35sR?v)y=4VB(iAokmmYs)jP}-y0sPw|k$P6yoi58VmgWJa+(C|uLMUrf zvA%k|6O#r<3287G44$`K1@&};@MbHSU;OG2R%TZ_!pbM=v6`QG;g32Gu5Xw84lWvq4Qy3 zlRS~_I3hFZBUbNZ=y}R^=gbthZi&kh*~7aKyc0KNJG_3sDQ!KVU0Uc)4@{EH$k{3v zxfBAR^kG*qZZMG>2E=s|Go3y)(a@5c*CMc~kv6Nx^@EDhry2d&5xuve*fNGtFJL;B z3Wt7C;=M`^mB_rx1F_^hR1wisJB?~b+mL13fX`MX>@aLPH8Ta1;6;wqco2M#_~P|h zySjJj{>tZs_p?c-f38059w`HrUNkqD@KuTQcO*se-*Nb-F_?$cR0qUKYBrLG4FJZGeolg5-1IWG9c~#5ji=4=-Pz$7^pRNwZzzDpd1nOWr^Ua#5jjB0Z#n z8n0hT5yzB|kBhQVnoxZoMJFtz8*2>jR<$X}%XzrkPGGeox8Ecrb>Vdozyjl24Ckky z=oc}fGD$O-V}bZiC)pwO2_r*@VGnuJ_aFg<3lKcKHU7>J)q5^O>CTJS@|RrP5-I?h z6&lF+8XGoKb(gCt8)F`!XQNjtVshMbDK=nBLEYq;Ty`qAT^;3dOxuN7#qbrY+S_9HiCPW;%x6XkaaBf7@`J*gF?C4QPT!0VACKw)Oj z3`lrVJTE!mQVTr;s>a(0xS_6rk3S_zKOOr{1>SuSe_~C(z^xn0O_hOBb$kcd0`0p| z$y5#ZP%+tr?T~(@3p-+-v{saaXKjv{fwv??N1Z)gx%L)rh+g^J`@8qUU}UVsCsfBq zoX{R9>1B(I9I7)%0!Kl0plT_rD!9smd!8uGU0CtVc15ax`k3_(b5)LlbeZ3`?tB&K z)}NbC#Lc6^?qcyMO_(X6rC4V>WsIDNn_S6SfbAvjes3q}O&2P^pKw~ahI+PP6t&^4 zZB`g4(ZNwlS|{vSayrI4kn3=7B#nAv#eA)J+;OT|-ORkRul);EwDzqLo+m0Qe?0wR<@t`wHT@$!_}-WMCiVW0w{-8B);u>Gx9GqqZ?X|$U{ zXp)tUqCY8g59W?hb{XA~U|(qwg9n%qc24(dWa}&&m3mZ}h-X~PAmbjnJB6hfmp{B6 z)Kx}15o(q?n^QxxrTWZx4C_j*H1OweHH@O#zYi1{kSo0mvfo?_MP;Rjh}|g|m1=;c zdz?@HkS0Fm1$zg^VmG{L9=Obw?3!lAu+u${0CdeFhT1H^fTX4DC1IJT>E2X_&5Ruz zO07f9*n>gu_m%kO;mE#6wL*e}PSQ^%j?#46HOn@>6j28z2t ztDuF0n6m1cGRSgVv1a#$c!!*Xk zU>e zZSS|?WU$gBdRnb}vA5Yy!+x0-CaVhUuu?O)EOA=T+gz0Fh2+bY}N4Hvta(w63T-l)p*x*Y?yY4~>TA zj8MeA^?ZlJS6J6lE}I%R-yMnkf{EP$+`9phO|C`mPGZy#6^<)JNO6Z)$P6-e^%l1? zqj`qQyofoFX#!tQfw^Pv{j$ZyX~HLOvsS&NgvT>be^}h$QoCU4lP8q;Hv6rKz^5e0 zNDViq$FLOlDqPd*?azw0RoYcOPXx+`%zv5AX{PtS2+ew1>}}uQlAF@718?;G6}LRm zi)(m7f)WPSO%8gtO{WY2@4Pf{RQTTtmH9_5mF>Uu|2p(%km@X;Glzy*4C<>yW8z5&mQ;e=jy+eaj7swlJ$m5fN+_#36yxzyOX|8UW6>n2>@#R zN5VVz$;Lsv0jiO196G1S7I_YJle{X>azaKv0 z=lgFu&i%9S!vEiX@9(+6I^PN-Knn72&GfI{{L4xNg8NohlYVIK7H_((`p}F$kRsaL z9`JxtG`cIsZFjN9~*0);0W;d#S{$Uxos<#mh)SbG0 zKEr*fOe%tcS@EPA*zp#t`;KPNxg)zQfGF#tH90KcPu&e+qYG7E&+SMA3Nyz*c!jn@ zS)j$hy?VlI#E)iN20t!%^s8A>OuR9k0JZD5NJpZg2kFuW@Q;@Ds8)*2?)BBKb_UOFplVs#{ zfy7~l{MX!6(4=^e-1R>!Nrk(ZodL79}D=G0cnyvB3C80e{MRuxkx+ z`GqX}_P}r{K-9r8(M}@jxnz9jTA2r6!utr~=78z06uAJBbKfOUw_WH#%B*dAC+Yj7`A5v2`)t1k3!V_el8To z7iw*pYq_ruSFf)6zOlR2up7%kG*X->t=aF4!+zmvB{l9jASOeb=ozz_zm*S%`ct?& z%*x^s+$OlTN4aUuTovB-ECk0Znk3=)ApSaPBN; z&ggj#gU~YWwGx_^*=87%t)(*TWL?WS1iLg!#IW!rN(P5(qbv-Z{LQfwgS`oyuv(?#&S|13#qTa|A3p;5o!|Aa=xqd! z7*?@%k|$>~G-cNbjxD)JgsSr|E)c}ltuxT(qMIcB0;c~g{ zGh>$5*GrQF+uKr?QdX>%K_TO<(w_!=YgJA=2l~v*8;d@~hFDN6?AgxR{wxza516QD zB~E|}U({Yp&lOoNPD#8SjZlqe=P^HE4N4_Je#J|*{$qSUzI>W& z=PZ$at)=wK;jW4MkCHE(hos)wd`bkU>DNaEe(4G8s?)Wo9OOf@h*tg-h){|vN+EKk%^$F{)N za$4Mzoxa%Pa(UH$+0JH^-DE$^3hETm!2xU*5ajo`RWkaxZBvStxBw4-HQ(nWYK78i zlkEv$e{iO){Wm0A=IQMJ8~a>EZWvW_6&eO0j3JzmkC6HkRsR#f4aZ3d+9AexmE7y2z;$gpi859amB@vI!nV3j?F7iLMnU?(}9M(?QALub?X;@GS5?X zaAl0BCt+@48{>%!KPCeSfMfI+Q~~#gMd8$qP`EhuFfd`(P_aNIF_~efEW2UBE~Y_T z?`8%0l%v*yXXm-F6xI9F=cK0ziAgWFxy)=E2GwXJlMMx2e4p{7#QDg;-u=olp`^!i z-gh$nk9Y#zCJz}rZn;KHn;wYnE6_{WFnB%YN7Cb3a_N@N3C|DzilfluNVgLTyuIYs z4P09{QRIY!v6tGq#Qt|ex`NHs!!?5diTdajVZjO)w@y@*o^A?Dn}6*$xNkGly_P|-oW^u$ zDMz(j6IXz^+02^7m{BU~OZ9_iWU_~d^WBzpwK>KQY^1-W2*w_AJ0ZWA1r?+|+BK_F z#H@)xbk)cfCc3szkFJ zS@*&yWBZyP4%`}c&-9o zbnVKPMmMfGMU9-4Hq`MkIf+a8t&(4aDTT&Y(c^Ro83wsFb+3M;TngL3lLbT7#Xfv< z+t$$eDea80!_76>Cuce5tYmy`?H*;JmqgkfWzzf*PvJ`9LyO?vl(%lTyN42SBJg%5>`RBqnI zv^Vw%g;c6Z2vo>mDfNF?3{l5+-H(_?fsb*$pTGYs;-oy|Tt95IIJ(R_6`Pl3Z%=|? zL>Ykm|NPU7{GPLNLls#s3R0_o1ux+h&~T=pTlkPf)l`?|xPGUdI48C5SQ;X%1R$-5 z{_*+CxCSiTTiU&92xKDJMH)%OL>Ng)w)p?$e^IAx(o92b7(h|{D2&5wjI9m&qp$mj zO;gJEaVncF&;?R46gzpo&dB)Ytv@VJ?(qgZN$`P2GNs29n#^Z(^gBTC#OMqJGE8Ug z+9On*!+61vYPleJ{HOkLty8MJ{G9K`Q0)#aQbIQ@YBcc8dDa1hLF-hz-{CPeT~`H# zj=c1x>|huZ;JB)qIWqsoToS-IPG@sYQKpbbyDaH%nmOuEK zLBs1SCTZ`vW0>;vQ`E$ne&p;?W;|D;!7O(a!AbCY0_q%R=8FsE^5v0iJ=YPE9WiI; z4(takX>33)!j0KFy6Y^Go$6*?hnl?@n91Dw-J9b$q{jMNth3PttoB+GWfu zAcJ>!igksJw=L8x;+DoCa`dz&dh){T=}ClOnIdql4o%PtpBqYrZaxZ6bl&0SG_s|Ao+EbskDf?gWGXY_ zbycXj`UEV;nwDvO&8Q#O{eHO4^1cnxQ(U)p%>PT>8MA_ds4-f1@3IfHG6;0J<_EP* zpr2jKNwKD7BV=VibbSY4rgmW5ebyIuQ_PA6?d*ns?>o4MpQGPrJP$uNwGx~}mn3rX zQ8k%-Nj%FMIh+&*Av?z1vHJDGTfdL zs73zvoGc95B$*9*`Ob$KPD3VCa1mExbEYq{&ufJT-Ar6d|J?6i4~%PJ)+cUs>1LvN zKHRUd3fHhq-_!b3fA^xg8z{3nZ30`>*oezi#F8x5aM&@{4@c@GQIhFR1;Iu$h(x{n z14K3(YX8Sj$dG#0KD;7aWkviPj#6+?Rg~=ET;~3I$v<4W=DDPy2q89*YsCMz>5}Ar z)M3lHhR{t4ipk!&nfn=e{9D$?z-u&{k?m%q&;2t495B#slxz$=3`UN{fEL@uqM}e6 z>(bfYUx|xB-Y<=nz@7-q;(Xi%P1d}epSTu07XVtaHPg<+%5+11`!(#F0MA;Xt)(*f z9~Mpg;g(a(??k7_S#I7?0jg_36Q(TGl^UWKDLDqmQq|b$1|K~%i6I;rR_!_oNLTlD zyFqYIz+u+Fu_H~CF*+Y+RxDA(aCldd;rrTAaoi)>NvEH2SW@!Hr$ec!&tI8eD>3c8p!mj~c_hqlofH-V zt-m_av-cE96~Ny@O?{vrgeozjm`Z{AHz}b9wQbOoo*?DtEwW1ulab^^I9+2>n%Nt6 zY~^-i4$ZncLQ&Nwy=yDb1)KS}cK8O*@9d)EmfQSIa>vr+O76lP_JH(uCTV$M{prpX|zEL4<|6ypSpK6YUcqpe=z;N&?e{K>~1ZkRN4 zMM_RKWfGD?Ds|k)t(Ru&)UJvG&4}cup**3NDJMTR>?Fmhi`n#0BTEBq9KNf8A!ZK4 z>lM?vrZrU&M1rssE@izJ%3W@+QJtM3_ce{dItXqnOL&vM46qu_&Dt#t(v!PBon21Q zh+j}OT$t_(QWMXPcjb=PI|JWYb0)h*`65^y#bC2KFQx>9tZhfMudA;#gDoqoGDTgP z-*4461}?n)8Ty*THpKetECL;<1!QI8iKZ`ck)AAUj)z`hmoqQVSpmd1D!l2{ z@(Ln-;%-jSXqjpFpjW}d*1{BIV*zg&qfdD>yDLN;oW}En>a|dPrkf?hjM~L7j!AyX zvZ)_Zrpp&nfsmO_M5)1%B~+#Q8oIOjIq#33i~anBt6qS8sUvq@@CmREoi-Z9KIq}8Vis$1*yN1NNk^_W{lb-s7Cg!nmO0f z{Ng!CVS1)xLhYH^3A;o4$)0v1$(a*qjDpMEzyzPJ1!?bAS44Lt#d((PB~uMB;pZ?V zSH62PfyCsuYnuZ%8(PfmZE7f%QGOYIc&?V>05>=s@gzj;X;Gw~?`D^>wVTR*j*Sk| zVDxm{`Q*a8hq4a6J!x^xWv?r6z_ipd-4v3|^sg;OA3!{Y7BuCa27x#6bO9Lb5mG19 zXjOPKxV!xz@h#Qn{S*~>i<(5X6kco*+BENuL8z};u4*=9EmxY9gFnBX0!a%c_uTjM z+3oLgpF?>VmS=M_*(*1@Cb{D^K_B99z7k@>`v@tE$JB?8P0ym+oM z-q8as=}xx9O_(Fx33DV(l;lTdy!*!&*5=@RD3$jP@`cq^-0*M9bvULIza+KkPh+w- zyON=a*A_Z}fcO)XocP)qqjsQu#lsErt@8QGVXX1{7)RO_0|^(JsY1Mh3S1~d2J)*K zij^u<6ewS=M@#}RCY0v6|Fg%yV=iT1W@=erfQtJLrGl+1uZ4O|*oI3v0UnoU zzIsA|W*s6DZCnAJ!EP@r3UEUy`|9%Y@-$kQzGaLlgOQL9@-7;?6qN23@9T1R2Ol*> zxvk7ajWI*DLj~ws+QDY@)0RgTg>+?GDhV|MCzmxrxattS@q(&o(bGf4)DnzG{e+Ll$6Y2IH7q*?{EeF0U(*1F?uf1CBdtt zCa)apq%hp%$`)56rf2;IG>$zG%Zd6xYS%VJaUrw?&Cj=Y9QnMLdgW-$Rj3%yYsB)U zgs@uD@t^Mak>WyTI$ z6){@agc`{HdZ*Mo-I4pUAqf@A)h&C5uqF)j%vq2jAwGT(;LmF~g!#fyX0kJ8z<5@G z_wHy>8QGaI>evHeI!FM+&J=4Hods;bNRzD2RL^AMB)FZc={Cy8VbaU3KHbOrQC1)E z-q=dCfv`ZLsi~>Sj)k5qjA=NsT!$Sq%>9{3s~Mr6{cC>U?@dD#IZUEWf-4+Ibno!j zlsNW*iYx}1BTr^g3G1)+4$B|Y^{TNlnt30b-9CzH~9iGy8J8p#7(I80K*LUg!t zs78ESgCTk*SHwcyTy^$EX4*TE?DcoaOHPlTKUwJ1{oqooa?jEkcjeb7o?dBR_|^x; z?3aAW+7ew4+f=B)#_v#Ku;y=&7Mk;oy@Ka%&yn%Pi-Q0f;Zqwf9Z7OV#^tSg6Q*R5NGSvXyY|O+wTxEbQdL22Dr(%63-7Z{qnu6Dgo1g?v?(eET)7^w-^Z z%=}b0W*kUUpbnDpe02L!`r-JQHDq^WM~fik^C(uwdcX+GebV3qs0EKGZI?+Z15asN z?AhVSg*Azz=;GfaHQ{S6chWwDDrpWj(iu|&$;y2YbDjJqsLNQ-lqEfb*&y-UgvLS- ztY;MLY{WWg4kMInxU9|4KKh%=(K~R-0i$8A?kG_sW_#x3dS}yhqKjs@%VW|Fe=p+I zlz>|)8UhEywDWWz{cd%IrX6TTc8bkg3Kr|ie(x~|vpcX>MrnLZ|I{6zo$`^(N3H6E z@{@DQvaG34Ls$1NXfgYZ^Mt9WwS6WBdOX$s7v7Aif&WFV1X^L`OI8d276K@NXeX6T zWH;u;r1tmx*zE!K70|DB+}3R=-#D6KL)Eixan%c&ydTeoeK)h8R$jYG(+MATV*9Rp zo*qnXCr=0&rrJ{~D#;-XGvoTfH8r@U1EI+w|EGH`Upw3rCI6ut!49{cg*0a1nihM` zD0SjXLgQei*WSE7e7u$I16yVJFUv`5iveh-1APs*8=Pf>hRIY^=C zY-y5-k8#N;u0bjyMIpiN;T+VYT`?-@Nze>wWgt5I=mW@Q(Y(1LR&u9tWUI-l;S13E51kg745Zz&!q+C~5=q~ir@xcEM& zLfV7e$ni;nN|_A?HFbA2%H&dr^YcbHar1Z8Ri7Up8QTy9?rzv+je^D!WjZ_9@>W_S zh*s-o>U}1CEUgGrF*Ze5XKc)&_eW{M;XTCM)$!So_zA6BgWHN1fho0rz74Su6F2Cb zip3(<)J7^Qc&@8{8~h0_X->WD$fBt-4$T_@HuoLt{M5$ZbTG0a$M8~7UU?Iq6}5zU zi~UQ_HivXz|AQ`LS^n}}8ld=0l{d6|CKz`cEev7-UAT;ObwA`@&^!45{g2G9|C1c> z2pw5)mM5AeN#6-5n>muK|FGnWB5A-ghipVw(lKL(0tL|qa9@i;e)z+(V!^Qwlub;d zeKTf>Gcq}Bg+9=XcS8`Ry7a1*6P2|6rXk90nT4bMkxN5hYPC`1JuFXS*}Czg&n7pn zA1PsDjco1onG$8HP*JlB9a$T3PD+*EeODMauSlg2w}X5^b~zD~Dk5*!9_;gE*ei|q z)ylL0BEYED5`g84!eb7AM!Z>nH%`ZRtv&&_SZIgmSaoxwS1@mnk6ZLT!#^y(+516h zm?@tfStk1+ph?@KK$*QxQGoFAFn*{Uh@XZaDYrM`0uC^AexvE|ce{@z2R4na$7Z`c zy$(wg7Cw#H;Rn1=n$)$7I~}u+tMb<0i2Pyc$zUM1J$hFjJY98!$!p$GFGafgANf8e zKaJSuSDQNc9lvjH`}Y6`U4fYj7>4PpRn!j!H}a=EnbL!V$wde|UA6S*lIF>qm%UQJ zZbpGEUaPXA&vdS1_<_5kK4pW!Eg^`cU`K)OyVs^G6 z2i`Ki{F9%VwB}_=XCOnk|MrQw+FUg4rO88tZ;eHOua;{ydSdPCwS|`zJ79Q)ZTcxE z;SaoAc=@%T?_VG9!9ai`PB@JjagfV2`@^!O<2z!1-t%{fl4nZeZ!Px+OG8PDR|=Y5 zxiNWY8wVWA$bCSB{bd{3rU&T%Z-iL_xE!EU5^wO7mvYYdySI{{Buyc|Lv4WhL^6#k zGUHh0kz|P$AjP@`aI*pzhMr_Zm)e`!78(IQ845&Lt`y}xyP8Wi+Q2U=1FLZk52W$~ ztLivok8yNIj}GWR=7N_#zCZa7M^`%?!2I_O z?fnz7o3rCVV&5ZI@vUQ8rB>Tu}5~6wZG+ z`LieB%oFWzl1b;M&OHvd#9)E9l_2J~=~cOPi5UR_N1?*bvl}M@QEhuUCpMwch=Q%o zuC$Mq9@_nazuYI?CkKV4-T>P0a(k_q9d9jSh;WER91%{1njPe}37dY{$pk8FW%LFj zyQVh8GrVkh3OK8@_aPF(FV+F_v3zk9Ww(9TcUb#@dUWcn=ml+~!bLtynSs*=Pxq@E zHd!|hV_bhdyM5)#PqXl=%rRv3ZW2v{4z)Z&1w)X$?x5o6Z9QSCk3-Jar9IUpami}4 zm2*CxPB)H@JpRJ#YwvpU94@PIxg0dbmr4NkP{J1Y>g$qRd8+PH1>fe;c-)Is`=prs02p$BGsl1=fwV9 zaXU=5jVVm2N$7u^>7V!kteBmksCsDL^z4oKONV?ue-YLdv{<3JWQI1C0%$Cz5QRtY zk(iT&?y)D5Z43wOn4VTmy)vv3KS9HhjL=Lmr2yoA;^+rdV!h5l=*1tF3+Jc+kmCw& z#~;|%Fw1A(V+NZ3r>DP+dY(B8r6b?}T|d}{{C|Jkoo8D!vNb*;gHqoIPD)2(YQ~iD z&KYakdNY=%r~q4GvKN`c8oK`s7G(^8; z1hQs|&R>d1VscZTP6`lo#REn3SQcwD10v|IZY9+CyUm}n+Gju zKlsCCc@$-|rlznZSIwpgS9Ws+(@coj9f8DIiJaGuPzA&IRaE+M}Z!Ux4ur11i!*{-`VOXHt zf@$@`Vl_G^`jwilibtmsmqR!?9!q?!<>K&k@*fvHSQvhkK5H2fM7R|YB{t;BarWW+ z*S!T%_XE%V5BAngArTQ%Aqr_Bg-oj@Ar*?4Q<4xn>8ios8^XB_tP##OWCLS0g_8qc#YyYsX#BFh*9&5Ku-8FL4dA+9Ol zR?})g&zLDRc`|&1$sE8J%os}Y((0HCX=B>bfo`ltCmQy;@imN_yp$mwu zHjEgxyBYys^xB=~T{1FKLJ0@l#Ilc$j2O*@3l^e{3sb1}!~PC6{gmLkuDiqz)}XtD zqG;@3(@szNO{z2P9Jt@-N+N5=T05SvH4}XEI^9IZWeU@!<))wfzBau4f(0rSsaf~h zA1x7`abQ!wqM+7f)$VSzH!@{oy1Ti&y&;|&t70BuTcSpCeVx0-gwQ2sX0D?utCl(w z(Y)?f`{?tRo0T@kS8;6hi#XRpc+ojgN{fdb<%S|gY6pf=xh3dfgv5%*#*s|I_+r+A z+QU62NxN(^kMkLKu-)~}wRUX!!?gLKI+;v9wfN&^MzKnQgF1)CUNc;XdRcA(=M6Ki zdvcz>&ZmH&R?)86M&|Qk3i)+swUTAE1G}E%rIbj+w;F}&<(1-0(rd3AY%a3qQ;N=8 zQ>Lnw{^gVKo}_oaZ}yw-zV8J3WF(91e2PY1cldVnJ7w0OpLeWNIP6q=A78+>kiYP|B#zTDsYOY(;;w#X*0$lcmtLo4ti2>u?ikVKWE%I{0YjbZ)Ymvo zb?hlvY;i#0xT!M4byxnQ1Ee`t`uLw6$&bW=Q$Q#;i-3kSeq#NGSSl+?;TxW_{{66U; zZ;OM$#X5V1r5N!{y6NXB<@^?#XIrd&35L0wN_wd9)9bR3#F`37IqIc3H+I9*mm}#n0GH#kuYiOJbHY+#-@#IRIo1c6!ac^; zN#Lh8(Kuk3yXI9>UkYhtaHpDV$B87;bW_^>!TPhPoad?6mCs7tHFPZAv@-L#Sdq1) zH=2ij+_9^Ja~Pyrt?sW4#;XTG?Au9xdTC}I8&xHWQZj$2`y_!`8)fLaJ1RxQ%5~$| zp1IVn4E&8@D-{&Vt_4$7rQvmQ)|grC!tewY(zP=q4Lfz#_@TXtN@Fs#o6WX3%?5?- zL$zOuVr);D>L_e_|IQM+jgL4k+ip^BRaTQ2(Gc}SJ-~G1w%d_KlpBFEDuxB9t%O^6 zUh08a&DxnT?V4;<1hwi?YPQ-+=GRTVHRErK16O4e_5D+9_R9~2>dBex<=)71!X?x| z-`pvfUaCH%Fy7HLLq)!q(izbBi1)#zuz4u#w2d!lqDEqQcTf za(3pK_OxhY4tyv^ylMaEm#W*yu9G8u$W>0ws1&W>hfvqO$~VK|lGRpf936!Q2kZJs zz4y_9;=!|9Bwh~%tlKfw{L)`8b*|r%;^1-Uiul#?P;Cr;X4Y=3c2+Eq){btPv7BYw z8ARZl&O#EM3vrvIz;!s4Rt)cUD+~gCyHa`;!S8AkTwDWVu22nzReImwcxQImoH!Si zx~ps8Qt!D+o+po5!UY%+kg0u}Wx>7?f75rSb#kkcrn-8sV5kE*NbXs@DDsxV zosw93a^=OBjbQLenET`Uz@`~%sQz)A-rMdknRDJUdo1|eXI8~+yRYT#eS=WeK`Fz+ z8&0C=T&hK=7FF^Zr+ox$BGkpbWhKeL4l=(J4T_q0V2#nPb>1hgF-n48kTVIj_#O-v zRL5|4K@a(K8zkkAN>5W7!{x~eZqd6^4~6a=3OG|Nb`@jI?mKY7XZwLrJ1#BK${-zd zefC*W(d~ZZD);$bGskQ4jh^X}1!?#Y$xsy&`=e|xD-USIE568y2tJS-CV|JbJ7y`K zcG|5AgTzhBtQTDgxM%m0Y~Jk6*oAx9VydtvjWp-l+B{4YE-H?+ktVqeyw7~3vDdbn zcK#WfUFz;u?rkNmu!2)Vy8u-zYN(y1!8+{Znbb5=v};WFnAr(g>vpd@hXQS_nRog` z8EoB|o}K<)>4SfMUnK3M;w1M@1Pv0P;kO{?pQ7j^Qx-_BMpsYO%Ej!ahXrA z>vVT`qfAl6Nl7k(Teb{8n1V@ish53v`(~BR>y77+vN8`!s~zP#t(se5GUm zQ;(g8k}}ug)yae+3{kHO?=?o3d}AAMxlw$2g70;RTDS{lOKRd5gUf@no7^`HSq}N1 zHo9YD%{fT89;6&Bc3a4$&{^BxYXsj@0dl)_)YW#Mk`%BgDXkykcP2f8d-obtm3T~B@}bdG0x)lvMII`EWx7@-cf3y774F~x%UvT>x3&H@-p zaB~z56^NTN>o){DH{*H}Ga=ex??(USzv}ao`F?@`_ zkRwZpFYfwb>(v}Ft*iW~^$oB+lBE2FV0IF?W;;pGNDs)DveE1TWOf5a?270qMl>UC zfCWQTuR&D~jJ8a5Hc^yl+79~E=1!#hJz1p(01Qgb^3_TEm&8jqA>uo#*A=sASGB$4 z(4LdV8;It&ODPky#;4`y*PiTnZbZrV@n}-<=eEd zt_UTN6MsiHF*J`+F%sJHh8Ty_@Vytl%l^U0X5|yxtt|R6L_SJU7MaOa3dd@&b1?mW zZX~N;H*)Q=vLC#qj6X$A#HKcZjtAFv z#$M=fWy(NI0mFo0(pWMYbefAg)5S()qeM}o$y;XgGuJeESREDAB=7a&Ffh0oG}SSf zmM$Y(Z0F>pnaY1~nnzDAom|H)V@OQUmeQp*|-IINpYD zORFn>;(R_uud^wnaB9fXF`PJkocW|gSgpI!)y>4r(!_Q4HRtuA(-2i4i%Y1^EPdGN zttQ$_L5u-{DicL&QqM*aHK6X2LzNs=HQtf>!b4f(%q)fWnP7I;32)idW@?iueqh)} zCW#r>uCh|`iTPzt*cGl&I7E)|&E|?nQxhpPos?9vaG#r-{P8%G0@cihl-Sy}zQcM( zuI>A@ZNgUw-98l{e&X~ozL3WjiCMcH$J>MHGijeRG;)DOi_x!{uH@7_|4UflfB5HD zUg)-G*yoz>EndHUv&shfCPf*;%_+CexL;v=uC8F{=RBrwWGTS4fOxVWlyr`CF!qEz z*prt?b?xi*ZLiSqi;y$ANN7Db4_Yi@Mzq&PcSjtM8j-)x0I7hjb+EVp$R(71&d}TOyw<}F8Na7g!jKgL7LqX}g%3>Y~C$&D* zv%HoYKxz-B-J@gK3^T@R-iPJ{Qmw*Hncj2lj0q|ef2aJsbyfbTkMl#n4BNw7`QxW%;q_stF=3HB zey1i%kbtmE8Z6tela!q+;d`;(h%|NDL^v`dV;8xp(*O4BiOmlVec(HWw4v(2@)ppj z2)L_==FooclQ4*D$Im9_8XwUsTG}PIe4b^=nJ0_=@u&rWsW}+o34cIUky}FRr;85d z`jOH{u^<;wm4B~&A8FiWTXo0KMEsDfLptTQDKX6Z+KHw^yw{VXXGVEE%Cq;SF=Bo& zoy$yixl}$ZA;;HPaPf7(3*Fo%z27#m6+k~q{|#pa#{Q<^U$L=QopB$#18#n{qYu@e zTWQV&oaq;}w+!a!RdZ6qqANnVI}-1QI9|1lEIcH@L28WbPSAmxttz(8Fyq86pgrw} z{3zbuX$^epVUv$#28^#p($+BztHIuJN|eKGiZQ z`-FKfPyIEbxfN4%cGXQIo%5i*;0NbwXJx;rGaUO6{0N>^lPhtj#j-B3NN%zp; z&6JSh`P!?``qRT$K$Adb=Kk|XH=3NpaIz<4pETY1_}uzV(U1I=ALC+Qt{oFtK!mM0 zHbWn3c-ZZuq+4qx26SfQ?$~z4mwCp*`9z_|@0hGUCkkp^N#=ih`|xVAN|tGH;Ity? z9BQ84^vo2w2Ie*m-&T{2jJ+9jV8}oqD?PVny6L=mLQIO#$BHduKE3Im9v19*#{#gH z{A)o($4n^T7wpz?ADa-byOb4_-}pIyx3!9JxZoY!c4A?ryafN-0HuA$64*Bun+65a zF3|a#B&8S!RkG`MWi}Z5N+?vD1ba!5D>kSHM44|$O0(71H=;$~J>J!`XnehAUCCs0M5xUw zW_8kM=O|O6YsPD0(VF&n@e8I0i}%fPoa8OwQ?`HVp|xUqRc#uD#Sr&t0pSCl`J^qf z17t5oY6Djf;64X=B6_57Zn+z4XE%nAlj>!VhUs2T{9HG1xOkV?>` z374o;oMyfJ@pz(1cYDKAJ~=&)+BUoe^X(@};&D9iZvIF36UQ6&&U3}rPXpW^Tl6uK z?^7v_jk+g|Zi;-!+(PLwU2oCv-)+1~e(SElUX1oNGs~N^^eSqT8?^+~8JTrL9F<-C zrfS-%RNlkwVjK5T$PPoV?4NaU9A_@BJDPp|jNrzsYx~Fa;I-6>PNWBKWxrQvRvbPg zpT{WUgQ`hTvjep!NNOr<Ch^ zd7)E9V1aH*vN@Ol0K#l2HrJ&bFi^EVXPlxG!u9$3 zX}RO_Iv;FNMfY`l=yK-WKI~V#^irm_%bd%}_I|$oP}$0~#$v4`SDx)pPHR1Gt^%165 zrr@0Mxm$*ioYG$`5`Jj37#Dy0vhG^2_Q9NVBD0Ex`8DH^wysovtEa`!-HjfF?>-`7 zLE%qR@QTR24}5J?g) z@5&xhaCu|m2{53~f1lW%0EIT!ge)?~Z1c)Ycfd7*)Hj0uV#cfB5vgQj$+BrUU}Wv_ zZEr>fq;s-w?3R&>yUYG0Y(=#fT60vnn$hh6k$C=rnbT95)e}Gvz2;`!v==o}5*n0KWoUthr zLT!7@u%jiKgl*0LlzA`G#%#0lq`RT=R;{ROHbRp>Ur zp2?#k6xO)v;aS4$FbFvOM*2}-VkiLm#Wjz5L8M4@+Mb_7GTkvmOibUR?sCY!>LQuI z`)Na*nfG3;2q@1?%J499JuN8y!H~j#CAH)-^zQ9#T$~9shuVdK7hrW7qpBUM6 zwl2sGN{#V$*xxy#cH?s%Mq(o9sCAI)^Ie2A*I~Pch8HU4cjT+2v&MxZtQGQu3np69 zDW9ntU0#?q081{i$DH|0vWt1bTd7KQ<*UA2pTB(EHfI))x_-~gPK4M%6~FClamQx* zW&0k2du9Th{&x7ldQ7UC`i=1Zds^$F+cI`&o`~hy@DhRG>Nd)_vD0^_^`48_yzg~~ zCYv3wZ>kQq3W%&%4+O=#o3o>h5<`YlG4(W1;r{LQ&pTK-8(OmurrOaSbIlO=5+z0!3 zH$B^+b%F;oyB85AzWYZ9Oqu>jL>1zgI&1|C zISfjSVhgut9^yWrd$wkr<)Z2)9W4j*+iahr;!1aA?b$6A?rqX?(Jlv@Kfy}8nW^q! zDZ%KyFHsnL=s;U*a=DMt=9VAcwPbAJb31inUnI|<*y9F}>4IE!>kkFlsKzmyp+`Rv zOYHX)Nm5Sixjg4A6u^FbYPYt0?~9OmR?qn1z9xFwnuzy^{k5PucRK)&{H}7wi<2kv zG+YRqrTLB;788YiX2u>>kYq)Ra`HE_S|!RuqOL$l$A*T@S3Z(2Rorh4$R$n>RBsH z`q6HaO@rFNvSir#GxK;~Kjqn$%<&@W8tXm34+CFrzA{Oli6%&_7zW*ei3EV2@O?5b ze!eM$Okj0bPX!}R(B~ViN)u;?wbKtNTx>dDwlN`fwDOK6*W2r3=_&)5{UG_e|f_Vjf_g{m4@^ z?z=Pwv|r^%u6ksJayQ_`<>&6Wtd=stl9kYV6X4+3oxgeArzz03&qrV5tncw{I=iDB zxJe4wh)aXFm8^GOGUzZ>k(xaqU)wgU*{WnMg?~{x=@qB)86m#m;X~}@hyYKMnEL2L z3C2mCvV_rQQr+eB^rTF+)8;RO)dLeEvjYvE4QZBe#XLJARG1|BDf-e2e>>@kdd-Wy zEm=aM5xT8mGMfI1E{es|oU#E8OaO zxJ~O4ypFEmR&XEVBAE8<@{qPp+Lm~Vhs|jHHQU7J9ruQWpDBN;-7k^j#cI|h`bvU# zT<2h>v8x$6xv8MdUNfy;QZZsnhh}nvT8fRo_<`0T@^P-@%$ZLr7S4S}RiF5FJVROz zo#q^;z=VW#??JTsB>!vMw&+(?y%PL=_4a*x?8#_Ohijm7afQPzs!gK90rD|2S9oZ* zBr=aIsRb7{PDstgG)Z`G)1WA8Aw5omgQGjiNrYkeFe97Nx@Ff>k2F=3kP^74r&l4mg-ih zu&2JiVMhOincStf3E8wb*mw)&U6k^^!&|R?Qo%>YM!Y$Xfrg1Ygo5(GXpUtj&+*&l z=YOEf4o19fZehHi8@8I>L_)z%Oa^oyCtQ7okWRc&f0Z z3w^)LLn)0%vf}f-1{=v-W8qnGwjurfmNu_iOxK8nG-C|ST>bSVCim=d5xIv_L z57@eO4tP4;$n5xtV;gSgA<;9nUvMiorXt5wpXTUUar=Au#663_S^2$2-HXAzBTaf! zcuIsf+|{1I)Z~7ePu$rh``qg4onLp9VC%ee&Kug#qZVq_&spL%4GoAlY%K?wG&2l z)@H~xdROvxc$(Rbcls)SftVxJw_dVYy$XGm(26>V{-{CcJX2n-(UROv7G?@ z?wuu|fD;KooW)RI%wStvZUDWVK$zQ!rE25eb^0YNDM#kLYG!5XkcNQcD5z{P;F=4|Bb7(@+0C4<8_H|q>Dz)CkO(J!Du!Nkcq@O5|K z&{ltU3mRWAfA%eW!v*MF)9=TH6oxAhj6a$Uf7~ACR7g#fZCb7GY&Rd_$1KVz5c>^P z6yBd75PYZDGT}5HT0l%qFfnLoco&xXI+qD{11X@qO$&&=S3tN_(6CDl_%}ElPf%cL zs`e6^BT2w3x{9Cc>O|97(?D&wC%lrdR5+KOK7ce})`=Q~Q1nemSQkl_3S2-8>Lckq zK%Q68X1*1D&Hu|DR{gbyzxV)T$%F>f`Ooy|?=MU!sy5+WKOMKY5HO-F?N z2Shdjb(QcZKXre`Ei5tne^TYQY!81rHb_y;Xsic%^E%a|?l_u{fa*5RiDhQ#PvMaL zah|~Ueahv^@b_>k|F}KSpnTwvl_>w+kq2SPZ+B*X%@J637XJLs%zl^`E-=jsVq0n4 zKED`Eh*38#iFlApBNccz{=ov`6}XKior30WLWQ+D;Gb3y3FMV~*(RF4iKmp72SbDh z_)!K5cI;e05X=CEL(S4ir8|VmHjk_V6@p3&h#)t*nlLoa4gR_o1s{VMP}gO+06kUk zkwE1)2Th8c(6V4ijJo0XUjEWR=)ZjVF{3g+k7oa({{NrCqXt|`Gup<9mR{SU%;enn?z$Q2()z^XpaR#UAfs(}!Z+S`qaVPIf>go@Uyvzq z3SC4L?5%>=xtW=_6oOZ_>YQwCN(FRPMn>$W)rXd)kv^5xkPTvP`r)%xdUF z%)v*_vo8YQMoVPa5cIKwNET|h!||YOhru_P=f!1Hw*qoTvnUGp!Og5DBMddD3DOc1 zbrAU?R3Hj}#!ZWYjNR-If%;~Cizj^>W8K^cyj6)}1Pcm4x zy$y9+z4D;J?$eNNJyuFrz5OP|@otXlVhosfWv!0Bw$JLb|5JC8RyP4;u>_o{e9VwEj+)3d{wjc4S92Q0>N!>Y2shrcen0d&k^ zslrwCD7Q`}Mj(cD0WoyGX+9YMUhW~EWc0gHA#JCL&S07kM1~(MM}Zi|p1gv8Na#IR z2aSA|>_b08rt<%7^zk}HMKSV)o*4Qj333(1dr&5^{LEWVd^%pFW~Wf_STJi50{9*<-^VG zGU~*bhF!MXPUbf|nW;-zZrB5g7y@#A8*_cEjEit&wvy4FMbro^GzqW_N~QB`?csI7 zv`y43bQUSbMM1*W`9g!#WcLC>tuA#VNB{7b;ugNou3|U*CGeAB<@;>qn?`3x^T=6f z>I^AegPiV8N4>eJBeNyQMuSo+=vv#`b}HE->3UX{op}$7-~naB8$YNQer|wG75JR0 zP$@v*$YgoAj+?+D&9G`VpgV@<$?81FmbcR2gv?FnQ|W2@sLt9u?GmK`lpoujNd0wN zL=zvc2vWSM<`jvH#<_=N7+Z@6KrAwL-!IzXg2qEo zqe~j0nZja@XHa~m+%;5#by74R>nLvx%a=rbHZioadxmNn(gbO&qMa5qoj~R)D;fxt zm53X!M^8fc+wZE~dR`*A{;+9{e?^aRCB5xPFf`s#E^*Yt9tnDTX@?gO)#RQI`fF%% zBk~)b)B0u4HzLW0K%ZmvcuX4`d<>`=cew)8^a^Ayy59-hJQSjk$=IJ!59lR!y3=RW zw7l&sI7m3UGdlMvNIh*f=@wzI4m*PUeD5lF`vePk@oJ|z=s6Xrmu0_DTD~%;c}*p{ zT8ukxS}tz(BuK`xQJu37E+CF=qV>VXApQF5I~h8?WFM$*R#FuMp)?UZhkFPu-?$XC zd|OQu(VS2r(bfYTiAH?uUeF8twG@KAaB2bcv=M?kA!zn87~0x3wZ**fSOxNa1@P%chwbY%=6dNK)$<_iGI9&E5Jut z$Cw0zqQMQ%((hxbZatJczzFn|5a!MTM)ESIBe2v^jZLUQn+xM^V6{jxldUoQKybE# zUIpTs<=;mwZlz?&R%$&_-RAPAz*E~aGLiHLP$7R8ZXSW{keot+JD}x1xcSYl@EvDF z0U1($BN+()Z;@gBBHaka{4`Y8Ve$lE-ls)LGU-?@`qH!xa1#hF(@>4kKsc)YSmC() zb7A^RIKG~#{~|K{>#9N+)!Bh>E^<}!mA9|Q&bi)OK&129EZ56Uv&U!IXeD#?^-y6m z^`Gm~KftI=EHQV#Zd8_j|2yXKKVnq=075La`@c?zrQiRz3DE?gqk%5s)GsL!I64N* z_-8PSw+Efuu(4!5at8Tn(Ooe9C3ivaf5%<;AK)zEZ$ZM*MRW4I8Rvgp@htaIK=94Z zaQZAqf%5P4wDzE;fTIcg6iO`A*8{I$fEE6t!?fue*8(_9v;U^U)bYRJF#Wq^_|N){ zJ0T+iRahr);!9mmq955lPeGWhp5R1jA|6g{s0?l{)Hye9!zg-to?GC&%Un`fl_}$1 zqFfp`Xmfky4 zE!seZR{bc!P{Su&Fq0JSM;hKJQxp>8<*><`ymBL7L2%L)gP}~VP_=;t#LeO{_SEph zuMXOs_TniIEZeSAZhBJmYPB~u8ag&_15C4V2NuUpN0Feefr5k$7#>?%ks`*!iI1v% zquN39)yZoDOx5%wZ%t*LO+vz=s!RGc*6ll$z~jaGavz(P-gLEn4WS#$ZDD@!l*iLW=lCO#s6r^o=-Ilr8^IiPGc_4Q@ig ztMxzyoOPq-v28I+KjOUqnUIY9GQ|KH7Rf^c1lmkfQhvao*c_e_L+DIHZc|tV*9aV> z9_#}B=2Z3hiUL{syjp5sU&FA5)SW&1?6_Dz7G{(hh|M53(6?3DR&ndCc~a{=e8Elr zX$+69jApxAz^tpDI0Z`_{ucN;6GH6l>2;+ex#Rnzp2${asLio2F{ zTq5Nr^Zpz8@rVP30VP7978ye`E<{eR10_@|K`21YgfBzg+i^re_%YiWsPo1v7zD`2 zQ7KA{bpjw;_ObFL-E0;`^^@F07-Yr4tkK{y3kQPH%}fQzDHu&%4eFaV&p(Ho$xMuu zBH#zlP*l(tgJXxPcS7{_&R~oKzAqTwxtHq%Z))ek%Ibe4Kl zF?Q+m-RNhLEKt=P)Cri;t1)awGb=>>Y{{s!Y>+581_YV36GSw=I+GOuontjzgPh)O zD+W%AG#fecn^!6ayLU`8X#k9b^Wdok@Gdr2)Bv48S`R7(T-0YasnS$hj<~{i4-<1l`2Wv&(aV zrG)^PaWH>XxLby?q7G!T>IrlEfYo#E!h#w}N%~5N?!i!lmdf!mk^N7+r2N-~YSCD8 zkon*TL*Tp(qv_9}!aPz5xULO`SD@fiFeu|)%4M%8m`G;(QO3L=crPIcO?4NcxPvHj zRUVqUH5T$glG!!JfTcgd^oT$@cy0l~i|Rg=tq9mC3sGf*7ZAIUP ziUDCl95i`#xgrT!!sH`=$?iXsUHlUL(LK0?cE8sFAuN5#&@$!w?`wB7Fh~Lx(0dUY zVQLC9GL(y3rivkP9GP8dj4jKCzFlC$XK;RzKo8b_PgnkPl|opmQG9_NxJ7mTnaM|2 zhUm1|My-Hw>l=SzdF1e4=f>+R?M?1p`IvvFQzUF) zUINXI%$a=S3DxO+bt0dcV-U7t$eFoV&0lll)tCB++`r%oE#>p?8!P-CZu~Nv>-8^+ z@884E6TdAsT&_<3h`8-PVMmtf_&+K(RQ;Z-^-T=5JU!rH>jT1XP=G|TmW0oux__V= z!Vn0-5g@6s6S{z2cCKrdzU1!j6uB)|CsiQnV3z@x;>JO1OeF#TdLGtAElq7m&x7lU z)gKwT@OHTEo|AMgdE7bjrg@j$^alwYrI2pTef1oDtz`oPYgiC_+cS=gq4~O*YEV9h zjd$b2`l&aS*i?g`ze(wBl8Rgtb-iUaDyZE!qab|W^~-4o!dI=)R*#yv-X;A0?){XY zG91X1EG0yV=n@D0h`tdQo#+&P7h3wYqGj~bw|;t#Y4=hZ>WeJ#(8@ zk)da|VCQUt8M)Ag;pJ=aid{_S>=@e&6F>-AYiE}9f82~$qHD34mzP$^@Rsx^16F6t z`)+Gk3#%*1^QNChEN{Zj%~+PoMIQq|G{=w&b+GYpL*UFffUSYUq#CF4^SZ;^6+KT~ ztqH7--!1UcS?eZ##F@H6?v`FMfl!ARthbsQNdMqLFHPL;7ZJ!uD<9Nq$jMv#Cg_In zGk=vbpbG zs<1yfW+Ze6-MZ=U>ZTrKiDXNBclrY`tys}Zz0u12P)ojZVJ4ef_Fm$7^p#dqU6c#D zaG$|&`gzlP$;6Y^9s)JnQ*4D?SH4%r?z%8r6*+;ul(^zk(xuT|9MXN-0bb`Vb_*Xa zAFj}`7qg?BrOy^(SmW*pZ1ELHHC8)eld017p-fLdFma+65+{(#pj0$tGtwPRRUbwz zPWL&09@AMsoGR(&hBs$o0TfNsOb=5bo`^qR1GBX1+19DmMU%0fiB@cz5~I6vWcy9J z)llafG{-)N;oFFMhbrau9wdK0Q+nMj@4ruo`7pRJowyHE&=rjT9- z3Z%aA;Z}9+?`IATkQk#Kw=(a#lyIH-{d$4RA%_F+&LJB&Gop?Z>PP$6ws#bU&)eBW zlD{yx-)Y>kq5$l))Tv2hQVHlh52k4?AZqqgT|l+icqB*)O+y93a04`b)@cXSB?t0P z09sKa9HfgL#RC*&(v4SdsHnV5r>DUJymnHKXE2-e3_4z?!91abo_c4uyun81zhi`@ z?)tDP!+-Z``>km+Ad|M{B>9u>Yqw50$W`eI`Sj<6xJx5j!ncxH?z^}(Psd3;dmVf> zm&UT+l>10SkL0R^%u`nbd2hM2N)k`Lu*`(Mq$K)7h2fQ{EfwgCz!~%S;p1eSqtiX8 z4Qn=LKb$sgJQ??|Zsn4$F`aAGIWmN$YXOX=80VA(NV875lKU(vr`Ktxf6Lhh^O-PH z+R1}e@|ilJHz_*04hd3sj^9PH_+=gF%aQ9+8lyRqxx1AfW$#F!9=FRYQaw`zhigP+4Yf?` zg=h?BJo1j_#)kEn(UNM%EAZ$ViZK~vkN-8sT!oNO=ga%S`D|>qP!X8YEE%0rpQ2_{ z7?Okco?)Y&mX;ISxYBH$k*YnPm)>(pOd@g4sq7Z7=#{qa}=y*pc?ZrNyp10N$e% zI{IE0NIO1YXtJfM_x!wg=QM_23O8%mG=Q(rGfSes`c|5d%g+qDVWJR46-5;xAEG(s zHAD)ln1@1QoTn>gj(94p@)i0|gV z0BV~!JF~ZD0wlm^ zMr3Hp$<~oyUjnkb%!7o7rXwIJyB5_Ch3kB)F>1cY{u$UcONlVBfM|*YaCi%d`DsFT zlpTSwesSsGA{^AM>AXWv|ND=Hij=3Ic@99VyZ}&M2acOT9#p;PR)O5{bJ$LVqQ-dh zpp0rnUJbS8d{q#V-|nGlp6Bd_lEBhl8wG-H@J}oLwMV0ui^liV={)Zu-X!qp419h> zPXy%R608K=Q3XlvwHkD1SlE6I7}W-DB0d_z#&doDiVO~#rP#v|pNj7TWC^uvAl4b>V4mS||J^oDNmME}^8d9X|0?lEq52Al2p~a` z!P z*aKTANf)t3!bqM zpFL(T+>}D|T6H_=w3E<&X7`>0>ndWeWfUvkZ<^lvqZV^#(eQN|n8AOdU!A|zFOM$x zkSKr;x)8mgHTbFVt|npSdTlj59X^Mu_)gr)eHnX=RS(2?KJ8;Q-*z5g;>yv-HBNKt zkk`{ne*xojMG4gnK1mc|I8`hlq9&WJUV+@9-lGvv(eo<#v+Lmj|UvGhz3A+3#vvv~3xMnexwR z?J0f{6%m@+UQvmwo*e~tXLUH$hpY;(OG0`;#uIByk{pbv0o_|dXqU+jTPVRMUBVxp zbBL{1V$%r5Cu(@lV?ScKhKUOJgu#1hX82qe z8KSxP%E&u7kVd?p;AkD%HvNRP=Kmnye#flBNTRI2dyxAlzDz5Lum__>FW|6Pq;+1s$p-d&IaeUX4sfAwq*%N`-5>d$t-D6Ce?Yz)73<$ zayb3lVq27UjkjRvxq$GG{&gTm*$wo8zXScUJwlI518}E#Fg`GlBt!S)#W3uO=<#ok zZoHej7HH{{Y9K;8ObB%P!KvQbu4tE1j`6Urnk$|oFLu|Ac6+3y{!KMfVj@{rFs>}uA#iAY`!@WZ!nZr*Pi;gd`LJhTsh z0Qqs23kV3Li@S#p)3Ug%X03K#TT+44r%ss6<+PY z9&nSIn?}6&%KF5wMfB=w17Kpxx(62M-tOvb{xP{%m7{B z??E}L74&m1#D`5lBhaU(!85Z?x;#WwKH9}Vph42QlBqy)kEaLUZV!PZpzA<#IV3CyzfQixtmH_$I{L8t{FX-fGQr zzF$rx+n;!Om+KL&TDd2Uez(Go`K|4IyXw>qNi=T@Uk9p+YBttym+s@ z4k}{9v%H0~o1#`5sMz#xedXIIn(*o|c8{pwrJ57oIdVi|v`Q!>#FW3cM@NlS|+ zs5XZnGy8P9%9X&Hq9^Alr&MiarfPpG-M5k1jAy4WSBl50hlZ!T_pIA-&v*OFTZ(%= zoIkBu4R3M~B^5sk-$LqebwI@o`JJ@J2!t8FZaHr1hi13%yXq_KEu5!yQ1pt*!>q#z z_+=6T5Y06JG-fL(vuK8>nHCyxyT1%8Hmtb#11n->uwZ)z`k(*cnDu3W>Ff4QV#^+c z>RL}~qSJ6Rnk1-TKE|)Dl(SV}2Ds%v7mVfPU;=?d=Sn+%Y=WBK#1-#c77}q+7p4`e zxNMJ`9oAU7l4>R`zX?U9oV$c2r+FFMtyVN#x3BS=FtEoDq1oicqSU(CHdmOu{-P&p z=G8(=#$ecQY+x=E@^oirUmWPC#XFRsD6Y9iU$C)ceps&p0~3U%Fbs^Fsl zfivoCHAGD!(?&qqvem?Q$UI|;D~J1$BfIoL3h^SydV+rS=hLoVC3Xs;OF+$N*bT)5 zb;2zG?g<2Z-!NM{j*cI{3cok%nC8*|W#1rWsQgvz&@x~FO$=av1}#Z8FxwbmZZGz) zCH&c_a7A8ZJENziJ$g~HE_9cSzEjjIHrTi=Jml*^ts``q!9cf7AEC;Ds(wlbJy@yX z4J1Z;2EBc5!&n16?vFjNaqZTxsuJX_zflg6@D{=-9%K!0&IA%<4q6+m9Cc$R2^ntl zWx-Ij7K+Y_>UZ4nRpB{+yP|LO0_v|7Wd7%9psDuhI^w=>dv`9lBw4Nvq+xNViWPTr z50ESzXBIX8_m!^rKNNq*UwZ#9`-i>n1?!FH6}K8QHGWU5WZ*3&qsas$>A9^8fJSp0 z%LFcjKJq)JN9BLp^bF~A<450p;&L$R>{danz_(lvsDlj#=W~F2b$f}zt(p6(aQDb> z?usl09#!eLu}RX=$Twe|xLMRW_mPR@cc95B5DINeY6})LkKJ^Rm zjGv7t@KlebI_@FztWz0}^408Dwh?tzToqJWfVjKAt~T>;el6%<+wz-Y)}V~P{q(1J zuj#mpdj7@J?TPIk5-sv?JQ;2Q@H)`Bwnk5b=L%IqVT-CzW7?{7W;2ae zd9OJ9f_ zA+{>b`ky+=x^BDoqjg|t+TAWomZLWsc#{HCYhvd3zl`FzT~)eomaosUuQ4+qMgni| zJMdcge_GhoYw-8Nra_JWQ(@B{9)+$2ge{m`7xKz^965`fdbvLKr=q9D!l&>3as3Hq z=M6J<26Bh3k0LlVPa_=G1JWsKL0`nW_k36l8DGAws6>A(Ax6!$H#0Ap)wG5IPhBAU z4gn_?TRn;^RVl%Nm;AP=ao!+f^*Q~@*UElZ-V{k$#`*O6Uc8XIo%I|Um*9j3K}H@I z%CJ(=-u;$?QjcA4DsFUEg*EP4|JHl)b9dLg6Gj@wo8RJp%ewEhBaYSh>An=B4GC>o zgV}!c9n(?s3U|1-41Bm%Ao`g9u%!hl95c9S> z5#GrM5lj~hxMSdtD3VH>PM!>sBc;Pfz{V{>vZppNwUmyI;9G~ealWRd><$I9=Sc7I z48a>L6{V|ns-Xm``ouUYgwTD)wgYxtENJkeV+Wwo#*V3SNm>__kNw#i;m3PXS8-G! ziLk;g!O*s`YeB=JdnF=*i$gZZCtZ1P)#?I5-2c&reXI9v_in0Fg4emwc46mys5DS> zu*G-*ac81TTvrAepLC%s@QnuH$?M^ksAMgvxx zOA#lqTANgp!%kEoR`{#HRfB6lFbD#_PEgeGE~bdEY6#-U<+Uv+)@^^7peqaq#1GAf zJxy!6H|bcm^G-}xO;R4`UWxn^B1U5`%}HaNjq0BWLJb^%mAwWE(B|C#@!H6LY@YB2 zPrZvA4fO^M6;z<0C^QtT1Bg`cgmAqb*W#oA{M4U5(t8)y(TXGsUjebL0tk6Ji(sY6 zIEoo&u8*5RB6KM=ClCgnqG$6Lg;2}zD{XLqHlP%69!drfQjT|E;0r%G27_;~$V*Fl z=kA~u6!Y#4Tuo_AHs$q=H)L5aF-q2LiIg+xTpQATv->#idFzTs>`d1BHbx$}Bh)Ge zV-tqK?H;-p+K0OV0`YK7sDl~ibPQ>M?M^Qs93-haGZ#^v?&$X>0LQ8WOCTv!Eg%A? z38Y@hW*OM|?;EOt)33@AKA^hKd;nqr;_5}Q=)3R_$}AC!H>j>s+#>bQZ)q({4FP}#>j*8gKGSO9Z_&}*|CYrh*@}s|D{9}EZ|DYJUL5F^X zjiNThNbi}TvweN$T@EO3JK?H(wj+0bv%G$fG?{b$R3=?^fBk4~{k+v?C5&(>mL|80 zGyI}=`s_2->AidNWy}a!19PT;H}aSpj6lh+GI#ao=WU71#e&VU6Rv|j?~2_q(-^xmlhSC!-oe=OXx?KDQd_mKJo$r zW@-k-hPlgFKK>)i_Yi)GZL1^S~fUxa^2fhK?Z08%eov)qqIJhAp?S1L-H7E2LRAtIC|Wt%o3Au6&?rLs&23B|lB*<*@3iZqo9i79)w z$zDmaXDnmM-ekryhM9TwJ)@1f@B8z<-|x@o`~5w>zwaL{W?o+BoaO;^~#(fw_EBvPKNJ{7Pu z$Z_MUbzG4Pu`VX)rqujy`dH)D^o@y=qIu4f147nVKH8eQ@7m91MW zRTNL+?(rlSW1yBd!{wGA?{a50w%TM<U*a6<{rmBmE`wVoKOO3pW8k)jyOlwdhH)MG=lgO=K{d) zeq;9+-H!X=AFhQa-<9C*NPtoA5pz(yZA|7SKa6qH6H*>zcxr_2BMJI#Yj?5EUSnw@ zv8V0X3y^iS2hr7>;ZAkTI)o5Vkk5qEDSFseBly(2%a#X{Y2B@EqtoV*j^%WhET2@tH$3za`qzTUg8X9()0A)x90onirkZ+<)iJO;EpRF>u;P-&S< zq{>5#b}r9z=9}2b-T3K+KxtmaeBr7X_&S}Q+0(&;=@5TR=h2@`_@6$Jz&uor9_I@K zkWtNFxWeD_Sm?Mzq`R`fgt105gorE`9K~1=dat0!;zPq~JH<}@x)9@k_Pu=NG_efY zdZ-z>ExG&3Ep=-@T~^V$kuO}i%E(AycPt+_d7L?ESr|X@Q11S|FXKHsx>AkCk!PR)q$e~u`6>kq$J=|HoTK#kWo-67rm>E~J z9Jo7;8M2BcD5S45J0^2jk(XgPTwz91i(pChAOu-jp?Z=1WmsfwmW<1U{H7JXR+x~7 z0eteA=|1NC$UpR#Do-iW_LtJ1F$h$qS@T0G0w=G+&qKdNH4f2~+ZaQwG^%46i-#}6 z`zoGP7pQA1PJSFC`OGr4x{g&9eH1o9*Xz>pp?bx2KbV$GQC!u7YykjCot6+S@2<^F zxGtu8J1W1;7GtGlVS2NP_@-yn$>1Zq|KoUGEi!Vs@A3wC1NJ>Jy{7ykd7)Et_673* zO*dqiCnVr)XI!Vsr~i4JqAs^0#lmWDHeIIdHSJJVLdOTxR*B*>ESoK6h7 z;96mL$Pl$uQ665CQ{o5u={jq4o3|Rbq)jyDj!LG&GYT9S6XXVR=$n~RIdOGb{0{$AR_G5&M6BPsQ-<{~XG9)j`)OSnFda}?v;d*LQ6M9&<^sZPA)pt6+IY3kXkz+c> z@{ibe1gUG+OPX^=AG*Ev*r9hh+ABh)@!^^0M-CcsrTyV#E6vTlKM0(-da%-l-9vbZ zp)3kkS4>!MsMocxIk_N3;dFOqQB21hYcfOFfUbWIcmjiM7qRo!v7_a+S9?nvGujEo zPrRebW7GUo^zFLCC5}?ttBY}zsD%V3dnC3yDH&`Q34EUX+Syrl`_}zeR8e{Z zsQE=1xK$V@(TAchqgYUpm zT}hoD-j~s^KX73e7}qmpkt{xASdGJYTR~}4(pLfId9ciV8|7lI7WBgPU6WsP$JwTY zn$;C_eSOE~SOA;lWrM#N$=byZQ$><^I-4sAp{BliRD6Z2h_}blrZbdGvheVhp{AG$ zKRe6(%c-J!iyEct`5)gnQ>Bct+?f;nA^7Hfr7Lp%7_vW}YAT0AFfxj<62Mg!;&z-G zWKQkBR4@WO|=%q^pIKvlY32@a+f;UCh8n zx9*t=ea=_|+{xlJ#WLucmnAu*NZzqXny<&%Tt7WCqfvrNPg9|okxi&~%r2BVjSjkT zV_c%2wGNUzGjRg-rQV_oBZ4loxWOy5Y0)o|h!uSJu%b2ZeGNNnR{3J%4pQ3)UMb%8 zW!f9+?dvwPRwTCE-e8cNsi^9?dh2LZ=EhCIk7RyW0RDg%Jcs{;o}WVz>vH8{gjGU|@dxv0+Bp}BBv!t5O~Q-Xw%8bW zx_My|+KQ#FdFZbVbJ?gszwx*YH@K%<#DGEGRYZlZ`Kl&5joWL`~VP)Py=?`iMBn}_ee&y3W-}rRsY*w$q%~Z=G zg85TxY(LgfF$?8G-(*Q?)pb_4Fo-i|OP$B%;rgw*15=oXS;uI&Fj&GnznGlqi7$~y z_XtIKKwXObS$#LUTMCpB{$?cU^z4knpo2N}P?yv$sgFWY?odu1d0f_b~(}lNOud%>iGWP)PQJW7e#wo&}~DxkBaiTv%E`%^epyp87))Z zA8%{b;u1L}Q-QkJn~!z>%s=LfZb~y4%Xwg9!h*52NPblifFeoskSS}ZiDtBlJEZSvE@By58l4Yu6?80H)z;xS8MB2Pt|asMQC37-v8;eY;_W{!OWvMZRo1bMeGeRlUDVtu9QciwRI$J?wCZ?CQ(mOvf4mG32qTu}&h z+s4stB)V#!6;VIO`=kz}^JWI%%Fka&AUV-v#Zuawu#S1ra7nkBwsmx*>_SVXKfHKV z7ZGQ>BAXF(-}huayxy9Wmhjj(fzjHnjM=7sx1e6vP3l$L-bU96-gC^9jVB}e{I>Qy z&p+{4rn!pHG(thGnt;@&ulAchgS;HtL6o%Pq{AFpnWm z8)^jEf5;pul-zxZILe1h1_+U5T3@(^;|;zV;pN|raPE``FHo~BT-`Y)_`PN6p#XRz zOLLZ8ikxFUUs8V~7xl}%fjDP=m(=s*{@|GMe&392&JaH@9ru={h4y~GEr3KP&UFC{ zF>8U}UoiV>GWW8LVtCg`SAo;!Ym?jdHykT;kirtcFp*?OMi6{`Lmsd=Fs{-y{h~| z|G{N5^KSq%+cf6EjzPAAAY$3X_~-!lvj;E06-NL_#cvy} zKbMFD+)saf;%`{ttKB928$15sm9G!{IjhXR=l;$r#&g#K;+fxXw=>tU@7$!7`R(@l z%OrSz#e5;iG-oNUiucfwHo(ECQ-F~gkfS4tY~}s|xu-ubv4F^bcI*B#8GxU20P^_O zxFlv-xSm+P-y?5b_s$*Lr!H}Q^r*YEg9uLitR`q^>mVSv8V5n&0^tm>55s9fj z7QvThkgLJ1xkPSU7+&nflpaQA8&gs`$ye!l;_=;D8=4OWm5D3VP8jYTyxzvZzUf{_ z+@%5$?KdfPZ`Zxmvz?TG8+z)tJ-9YeHsgZMHdvQx#?7Ot<8bhC;h;)PO}n-4Bx-OFW+8BrZ$K5dpVm?=?a&}Ljk9erIYRY}(2gd9)_glz zYsiTm(kbtvhWcYDG?zDNf)xdKKpDs|;zAS z_hwbx4LjSZ*XwM-y_oB)?pqB!QnsciBxz9HWo3?%5vlxBBfcXKsf#~*je7WqcVoET z*mnI2_)}eeSRPus`S}MBdy+O z3S#g~Bs(cixdKcpI7^;Oa15e&(W1)G8Izy7x@)dsS7oQ?H=Gkwv9)Y<+PyFP#@kxm zkOjtT1I~VAJzeHj?YAB+3Kj42UEfU*hbv%hYS5K)!|hf_ifW@p+DPmK z3(6djg1UsM!aT4yK*EVP1n$osxE?<)=gkMRIG>>L&P-rvFAM=!A9PO|5kP0NHW8`i zsB9iKaDoB(-1z{a5`bpTP$Ku@Kw|3xkiCEog@H%s{XRu--$21;Sbsc2L6@Qq^_f$J z*;4>^LmkJ!8dKC!eP-W3AZnw!LZLH=X(4C*wAl@LP)&ys=P zRzcCI3m^~P}btwrF4|1bBl9iebXO9sPxC zGj;+7PUIEna@T2IAc{H@K|Vswdx(M6r2q=-&bjpB3)f?H+!rosFnh#ncA^`Op;G{k zFBa5k8xd-Ldo4)QIK(^#)hg1+@Dgc|!h@RGlMdorRJR->6~wGp6InnVOj%4Lz#Ct& z6QO}08v^LO5PYJ`debo<)Bw$lYu;b^>lXY81zT>rj#iGK`;at2mKiXEuA$*25bG(M z2Qbq|Y<(e1{s(bp$mInGxXy#f@w(ueo-)=0uTG@D1Qprn9xyp$Sm1Z^eR`WSWj9F} zLoh;NIp`->T_Lc9Md^2TUG8$72WTCwj)RzfiBRb%Q| zEdXzk(4E6x0355-Lm&;tegMTnNzmiLZGe|}i8%5u0pxZ!;5i#VcP0J*{!08o1`n7u zI7Lc1bBFLqWNnzeAvOQ`h9tnJe$XErm7tGs{p^qaqb-?P2+Faxz6AckMc=sB1OMO3 zz2H2|;$9$5zMqL7H2(8}cj#K>>r_s>TGX9S4bpQE6?J=rVU9dO&xrJYAWwJ!ul8c= zi9k8=d{s{RU*#fd;0P7*RjvHD5bF+DhWSg>*0fg_opTd{;ADnd#vB1IRX!38w(&Z?`JA*)q=$IJ zqTeA{E12{P3>r0#9?=Y%zwIe5lB7;tg_qoYCt7{F!ocBVJmJipy=grQ-$-n+&0Wwu z?uTAjrs7b#vS)W&wiPF|QX8l7E@PQcVn|K@juSTfL-KC)-i)d@kk@bkVODSZnT%+z z!>Cap1c2G43cOm1-0V_TIhP@5O3Z0Y{AYi7kln~BgadwpsR&0^*#x+CtOs%Q+1cRl zzyBJ~dJqDcRjpqBD<)*YxAw>HIqi=jd&O(qbSz`Cb0nPdSOK{8XkrmX(;DptCv&Ro zWujg=c2||9FV{Y{+U-qxuW=X%$7XFxF0ZiCLUis|ePra8%=5&{HVMv^2X;dDNDEp2 z8IB%b2XlK{`+67A*5;jl?tY#3FyjEvXp1UY>a#8T!RS5DEmt)xLO+pivQm9bXfGHs zG-)&=h`uW^?M15XQ0-uqn<|MU)m9_h`fzgT*qiG-yr~jG>y|bz*73RRq8n7QWB*l4 z*L+u_F!S_*i?^Ev?38&+R^Q&3kQO3uOKEYL#(V$69y=C4O!nq~X0j)!aGB`GD#^@q z`jMO+qbe{{(Zw;`Hz~=yy0`E_l*;K#LZJsj!E+7|UnOkd`~WI+zWyZ9!TAxCk?!Qv z*{HKdU>@ipHxn0ZQ2tnX31Pw-vJU04+@!>*5a7H^qT?Fw!oTwY4z>ajR8*>g2~_3$ zVSDRmk09|K2xvQtzMYpmSvF{Lyl*qo$jKqhm zU9_inH`HnF=oa-u?yZZsVRy_qA;12q>XsU@Q3#b?0XeY-tt9a23Jtyy3W?$ zuj2_)L8e=^XR!-&=$<(@#j37bo-~$odQ-Oa6_?$xO~baP?9nQf6%RDuil08hmA`Na z-*Tlg8E$D!E;)l?PqstM0L=7vYRZ`ak<`OEEf#}zqpkF-KWPj8pnS077!1uhxNpwZG2e_F{K zd5MaxgXCor{s!Vw;v1f2(+(LY4D;C~ctb~X{P#5)#Yz@7ZE!X<)U?oD@-Rjt#8-ib z5_AouXd^1=UfEdrI*KAHQ{`ge{JyRSeD3F%A00lgvkI8+-E-sUM+t)FL{cU(0OHTI z;W40A+`c!sd~jOgVV>cGmOZ@|uaWJMf}=D6|FrRW8SgyI_qm1oz8NvDL6_1f$D7=o zG@mpTvc}C=u26Gg?*WK<7$wr7+r!?EUgUT|;dU4MfP=PJQM0Ohx_%0Td3-}GBGqw$ zsr6u8M}`W0RDh|T)T)vk-*V;Pf;Uubu<|Z(HY_%GIB?vz;^lzaT2IwK zumX4CkuI7aTf;G-X>60+>qiNN1q93mQmbeF=Txfer+TIfBc?~K#M{>~jiCgOSQ!hW z0#w`iUVe_vb5Qm<_>P~_UKVZEyh3;BK#fP@YY*;43s+}k7oW*n5hzU43>uf05gi^?E$XmfdV9ye-GWAbo>Gw3W`mYa>E7m4muLzFWHp zyn~MS8Ro|6^=A@Jds9@H-jw@h9xp;GS3a0u{d%4Ih2}p7P#L%qdA@S26fI-0e&|9> z*on|)OMIDdjAcpTos|YEW@pR&a??yG%SHx_Stel>w06@3%K{Gj%=c=lht!+B;==k4^;y0(g5yhwLr7roy-mY(;T?rm>5X_OJuj~5#*Wtz^^{&Cb4r+j^!>mPh@unl^G zGGUb)+!*;<--_##`KliOGo5A`1lyuIkxig@I*Hg@bk;3n`u&22e}AHDgXUlY305*L zts*$ws$pTPXF!9{U)A)__1C@s)!qu)|Irr>6O2KzO(7l*vhkZKxq`mba%$j0;apS= zl>;C2&$7IF?h(oh4u0YCxep^7@pT=Gq1aiU*%G%aad^$kN5 z*5Ox!3%cPWQy-E&$6WxpeJ7YN6S#T<$B%6U9#<#_j^(h}p?RGbe2f1?E*j zC4yb3X&w;gB!gn-#Z)|-n*xord#W7d0j3?VIZ%ZINDOfCl(*4r5^|R5S_H)*R z^}qIF2Pbr9uYRnA!yi9+>N|hNncde!oI%5t2ThrwR+a($J1*v3)6rL?ayn<$!4I)`L(I4gT$EOg~ z1ntsMWEZ*_Pm{TA&*o_WH-Qs=lhghNFqZfNIb09?75qTWfO3dh3_}XIpIvT%kq$$p zrBG)yG;RqmhPtdCLZe5Z>)C!mA5kUOn21+U0CLvrkVE{6Q`7KAh+xjrRE8JZf8o*r zvx5cmfIn5r^xuA!+OMxVi0ZtLry8K9<)H;cYHA&7N|yM_d}e|BX6A-t{Ba-y_8)>e zw<0DM#jx0rJE}Pm78eCEfDGW!ZKeXPzocCPAg&}sU0vW^nmw>Ex>%7VcMb>dCyWZ# zJYxkhOukYMmRbc0EACC>8B6!DcRd2YyXZFD1PHVQfh4x<_WmZ zFI@UNC6B2&5xDE z{OPn)SOFe0@3H@Y-NTXr#U6XBPy^bDF!%B=Tn8PYn&wFf_66e1+Gf)!>aX6G@$bCt z7GSu8fZ-mKV|viVn}J6|`ScC8LJBMJ>mp|+x9RVedKdZ#h&K*U*uGMa`B;&Mq3s>* zu=z~8*d96A`MeVAuBczxAqYziUMXGYDHfDOC(y}MA}cH!InnD1HnWMyL(4B&s<`B- zXz2;5L!>eCZ7CO91IzX)+^y~dS5_nKUvHcu5N{ISg7Yy;=5VC$54{4UF8VTZ^M4{+ ze@d1?R2MH5&*DJwkdf>QLrM3Rfy#c|;tA$Lpzc8407v-g0_E6W(hP&h&v5e*Al3oY z6i|D<^3Vtlz;Mt0!SnDz9y#zUs9C=<`xR^sDEt$kVbP3TEFeL(g@}`t0MPt4_#JeB zW;MQrp4*ClF){&kHHbLlK-}-%GFQ%wKw)VBferSp8AyC+){kjVfKG;fE`vCGN+6VG zoA*S9`zD~EL;KR;*lW{)@Nn8@6QK{~2Sku~z3R4CeQ2e$_o+GlX`o`U_M|4!*+-#slzLB}^dS z9&Y^ppN8*W{nfwogjqQNQW>(0egHybK$IYcO@DYXzk4(1{ELnQ4JMF4Kz=F)QdSwj zP}(u~zXs*qRBO`Bch`M)BrC*^IXSvyOqJieqsNtnvu|%5^Qgu~HuKYjOJa%PnsVCy z_lNT;gIsJP^(M4??tn!M;;QWcf$++q)V78R~dR4eP;y1cw|X#3%CisYxp zdZU`!Ow7K_!PiF1s*&#skW?oeYbB{q>h303THyE#3t#;eOA)(ypA|1~=OxT%l!BT& z7SIj6MEmN~mEfnf@OlQWL@n=Te#g;P@30E`d4y_#(eu|zAx~o_H%Z7A1ofKklr$eU z|HMsO)r~4e2zp1+_c8nv8BD8~Qt=JZf;5H1$Ls2QoDUczEzFD+$cj)zSnVBjOx)kp z)gQSd?u^Vz!3s7ZlN>6(l%1D}zNW5LOce07qV@ts57PY^i!O>S(vmeWNgh&v7w^HJ zc8~Ok^7Jgpde?%xXYx-S+36lWY22nL%(y!UIW`W{@p{=Y)sK+32TQk?tF%5H&*_oG zx5YscWBEXr`#YdIBc?$DSsGT%A-2EFs_=BZPC8A1ZU#$Q@YZ_@$7)#J^sK*TEKuVr zV;W&x6}e@X_GK`O$G|O%>dz@huw43Jc@Tc&{VL?Cuwls>xuA}ZG87_HortWJMWam`!y3ep zs68r}&4+Zl_2R`=pR%wxH@f86$K_{r_kZ*xwj|KWj8^7O*6{)w>awqSS39Qz3~XmNQ0*TU>XMPx#u_=Ww9 zC@QY!D0N>Db+R;8(LXbIqAw7^!zd{|m@Cpw&vz6m3Rlf)I88F}d3f?9M4k#dPn>>Y_NcwL4yTLkT>X3p;JE zAKTuG0eY_;la{*?OgD7L2Qt~!;BE6|~?svxO_J;+ZyR0?0B+$yD_tziMv+5hbhxr;(q*nKNo=Jn%>fFN#%{tZR zvJ4$U%j}QlIij4}+cSZ&hN(BVCp8oI6c>O?@S;g@ruj78X!fRz6`dnUZ~S3e^o{;6 zT#nvOr4`XpLqdHGNx9|$n+fDdyM zUBb0TdZbHmAM8t!D>bFNoGU2SQB`gkn2gyPQyg3VXw1nK5qwOTAgT5) zgj+@DRL*$~b>wC8bLmROH6MHxsKViWjB=h*8qKOKwmIaSypaz}TH*EsRNxW$@jjuu z8yY=%YF)CjRtA&UZEeC!R<2xJ5_4^>u_Up}h){Aj9b>*RHak0u&}^WcE#^BcY`ASG zC&d!;z}&cRe6aZDmFLB4 zj>Te#)ti(2m-uMkP8)(4s^;)wJ7>POwlO+(q?Th#!PM;5BWlW`waW{)e!(BzEjH)c2&t!~oX`gCvv8Te^qqi!&_ zW!%QCo0nS+F0Xb~Dw>K_vp+NBU0J?F!R_r6qPuC1a#2jCv)72Ky>nCkhatG3>|#o5 z{)^GVBdMoGE;uzZ2k%#TXTbc4fR$Ceh4nL=S$5r!=wP(^)m80BJ8T(wBpH|W+1Zh! zjvH<~JzwBP+B=r9Y2Cg@W0zyIdJv3lXn66w&ei9_aKgjLm7DFFC*n|uff2>v%fgm{ z_l{aw-M3LK@(GQ9^sK;L#|Kre$dFE?oT9aMNf)Dn@O&V}N^y~c_*#aaYnPwB4()vM z(@M|dpo>u;K5gVyAf^3?oxrBebnZy7 zql#9-(llSvejSDT$e;py%!T8j?!y9!Z_>|{%Iz!JVJ_2aRCm^i9rQ_q>B&0G*waI} znoxRItIpRr)pt{KCO(vuh|@H3DbVU6uYsIfY#b~1BS~tz`7rr>7e8R@1q<(-wX%Lx zaD%#==~~W4!AlHi{au7|J#O0%;%du*yKLOqm6|b zwXMxvt##AW4!w?+56M&v4{K#~2h1-Eh}yl^%UoYFx~Zny+sx5atjKPIDA(KPBZeQB zDs#Z9HALJ1mPdteVv0cn)q;hwUlqoLtwWhhmcTub= zV|5ao?hTq_SA-qTuYX(%*C!%Ykhi|?+G4vp7L)R#%;<*Un!VxX(}(bElWAO^oPGD? zs?kJtqB+Q0hni|^t6endXTTlW2Givf7xn8`$;>@&GChdDV13yl*=bYSSh@ zk`vvY$=v?**t=JDN2AB#&T|3{|PfAB+XA!5l zZ6WfoE39RsB{1(TS^-4mus_uI%?!Q?24YI*w}ox=+{oto4_MGFV-;G&Hx`h2_`iGUk1VlK#r$E zjEQsO2YI%#vr?h=;$!d^E^s-b4T1R(rnx)7e9&L#Qw3@cXHO`GiLnB{60LqVr*O2V zu^u=xD}RmyE{b! zB{S&v``faKbEl9D&Ftm(HG#eSm3Ds&&)&JTyP;ftU-$oCUK40@I){vd1NWBM&7E4q zFU!6l!+qO!b|6e{#soY2u%=Z4LgGKTiO7iKoEV6Y%{d zb2;2jVvTob{w6l#$JJ62z8E`0B0=d;PdO-ku>=>a8!4O0dpR>Y9kR721wvB`@$sv zc}ti#?K7sUeyFiFz;FbWNYOK**S`GKw_vGtVbh2f<^U$`WZL(wSM50-(c zfzO{_16%xYf=wX+-SRlx>gbw^aIYG4#|%Hh2w>a%PVd#`6n>B?Z(odNXmZ+G;uz7N z<$;dS0VD)TY_EBy9|2ClL!qbwB*o7TzSQjNj=IR1aB@0Ey?#CaL)&G#tnBQGFI;Jj ztaZO{G8_*sC*ee@31JP`0b}TtycOM_Rug!RxTJlsL%oQ4=fe3YFV&BV{PAgc_YCEC zH*(KoDopPkTgbCR()=yW)&DlEf7Sa_#g%ZCI-;g6!Fko0p30NkpK5A7aW9wzMOe?AqW?lzgW>;NSQimLd^ZaI zC7p0)_`6X-WL?PnmvjPfEX`JE2f&Jb+rDtwb=bQatG7M|D!|hdRD?kw;f%|J{DXx5 zfI7d(>bbP|U2FVe`TVLhez8RUN)B_B25!p#r^*HU@WVu!F(dp_XB#}f&%-A=>xok& zM&xkhV2(+qm%EBv=Ji#Zw&-n3UU_o?;@0PN$BwPNVWalbI^eMS6T+Ol1C{?;N&d3j z)YeV?x!D$J$1i&Acxx(n+FHDKP0ZZ_ku0E2#T7w)5V*R2u&n_o@M*vG1h-ZHmJ!P7 zX!L&|MdAPBUT2V!yWktR&3AYBp@Z|#u_)@%8a_r`1(A9{F}y>N>JpdaG!4Roms}~I z&b2F$tn%|ij#dVDT@my->Cs}MPbbt3jiPbTiqF_(tmU+z@)Qk4${LUx7JjH~#ZlL- zVL|njQ@KY$U#!qRP2~V7b4l)N=GY^*3xs4Nw(f)9nZ)OhW>OfUU2#X&^%T~^s8P6- zoPGO7Fat$96pb3&gh8a^eQ$0(p}?i&=qkr z%|V2on&f?j>K!>*3a`d}s{K&@aDeo5aiU*J`Boi|Q=8TAdp2l}ZN(j9ig#itS*T|5-`J))5wpFAY)yF=>*jx86ZG+HTpRa ziYid)GMI!P*M((4N&Zu4lpmxImDjX(BLeWEp6EcnF0%%e!bROx22H9E<(51l#9KpESx+q01ZdR;>?jc~icjR9X7?rlNN{X0~-2YnSas(6!p*!Res$H4xlVzLc>qE)a#I=h+?p3+|(J0 zs?BM!eN_i+wjCG8dq~}EZ!hoLoS>8EbAI{oD#3+V5Y;U{;Hte9BhH84&G7R=1hI_r z3?C<^RHyA!P-#D89%Xe!?r{3Hp-Hz&?T`)n8v2Wbj(%8Q7J40y`OpC3D;f7wQ%9;& zDfOvDg`xiGDce!o*EY7(bn<65(%y9xT=0Hp8&p>i!nFZ8%yyj`FPemP8Fio#>(?`JPnRVEV&UEPC1{2;07_D(nU}_bkyJNnD`13g z48j}g)6P}JJ1(tQuBB^xF_eK= ziM_u4u4I+Cl{t@y;~S;CC|Bc+H+t>Y9yzo|_rsf5n{y|}O>`hFq$~$T{IC89BEH4- zVgY5oMw|Z@r@V)?>kHR<$|{(63INaeV4kIEl!2Z)wWwh_nk^ljFY*J~{G;aV#JaPC z*0``aHOz#2^Fw9sA-=-R|L;$=YwQ=eEj|IDTFfjn4fqG#pR`uI7FWKrqH&G>@cTPv zYOM7acqXNR90WVfpm2HanCTz@Y++v8%k0-60J;%^H9+RmuTM<~Xs&;S@}^IwfuMaM zHxmR9I)R*LoqrHi96N7Ma6fV@AaooKxOaAnou6qf1d`8$??LVmYcc){R|H`H*+c9U zGx~oINwO+-@+5uo(=oVVeGOjveu2-{cRp)HXQ*h(Noo5@zV|$bzIFoxnRf%oHbBw& z_Ta{uIfuso@1cnIJo+Hp5A?~(1*Ij{xCgkNrjvJ#V93%*vcXkWg)XO?&D4C2nhb`cR`| zZbQ&us==R}ccr}%4x~+om7dAIDP8AZyg;0H!wWoP_c-FSG0(-obvyK7jc&AV;DJ4aG?~9Oww%pVpx^ZBlLHo)&XOTC1GYj|bRlBjsf3bI$!0WU3 z^gf<^PO$iUz6&1AnP?vKl0D??o|8EB5Z@U&{=UZ(bqPJ+vv7}P;{<-8x6$X3vg=Yg z!D(rk7k8X)Tpxa;PNUh|z-Av#MCobyCE17W3(pLq7Gnh+wk>zGL@m*0ez7D@ zPfD2roW`3EAhxX7*h3-Q&JgE9@%8$PI;7~sm8>Xs8YPJrY?Ywpd9{GolqFxc#z*PpLW znEBODrQ)9`GF(e-MX4_Sj$N+2j&w`2+sBu*9vvh(=UG{{mf+8CzaL_{z4oGP(){dE zt>L~Zj`B`#d{h>lW4oRD(7jwjXS2I^^um+pq%4k#z6}K6W?D^oD^im4et=}O&A<)Lr34;>^ajI7^ zb}?)3-HWbV!_!7NQcomfwhoHCDs+h6y*u*?UyRJ^Y^1+GmTtUycUX|4*J8Qk-(m z=MhCM%c`EMiyP3?KnHOwS;60tO|3vVFb|DlqCDJyqkD9mJ>ki*V^BZsn0XB|$Hy8} z9n3a8k#nsb&Qo8d*Jnl&k3MIgVs=cx<-m!Ly>~Vb6-%G-O#=-n6lvRDTnTGo50;8H){>-;_pTB zY$T?(8kp6L)r;LaSW@TtTvdWszIWq;n3VInF|}x55SNOf&WQZy|dbWFpTRAILY&cCoP;I-iK2&H0JWG=N8p?CP>TAvAfqM}Iii9poz9s4hh@ZH zF}43W^3VEQP^^kgmU`QwHKGBy76|G!`o z4LSYxlpD<2=|R`RB&`GH!+tOy4Y)SI9l_X^^g0&BtL2*!R@s0%s9L}mG4g_+=f z?u(-mJ|oM)y~hRp$-WyeLdIAoMqnlOKuOoK4u7nCtH<%uJ0m3} zcZ?#uK9(I04V-6y+vR0TV7$ntj5}%S`X3DS%SVp$-^9MJ;oo%Jq3-TRf{LBM7cLL2 zIz3oo3$fjp`4I|OjsBd-9z~T9pI<~X*4409PES_k;Q7zMs*DKqiorpZyr7_N8Y0Nb zG~3{myt1Lw<^#0gKZLHQ^ZQ4?OtXMGLm{f3S2|p0Ojqg+L% zYZaZYiF5JDxpN}Z+TaR&V*o1O%TfZtj#x5dqy0^&_}M(&vXQW>3wfA4I|sPHtu2MT zA1(v8iJ4twaX1S^KZ~`ACr3WG8qyRi&@I3gi^YXz8n1GEU+(vGDE97cpT|0jqBegX z)$y=%U9fXs&AQvSTg)BbZNW@9HJjE?ge&ly>)#bku}VtF&3=*N1`-d~EyanZ9jR@& z^7P>qUWrh~o$dmsZfR3KLjR`3o$U8Ukb#wW z%dnK(pXPOsEIrCnX&e`&?5^t=)pGm% zR(O@9VXywu{b_3HA6>#PSMF&drnsd{+E2-iUrJEFqjuy_nCo3p%%hseEG>|}ms7MU zvp$P0TR}ExoQOBxQ%gAWxi|J>o8uL8l+zVGiM7P-e+cE70vB6>~eN@0bw%FX!wqe^c)jARq? zhbJ#6kthvH(SM}z=1Y^ zIuD?FbZ5QuD`5QpUEdmhPk3=(>C_={vea?&B3)MDZVk30NYR)@2M!oMaA3_L9`gV2 z16!f+|NT=yeJ^QvfuOjqn2IB&TWzmo`R%D6OZ%~6RNYq$bx0g38QkoH8l6B8n((u!@n4+Szdg?XGha>thx|f5h96D(rQsuNQ_=iJ z?e!Q?r!|~iv>sm~jx4;ileyx|mDLX;B$kB=+2=RC%5@Yc3GX#2cr8p8N6^rrMY6uq z_(O(jN2nDkp7w4SXZCZkOw+}0b?oxPH;-hk4#8+#gqHAr6o(?Q@oNr}0P_n(};Ejry~~JLo{> z7^TnJ_^em(@U$)bfhNtK4^oBZp8#>0a++8sy!pLifqXZF*yHzj$SmrplaQK6v81q2 zGp}9jniZGLB96A)Xt=^FX{H?J`)<0(wOq1jSOY!P+v|LG;;_`B#s=p^G89+ezj}&1 z6~T@>#m*>N=Pj&5GxwKZ-%~~K4>cjVvj{7!ef8DMy@a);K}gSamCeBL*^^Ps)dO*o z`IuwbJNmXD{Vpm3xJG;95LnWc3;uv!<*Q6PA#Yrc$_$^3@I^oHmSFDVhYYAB$DC>D zz~e<3^(2j)(2Ayx+Q(e@40d`yYBLV~9<)47)0DpVfnilDE*@BFSiiZeew-<4(HX&q z!JIqV9VlrntGm|e$&>9|TyoY!u!N@AdLnhB>`>X^u&A?UlT$H?DpFO0pC$A64Sc+Y zE@5I&otD`Ka9=nQIQT0Bx zJ9mOPvbOvlDzLj{eUNc}HqAO#o$>Gu^-0^JB}zk0aloImi=_5xglbnxT>2TibO627 zSJ}SHpxS_WdaSgGYEfzDhEbHJVwO7gRnH zEaF+?pJ{+t)ix}GE$rf#-V-gq(kt#zZn%OJ6;7JW$aG8^oJ>;XRixj#^FXP2e6RWv z;Psw$LO`CVCh*_R9G6>AMNYDL+=%N4OcZZ2h~AP_xROaE47L}|U~SKN5J$lQv*|rJ zaT}4er(rW#+aG9(Us)J9@UM>XTcz=9##R@vleD!{^ z7V*$r_up1}&3o)rtIMBM`5V^o)LMg7qPp(@mA&9fJv9D>i^&J>jsu>}zg}d% zm6!tfx51~uU%32^JZEhs;%S@QX~5P_vF|<@axup4tk>3?F{9 z%=tx>IkV;k2#m&{MkA4zk@O=48vm=aGl6C+-{W|+im|+)RLK}UC|bLr?buSK3?52r z7kOG#q^M^n6;)fSXel}r9j(1-MkQiRhA5q>Jp|F(>BQ0|L71FqCFk7a zo_lZZx#!;h`QP8~`}_SqzxSXDd-!sH`vLU59@#jP5&34psAvIw1T+JfoL`B}NopUqerDMR$=S47hTgBK-CL>t+~FVg z?fZ?-eD>cYjeS6r`*VxPujJ_8F0ArTj|LBfagL3xCCulHquCcm%+0uF6vGm4mjL7d#4iGTGWD>U)y&jSXRD%Isk7sJiCZ6X5iEE zV*+=;v@N@>_1D&OaNK}iXNJtv!N1x5WJby&abn(s)+j*Q2y!lme^3Fi_Eg+iz_-vF zem{WPNfeG_g(jqy*Q|ISer>Hh zkj(I*?LcMqo*=|I&vpJ!g9=oQUY!4+;st-4Z9h8<_vbLdpLFzVdI#hdy-2VE@@OW| ztWvMlpuHiv{|*zq1JGPL+2|d`sQHRN<`FhU0ysjTY~lmiXl2WUFEK>mgh9EoN}WA( z;0bXiC&4F=EEX$SL$&&1@yhg%_q`zn89BvQA#qqTVy1|8nb72Bs}h|xlqUKtS43q* z>W0^p0#i2w1(|~=9pc#Sy!-p8;mam0Yt6dTEqRH$Bn1Al{@WCTZ*L2@2};37Jqr%5 z9FD&ygXt|I)k0I&eLDQ~?}Z!W7^LJ85=_eI>N6`O&M`8&!-Ja;G8?p}#Ew~KeI2Kp znfU}6!7GuY38(e`qj$_aPHjeSS7#UXY6#Y2WyW<6M zs?l1jT?MLneJHHN)kTzcj4$ChPDVOw6W>k`d73^QPLTK0A9qb7NBq1I1u1EtoPM2! zt!y6v^xyw}?i+iE&1PEimeM(Bx5S$>g}Yg3CO;`&VG#wGKFVpu0OlYZeSA6RlOBTb z`$~94*d9~hyZigV}bo&kovz$I@;U*hs%mG`qMNe)b6ee zr*1AinJ5^0qpM8F+r~-Sn+&3O`!prfM&b%3d-SDY_6M-Ev#qvl*QTOVzVk0V70>0t z%h00GE4nw~vWbSMlMGU7o2}VcdX(;g#(Ip|+waQhTU)gJ#`@slIk-<=C1E#Id*l6bHqit`@y=aeH0(6&0bz?gX@M2O) ziwtcsm^QfyUqrH6p~sA%lM!>I+vq@ea-uQADI-z?WigKlqVqMIhUILFUj*SWj@%BQ zIJyLr^(5=)13xPA#u0}Osj{lNAfO|VA=h=3uXQ4e2__TeS{bQq!t5HPBkQp@Jopa! zl8yM#@>b%t5tm-018PBY*+zsz>pl3F$#33PaReATu;cS*@;XXokeaQVqdFLf%`?5e z*z5VBsyQ#lH)eW;tEOqo50syIkH6ZZvyH52b7;B9bog zJVQ!Hqe9f3E09xD?syFS^49b}9~3qR;IT`2=n^W?ZKBk6R|9CUHKolfYLlBM-E3(m z5-=-#g&&v));2M@2uA1#X1-O*MlkGn?U|!eHB+9+3lH+(cS%yDE7Oyj9ZJsv)0xkx zOG_)BOu39G`?`cc#!>-O;%mF1%@X?%ob&SJV@hvUwg_RNK$nLHnE+hO9 zvZ?o+$%6V%&XJ{4tgHlFT`wGPJqH3lGj3K>5XR1gMNs33i<)Pq12QbWdVzW#H2o6R zvpp554}8z+(4;z(tmb-<}vNJhvuyXsX&cb}#S9f)cy z3N5>|#7jH;{l+5VjlS~tx$dB^8wXE*0}9Bt{c-7snAS!rVG-Nkx9$7V(!QJ8NB|H@ z8{T0^d`#5>I0=}yD#{M?k9yNv)KkpAinW{X@;x$wg8nTtu+-3i&hT2H?(jOZ9M`qK zvc`{K$_Kyl)KW=yI~K0gFN*hKmT_evT}t4#vsgM`i=k5-rj*W~d_0Y?or{ITZg^2x zt!8$CO%1zp`G- zTgBJqGhW%MO&fSu-p-GE^_|DT=4{9KOl-QZ0FjDC*~FD2q?Z*6%6NIzaQlG$uV5Yk zNdNIYWJP;$Zib<}=2^i6YbhQoN;bLVC5IlHt0b;e1yB3{%e}g@Lna zfRZv{Zp1w{#FwA&)a``` z&eO}B74Ez*bpHrrAf2)+`yK9DOGx#unEsjWNlE@pp0s}t!Nh&=!aC|0$IsAs6gQwwC6zZg zXtyl4hSdLrI>#*QlGx8iQ@IH<2{7r6u7Xmz=20!}fVO;AjjnPd;n7L=e#=v&T3PKQH3;1?q+2;mwuN!Ey_^+#f0n zR1YCSZ)V+j;wO`I`bG8Xdr+WKa(q}9Ol~gpyK)Oe!t6L!C_TWxFQj6^XBk3W-qALe zF0gcNx@D94PTV@;pkYF1!@e`lF0EdOs*1O*D^VNvk(MNNU0BpENi%wx{fhe&P}Gx* zi|wB3Ts~~B^vjrW)+R3%n^sQYI&3mWDwwml3%8FoVIu%81w4HVD#ikf{!%s`Low^G@Qb=EF%m;DO*^kWT>{}fg}pIZ znj-Hn)oiDjLr|teZD&kjkAu97p4K)>(J!5EtWVagzbvxPB;45g=(;56koKeu2qasg z9)Ji2d`h>*%`uks_)Uv`$0A((wH4OzR6_e$$XOW|{omruAMMb}kA`W`XB*g@0=O{f z-`k&>*`HddU4I#)TlvI5C0Bs7WyT`2#B(KUALjNXftG{Q*+d`^1=T}OBj1CbpA&vR z%mv=2$>p3Fpy+{zZ$nmcLc0f?2*$Ye`&xeAs!Xjl7yAd-HQ!Wy;Gfr*m@q~c=E|n? zopQQK?1P2dx46^TeK#n{QU1JO#48>cK_*ocap5ZhPdAz-0%PX$-3?JlJPf{h@e09Pxm}r z5mf@A=t4oz5KW~;EkN**Z&?}GOJCfMmTG0E6 Fe*xtRokRct literal 0 HcmV?d00001 diff --git a/docs/guide/install-and-run.asciidoc b/docs/guide/install-and-run.asciidoc index 40ed0f789e0..707bae50e2b 100644 --- a/docs/guide/install-and-run.asciidoc +++ b/docs/guide/install-and-run.asciidoc @@ -8,6 +8,7 @@ you need to have: * APM Server * APM agents installed in your services +TIP: For best results, ensure you're using the same version of Elasticsearch, Kibana, and APM Server. For information about setting up an Elasticsearch cluster, see the Elasticsearch {ref}/getting-started.html[Getting Started]. @@ -102,7 +103,7 @@ you need to install an agent in your service. === Install and configure APM agents Agents are written in the same language as your service. -Currently Elastic APM has agents for Node.js, Python, Ruby, Java, Go, and JavaScript. +Currently Elastic APM has agents for Node.js, Python, Ruby, Java, Go, and JavaScript RUM. Setting up a new service to be monitored requires installing the agent, and configuring it with the address of your APM Server and the service name. @@ -119,146 +120,50 @@ you will see the old service name and the new service name as two separate servi Make sure you choose a good service name before you get started. The service name can only contain alphanumeric characters, -spaces, -underscores, -and dashes (must match `^[a-zA-Z0-9 _-]+$`). +spaces, underscores, and dashes (must match `^[a-zA-Z0-9 _-]+$`). [[nodejs-agent]] [float] ==== Install the Node.js agent -Install the elastic-apm-node module from npm in your service: - -[source,bash] ----------------------------------- -npm install elastic-apm-node --save ----------------------------------- - -Then configure the elastic-apm-node module by adding the following lines to the top of your code: - -[source,javascript] ----------------------------------- -// Add this to the VERY top of the first file loaded in your app -var apm = require('elastic-apm-node').start({ - // Override service name from package.json - // Allowed characters: a-z, A-Z, 0-9, -, _, and space - serviceName: '', - - // Use if APM Server requires a token - secretToken: '', - - // Set custom APM Server URL (default: http://localhost:8200) - serverUrl: '' -}) ----------------------------------- - The Node.js agent automatically instruments Express, hapi, Koa, and Restify out of the box. -See the {apm-node-ref}/index.html[APM Node.js Agent documentation] for more details. +See the {apm-node-ref}/index.html[APM Node.js Agent documentation] for more information. [[python-agent]] [float] ==== Install the Python agent -Install the Elastic APM module from pypi: - -[source,bash] ----------------------------------- -pip install elastic-apm ----------------------------------- - The Python agent automatically instruments Django and Flask out of the box. -See the {apm-py-ref}/index.html[APM Python Agent documentation] for more details. +See the {apm-py-ref}/index.html[APM Python Agent documentation] for more information. [[ruby-agent]] [float] ==== Install the Ruby agent -Add the gem to your Gemfile: - -[source,bash] ----------------------------------- -gem 'elastic-apm' ----------------------------------- - -Create a config file config/elastic_apm.yml: - -[source,bash] ----------------------------------- -server_url: http://localhost:8200 -secret_token: '' ----------------------------------- - The Ruby agent automatically instruments Rails out of the box. -See the {apm-ruby-ref}/index.html[APM Ruby Agent documentation] for more details. +See the {apm-ruby-ref}/index.html[APM Ruby Agent documentation] for more information. [[java-agent]] [float] ==== Install the Java Agent -When starting your application, add the JVM flag: -`-javaagent:/path/to/elastic-apm-agent-.jar` - -For a Spring Boot application, start your application and add the `-javaagent` JVM flag: - -[source,bash] ----- -java -javaagent:/path/to/elastic-apm-agent-.jar -jar my-spring-boot-application.jar ----- - -For Apache Tomcat, add the `javaagent` flag. Create `$CATALINA_BASE/bin/setenv.sh` (or modify if the file already exists). Add the following line: - -[source,bash] -.setenv.sh ----- -export CATALINA_OPTS="$CATALINA_OPTS -javaagent:/path/to/elastic-apm-agent-.jar" ----- - The Java agent automatically instruments Servlet API, Spring MVC, and Spring Boot out of the box. -See the {apm-java-ref}/index.html[APM Java Agent documentation] for more details. +See the {apm-java-ref}/index.html[APM Java Agent documentation] for more information. [[go-agent]] [float] ==== Install the Go Agent -At present, Go applications must be instrumented manually at the source code level. -Where possible, you should use the {apm-go-ref}/instrumenting-source.html#builtin-modules[built-in instrumentation modules] to report transactions served by web and RPC frameworks in your application. -If the built-in modules are not suitable, -please refer to {apm-go-ref}/instrumenting-source.html#custom-instrumentation[Custom Instrumentation]. - The Go agent automatically instruments Gorilla and Gin, as well as support for Go's built-in net/http and database/sql drivers. -See the {apm-go-ref}/index.html[APM Go Agent documentation] for more details. +See the {apm-go-ref}/index.html[APM Go Agent documentation] for more information. [[rum-agent]] [float] ==== Install the RUM JavaScript agent -Install the agent as a dependency to your application: - -[source,bash] ----------------------------------- -npm install elastic-apm-js-base --save ----------------------------------- - -Configure the agent: - -[source,bash] ----------------------------------- -import { init as initApm } from 'elastic-apm-js-base' -var apm = initApm({ - - // Set required service name (allowed characters: a-z, A-Z, 0-9, -, _, and space) - serviceName: '', - - // Set custom APM Server URL (default: http://localhost:8200) - serverUrl: 'http://localhost:8200', - - // Set service version (required for sourcemap feature) - serviceVersion: '' -}) ----------------------------------- - -See the {apm-rum-ref}/index.html[APM RUM JavaScript Agent documentation] for more details. +Real User Monitoring (RUM) captures user interactions with clients such as web browsers. +See the {apm-rum-ref}/index.html[APM RUM JavaScript Agent documentation] for more information. diff --git a/docs/guide/overview.asciidoc b/docs/guide/overview.asciidoc index 72033d2e803..87c4903765f 100644 --- a/docs/guide/overview.asciidoc +++ b/docs/guide/overview.asciidoc @@ -34,14 +34,6 @@ You install them into your service as you would install any other library. They instrument your code and collect performance data and errors at runtime. This data is buffered for a short period and sent on to APM Server. -Out of the box, -APM automatically instruments web frameworks, -database drivers, -calls to caching servers, -and HTTP libraries for requests to external services. -Additionally, -agents provide an API to manually instrument anything else you find interesting. - *APM Server* is an open source application written in Go which typically runs on dedicated servers. It listens on port 8200 by default and receives data from agents through a JSON HTTP API. Then it creates documents from that data and stores them in Elasticsearch. diff --git a/docs/metadata-api.asciidoc b/docs/metadata-api.asciidoc index c5412e71cfe..afafb6efa9b 100644 --- a/docs/metadata-api.asciidoc +++ b/docs/metadata-api.asciidoc @@ -4,9 +4,11 @@ Every new connection to the APM Server starts with a `metadata` stanza. This provides general metadata concerning the other objects in the stream. -Rather then send this metadata information from the agent multiple times, +Rather than send this metadata information from the agent multiple times, the APM Server hangs on to this information and applies it to other objects in the stream as necessary. +TIP: Metadata is stored under `context` when viewing documents in Elasticsearch. + [[metadata-schema-definition]] [float] ==== Schema Definition @@ -57,7 +59,4 @@ include::./spec/system.json[] [source,json] ---- include::./spec/user.json[] ----- - - -TIP: Metadata is stored under `context` when viewing documents in Elasticsearch. \ No newline at end of file +---- \ No newline at end of file diff --git a/docs/span-api.asciidoc b/docs/span-api.asciidoc new file mode 100644 index 00000000000..a0321b4bfd9 --- /dev/null +++ b/docs/span-api.asciidoc @@ -0,0 +1,27 @@ +[[span-api]] +=== Spans Schema + +The following section contains information about: + +* <> +* <> + +[[span-schema-definition]] +[float] +==== Schema Definition + +The APM Server uses JSON Schema for validating requests. The specification for spans is defined below: + +[float] +[[transaction-span-schema]] +===== Span Schema + +[source,json] +---- +include::./spec/spans/v2_span.json[] +---- + +[source,json] +---- +include::./spec/spans/common_span.json[] +---- \ No newline at end of file diff --git a/docs/transaction-api.asciidoc b/docs/transaction-api.asciidoc index 6d1f34338b7..8865624ce0b 100644 --- a/docs/transaction-api.asciidoc +++ b/docs/transaction-api.asciidoc @@ -1,18 +1,16 @@ [[transaction-api]] -=== Transactions and Spans +=== Transactions Schema The following section contains information about: * <> * <> -* <> -* <> [[transaction-schema-definition]] [float] ==== Schema Definition -The APM Server uses JSON Schema for validating requests. The specification for transactions and spans is defined below: +The APM Server uses JSON Schema for validating requests. The specification for transactions is defined below: [float] [[transaction-schema]] @@ -26,64 +24,4 @@ include::./spec/transactions/common_transaction.json[] [source,json] ---- include::./spec/transactions/v2_transaction.json[] ----- - -[float] -[[transaction-span-schema]] -===== Span Schema - -[source,json] ----- -include::./spec/spans/common_span.json[] ----- - -[source,json] ----- -include::./spec/spans/v2_span.json[] ----- - -[[transaction-api-examples]] -[float] -==== Examples - -Request example: - -["source","sh",subs="attributes"] ------------------------------------------------------------- -curl http://localhost:8200/intake/v2/events \ - --header "Content-Type: application/json" \ - --data @docs/data/intake-api/generated/transaction/payload.json ------------------------------------------------------------- - -Example transaction requests: - -* <> -* <> -* <> - -[[payload-with-transactions]] -[float] -===== Payload with several Transactions - -[source,json] ----- -include::./data/intake-api/generated/transaction/payload.json[] ----- - -[[payload-with-minimal-transaction]] -[float] -===== Payload with a minimal Transaction - -[source,json] ----- -include::./data/intake-api/generated/transaction/minimal_payload.json[] ----- - -[[payload-with-minimal-span]] -[float] -===== Payload with a Transaction with a minimal Span - -[source,json] ----- -include::./data/intake-api/generated/transaction/minimal_span.json[] ----- +---- \ No newline at end of file diff --git a/docs/upgrading-to-65.asciidoc b/docs/upgrading-to-65.asciidoc index ce7174c9b83..69bbffbb708 100644 --- a/docs/upgrading-to-65.asciidoc +++ b/docs/upgrading-to-65.asciidoc @@ -1,20 +1,29 @@ [[upgrading-to-65]] === Upgrading to APM Server v6.5 -If you're using an Elastic APM Agent, upgrading to version 6.5 of the APM Server is easy, -just ensure you're using the most recent version of your applicable APM agent. -Check the {apm-get-started-ref}/agent-server-compatibility.html[agent/server compatibility matrix] for details. - If you're an advanced user, are implementing a custom agent, or provide custom data to Elasticsearch, -there are some changes in 6.5 that you should be aware of. +there are some changes in APM Server version 6.5 that you should be aware of. -NOTE: None of the changes outlined below are breaking changes. -APM Server 6.5 should still work with your currently implemented agent. -To take advantage of new features, +None of the changes in APM Server v6.5 are breaking changes, +and the APM Server is still compatible with your currently implemented agents. +However, to take advantage of new features, it's recommended you read about the changes below and update accordingly. If you'd like to continue using the deprecated Intake API and Elasticsearch Schema, -you can view https://www.elastic.co/guide/en/apm/server/6.4/overview.html[previous documentation]. +you can view https://www.elastic.co/guide/en/apm/server/6.4/overview.html[previous documentation]. + +[[upgrade-steps-65]] +==== Upgrade Steps + +. Upgrade the Elastic Stack: {ref}/setup-upgrade.html[Elasticsearch], +{kibana-ref}/upgrade.html[Kibana], +and APM Server. +. Upgrade your {apm-agents-ref}/index.html[APM Agent]. + +Check the {apm-get-started-ref}/agent-server-compatibility.html[agent/server compatibility matrix] for more details on which APM agents are compatible with APM Server version 6.5. + +IMPORTANT: You must upgrade the APM Server before upgrading your APM Agent. +New agents do not support APM Server <6.5 and will not work if you upgrade in the wrong order. [float] [[intake-api-changes-65]] @@ -25,6 +34,10 @@ This API has been redesigned in 6.5 to increase the memory efficiency and predic Notably, most of the endpoints associated with this API have been deprecated, and replaced with a new endpoint: `intake/v2/events`. + +Similarly, +the <> endpoint has been deprecated and replaced with a new endpoint: `intake/v2/rum/events`. + Details about these changes, and the information you can send to the new endpoint, is available in the <> documentation. @@ -33,7 +46,8 @@ is available in the <> documentation. [[metadata-api-changes-65]] ===== Metadata -Metadata is now sent only once from the agent to the server. APM Server will retain this information and apply it to the applicable documents when it writes to Elasticsearch. +Metadata is now sent to the APM Server only once per request. +APM Server will retain this information and apply it to the applicable documents when it writes to Elasticsearch. Metadata should be sent to the new `intake/v2/events` endpoint. See the <> for more information. @@ -94,7 +108,9 @@ The <> api has replaced it. [[assets-api-changes-65]] ===== Assets Endpoint -The assets endpoint, `/assets/v1/sourcemaps`, remains unchanged. +The assets endpoint, `/v1/rum/sourcemaps` has been deprecated, +and replaced with the endpoint: `/assets/v1/sourcemaps`. +The data format remains unchanged. [float] [[metrics-api-changes-65]] @@ -109,23 +125,26 @@ See the <> for more information. [[server-config-changes-65]] ==== Server Configuration Changes -On the server there are some configuration options that are worth mentioning: +There are some configuration changes that are worth mentioning. + +Deprecated configuration options: + +* <> +* <> +* <> -`max_unzipped_size`, `max_request_queue_time`, and `concurrent_requests` have all been deprecated with 6.4. +New configuration options: -New with 6.5 is `max_event_size` which has a default value of `307200` bytes. -You'll notice this value is much smaller than the previous `max_unzipped_size` value. -This is because in APM Server 6.5, an event is much smaller than an event in 6.4. -Instead of getting the whole payload at once, and event is just a single line of data. +* <> with a default value: `307200` bytes. You may notice this default value is much smaller than the deprecated <> default value. This is because `max_unzipped_size` referred to the whole request, while `max_event_size` only refers to a single event within the request. -`read_timeout` and `write_timeout` remain at 30 seconds, +Notable options: + +* <> and <> remain at 30 seconds, but if you have a load balancer in front of the APM server you may notice connections are a little bit longer lived than they used to be. -NOTE: If you are updating APM Server from version 6.3 or earlier, -you may not be using an updated `apm-server.yml` file. -How APM Server stores data in Elasticsearch has changed overtime, -and in order to take advantage of new features, -you must be using an updated `apm-server.yml` file. +TIP: If you are updating APM Server from version 6.3 or earlier, +you may not be using an updated `apm-server.yml` configuration file. Update your `apm-server.yml` configuration file to take advantage of new configuration options. + [float] [[es-schema-changes-65]] From c212c33c13a1ed8361f6970e53b90f61fac0627b Mon Sep 17 00:00:00 2001 From: Silvia Mitter Date: Fri, 9 Nov 2018 19:50:29 +0100 Subject: [PATCH 09/12] Update example files for official docs. (#2) implements #1532 --- Makefile | 16 +- docs/data/elasticsearch/generated/errors.json | 434 ++++++++++++++++++ .../elasticsearch/generated/metricsets.json | 85 ++++ docs/data/elasticsearch/generated/spans.json | 268 +++++++++++ .../elasticsearch/generated/transactions.json | 282 ++++++++++++ .../error/minimal_payload_exception.json | 16 - .../generated/error/minimal_payload_log.json | 16 - .../intake-api/generated/error/payload.json | 254 ---------- docs/data/intake-api/generated/error/rum.json | 88 ---- docs/data/intake-api/generated/events.ndjson | 5 + .../generated/metricset/payload.json | 62 --- .../transaction/minimal_payload.json | 16 - .../generated/transaction/minimal_span.json | 26 -- .../generated/transaction/payload.json | 272 ----------- docs/error-indices.asciidoc | 6 +- docs/events-api.asciidoc | 4 +- docs/example-intakev2-events.asciidoc | 9 + docs/metricset-indices.asciidoc | 10 +- docs/span-indices.asciidoc | 6 +- docs/transaction-indices.asciidoc | 6 +- 20 files changed, 1109 insertions(+), 772 deletions(-) create mode 100644 docs/data/elasticsearch/generated/errors.json create mode 100644 docs/data/elasticsearch/generated/metricsets.json create mode 100644 docs/data/elasticsearch/generated/spans.json create mode 100644 docs/data/elasticsearch/generated/transactions.json delete mode 100644 docs/data/intake-api/generated/error/minimal_payload_exception.json delete mode 100644 docs/data/intake-api/generated/error/minimal_payload_log.json delete mode 100644 docs/data/intake-api/generated/error/payload.json delete mode 100644 docs/data/intake-api/generated/error/rum.json create mode 100644 docs/data/intake-api/generated/events.ndjson delete mode 100644 docs/data/intake-api/generated/metricset/payload.json delete mode 100644 docs/data/intake-api/generated/transaction/minimal_payload.json delete mode 100644 docs/data/intake-api/generated/transaction/minimal_span.json delete mode 100644 docs/data/intake-api/generated/transaction/payload.json create mode 100644 docs/example-intakev2-events.asciidoc diff --git a/Makefile b/Makefile index bd0015a4644..ef55189863d 100644 --- a/Makefile +++ b/Makefile @@ -46,16 +46,14 @@ go-generate: .PHONY: create-docs create-docs: - @mkdir -p docs/data/intake-api/generated/{error,transaction,metricset,sourcemap} - @cp testdata/error/payload.json docs/data/intake-api/generated/error/ - @cp testdata/error/rum.json docs/data/intake-api/generated/error/ - @cp testdata/error/minimal_payload_exception.json docs/data/intake-api/generated/error/ - @cp testdata/error/minimal_payload_log.json docs/data/intake-api/generated/error/ - @cp testdata/metricset/payload.json docs/data/intake-api/generated/metricset/ - @cp testdata/transaction/payload.json docs/data/intake-api/generated/transaction/ - @cp testdata/transaction/minimal_payload.json docs/data/intake-api/generated/transaction/ - @cp testdata/transaction/minimal_span.json docs/data/intake-api/generated/transaction/ + @mkdir -p docs/data/intake-api/generated/sourcemap + @cp testdata/intake-v2/events.ndjson docs/data/intake-api/generated/ @cp testdata/sourcemap/bundle.js.map docs/data/intake-api/generated/sourcemap/ + @mkdir -p docs/data/elasticsearch/generated/ + @cp processor/stream/approved-es-documents/testV2IntakeIntegrationErrors.approved.json docs/data/elasticsearch/generated/errors.json + @cp processor/stream/approved-es-documents/testV2IntakeIntegrationTransactions.approved.json docs/data/elasticsearch/generated/transactions.json + @cp processor/stream/approved-es-documents/testV2IntakeIntegrationSpans.approved.json docs/data/elasticsearch/generated/spans.json + @cp processor/stream/approved-es-documents/testV2IntakeIntegrationMetricsets.approved.json docs/data/elasticsearch/generated/metricsets.json # Start manual testing environment with agents start-env: diff --git a/docs/data/elasticsearch/generated/errors.json b/docs/data/elasticsearch/generated/errors.json new file mode 100644 index 00000000000..f772c26ed63 --- /dev/null +++ b/docs/data/elasticsearch/generated/errors.json @@ -0,0 +1,434 @@ +{ + "events": [ + { + "@timestamp": "2017-05-09T15:04:05.999999Z", + "context": { + "custom": { + "and_objects": { + "foo": [ + "bar", + "baz" + ] + }, + "my_key": 1, + "some_other_value": "foo bar" + }, + "process": { + "argv": [ + "node", + "server.js" + ], + "pid": 1234, + "ppid": 6789, + "title": "node" + }, + "request": { + "body": "Hello World", + "cookies": { + "c1": "v1", + "c2": "v2" + }, + "env": { + "GATEWAY_INTERFACE": "CGI/1.1", + "SERVER_SOFTWARE": "nginx" + }, + "headers": { + "array": [ + "foo", + "bar", + "baz" + ], + "content-type": "text/html", + "cookie": "c1=v1; c2=v2", + "some-other-header": "foo", + "user-agent": "Mozilla Chrome Edge" + }, + "http_version": "1.1", + "method": "POST", + "socket": { + "encrypted": true, + "remote_address": "12.53.12.1" + }, + "url": { + "full": "https://www.example.com/p/a/t/h?query=string#hash", + "hash": "#hash", + "hostname": "www.example.com", + "pathname": "/p/a/t/h", + "port": "8080", + "protocol": "https:", + "raw": "/p/a/t/h?query=string#hash", + "search": "?query=string" + } + }, + "response": { + "finished": true, + "headers": { + "content-type": "application/json" + }, + "headers_sent": true, + "status_code": 200 + }, + "service": { + "agent": { + "name": "elastic-node", + "version": "3.14.0" + }, + "environment": "staging", + "framework": { + "name": "Express", + "version": "1.2.3" + }, + "language": { + "name": "ecmascript", + "version": "8" + }, + "name": "1234_service-12a3", + "runtime": { + "name": "node", + "version": "8.0.0" + }, + "version": "5.1.3" + }, + "system": { + "architecture": "x64", + "hostname": "prod1.example.com", + "ip": "192.0.0.1", + "platform": "darwin" + }, + "tags": { + "organization_uuid": "9f0e9d64-c185-4d21-a6f4-4673ed561ec8" + }, + "user": { + "email": "foo@example.com", + "id": 99, + "username": "foo" + } + }, + "error": { + "culprit": "my.module.function_name", + "exception": { + "attributes": { + "foo": "bar" + }, + "code": "42", + "handled": false, + "message": "The username root is unknown", + "module": "__builtins__", + "stacktrace": [ + { + "abs_path": "/real/file/name.py", + "context": { + "post": [ + "line4", + "line5" + ], + "pre": [ + "line1", + "line2" + ] + }, + "exclude_from_grouping": false, + "filename": "file/name.py", + "function": "foo", + "library_frame": true, + "line": { + "column": 4, + "context": "line3", + "number": 3 + }, + "module": "App::MyModule", + "vars": { + "key": "value" + } + }, + { + "abs_path": "/Users/watson/code/node_modules/elastic/lib/instrumentation/index.js", + "context": { + "post": [ + " ins.currentTransaction = prev", + " return result", + "}", + "}", + "", + "Instrumentation.prototype._recoverTransaction = function (trans) {", + " if (this.currentTransaction === trans) return" + ], + "pre": [ + " var trans = this.currentTransaction", + "", + " return instrumented", + "", + " function instrumented () {", + " var prev = ins.currentTransaction", + " ins.currentTransaction = trans" + ] + }, + "exclude_from_grouping": false, + "filename": "lib/instrumentation/index.js", + "function": "instrumented", + "line": { + "context": " var result = original.apply(this, arguments)", + "number": 102 + }, + "vars": { + "key": "value" + } + } + ], + "type": "DbError" + }, + "grouping_key": "50f62f37edffc4630c6655ba3ecfcf46", + "id": "0123456789012345", + "log": { + "level": "warning", + "logger_name": "my.logger.name", + "message": "My service could not talk to the database named foobar", + "param_message": "My service could not talk to the database named %s", + "stacktrace": [ + { + "abs_path": "/real/file/name.py", + "context": { + "post": [ + "line4", + "line5" + ], + "pre": [ + "line1", + "line2" + ] + }, + "exclude_from_grouping": false, + "filename": "/webpack/file/name.py", + "function": "foo", + "library_frame": false, + "line": { + "column": 4, + "context": "line3", + "number": 3 + }, + "module": "App::MyModule", + "vars": { + "key": "value" + } + }, + { + "abs_path": "/Users/watson/code/node_modules/elastic/lib/instrumentation/index.js", + "context": { + "post": [ + " ins.currentTransaction = prev", + " return result", + "}", + "}", + "", + "Instrumentation.prototype._recoverTransaction = function (trans) {", + " if (this.currentTransaction === trans) return" + ], + "pre": [ + " var trans = this.currentTransaction", + "", + " return instrumented", + "", + " function instrumented () {", + " var prev = ins.currentTransaction", + " ins.currentTransaction = trans" + ] + }, + "exclude_from_grouping": false, + "filename": "lib/instrumentation/index.js", + "function": "instrumented", + "line": { + "context": " var result = original.apply(this, arguments)", + "number": 102 + }, + "vars": { + "key": "value" + } + } + ] + } + }, + "processor": { + "event": "error", + "name": "error" + }, + "timestamp": { + "us": 1494342245999999 + } + }, + { + "@timestamp": "2018-08-09T14:59:05.999Z", + "context": { + "process": { + "argv": [ + "node", + "server.js" + ], + "pid": 1234, + "ppid": 6789, + "title": "node" + }, + "service": { + "agent": { + "name": "elastic-node", + "version": "3.14.0" + }, + "environment": "staging", + "framework": { + "name": "Express", + "version": "1.2.3" + }, + "language": { + "name": "ecmascript", + "version": "8" + }, + "name": "1234_service-12a3", + "runtime": { + "name": "node", + "version": "8.0.0" + }, + "version": "5.1.3" + }, + "system": { + "architecture": "x64", + "hostname": "prod1.example.com", + "ip": "192.0.0.1", + "platform": "darwin" + } + }, + "error": { + "exception": { + "message": "Cannot read property 'baz' no defined" + }, + "grouping_key": "ae0232fed4cb40e7ebc62a585a421d60", + "id": "cdefab0123456789" + }, + "processor": { + "event": "error", + "name": "error" + }, + "timestamp": { + "us": 1533826745999000 + } + }, + { + "@timestamp": "2018-08-01T10:00:00Z", + "context": { + "process": { + "argv": [ + "node", + "server.js" + ], + "pid": 1234, + "ppid": 6789, + "title": "node" + }, + "service": { + "agent": { + "name": "elastic-node", + "version": "3.14.0" + }, + "environment": "staging", + "framework": { + "name": "Express", + "version": "1.2.3" + }, + "language": { + "name": "ecmascript", + "version": "8" + }, + "name": "1234_service-12a3", + "runtime": { + "name": "node", + "version": "8.0.0" + }, + "version": "5.1.3" + }, + "system": { + "architecture": "x64", + "hostname": "prod1.example.com", + "ip": "192.0.0.1", + "platform": "darwin" + } + }, + "error": { + "exception": { + "type": "DbError" + }, + "grouping_key": "c3868d6704b923014eaffea034e70a3d", + "id": "cdefab0123456789" + }, + "processor": { + "event": "error", + "name": "error" + }, + "timestamp": { + "us": 1533117600000000 + } + }, + { + "@timestamp": "2018-08-09T15:04:05.999Z", + "context": { + "process": { + "argv": [ + "node", + "server.js" + ], + "pid": 1234, + "ppid": 6789, + "title": "node" + }, + "service": { + "agent": { + "name": "elastic-node", + "version": "3.14.0" + }, + "environment": "staging", + "framework": { + "name": "Express", + "version": "1.2.3" + }, + "language": { + "name": "ecmascript", + "version": "8" + }, + "name": "1234_service-12a3", + "runtime": { + "name": "node", + "version": "8.0.0" + }, + "version": "5.1.3" + }, + "system": { + "architecture": "x64", + "hostname": "prod1.example.com", + "ip": "192.0.0.1", + "platform": "darwin" + } + }, + "error": { + "grouping_key": "d6b3f958dfea98dc9ed2b57d5f0c48bb", + "id": "abcdef0123456789", + "log": { + "level": "custom log level", + "message": "Cannot read property 'baz' of undefined" + } + }, + "parent": { + "id": "9632587410abcdef" + }, + "processor": { + "event": "error", + "name": "error" + }, + "timestamp": { + "us": 1533827045999000 + }, + "trace": { + "id": "0123456789abcdeffedcba0123456789" + }, + "transaction": { + "id": "1234567890987654" + } + } + ] +} diff --git a/docs/data/elasticsearch/generated/metricsets.json b/docs/data/elasticsearch/generated/metricsets.json new file mode 100644 index 00000000000..da6753b3599 --- /dev/null +++ b/docs/data/elasticsearch/generated/metricsets.json @@ -0,0 +1,85 @@ +{ + "events": [ + { + "@timestamp": "2017-05-30T18:53:42.281Z", + "byte_counter": 1, + "context": { + "process": { + "pid": 1234 + }, + "service": { + "agent": { + "name": "elastic-node", + "version": "3.14.0" + }, + "language": { + "name": "ecmascript" + }, + "name": "1234_service-12a3" + }, + "tags": { + "code": "200", + "some.other.code": "abc" + } + }, + "dotted": { + "float": { + "gauge": 6.12 + } + }, + "double_gauge": 3.141592653589793, + "float_gauge": 9.16, + "integer_gauge": 42767, + "long_gauge": 3147483648, + "negative": { + "d": { + "o": { + "t": { + "t": { + "e": { + "d": -1022 + } + } + } + } + } + }, + "processor": { + "event": "metric", + "name": "metric" + }, + "short_counter": 227 + }, + { + "@timestamp": "2017-05-30T18:53:42.281Z", + "context": { + "process": { + "pid": 1234 + }, + "service": { + "agent": { + "name": "elastic-node", + "version": "3.14.0" + }, + "language": { + "name": "ecmascript" + }, + "name": "1234_service-12a3" + } + }, + "go": { + "memstats": { + "heap": { + "sys": { + "bytes": 6520832 + } + } + } + }, + "processor": { + "event": "metric", + "name": "metric" + } + } + ] +} diff --git a/docs/data/elasticsearch/generated/spans.json b/docs/data/elasticsearch/generated/spans.json new file mode 100644 index 00000000000..4f3b3d1bec5 --- /dev/null +++ b/docs/data/elasticsearch/generated/spans.json @@ -0,0 +1,268 @@ +{ + "events": [ + { + "@timestamp": "2018-07-30T18:53:42.281Z", + "context": { + "service": { + "agent": { + "name": "elastic-node", + "version": "3.14.0" + }, + "name": "backendspans" + } + }, + "parent": { + "id": "abcdef0123456789" + }, + "processor": { + "event": "span", + "name": "transaction" + }, + "span": { + "duration": { + "us": 141581 + }, + "hex_id": "abcdef01234567", + "id": -9175013389437287000, + "name": "GET /api/types", + "parent": 3156441702022342700, + "type": "request" + }, + "timestamp": { + "us": 1532976822281000 + }, + "trace": { + "id": "fdedef0123456789abcdef9876543210" + }, + "transaction": { + "id": "01af25874dec69dd" + } + }, + { + "@timestamp": "2018-07-30T18:53:42.281Z", + "context": { + "service": { + "agent": { + "name": "elastic-node", + "version": "3.14.0" + }, + "name": "backendspans" + } + }, + "parent": { + "id": "0000000011111111" + }, + "processor": { + "event": "span", + "name": "transaction" + }, + "span": { + "duration": { + "us": 32592 + }, + "hex_id": "1234abcdef567895", + "id": -7911509744411052000, + "name": "GET /api/types", + "parent": -9223372036568445000, + "start": { + "us": 22000 + }, + "type": "request" + }, + "timestamp": { + "us": 1532976822281000 + }, + "trace": { + "id": "abcdef0123456789abcdef9876543210" + }, + "transaction": { + "id": "ab45781d265894fe" + } + }, + { + "@timestamp": "2018-08-01T10:00:00.001845Z", + "context": { + "service": { + "agent": { + "name": "elastic-node", + "version": "3.14.0" + }, + "name": "backendspans" + }, + "tags": { + "tag1": "value1" + } + }, + "parent": { + "id": "abcdefabcdef7890" + }, + "processor": { + "event": "span", + "name": "transaction" + }, + "span": { + "duration": { + "us": 3564 + }, + "hex_id": "0123456a89012345", + "id": -9141386494764572000, + "name": "GET /api/types", + "parent": 3156442435030055000, + "start": { + "us": 1845 + }, + "type": "request" + }, + "timestamp": { + "us": 1533117600001845 + }, + "trace": { + "id": "abcdef0123456789abcdef9876543210" + }, + "transaction": { + "id": "ab23456a89012345" + } + }, + { + "@timestamp": "2018-08-01T10:00:00Z", + "context": { + "service": { + "agent": { + "name": "elastic-node", + "version": "3.14.0" + }, + "name": "backendspans" + } + }, + "parent": { + "id": "ababcdcdefefabde" + }, + "processor": { + "event": "span", + "name": "transaction" + }, + "span": { + "duration": { + "us": 13980 + }, + "hex_id": "abcde56a89012345", + "id": 3156431159584433000, + "name": "get /api/types", + "parent": 3146835049025875000, + "start": { + "us": 0 + }, + "type": "request" + }, + "timestamp": { + "us": 1533117600000000 + }, + "trace": { + "id": "abcdef0123456789abcdef9876543210" + }, + "transaction": { + "id": "bed3456a89012345" + } + }, + { + "@timestamp": "2018-08-01T10:00:00.00283092Z", + "context": { + "db": { + "instance": "customers", + "statement": "SELECT * FROM product_types WHERE user_id=?", + "type": "sql", + "user": "readonly_user" + }, + "http": { + "method": "GET", + "status_code": 200, + "url": "http://localhost:8000" + }, + "service": { + "agent": { + "name": "elastic-node", + "version": "3.14.0" + }, + "name": "backendspans" + } + }, + "parent": { + "id": "abcdef0123456789" + }, + "processor": { + "event": "span", + "name": "transaction" + }, + "span": { + "duration": { + "us": 3781 + }, + "hex_id": "1234567890aaaade", + "id": -7911603569559950000, + "name": "SELECT FROM product_types", + "parent": 3156441702022342700, + "stacktrace": [ + { + "exclude_from_grouping": false, + "filename": "net.js", + "line": { + "number": 547 + } + }, + { + "context": { + "post": [ + " ins.currentTransaction = prev", + "}" + ] + }, + "exclude_from_grouping": false, + "filename": "file2.js", + "line": { + "number": 12 + } + }, + { + "abs_path": "net.js", + "context": { + "post": [ + " ins.currentTransaction = prev", + " return result" + ], + "pre": [ + " var trans = this.currentTransaction", + "" + ] + }, + "exclude_from_grouping": false, + "filename": "net.js", + "function": "onread", + "library_frame": true, + "line": { + "column": 4, + "context": "line3", + "number": 547 + }, + "module": "some module", + "vars": { + "key": "value" + } + } + ], + "start": { + "us": 2830 + }, + "type": "db.postgresql.query" + }, + "timestamp": { + "us": 1533117600002830 + }, + "trace": { + "id": "abcdef0123456789abcdef9876543210" + }, + "transaction": { + "id": "aff4567890aaaade" + } + } + ] +} diff --git a/docs/data/elasticsearch/generated/transactions.json b/docs/data/elasticsearch/generated/transactions.json new file mode 100644 index 00000000000..df1c5712a14 --- /dev/null +++ b/docs/data/elasticsearch/generated/transactions.json @@ -0,0 +1,282 @@ +{ + "events": [ + { + "@timestamp": "2018-08-01T10:00:00Z", + "context": { + "process": { + "argv": [ + "node", + "server.js" + ], + "pid": 1234, + "ppid": 6789, + "title": "node" + }, + "service": { + "agent": { + "name": "elastic-node", + "version": "3.14.0" + }, + "environment": "staging", + "framework": { + "name": "Express", + "version": "1.2.3" + }, + "language": { + "name": "ecmascript", + "version": "8" + }, + "name": "1234_service-12a3", + "runtime": { + "name": "node", + "version": "8.0.0" + }, + "version": "5.1.3" + }, + "system": { + "architecture": "x64", + "hostname": "prod1.example.com", + "ip": "192.0.0.1", + "platform": "darwin" + } + }, + "parent": { + "id": "abcdefabcdef01234567" + }, + "processor": { + "event": "transaction", + "name": "transaction" + }, + "timestamp": { + "us": 1533117600000000 + }, + "trace": { + "id": "0123456789abcdef0123456789abcdef" + }, + "transaction": { + "duration": { + "us": 32592 + }, + "id": "945254c567a5417e", + "sampled": true, + "span_count": { + "started": 43 + }, + "type": "request" + } + }, + { + "@timestamp": "2017-05-30T18:53:27.154Z", + "context": { + "custom": { + "(": "not a valid regex and that is fine", + "and_objects": { + "foo": [ + "bar", + "baz" + ] + }, + "my_key": 1, + "some_other_value": "foo bar" + }, + "process": { + "argv": [ + "node", + "server.js" + ], + "pid": 1234, + "ppid": 6789, + "title": "node" + }, + "request": { + "body": { + "additional": { + "bar": 123, + "req": "additional information" + }, + "str": "hello world" + }, + "cookies": { + "c1": "v1", + "c2": "v2" + }, + "env": { + "GATEWAY_INTERFACE": "CGI/1.1", + "SERVER_SOFTWARE": "nginx" + }, + "headers": { + "array": [ + "foo", + "bar", + "baz" + ], + "content-type": "text/html", + "cookie": "c1=v1; c2=v2", + "some-other-header": "foo", + "user-agent": "Mozilla Chrome Edge" + }, + "http_version": "1.1", + "method": "POST", + "socket": { + "encrypted": true, + "remote_address": "12.53.12.1" + }, + "url": { + "full": "https://www.example.com/p/a/t/h?query=string#hash", + "hash": "#hash", + "hostname": "www.example.com", + "pathname": "/p/a/t/h", + "port": "8080", + "protocol": "https:", + "raw": "/p/a/t/h?query=string#hash", + "search": "?query=string" + } + }, + "response": { + "finished": true, + "headers": { + "content-type": "application/json" + }, + "headers_sent": true, + "status_code": 200 + }, + "service": { + "agent": { + "name": "elastic-node", + "version": "3.14.0" + }, + "environment": "staging", + "framework": { + "name": "Express", + "version": "1.2.3" + }, + "language": { + "name": "ecmascript", + "version": "8" + }, + "name": "1234_service-12a3", + "runtime": { + "name": "node", + "version": "8.0.0" + }, + "version": "5.1.3" + }, + "system": { + "architecture": "x64", + "hostname": "prod1.example.com", + "ip": "192.0.0.1", + "platform": "darwin" + }, + "tags": { + "organization_uuid": "9f0e9d64-c185-4d21-a6f4-4673ed561ec8" + }, + "user": { + "email": "foo@example.com", + "id": "99", + "username": "foo" + } + }, + "processor": { + "event": "transaction", + "name": "transaction" + }, + "timestamp": { + "us": 1496170407154000 + }, + "trace": { + "id": "0acd456789abcdef0123456789abcdef" + }, + "transaction": { + "duration": { + "us": 32592 + }, + "id": "4340a8e0df1906ecbfa9", + "name": "GET /api/types", + "result": "success", + "sampled": true, + "span_count": { + "started": 17 + }, + "type": "request" + } + }, + { + "@timestamp": "2018-07-30T18:53:42.281Z", + "context": { + "process": { + "argv": [ + "node", + "server.js" + ], + "pid": 1234, + "ppid": 6789, + "title": "node" + }, + "request": { + "method": "POST" + }, + "service": { + "agent": { + "name": "elastic-node", + "version": "3.14.0" + }, + "environment": "staging", + "framework": { + "name": "Express", + "version": "1.2.3" + }, + "language": { + "name": "ecmascript", + "version": "8" + }, + "name": "1234_service-12a3", + "runtime": { + "name": "node", + "version": "8.0.0" + }, + "version": "5.1.3" + }, + "system": { + "architecture": "x64", + "hostname": "prod1.example.com", + "ip": "192.0.0.1", + "platform": "darwin" + } + }, + "processor": { + "event": "transaction", + "name": "transaction" + }, + "timestamp": { + "us": 1532976822281000 + }, + "trace": { + "id": "0acd456789abcdef0123456789abcdef" + }, + "transaction": { + "duration": { + "us": 13980 + }, + "id": "cdef4340a8e0df19", + "marks": { + "another_mark": { + "some_float": 10, + "some_long": 10 + }, + "navigationTiming": { + "appBeforeBootstrap": 608.93, + "navigationStart": -21 + } + }, + "sampled": true, + "span_count": { + "dropped": { + "total": 55 + }, + "started": 436 + }, + "type": "request" + } + } + ] +} diff --git a/docs/data/intake-api/generated/error/minimal_payload_exception.json b/docs/data/intake-api/generated/error/minimal_payload_exception.json deleted file mode 100644 index f616c4bc552..00000000000 --- a/docs/data/intake-api/generated/error/minimal_payload_exception.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "service": { - "name": "1234_service-12a3", - "agent": { - "name": "python", - "version": "1.0" - } - }, - "errors": [ - { - "exception": { - "message": "" - } - } - ] -} diff --git a/docs/data/intake-api/generated/error/minimal_payload_log.json b/docs/data/intake-api/generated/error/minimal_payload_log.json deleted file mode 100644 index 79f51e395ec..00000000000 --- a/docs/data/intake-api/generated/error/minimal_payload_log.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "service": { - "name": "1234_service-12a3", - "agent": { - "name": "python", - "version": "1.0" - } - }, - "errors": [ - { - "log": { - "message": "" - } - } - ] -} diff --git a/docs/data/intake-api/generated/error/payload.json b/docs/data/intake-api/generated/error/payload.json deleted file mode 100644 index e0c6fa0e763..00000000000 --- a/docs/data/intake-api/generated/error/payload.json +++ /dev/null @@ -1,254 +0,0 @@ -{ - "service": { - "name": "1234_service-12a3", - "version": "5.1.3", - "environment": "staging", - "language": { - "name": "ecmascript", - "version": "8" - }, - "runtime": { - "name": "node", - "version": "8.0.0" - }, - "framework": { - "name": "Express", - "version": "1.2.3" - }, - "agent": { - "name": "elastic-node", - "version": "3.14.0" - } - }, - "process": { - "pid": 1234, - "ppid": 7788, - "title": "node", - "argv": [ - "node", - "server.js" - ] - }, - "system": { - "hostname": "prod1.example.com", - "architecture": "x64", - "platform": "darwin" - }, - "errors": [ - { - "id": "5f0e9d64-c185-4d21-a6f4-4673ed561ec8", - "timestamp": "2017-05-09T15:04:05.999999Z", - "culprit": "my.module.function_name", - "log": { - "message": "My service could not talk to the database named foobar", - "param_message": "My service could not talk to the database named %s", - "logger_name": "my.logger.name", - "level": "warning", - "stacktrace": [ - { - "abs_path": "/real/file/name.py", - "filename": "/webpack/file/name.py", - "function": "foo", - "vars": { - "key": "value" - }, - "pre_context": [ - "line1", - "line2" - ], - "context_line": "line3", - "library_frame": false, - "lineno": 3, - "module": "App::MyModule", - "colno": 4, - "post_context": [ - "line4", - "line5" - ] - }, - { - "filename": "lib/instrumentation/index.js", - "lineno": 102, - "function": "instrumented", - "abs_path": "/Users/watson/code/node_modules/elastic/lib/instrumentation/index.js", - "vars": { - "key": "value" - }, - "pre_context": [ - " var trans = this.currentTransaction", - "", - " return instrumented", - "", - " function instrumented () {", - " var prev = ins.currentTransaction", - " ins.currentTransaction = trans" - ], - "context_line": " var result = original.apply(this, arguments)", - "post_context": [ - " ins.currentTransaction = prev", - " return result", - "}", - "}", - "", - "Instrumentation.prototype._recoverTransaction = function (trans) {", - " if (this.currentTransaction === trans) return" - ] - } - ] - }, - "exception": { - "message": "The username root is unknown", - "type": "DbError", - "module": "__builtins__", - "code": 42, - "handled": false, - "attributes": { - "foo": "bar" - }, - "stacktrace": [ - { - "abs_path": "/real/file/name.py", - "filename": "file/name.py", - "function": "foo", - "vars": { - "key": "value" - }, - "pre_context": [ - "line1", - "line2" - ], - "context_line": "line3", - "library_frame": true, - "lineno": 3, - "module": "App::MyModule", - "colno": 4, - "post_context": [ - "line4", - "line5" - ] - }, - { - "filename": "lib/instrumentation/index.js", - "lineno": 102, - "function": "instrumented", - "abs_path": "/Users/watson/code/node_modules/elastic/lib/instrumentation/index.js", - "vars": { - "key": "value" - }, - "pre_context": [ - " var trans = this.currentTransaction", - "", - " return instrumented", - "", - " function instrumented () {", - " var prev = ins.currentTransaction", - " ins.currentTransaction = trans" - ], - "context_line": " var result = original.apply(this, arguments)", - "post_context": [ - " ins.currentTransaction = prev", - " return result", - "}", - "}", - "", - "Instrumentation.prototype._recoverTransaction = function (trans) {", - " if (this.currentTransaction === trans) return" - ] - } - ] - }, - "context": { - "request": { - "socket": { - "remote_address": "12.53.12.1", - "encrypted": true - }, - "http_version": "1.1", - "method": "POST", - "url": { - "protocol": "https:", - "full": "https://www.example.com/p/a/t/h?query=string#hash", - "hostname": "www.example.com", - "port": "8080", - "pathname": "/p/a/t/h", - "search": "?query=string", - "hash": "#hash", - "raw": "/p/a/t/h?query=string#hash" - }, - "headers": { - "user-agent": "Mozilla Chrome Edge", - "content-type": "text/html", - "cookie": "c1=v1; c2=v2", - "some-other-header": "foo", - "array": [ - "foo", - "bar", - "baz" - ] - }, - "cookies": { - "c1": "v1", - "c2": "v2" - }, - "env": { - "SERVER_SOFTWARE": "nginx", - "GATEWAY_INTERFACE": "CGI/1.1" - }, - "body": "Hello World" - }, - "response": { - "status_code": 200, - "headers": { - "content-type": "application/json" - }, - "headers_sent": true, - "finished": true - }, - "user": { - "id": 99, - "username": "foo", - "email": "foo@example.com" - }, - "tags": { - "organization_uuid": "9f0e9d64-c185-4d21-a6f4-4673ed561ec8" - }, - "custom": { - "my_key": 1, - "some_other_value": "foo bar", - "and_objects": { - "foo": [ - "bar", - "baz" - ] - } - } - }, - "transaction": { - "id": "945254c5-67a5-417e-8a4e-aa29efcbfb79" - } - }, - { - "id": "8f0e9d68-c185-4d21-a6f4-4673ed561ec8", - "timestamp": "2017-05-09T15:04:05.1Z", - "exception": { - "message": "foo is not defined", - "code": "35" - } - }, - { - "id": "7f0e9d68-c185-4d21-a6f4-4673ed561ec8", - "timestamp": "2017-05-09T15:04:05Z", - "exception": { - "type": "connection error" - } - }, - { - "id": "0f0e9d67-c185-4d21-a6f4-4673ed561ec8", - "timestamp": "2017-05-09T15:04:05.999Z", - "log": { - "level": "custom log level", - "message": "Cannot read property 'baz' of undefined" - } - } - ] -} diff --git a/docs/data/intake-api/generated/error/rum.json b/docs/data/intake-api/generated/error/rum.json deleted file mode 100644 index 38bdd58dd32..00000000000 --- a/docs/data/intake-api/generated/error/rum.json +++ /dev/null @@ -1,88 +0,0 @@ - { - "service": { - "name": "apm-agent-js", - "version": "1.0.1", - "agent": { - "name": "apm-js", - "version": "0.0.0" - } - }, - "errors": [ - { - "id": "aba2688e-0338-48ce-9c4e-4005f1caa534", - "timestamp": "2017-12-08T12:18:50.291Z", - "culprit": "test/e2e/general-usecase/bundle.js.map", - "log": { - "message": "Uncaught Error: log timeout test error", - "stacktrace": [ - { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - "filename": "~/test/e2e/general-usecase/bundle.js.map", - "function": "", - "lineno": 1, - "colno": 18 - } - ] - }, - "exception": { - "message": "Uncaught Error: timeout test error", - "stacktrace": [ - { - "abs_path": "http://localhost:8000/test/../test/e2e/general-usecase/bundle.js.map", - "filename": "test/e2e/general-usecase/bundle.js.map", - "function": "", - "library_frame": true, - "lineno": 1, - "colno": 18 - }, - { - "abs_path": "http://localhost:8000/test/./e2e/general-usecase/bundle.js.map", - "filename": "~/test/e2e/general-usecase/bundle.js.map", - "function": "invokeTask", - "library_frame": false, - "lineno": 1, - "colno": 181 - }, - { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - "filename": "~/test/e2e/general-usecase/bundle.js.map", - "function": "runTask", - "lineno": 1, - "colno": 15 - }, - { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - "filename": "~/test/e2e/general-usecase/bundle.js.map", - "function": "invoke", - "lineno": 1, - "colno": 199 - }, - { - "abs_path": "http://localhost:8000/test/e2e/general-usecase/bundle.js.map", - "filename": "~/test/e2e/general-usecase/bundle.js.map", - "function": "timer", - "lineno": 1, - "colno": 33 - } - ], - "type": "Error" - }, - "context": { - "environment": { - "browserWidth": 150, - "browserHeight": 726, - "screenWidth": 1280, - "screenHeight": 800, - "language": "en-US", - "userAgent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36", - "platform": "MacIntel" - }, - "page": { - "referer": "http://localhost:8000/test/e2e/", - "host": "localhost", - "location": "http://localhost:8000/test/e2e/general-usecase/" - } - } - } - ] -} diff --git a/docs/data/intake-api/generated/events.ndjson b/docs/data/intake-api/generated/events.ndjson new file mode 100644 index 00000000000..53c41863d61 --- /dev/null +++ b/docs/data/intake-api/generated/events.ndjson @@ -0,0 +1,5 @@ +{"metadata": { "process": {"pid": 1234 }, "service": {"name": "1234_service-12a3", "language": {"name": "ecmascript"}, "agent": {"version": "3.14.0", "name": "elastic-node"}}}} +{ "error": {"id": "abcdef0123456789", "timestamp": 1533827045999000,"log": {"level": "custom log level","message": "Cannot read property 'baz' of undefined"}}} +{ "span": { "id": "0123456a89012345", "trace_id": "0123456789abcdef0123456789abcdef", "parent_id": "ab23456a89012345", "transaction_id": "ab23456a89012345", "parent": 1, "name": "GET /api/types", "type": "request", "start": 1.845, "duration": 3.5642981, "stacktrace": [], "context": {} }} +{ "transaction": { "trace_id": "01234567890123456789abcdefabcdef", "id": "abcdef1478523690", "type": "request", "duration": 32.592981, "timestamp": 1535655207154000, "result": "200", "context": null, "spans": null, "sampled": null, "span_count": { "started": 0 }}} +{ "metricset": { "samples": { "go.memstats.heap.sys.bytes": { "value": 61235 } }, "timestamp": 1496170422281000 }} diff --git a/docs/data/intake-api/generated/metricset/payload.json b/docs/data/intake-api/generated/metricset/payload.json deleted file mode 100644 index 25ec0583ac6..00000000000 --- a/docs/data/intake-api/generated/metricset/payload.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "service": { - "name": "apm-server", - "version": "6.x", - "environment": "staging", - "language": { - "name": "go", - "version": "go1.10.1" - }, - "runtime": { - "name": "gc", - "version": "go1.10.1" - }, - "agent": { - "name": "elastic-go", - "version": "0.1.0" - } - }, - "process": { - "pid": 32472, - "ppid": 26110, - "argv": ["./apm-server"] - }, - "system": { - "ip":"127.0.0.1", - "architecture":"amd64", - "platform": "darwin" - }, - "metrics": [{ - "samples": { - "byte_counter": { - "value": 1 - }, - "short_counter": { - "value": 227 - }, - "integer_gauge": { - "value": 42767 - }, - "long_gauge": { - "value": 3147483648 - }, - "float_gauge": { - "value": 9.16 - }, - "double_gauge": { - "value": 3.141592653589793 - }, - "dotted.float.gauge": { - "value": 6.12 - }, - "negative.d.o.t.t.e.d": { - "value": -1022 - } - }, - "tags": { - "code": "200", - "some.other.code": "abc" - }, - "timestamp": "2017-05-30T18:53:42.281Z" - }] -} diff --git a/docs/data/intake-api/generated/transaction/minimal_payload.json b/docs/data/intake-api/generated/transaction/minimal_payload.json deleted file mode 100644 index bfdc2f54e87..00000000000 --- a/docs/data/intake-api/generated/transaction/minimal_payload.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "service": { - "name": "service1", - "agent": { - "name": "python", - "version": "1.0" - } - }, - "transactions": [ - { - "id": "945254c5-67a5-417e-8a4e-aa29efcbfb79", - "type": "request", - "duration": 32.592981 - } - ] -} diff --git a/docs/data/intake-api/generated/transaction/minimal_span.json b/docs/data/intake-api/generated/transaction/minimal_span.json deleted file mode 100644 index 9bf32c2e2bc..00000000000 --- a/docs/data/intake-api/generated/transaction/minimal_span.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "service": { - "name": "service1", - "agent": { - "name": "python", - "version": "1.0" - } - }, - "transactions": [ - { - "id": "945254c5-67a5-417e-8a4e-aa29efcbfb79", - "type": "request", - "duration": 32.592981, - "timestamp": "2017-05-30T18:53:27.154Z", - "spans": [ - { - "id": 100000000000, - "name": "GET /api/types", - "type": "request", - "start": 0, - "duration": 32.592981 - } - ] - } - ] -} diff --git a/docs/data/intake-api/generated/transaction/payload.json b/docs/data/intake-api/generated/transaction/payload.json deleted file mode 100644 index 6c87193cdd8..00000000000 --- a/docs/data/intake-api/generated/transaction/payload.json +++ /dev/null @@ -1,272 +0,0 @@ -{ - "service": { - "name": "1234_service-12a3", - "version": "5.1.3", - "environment": "staging", - "language": { - "name": "ecmascript", - "version": "8" - }, - "runtime": { - "name": "node", - "version": "8.0.0" - }, - "framework": { - "name": "Express", - "version": "1.2.3" - }, - "agent": { - "name": "elastic-node", - "version": "3.14.0" - } - }, - "process": { - "pid": 1234, - "ppid": 6789, - "title": "node", - "argv": [ - "node", - "server.js" - ] - }, - "system": { - "hostname": "prod1.example.com", - "architecture": "x64", - "platform": "darwin" - }, - "transactions": [ - { - "id": "945254c5-67a5-417e-8a4e-aa29efcbfb79", - "name": "GET /api/types", - "type": "request", - "duration": 32.592981, - "result": "success", - "timestamp": "2017-05-30T18:53:27.154Z", - "sampled": true, - "span_count": { - "dropped": { - "total": 2 - } - }, - "context": { - "request": { - "socket": { - "remote_address": "12.53.12.1", - "encrypted": true - }, - "http_version": "1.1", - "method": "POST", - "url": { - "protocol": "https:", - "full": "https://www.example.com/p/a/t/h?query=string#hash", - "hostname": "www.example.com", - "port": "8080", - "pathname": "/p/a/t/h", - "search": "?query=string", - "hash": "#hash", - "raw": "/p/a/t/h?query=string#hash" - }, - "headers": { - "user-agent": "Mozilla Chrome Edge", - "content-type": "text/html", - "cookie": "c1=v1; c2=v2", - "some-other-header": "foo", - "array": [ - "foo", - "bar", - "baz" - ] - }, - "cookies": { - "c1": "v1", - "c2": "v2" - }, - "env": { - "SERVER_SOFTWARE": "nginx", - "GATEWAY_INTERFACE": "CGI/1.1" - }, - "body": { - "str": "hello world", - "additional": { - "foo": {}, - "bar": 123, - "req": "additional information" - } - } - }, - "response": { - "status_code": 200, - "headers": { - "content-type": "application/json" - }, - "headers_sent": true, - "finished": true - }, - "user": { - "id": "99", - "username": "foo", - "email": "foo@example.com" - }, - "tags": { - "organization_uuid": "9f0e9d64-c185-4d21-a6f4-4673ed561ec8" - }, - "custom": { - "my_key": 1, - "some_other_value": "foo bar", - "and_objects": { - "foo": [ - "bar", - "baz" - ] - }, - "(": "not a valid regex and that is fine" - } - }, - "spans": [ - { - "id": 0, - "parent": null, - "name": "SELECT FROM product_types", - "type": "db.postgresql.query", - "start": 2.83092, - "duration": 3.781912, - "stacktrace": [ - { - "function": "onread", - "abs_path": "net.js", - "filename": "net.js", - "lineno": 547, - "library_frame": true, - "vars": { - "key": "value" - }, - "module": "some module", - "colno": 4, - "context_line": "line3", - "pre_context": [ - " var trans = this.currentTransaction", - "" - ], - "post_context": [ - " ins.currentTransaction = prev", - " return result", - "}" - ] - }, - { - "filename": "my2file.js", - "lineno": 10 - } - ], - "context": { - "db": { - "instance": "customers", - "statement": "SELECT * FROM product_types WHERE user_id=?", - "type": "sql", - "user": "readonly_user" - }, - "http": { - "url": "http://localhost:8000", - "status_code": 200, - "method": "GET" - }, - "tags": { - "span_tag": "something" - } - } - }, - { - "id": 1, - "parent": 0, - "name": "GET /api/types", - "type": "request", - "start": 0, - "duration": 32.592981 - }, - { - "id": 2, - "parent": 1, - "name": "GET /api/types", - "type": "request", - "start": 1.845, - "duration": 3.5642981, - "stacktrace": [], - "context": {} - }, - { - "id": 3, - "parent": 2, - "name": "GET /api/types", - "type": "request", - "start": 0, - "duration": 13.9802981, - "stacktrace": null, - "context": null - } - ], - "marks": { - "navigationTiming": { - "appBeforeBootstrap": 608.9300000000001, - "navigationStart": -21 - - }, - "another_mark": { - "some_long": 10, - "some_float": 10.0 - }, - "performance": {} - } - }, - { - "id": "85925e55-b43f-4340-a8e0-df1906ecbf7a", - "name": "GET /api/types", - "type": "request", - "duration": 13.980558, - "result": "failure", - "timestamp": "2017-05-30T18:53:42.281Z", - "spans": [], - "sampled": true - }, - { - "id": "85925e55-b43f-4340-a8e0-df1906ecbf78", - "name": "GET /api/types", - "type": "request", - "duration": 13.980558, - "result": "200", - "timestamp": "2017-05-30T18:53:42Z", - "sampled": false - }, - { - "id": "85925e55-b43f-4340-a8e0-df1906ecbfa9", - "name": "GET /api/types", - "type": "request", - "duration": 13.980558, - "result": "200", - "timestamp": "2017-05-30T18:53:42.281999Z", - "sampled": true, - "span_count": { - "dropped": { - "total": 258 - } - }, - "spans": [ - { - "id": 15, - "name": "SELECT FROM product_types", - "type": "db.postgresql.query", - "start": 2.83092, - "duration": 3.781912, - "stacktrace": [], - "context": { - "db": { - "instance": "customers", - "statement": "SELECT * FROM product_types WHERE user_id=?", - "type": "sql", - "user": "readonly_user" - } - } - } - ] - } - ] -} diff --git a/docs/error-indices.asciidoc b/docs/error-indices.asciidoc index c513cf98777..ce296b9df75 100644 --- a/docs/error-indices.asciidoc +++ b/docs/error-indices.asciidoc @@ -5,11 +5,11 @@ Errors are stored in separate indices of the format `apm-[version]-error-[date]` [[error-example]] [float] -=== Example document +=== Example documents -See an example error indexed in Elasticsearch: +See how error documents can look like when indexed in Elasticsearch: [source,json] ---- -include::./data/elasticsearch/error.json[] +include::./data/elasticsearch/generated/errors.json[] ---- diff --git a/docs/events-api.asciidoc b/docs/events-api.asciidoc index 60f94f20567..47e43e1edba 100644 --- a/docs/events-api.asciidoc +++ b/docs/events-api.asciidoc @@ -103,9 +103,11 @@ The APM Server uses a collection of JSON Schemas for validating requests to the * <> * <> * <> +* <> include::./metadata-api.asciidoc[] include::./transaction-api.asciidoc[] include::./span-api.asciidoc[] include::./error-api.asciidoc[] -include::./metricset-api.asciidoc[] \ No newline at end of file +include::./metricset-api.asciidoc[] +include::./example-intakev2-events.asciidoc[] diff --git a/docs/example-intakev2-events.asciidoc b/docs/example-intakev2-events.asciidoc new file mode 100644 index 00000000000..0d4be93dcc5 --- /dev/null +++ b/docs/example-intakev2-events.asciidoc @@ -0,0 +1,9 @@ +[[example-intakev2-events]] +=== Example Request Body + +See a request body example containing one event for all currently supported event types. + +[source,json] +---- +include::./data/intake-api/generated/events.ndjson[] +---- diff --git a/docs/metricset-indices.asciidoc b/docs/metricset-indices.asciidoc index 3ac705c1c6f..51d06a55bc6 100644 --- a/docs/metricset-indices.asciidoc +++ b/docs/metricset-indices.asciidoc @@ -5,9 +5,11 @@ Metricsets are by default stored to indices of the format `apm-[version]-metric- [[metricset-example]] [float] -=== Example Document -See an example metricset indexed in Elasticsearch: +=== Example Documents + +See how metricset documents can look like when indexed in Elasticsearch: + [source,json] ---- -include::./data/elasticsearch/metric.json[] ----- \ No newline at end of file +include::./data/elasticsearch/generated/metricsets.json[] +---- diff --git a/docs/span-indices.asciidoc b/docs/span-indices.asciidoc index 3d13729dd41..76320294669 100644 --- a/docs/span-indices.asciidoc +++ b/docs/span-indices.asciidoc @@ -6,10 +6,12 @@ Conceptually, spans are part of transactions. [[span-example]] [float] -=== Example document +=== Example documents + +See how span documents can look like when indexed in Elasticsearch: [source,json] ---- -include::./data/elasticsearch/span.json[] +include::./data/elasticsearch/generated/spans.json[] ---- diff --git a/docs/transaction-indices.asciidoc b/docs/transaction-indices.asciidoc index a81f1ba185c..29e77a0bf81 100644 --- a/docs/transaction-indices.asciidoc +++ b/docs/transaction-indices.asciidoc @@ -5,12 +5,12 @@ Transactions are by default stored to indices of the format `apm-[version]-trans [[transaction-example]] [float] -=== Example Document +=== Example Documents -See an example transaction indexed in Elasticsearch: +See how transaction documents can look like when indexed in Elasticsearch: [source,json] ---- -include::./data/elasticsearch/transaction.json[] +include::./data/elasticsearch/generated/transactions.json[] ---- From a54db059d2c6621bd38ca1af72a74dc3e98604a9 Mon Sep 17 00:00:00 2001 From: Brandon Morelli Date: Fri, 9 Nov 2018 14:27:28 -0800 Subject: [PATCH 10/12] docs: update structure and config --- docs/configuration-process.asciidoc | 77 ++++++++++++------- docs/configuration-rum.asciidoc | 53 ++++++++++--- docs/data-ingestion.asciidoc | 4 +- docs/error-api.asciidoc | 15 +--- docs/events-api.asciidoc | 6 +- .../guide/agent-server-compatibility.asciidoc | 4 +- docs/metadata-api.asciidoc | 23 +++--- docs/metricset-api.asciidoc | 4 +- docs/server-info.asciidoc | 2 +- docs/span-api.asciidoc | 15 +--- docs/transaction-api.asciidoc | 15 +--- docs/upgrading-to-65.asciidoc | 2 +- 12 files changed, 125 insertions(+), 95 deletions(-) diff --git a/docs/configuration-process.asciidoc b/docs/configuration-process.asciidoc index 9aa82473eaf..75f7557ea9c 100644 --- a/docs/configuration-process.asciidoc +++ b/docs/configuration-process.asciidoc @@ -17,7 +17,8 @@ max_procs: 4 ---- [float] -=== Configuration options `apm-server.*` +[[configuration-apm-server]] +=== Configuration options: `apm-server.*` [[host]] [float] @@ -26,30 +27,12 @@ Defines the host and port the server is listening on. Use "unix:/path/to.sock" to listen on a unix domain socket. Defaults to 'localhost:8200'. -[[max_unzipped_size]] -[float] -==== `max_unzipped_size` deprecated[6.5] -Maximum permitted size of an unzipped request accepted by the server to be processed (in Bytes). -Defaults to 31457280 Bytes (30 MB). - -[[max_event_size]] -[float] -==== `max_event_size added[6.5] -Maximum permitted size of an event accepted by the server to be processed (in Bytes). -Defaults to 307200 Bytes. - [[max_header_size]] [float] ==== `max_header_size` Maximum permitted size of a request's header accepted by the server to be processed (in Bytes). Defaults to 1048576 Bytes (1 MB). -[[max_request_queue_time]] -[float] -==== `max_request_queue_time` deprecated[6.5] -Maximum duration a request will be queued before being read. -Defaults to 2 seconds. - [[read_timeout]] [float] ==== `read_timeout` @@ -68,6 +51,26 @@ Defaults to 30 seconds. Maximum duration in seconds before releasing resources when shutting down the server. Defaults to 5 seconds. +[float] +[[configuration-v1-api]] +=== Configuration options: v1 Intake API deprecated[6.5] + +Version 6.5 of the APM Server introduced a new intake API. You can learn more about this change in the <> documentation. + +If you do not upgrade your APM Agent, you'll continue to use the deprecated v1 intake API endpoint, and thus, the following configuration options. + +[[max_unzipped_size]] +[float] +==== `max_unzipped_size` deprecated[6.5] +Maximum permitted size of an unzipped request accepted by the server to be processed (in Bytes). +Defaults to 31457280 Bytes (30 MB). + +[[max_request_queue_time]] +[float] +==== `max_request_queue_time` deprecated[6.5] +Maximum duration a request will be queued before being read. +Defaults to 2 seconds. + [[concurrent_requests]] [float] ==== `concurrent_request` deprecated[6.5] @@ -75,18 +78,30 @@ Maximum number of requests the server can process concurrently. Read more about how to tune data ingestion by <>. Default value is 5. +[float] +[[configuration-v2-api]] +=== Configuration options: v2 Intake API + +Version 6.5 of the APM Server introduced a new intake API. You can learn more about this change in the <> documentation. + +The following configuration options only affect users that have upgraded their agent to take advantage of the new v2 intake API. + +[[max_event_size]] +[float] +==== `max_event_size` +Maximum permitted size of an event accepted by the server to be processed (in Bytes). +Defaults to 307200 Bytes. + +[float] +[[configuration-other]] +=== Configuration options: general + [[max_connections]] [float] ==== `max_connections` Maximum number of TCP connections to accept simultaneously. Default value is 0, which means _unlimited_. -[[instrumentation.enabled]] -[float] -==== `instrumentation.enabled` -Enables self instrumentation of the APM Server itself. -Disabled by default. - [[config-secret-token]] [float] ==== `secret_token` @@ -117,6 +132,12 @@ Disabled by default. Configure the url to expose expvar. Defaults to `debug/vars`. +[[instrumentation.enabled]] +[float] +==== `instrumentation.enabled` +Enables self instrumentation of the APM Server itself. +Disabled by default. + [[metrics.enabled]] [float] ==== `metrics` @@ -135,9 +156,9 @@ Defaults to false. Overwrites existing pipeline definitions in Elasticsearch. Defaults to true. [float] -=== Configuration options `queue.mem.*` +=== Configuration options: `queue.mem.*` Data is buffered in a memory queue before it is published to the configured output. -`queue.mem.*` settings modify the queue behaviour. +`queue.mem.*` settings modify the queue behavior. [[mem.events]] [float] @@ -161,7 +182,7 @@ if the number of events stored in the queue is < _min_flush_events_. Default value is 1 second. [float] -=== Configuration options `max_procs` +=== Configuration options: `max_procs` [[max_procs]] [float] diff --git a/docs/configuration-rum.asciidoc b/docs/configuration-rum.asciidoc index 8a211e35655..16c745ea5aa 100644 --- a/docs/configuration-rum.asciidoc +++ b/docs/configuration-rum.asciidoc @@ -17,7 +17,7 @@ apm-server.rum.source_mapping.index_pattern: "apm-*-sourcemap*" ---- [float] -=== Configuration options +=== Configuration [[rum-enable]] [float] @@ -26,12 +26,44 @@ For enabling RUM support, set the `apm-server.rum.enabled` to `true`. By default this is disabled. [float] -==== `rate_limit` +[[configuration-rum-v1]] +=== Configuration options: v1 RUM endpoint deprecated[6.5] + +Version 6.5 of the APM Server introduced a new intake API. You can learn more about this change in the <> documentation. + +If you do not upgrade your APM Agent, you'll continue to use the deprecated v1 RUM intake API endpoint, and thus, the following configuration options. + +[float] +==== `rate_limit` deprecated[6.5] Rate limit per second and IP address for requests sent to the RUM endpoint. If the rate limit is hit, the APM Server will return an HTTP status code `429`. The rate limit cannot be disabled. Ensure to have it set to a number suiting your requirements. Default value is set to 10. +[float] +[[configuration-rum-v2]] +=== Configuration options: v2 RUM endpoint + +Version 6.5 of the APM Server introduced a new intake API. You can learn more about this change in the <> documentation. + +The following configuration options only affect users that have upgraded their agent to take advantage of the new v2 intake API. + +[float] +==== `event_rate.limit` +Defines the maximum amount of events allowed to be sent to the APM Server v2 RUM endpoint per ip per second. +Defaults to 300. + +[float] +==== `event_rate.lru_size` +An LRU cache is used to keep a rate limit per IP for the most recently seen IPs. +This setting defines the number of unique IPs that can be tracked in the cache. +Sites with many concurrent clients should consider increasing this limit. +Defaults to 1000. + +[float] +[[configuration-rum-general]] +=== Configuration options: general + [float] ==== `allow_origins` Comma separated list of permitted origins for RUM support. @@ -55,6 +87,13 @@ RegExp to be matched against a stacktrace frame's `file_name`. If the RegExp matches, the stacktrace frame is excluded from being used for calculating error groups. The default pattern excludes stacktrace frames that have a filename starting with `/webpack`. +[[config-sourcemapping-elasticsearch]] +[float] +==== `source_mapping.elasticsearch` +Configure the Elasticsearch source map retrieval location, taking the same options as <>. +This must be set when using an output other than Elasticsearch, and that output is writing to Elasticsearch. +Otherwise leave this section empty. + [[rum-sourcemap-cache]] [float] ==== `source_mapping.cache.expiration` @@ -64,15 +103,7 @@ Source maps are fetched from Elasticsearch and then kept in an in-memory cache f Values configured without a time unit are treated as seconds. Default value is 5 minutes. -[[config-sourcemapping-elasticsearch]] -[float] -==== `source_mapping.elasticsearch` -Configure the Elasticsearch source map retrieval location, taking the same options as <>. -This must be set when using an output other than Elasticsearch, and that output is writing to Elasticsearch. -Otherwise leave this section empty. - [float] ==== `source_mapping.index_pattern` Source maps are stored in a separate index `apm-%{[beat.version]}-sourcemap` by default. -If changed, a matching index pattern needs to be specified here. - +If changed, a matching index pattern needs to be specified here. \ No newline at end of file diff --git a/docs/data-ingestion.asciidoc b/docs/data-ingestion.asciidoc index dab9cab4644..b97bbeafac2 100644 --- a/docs/data-ingestion.asciidoc +++ b/docs/data-ingestion.asciidoc @@ -48,9 +48,9 @@ Increasing `queue.mem.events` can significantly affect APM Server memory usage. [[adjust-concurrent-requests]] [float] -=== Adjust concurrent requests +=== Adjust concurrent requests deprecated[6.5] APM Server has a limit to how many requests can be processed concurrently. -This limit is determined by the `apm-server.concurrent_requests` setting deprecated[6.5]. +This limit is determined by the `apm-server.concurrent_requests` setting. Increasing this value will improve throughput, but it can significantly affect APM Server memory usage. [[add-apm-server-instances]] diff --git a/docs/error-api.asciidoc b/docs/error-api.asciidoc index c14d85b7daa..02b337f0675 100644 --- a/docs/error-api.asciidoc +++ b/docs/error-api.asciidoc @@ -1,21 +1,14 @@ [[error-api]] -=== Error API +=== Errors -The following section contains information about: +An error or a logged error message captured by an agent occurring in a monitored service. -* <> -* <> - -[[error-schema-definition]] [float] -==== Schema Definition +[[error-schema]] +==== Error Schema The APM Server uses JSON Schema for validating requests. The specification for errors is defined below: -[float] -[[error-schema]] -===== Error - [source,json] ---- include::./spec/errors/v2_error.json[] diff --git a/docs/events-api.asciidoc b/docs/events-api.asciidoc index 47e43e1edba..9ab53f9b229 100644 --- a/docs/events-api.asciidoc +++ b/docs/events-api.asciidoc @@ -40,17 +40,17 @@ Keep in mind that events can succeed and fail independently of each other. Only [[events-api-errors]] [float] -=== Errors +==== Errors There are two types of errors that the APM Server may return to an agent: * Event related errors (typically validation errors) * Non-event related errors -The APM Server process events one after the other. +The APM Server processes events one after the other. If an error is encountered while processing an event, the error encountered as well as the document causing the error are added to an internal array. -The APM Server will only process and save 5 event related errors. +The APM Server will only save 5 event related errors. If it encounters more than 5 event related errors, the additional errors will not be returned to agent. Once all events have been processed, diff --git a/docs/guide/agent-server-compatibility.asciidoc b/docs/guide/agent-server-compatibility.asciidoc index e1022c328fd..cae7b66cf06 100644 --- a/docs/guide/agent-server-compatibility.asciidoc +++ b/docs/guide/agent-server-compatibility.asciidoc @@ -14,7 +14,6 @@ it is recommended that you update to the newest version of the applicable agent. [options="header"] |======================================================================= |Agent Version |APM Server Version -|0.x |< 6.2 |1.x |6.2-6.x |2.x |>= 6.5 |======================================================================= @@ -26,7 +25,6 @@ it is recommended that you update to the newest version of the applicable agent. [options="header"] |======================================================================= |Agent Version |APM Server Version -|1.x |6.1 beta[] |2.x/3.x |6.2-6.x |4.x |>= 6.5 |======================================================================= @@ -38,7 +36,7 @@ it is recommended that you update to the newest version of the applicable agent. [options="header"] |======================================================================= |Agent Version |APM Server Version -|1.x |\<= 6.x +|1.x |6.4-6.x |2.x |>= 6.5 |======================================================================= diff --git a/docs/metadata-api.asciidoc b/docs/metadata-api.asciidoc index afafb6efa9b..54ae02103ba 100644 --- a/docs/metadata-api.asciidoc +++ b/docs/metadata-api.asciidoc @@ -9,12 +9,6 @@ the APM Server hangs on to this information and applies it to other objects in t TIP: Metadata is stored under `context` when viewing documents in Elasticsearch. -[[metadata-schema-definition]] -[float] -==== Schema Definition - -The specification for metadata is outlined below: - * <> * <> * <> @@ -23,7 +17,10 @@ The specification for metadata is outlined below: [[metadata-schema]] [float] -===== Metadata +==== Metadata Schema + +The APM Server uses JSON Schema for validating requests. The specification for metadata is defined below: + [source,json] ---- include::./spec/metadata.json[] @@ -31,7 +28,8 @@ include::./spec/metadata.json[] [[metadata-service-schema]] [float] -===== Service +===== Service Schema + [source,json] ---- include::./spec/service.json[] @@ -39,7 +37,8 @@ include::./spec/service.json[] [[metadata-process-schema]] [float] -===== Process +===== Process Schema + [source,json] ---- include::./spec/process.json[] @@ -47,7 +46,8 @@ include::./spec/process.json[] [[metadata-system-schema]] [float] -===== System +===== System Schema + [source,json] ---- include::./spec/system.json[] @@ -55,7 +55,8 @@ include::./spec/system.json[] [[metadata-user-schema]] [float] -===== User +===== User Schema + [source,json] ---- include::./spec/user.json[] diff --git a/docs/metricset-api.asciidoc b/docs/metricset-api.asciidoc index 17828e7872a..84edd1fae5b 100644 --- a/docs/metricset-api.asciidoc +++ b/docs/metricset-api.asciidoc @@ -3,9 +3,9 @@ Metricsets contain application metric data captured by an APM agent. -[[metricset-schema-definition]] +[[metricset-schema]] [float] -==== Schema Definition +==== Metricset Schema The APM Server uses JSON Schema for validating requests. The specification for metricsets is defined below: diff --git a/docs/server-info.asciidoc b/docs/server-info.asciidoc index d13e0228f65..a20751ca9af 100644 --- a/docs/server-info.asciidoc +++ b/docs/server-info.asciidoc @@ -16,7 +16,7 @@ http(s)://{hostname}:{port}/ This endpoint always returns an HTTP 200. -If a <> is set, only requests including <> will include server details. +If a <> is set, only requests including <> will receive server details. Set the `Accept` header set to `text/plain` to move the server information to the root level of the response, removing `ok`. diff --git a/docs/span-api.asciidoc b/docs/span-api.asciidoc index a0321b4bfd9..31c3236681a 100644 --- a/docs/span-api.asciidoc +++ b/docs/span-api.asciidoc @@ -1,21 +1,14 @@ [[span-api]] -=== Spans Schema +=== Spans -The following section contains information about: +Spans are events captured by an agent occurring in a monitored service. -* <> -* <> - -[[span-schema-definition]] +[[span-schema]] [float] -==== Schema Definition +==== Span Schema The APM Server uses JSON Schema for validating requests. The specification for spans is defined below: -[float] -[[transaction-span-schema]] -===== Span Schema - [source,json] ---- include::./spec/spans/v2_span.json[] diff --git a/docs/transaction-api.asciidoc b/docs/transaction-api.asciidoc index 8865624ce0b..769ed41e71c 100644 --- a/docs/transaction-api.asciidoc +++ b/docs/transaction-api.asciidoc @@ -1,21 +1,14 @@ [[transaction-api]] -=== Transactions Schema +=== Transactions -The following section contains information about: +Transactions are events corresponding to an incoming request or similar task occurring in a monitored service. -* <> -* <> - -[[transaction-schema-definition]] +[[transaction-schema]] [float] -==== Schema Definition +==== Transaction Schema The APM Server uses JSON Schema for validating requests. The specification for transactions is defined below: -[float] -[[transaction-schema]] -===== Transaction Schema - [source,json] ---- include::./spec/transactions/common_transaction.json[] diff --git a/docs/upgrading-to-65.asciidoc b/docs/upgrading-to-65.asciidoc index 69bbffbb708..4a640f7c68b 100644 --- a/docs/upgrading-to-65.asciidoc +++ b/docs/upgrading-to-65.asciidoc @@ -95,7 +95,7 @@ The span schema adds three new properties, removes two, and changes one. * _add_ `timestamp` Span events should be sent to the new `intake/v2/events` endpoint. -See the <> for more information. +See the <> for more information. [float] [[healthcheck-api-changes-65]] From 97420d3de4028d3793028ce0f2eebbad2384c0a5 Mon Sep 17 00:00:00 2001 From: Brandon Morelli Date: Fri, 9 Nov 2018 15:36:25 -0800 Subject: [PATCH 11/12] docs: add redirects file --- docs/index.asciidoc | 2 + docs/redirects.asciidoc | 155 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 157 insertions(+) create mode 100644 docs/redirects.asciidoc diff --git a/docs/index.asciidoc b/docs/index.asciidoc index c3c30798cfb..50ce8792602 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -61,3 +61,5 @@ include::./fields.asciidoc[] include::./troubleshooting.asciidoc[] include::./release-notes.asciidoc[] + +include::./redirects.asciidoc[] diff --git a/docs/redirects.asciidoc b/docs/redirects.asciidoc new file mode 100644 index 00000000000..8c23133c376 --- /dev/null +++ b/docs/redirects.asciidoc @@ -0,0 +1,155 @@ +["appendix",role="exclude",id="redirects"] += Deleted pages + +The following pages have moved or been deleted. + +// Event Types + +[role="exclude",id="event-types"] +=== Event types + +This page has moved. Please see {apm-get-started-ref}/apm-data-model.html[APM data model]. + +[role="exclude",id="errors"] +=== Errors + +This page has moved. Please see {apm-get-started-ref}/errors.html[Errors]. + +[role="exclude",id="transactions"] +=== Transactions + +This page has moved. Please see {apm-get-started-ref}/transactions.html[Transactions]. + +[role="exclude",id="transactions-spans"] +=== Spans + +This page has moved. Please see {apm-get-started-ref}/transactions-spans.html[Spans]. + +// Error API + +[role="exclude",id="error-endpoint"] +=== Error endpoint + +The error endpoint has been deprecated. Instead, see <>. + +[role="exclude",id="error-schema-definition"] +=== Error schema definition + +The error schema has moved. Please see <>. + +[role="exclude",id="error-api-examples"] +=== Error API examples + +The error API examples have moved. Please see <>. + +[role="exclude",id="error-payload-schema"] +=== Error payload schema + +This schema has changed. Please see <>. + +[role="exclude",id="error-service-schema"] +=== Error service schema + +This schema has changed. Please see <>. + +[role="exclude",id="error-system-schema"] +=== Error system schema + +This schema has changed. Please see <>. + +[role="exclude",id="error-context-schema"] +=== Error context schema + +This schema has changed. Please see <>. + +[role="exclude",id="error-stacktraceframe-schema"] +=== Error stacktrace frame schema + +This schema has changed. Please see <>. + +[role="exclude",id="payload-with-error"] +=== Payload with error + +This is no longer helpful. Please see <>. + +[role="exclude",id="payload-with-minimal-exception"] +=== Payload with minimal exception + +This is no longer helpful. Please see <>. + +[role="exclude",id="payload-with-minimal-log"] +=== Payload with minimal log + +This is no longer helpful. Please see <>. + +// Transaction API + +[role="exclude",id="transaction-endpoint"] +=== Transaction endpoint + +The transaction endpoint has been deprecated. Instead, see <>. + +[role="exclude",id="transaction-schema-definition"] +=== Transaction schema definition + +The transaction schema has moved. Please see <>. + +[role="exclude",id="transaction-api-examples"] +=== Transaction API examples + +The transaction API examples have moved. Please see <>. + +[role="exclude",id="transaction-span-schema"] +=== Transaction span schema + +This schema has changed. Please see <>. + +[role="exclude",id="transaction-payload-schema"] +=== Transaction payload schema + +This schema has changed. Please see <>. + +[role="exclude",id="transaction-service-schema"] +=== Transaction service schema + +This schema has changed. Please see <>. + +[role="exclude",id="transaction-system-schema"] +=== Transaction system schema + +This schema has changed. Please see <>. + +[role="exclude",id="transaction-context-schema"] +=== Transaction context schema + +This schema has changed. Please see <>. + +[role="exclude",id="transaction-stacktraceframe-schema"] +=== Transaction stacktrace frame schema + +This schema has changed. Please see <>. + +[role="exclude",id="transaction-request-schema"] +=== Transaction request schema + +This schema has changed. Please see <>. + +[role="exclude",id="transaction-user-schema"] +=== Transaction user schema + +This schema has changed. Please see <>. + +[role="exclude",id="payload-with-transactions"] +=== Payload with transactions + +This is no longer helpful. Please see <>. + +[role="exclude",id="payload-with-minimal-transaction"] +=== Payload with minimal transaction + +This is no longer helpful. Please see <>. + +[role="exclude",id="payload-with-minimal-span"] +=== Payload with minimal span + +This is no longer helpful. Please see <>. \ No newline at end of file From dd62bce8f381c9bcd948b53e5945fa48b5ccd285 Mon Sep 17 00:00:00 2001 From: Brandon Morelli Date: Mon, 12 Nov 2018 06:32:57 -0800 Subject: [PATCH 12/12] docs: fix agent server compat --- docs/guide/agent-server-compatibility.asciidoc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/guide/agent-server-compatibility.asciidoc b/docs/guide/agent-server-compatibility.asciidoc index cae7b66cf06..7cd49f41a83 100644 --- a/docs/guide/agent-server-compatibility.asciidoc +++ b/docs/guide/agent-server-compatibility.asciidoc @@ -47,7 +47,7 @@ it is recommended that you update to the newest version of the applicable agent. [options="header"] |======================================================================= |Agent Version |APM Server Version -|1.x |>= 6.5.0 +|1.x |>= 6.5 |======================================================================= [float] @@ -57,7 +57,6 @@ it is recommended that you update to the newest version of the applicable agent. [options="header"] |======================================================================= |Agent Version |APM Server Version -|0.x |\<= 6.x |1.x |>= 6.5 |=======================================================================