feat(fleet): collect Docker logs and metrics

Mirrors the behaviour of the Metricsbeat and Filebeat extensions through
Elastic Agent, using the Docker integration.

Author: antoineco

.github/workflows/scripts/run-tests-fleet.sh

@@ -35,7 +35,7 @@ endgroup
 #   AND event.dataset:"system.cpu"
 #   AND metricset.name:"cpu"
 #
-log 'Searching a document generated by Fleet Server'
+log 'Searching a system document generated by Fleet Server'
 
 declare response
 declare -i count
@@ -70,3 +70,44 @@ if (( count == 0 )); then
 	echo 'Expected at least 1 document'
 	exit 1
 fi
+
+# We expect to find log entries for the 'elasticsearch' Compose service using
+# the following query:
+#
+#   agent.name:"fleet-server"
+#   AND agent.type:"filebeat"
+#   AND container.name:"docker-elk-elasticsearch-1"
+#
+log 'Searching a container document generated by Fleet Server'
+
+response=
+count=0
+
+was_retried=0
+
+# retry for max 60s (30*2s)
+for _ in $(seq 1 30); do
+	response="$(curl "http://${ip_es}:9200/logs-docker.container_logs-default/_search?q=agent.name:%22fleet-server%22%20AND%20agent.type:%22filebeat%22%20AND%20container.name:%22docker-elk-elasticsearch-1%22&pretty" -s -u elastic:testpasswd)"
+
+	set +u  # prevent "unbound variable" if assigned value is not an integer
+	count="$(jq -rn --argjson data "${response}" '$data.hits.total.value')"
+	set -u
+
+	if (( count > 0 )); then
+		break
+	fi
+
+	was_retried=1
+	echo -n 'x' >&2
+	sleep 2
+done
+if ((was_retried)); then
+	# flush stderr, important in non-interactive environments (CI)
+	echo >&2
+fi
+
+echo "$response"
+if (( count == 0 )); then
+	echo 'Expected at least 1 document'
+	exit 1
+fi

extensions/fleet/README.md

@@ -37,11 +37,6 @@ management UI: [Fleet UI Settings][fleet-cfg].
 
 ## Known Issues
 
-- Logs and metrics are only collected within the Fleet Server's container. Ultimately, we want to emulate the behaviour
-  of the existing Metricsbeat and Filebeat extensions, and collect logs and metrics from all ELK containers
-  out-of-the-box. Unfortunately, this kind of use-case isn't (yet) well supported by Fleet, and most advanced
-  configurations currently require running Elastic Agents in [standalone mode][fleet-standalone].
-  (Relevant resource: [Migrate from Beats to Elastic Agent][fleet-beats])
 - The Elastic Agent auto-enrolls using the `elastic` super-user. With this approach, you do not need to generate a
   service token — either using the Fleet management UI or [CLI utility][es-svc-token] — prior to starting this
   extension. However convenient that is, this approach _does not follow security best practices_, and we recommend
@@ -64,6 +59,4 @@ management UI: [Fleet UI Settings][fleet-cfg].
 
 [config-kbn]: ../../kibana/config/kibana.yml
 
-[fleet-standalone]: https://www.elastic.co/guide/en/fleet/current/elastic-agent-configuration.html
-[fleet-beats]: https://www.elastic.co/guide/en/fleet/current/migrate-beats-to-agent.html
 [es-svc-token]: https://www.elastic.co/guide/en/elasticsearch/reference/current/service-tokens-command.html

extensions/fleet/fleet-compose.yml

@@ -6,8 +6,19 @@ services:
       context: extensions/fleet/
       args:
         ELASTIC_VERSION: ${ELASTIC_VERSION}
+    # Run as 'root' instead of 'elastic-agent' (uid 1000) to allow reading
+    # 'docker.sock' and the host's filesystem.
+    user: root
     volumes:
       - fleet-server:/usr/share/elastic-agent/state:Z
+      - type: bind
+        source: /var/lib/docker/containers
+        target: /var/lib/docker/containers
+        read_only: true
+      - type: bind
+        source: /var/run/docker.sock
+        target: /var/run/docker.sock
+        read_only: true
     environment:
       FLEET_SERVER_ENABLE: '1'
       FLEET_SERVER_INSECURE_HTTP: '1'

kibana/config/kibana.yml

@@ -47,6 +47,8 @@ xpack.fleet.packages:
     version: latest
   - name: elastic_agent
     version: latest
+  - name: docker
+    version: latest
   - name: apm
     version: latest
 
@@ -67,6 +69,9 @@ xpack.fleet.agentPolicies:
       - name: elastic_agent-1
         package:
           name: elastic_agent
+      - name: docker-1
+        package:
+          name: docker
   - name: Agent Policy APM Server
     id: agent-policy-apm-server
     description: Static agent policy for the APM Server integration