feat: store localnet logs between runs (#4830)

Author: sophoah

.gitignore

@@ -101,3 +101,4 @@ cache_*_db
 
 # log aggregation logs folder variable file, no sense to track it
 test/logs_aggregator/.env
+test/logs_aggregator/loki

Makefile

@@ -55,10 +55,13 @@ help:
 	@echo "travis_rpc_checker - run the Travis RPC checker script, defaulting the test branch to 'master' unless overridden by TEST_REPO_BRANCH"
 	@echo "travis_rosetta_checker - run the Travis Rosetta checker script, defaulting the test branch to 'master' unless overridden by TEST_REPO_BRANCH"
 	@echo "debug_external - cleans up environment, rebuilds the binary, and deploys with external nodes"
+	@echo "debug-multi-bls - cleans up environment, rebuilds the binary, and deploys with external nodes in configuration 1 harmony process -> 2 validators"
 	@echo "build_localnet_validator - imports validator keys, funds validator accounts, waits for the epoch, and creates external validators on a local network"
-	@echo "debug-start-log - start a docker compose Promtail->Loki->Grafana stack against localnet logs, needs docker compose and started localnet"
+	@echo "debug-start-log - start a docker compose Promtail->Loki->Grafana stack against localnet logs, creates"\
+		"persistent volume to store parsed logs between localnet runs, needs docker compose and started localnet"
 	@echo "debug-stop-log - stops a docker compose Promtail->Loki->Grafana stack"
 	@echo "debug-restart-log - restart a docker compose Promtail->Loki->Grafana stack"
+	@echo "debug-delete-log - removes persistent volume for the Loki and host folder for it"
 
 libs:
 	make -C $(TOP)/mcl -j8
@@ -101,7 +104,7 @@ debug-multi-bls:
 	echo sleep 10s before creating the external validator
 	sleep 10
 	bash ./test/build-localnet-validator.sh
-	
+
 debug-multi-bls-with-terminal:
 	# add VERBOSE=true before bash or run `export VERBOSE=true` on the shell level for have max logging
 	# add LEGACY_SYNC=true before bash  or run `export LEGACY_SYNC=true` on the shell level to switch to the legacy sync
@@ -111,7 +114,7 @@ debug-multi-bls-with-terminal:
 	bash ./test/build-localnet-validator.sh
 	screen -r localnet
 
-debug-multi-bls-multi-ext-node: pre-external
+debug-multi-bls-multi-ext-node:
 	# add VERBOSE=true before bash or run `export VERBOSE=true` on the shell level for have max logging
 	# add LEGACY_SYNC=true before bash  or run `export LEGACY_SYNC=true` on the shell level to switch to the legacy sync
 	./test/debug.sh ./test/configs/local-multi-bls-multi-ext-node.txt &
@@ -120,7 +123,7 @@ debug-multi-bls-multi-ext-node: pre-external
 	bash ./test/build-localnet-validator.sh
 
 clean:
-	rm -rf ./tmp_log*
+	rm -rf ./tmp_log/*
 	rm -rf ./.dht*
 	rm -rf ./db-*
 	rm -rf ./latest
@@ -232,9 +235,6 @@ travis_rpc_checker:
 travis_rosetta_checker:
 	bash ./scripts/travis_rosetta_checker.sh
 
-debug_external: clean
-	bash test/debug-external.sh
-
 build_localnet_validator:
 	bash test/build-localnet-validator.sh
 
@@ -248,3 +248,8 @@ debug-stop-log:
 	bash ./test/logs_aggregator/stop_log_aggregator.sh
 
 debug-restart-log: debug-stop-log debug-start-log
+
+debug-delete-log:
+	docker volume rm logs_aggregator_loki_data
+	@echo "[WARN] - it needs sudo to remove folder created with loki docker image user"
+	sudo rm -rf test/logs_aggregator/loki

test/debug-external.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 
 ./test/kill_node.sh
-rm -rf tmp_log* 2> /dev/null
+rm -rf tmp_log/* 2> /dev/null
 rm *.rlp 2> /dev/null
 rm -rf .dht* 2> /dev/null
 scripts/go_executable_build.sh -S || exit 1  # dynamic builds are faster for debug iteration...

test/debug.sh

@@ -6,7 +6,7 @@ Localnet_Blocks_Per_Epoch=$2
 Localnet_Blocks_Per_Epoch_V2=$3
 
 ./test/kill_node.sh
-rm -rf tmp_log* 2> /dev/null
+rm -rf tmp_log/* 2> /dev/null
 rm *.rlp 2> /dev/null
 rm -rf .dht* 2> /dev/null
 scripts/go_executable_build.sh -S || exit 1  # dynamic builds are faster for debug iteration...

test/logs_aggregator/docker-compose.yml

@@ -5,13 +5,14 @@ services:
     ports:
       - 3100:3100
     volumes:
-      - ./loki-config.yml:/etc/loki/loki-config.yaml
+      - ./loki-config.yml:/etc/loki/local-config.yaml
+      - loki_data:/loki
   promtail:
     container_name: promtail
     image: grafana/promtail:latest
     volumes:
-      - ./promtail-config.yml:/etc/promtail/promtail-config.yaml
-      - ${CURRENT_SESSION_LOGS}:/var/log/
+      - ./promtail-config.yml:/etc/promtail/config.yml
+      - ${LOG_FOLDER}:/var/log/
   grafana:
     container_name: grafana
     image: grafana/grafana
@@ -23,3 +24,11 @@ services:
       - 3000:3000
     volumes:
       - ./loki-datasource.yaml:/etc/grafana/provisioning/datasources/loki-datasource.yaml
+
+volumes:
+  loki_data:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ${LOKI_FOLDER}

test/logs_aggregator/loki-config.yml

@@ -32,3 +32,8 @@ schema_config:
       index:
         prefix: index_
         period: 24h
+
+limits_config:
+  ingestion_rate_strategy: local
+  ingestion_rate_mb: 30
+  ingestion_burst_size_mb: 50

test/logs_aggregator/promtail-config.yml

@@ -15,4 +15,4 @@ scrape_configs:
       - "localhost"
     labels:
       job: varlogs
-      __path__: /var/log/*.log
+      __path__: /var/log/log*/*.log

test/logs_aggregator/start_log_aggregator.sh

@@ -5,13 +5,20 @@ set -eou pipefail
 DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
 echo "working in ${DIR}"
 cd $DIR && pwd
-logs="$(ls -1 '../../tmp_log/')"
 path="$(readlink -f '../../tmp_log/')"
-echo "Current localnet logs are placed into '${path}/${logs}'"
-echo "CURRENT_SESSION_LOGS='${path}/${logs}'" > .env
-echo "CURRENT_SESSION_LOGS='${path}/${logs}'"
-echo "starting docker compose lor log aggregation"
+test -d "${path}" || (echo "logs folder do not exist - ${path},"\
+    "please create a localnet first, exiting" && exit 1)
+log_path="$(find ${path} -type d | sort | tail -n 1)"
+log_path=$(echo "${log_path}" | sed "s#.*/tmp_log/#/var/log/#")
+loki_path="${DIR}/loki"
+mkdir -p "${loki_path}"
+echo "LOG_FOLDER='${path}'" > .env
+echo "LOKI_FOLDER='${loki_path}'" >> .env
+echo "starting docker compose for log aggregation"
 docker compose up --detach
 sleep 5
+echo "Whole list of log folders"
+find ${path} -type d | sort | grep 'log-'
 echo "Opening Grafana"
 python3 -m webbrowser "http://localhost:3000/explore"
+echo "Please, use {filename=~\"${log_path}/.*\"} to get the latest run localnet logs"