## Tracing Stack Management
## Distributed tracing with OpenTelemetry and Tempo

# Include common variables and functions
include common/Makefile

# Tracing configuration
HELMFILE_OTEL_OPERATOR := tracing/helmfile-otel-operator.yaml
HELMFILE_TEMPO_OPERATOR := tracing/helmfile-tempo-operator.yaml

.PHONY: tracing-help
tracing-help:
	@echo ""
	@echo "$(COLOR_BOLD)Tracing Commands:$(COLOR_RESET)"
	@echo "  make tracing-install-all              - Install complete tracing stack"
	@echo "  make tracing-install-otel-operator    - Install OpenTelemetry Operator"
	@echo "  make tracing-install-tempo-operator   - Install Tempo Operator"
	@echo "  make tracing-install-otel-collector   - Deploy OTel Collector CR"
	@echo "  make tracing-install-tempo-stack      - Deploy Tempo Stack CR"
	@echo "  make tracing-create-s3-secret         - Create S3 credentials secret (interactive)"
	@echo "  make tracing-create-bucket            - Instructions for creating S3 bucket"
	@echo "  make tracing-enable-monitoring        - Enable metrics collection (ServiceMonitors + Alerts)"
	@echo "  make tracing-update-otel-operator     - Update OTel Operator configuration"
	@echo "  make tracing-update-tempo-operator    - Update Tempo Operator configuration"
	@echo "  make tracing-update-otel-collector    - Update OTel Collector CR"
	@echo "  make tracing-update-tempo-stack       - Update Tempo Stack CR"
	@echo "  make tracing-status                   - Check tracing stack status"
	@echo "  make tracing-logs-otel                - Show OTel Collector logs"
	@echo "  make tracing-logs-tempo               - Show Tempo logs"
	@echo "  make tracing-port-forward-tempo       - Port-forward Tempo (http://localhost:3200)"
	@echo "  make tracing-test-query               - Test Tempo query API"
	@echo "  make tracing-uninstall                - Uninstall tracing stack (DELETES DATA!)"
	@echo ""
	@echo "$(COLOR_YELLOW)Note:$(COLOR_RESET) Before installing, ensure:"
	@echo "  1. Longhorn is installed: make longhorn-install-all"
	@echo "  2. MinIO S3 is installed: make minio-install-all"
	@echo "  3. cert-manager is installed (for operator webhooks)"
	@echo "  4. Nodes are prepared: cd ../.. && make k8s-prepare-nodes"

.PHONY: tracing-check-longhorn
tracing-check-longhorn:
	$(call check_longhorn)

.PHONY: tracing-check-minio
tracing-check-minio:
	@echo ""
	@echo "$(COLOR_BOLD)=== Checking MinIO availability ===$(COLOR_RESET)"
ifeq ($(DETECTED_OS),Windows)
	@$(KUBECTL) get namespace tech-minio-tenants >nul 2>&1 || \
		($(call print_warning,MinIO namespace not found. MinIO may not be installed.) && \
		 echo "$(COLOR_YELLOW)Please install MinIO first:$(COLOR_RESET)" && \
		 echo "  make minio-install-all" && \
		 exit 1)
	@$(KUBECTL) -n tech-minio-tenants get tenant s3-public >nul 2>&1 || \
		($(call print_warning,MinIO tenant s3-public not found.) && \
		 echo "$(COLOR_YELLOW)Please create MinIO tenant first:$(COLOR_RESET)" && \
		 echo "  make minio-create-tenant" && \
		 exit 1)
else
	@$(KUBECTL) get namespace tech-minio-tenants >/dev/null 2>&1 || \
		($(call print_warning,MinIO namespace not found. MinIO may not be installed.) && \
		 echo "$(COLOR_YELLOW)Please install MinIO first:$(COLOR_RESET)" && \
		 echo "  make minio-install-all" && \
		 exit 1)
	@$(KUBECTL) -n tech-minio-tenants get tenant s3-public >/dev/null 2>&1 || \
		($(call print_warning,MinIO tenant s3-public not found.) && \
		 echo "$(COLOR_YELLOW)Please create MinIO tenant first:$(COLOR_RESET)" && \
		 echo "  make minio-create-tenant" && \
		 exit 1)
endif
	$(call print_success,MinIO is available)

.PHONY: tracing-check-cert-manager
tracing-check-cert-manager:
	@echo ""
	@echo "$(COLOR_BOLD)=== Checking cert-manager availability ===$(COLOR_RESET)"
ifeq ($(DETECTED_OS),Windows)
	@$(KUBECTL) get namespace cert-manager >nul 2>&1 || \
		($(call print_warning,cert-manager namespace not found.) && \
		 echo "$(COLOR_YELLOW)Please install cert-manager first:$(COLOR_RESET)" && \
		 echo "  make ingress-install-cert-manager" && \
		 exit 1)
else
	@$(KUBECTL) get namespace cert-manager >/dev/null 2>&1 || \
		($(call print_warning,cert-manager namespace not found.) && \
		 echo "$(COLOR_YELLOW)Please install cert-manager first:$(COLOR_RESET)" && \
		 echo "  make ingress-install-cert-manager" && \
		 exit 1)
endif
	$(call print_success,cert-manager is available)

.PHONY: tracing-install-all
tracing-install-all: tracing-check-longhorn tracing-check-minio tracing-check-cert-manager
	$(call print_info,Installing complete tracing stack...)
	@$(MAKE) tracing-install-otel-operator
	@$(MAKE) tracing-install-tempo-operator
	$(call print_success,Tracing operators installed successfully!)
	@echo ""
	@echo "$(COLOR_YELLOW)Next steps:$(COLOR_RESET)"
	@echo "  1. Create S3 bucket: make tracing-create-bucket"
	@echo "  2. Create S3 secret: make tracing-create-s3-secret"
	@echo "  3. Deploy Tempo Stack: make tracing-install-tempo-stack"
	@echo "  4. Deploy OTel Collector: make tracing-install-otel-collector"
	@echo "  5. Enable monitoring: make tracing-enable-monitoring"
	@echo "  6. Check status: make tracing-status"

.PHONY: tracing-install-otel-operator
tracing-install-otel-operator: tracing-check-cert-manager
	$(call print_info,Creating tracing namespace...)
	@$(KUBECTL) apply -f tracing/manifests/tech-tracing.yaml
	$(call print_info,Applying tracing resource policies...)
	@$(KUBECTL) apply -f tracing/manifests/limitrange.yaml
	@$(KUBECTL) apply -f tracing/manifests/resourcequota.yaml
	@$(KUBECTL) apply -f tracing/manifests/networkpolicy.yaml
	@$(KUBECTL) apply -f tracing/manifests/storageclass.yaml
	$(call print_info,Installing OpenTelemetry Operator via helmfile...)
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$helmfilePath = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, 'tracing', 'helmfile-otel-operator.yaml')); Write-Host 'Using helmfile: ' $$helmfilePath; if (-not (Test-Path $$helmfilePath)) { Write-Host 'Error: Helmfile not found at: ' $$helmfilePath; exit 1 }; $$env:KUBECONFIG = $$kubeconfig; helmfile -f \"$$helmfilePath\" sync"
else
	$(call run_helmfile,$(HELMFILE_OTEL_OPERATOR),sync)
endif
	$(call print_success,OpenTelemetry Operator installed successfully!)

.PHONY: tracing-install-tempo-operator
tracing-install-tempo-operator: tracing-check-cert-manager
	$(call print_info,Ensuring tracing namespace exists...)
	@$(KUBECTL) apply -f tracing/manifests/tech-tracing.yaml
	$(call print_info,Installing Tempo Operator via official manifest...)
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$env:KUBECONFIG = $$kubeconfig; kubectl apply -f https://github.com/grafana/tempo-operator/releases/latest/download/tempo-operator.yaml"
else
	@export KUBECONFIG=$(KUBECONFIG); $(KUBECTL) apply -f https://github.com/grafana/tempo-operator/releases/latest/download/tempo-operator.yaml
endif
	$(call print_success,Tempo Operator installed successfully!)

.PHONY: tracing-install-tempo-stack
tracing-install-tempo-stack:
	$(call print_info,Deploying Tempo Stack CR...)
	@$(KUBECTL) apply -f tracing/manifests/tempo-stack.yaml
	$(call print_success,Tempo Stack deployed! Checking status...)
	@$(KUBECTL) -n tech-tracing get tempostack tempo-stack
	@echo ""
	@echo "$(COLOR_YELLOW)Wait for Tempo components to be ready:$(COLOR_RESET)"
	@echo "  kubectl -n tech-tracing get pods -w"

.PHONY: tracing-install-otel-collector
tracing-install-otel-collector:
	$(call print_info,Deploying OpenTelemetry Collector CR...)
	@$(KUBECTL) apply -f tracing/manifests/otel-collector.yaml
	$(call print_success,OTel Collector deployed! Checking status...)
	@$(KUBECTL) -n tech-tracing get otelcol otel-gateway
	@echo ""
	@echo "$(COLOR_YELLOW)Wait for collector to be ready:$(COLOR_RESET)"
	@echo "  kubectl -n tech-tracing get pods -l app.kubernetes.io/name=otel-gateway-collector -w"

.PHONY: tracing-create-s3-secret
tracing-create-s3-secret:
	$(call print_info,Creating S3 credentials secret...)
	@$(call print_info,Ensuring namespace exists...)
	@$(KUBECTL) apply -f tracing/manifests/tech-tracing.yaml
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$namespace = 'tech-tracing'; $$secretName = 'tempo-s3-secret'; $$env:KUBECONFIG = $$kubeconfig; $$secretExists = kubectl -n $$namespace get secret $$secretName 2>&1; if ($$secretExists -notmatch 'NotFound|not found') { Write-Host 'Secret already exists. Do you want to recreate it? (y/N):' -NoNewline; $$response = Read-Host; if ($$response -ne 'y' -and $$response -ne 'Y') { Write-Host 'Aborted.'; exit 0 }; kubectl -n $$namespace delete secret $$secretName }; Write-Host 'Enter MinIO S3 Access Key ID:' -NoNewline; $$accessKey = Read-Host; Write-Host 'Enter MinIO S3 Secret Access Key:' -NoNewline; $$secretKey = Read-Host -AsSecureString; $$secretKeyPlain = [Runtime.InteropServices.Marshal]::PtrToStringAuto([Runtime.InteropServices.Marshal]::SecureStringToBSTR($$secretKey)); if ([string]::IsNullOrWhiteSpace($$accessKey) -or [string]::IsNullOrWhiteSpace($$secretKeyPlain)) { Write-Host 'Error: Access Key ID and Secret Access Key cannot be empty' -ForegroundColor Red; exit 1 }; kubectl -n $$namespace create secret generic $$secretName --from-literal=endpoint='s3-public-hl.tech-minio-tenants.svc.cluster.local:9000' --from-literal=bucket='tempo-traces' --from-literal=access_key_id=$$accessKey --from-literal=access_key_secret=$$secretKeyPlain; Write-Host 'Secret created successfully!' -ForegroundColor Green; kubectl -n $$namespace get secret $$secretName"
else
	@read -p "Enter MinIO S3 Access Key ID: " accessKey && \
	read -sp "Enter MinIO S3 Secret Access Key: " secretKey && echo && \
	if [ -z "$$accessKey" ] || [ -z "$$secretKey" ]; then echo "Error: Credentials cannot be empty"; exit 1; fi && \
	$(KUBECTL) -n tech-tracing delete secret tempo-s3-secret --ignore-not-found && \
	$(KUBECTL) -n tech-tracing create secret generic tempo-s3-secret \
		--from-literal=endpoint='s3-public-hl.tech-minio-tenants.svc.cluster.local:9000' \
		--from-literal=bucket='tempo-traces' \
		--from-literal=access_key_id="$$accessKey" \
		--from-literal=access_key_secret="$$secretKey" && \
	$(call print_success,Secret created successfully!)
endif

.PHONY: tracing-create-buckets
tracing-create-buckets:
	@echo ""
	@echo "$(COLOR_BOLD)=== Creating Tempo S3 Buckets in MinIO ===$(COLOR_RESET)"
	@echo ""
	@echo "$(COLOR_YELLOW)Please create the following bucket manually through MinIO Console:$(COLOR_RESET)"
	@echo ""
	@echo "$(COLOR_BOLD)Required bucket:$(COLOR_RESET)"
	@echo "  - tempo-traces"
	@echo ""
	@echo "$(COLOR_BOLD)Instructions:$(COLOR_RESET)"
	@echo "  1. Open MinIO Console: https://s3.internal.ai-ops.tech"
	@echo "  2. Login with your MinIO credentials"
	@echo "  3. Go to 'Buckets' section (left sidebar)"
	@echo "  4. Click 'Create Bucket' button"
	@echo "  5. Name: tempo-traces"
	@echo "  6. Leave all settings as default"
	@echo ""
	@echo "$(COLOR_BOLD)Recommended: Set Quota (100 GB for traces)$(COLOR_RESET)"
	@echo "  After creating the bucket, set a size quota to limit storage usage:"
	@echo "    1. Select 'tempo-traces' bucket"
	@echo "    2. Go to 'Summary' tab"
	@echo "    3. Click 'Set Quota' button"
	@echo "    4. Type: Size, Value: 100GB"
	@echo "    5. Click 'Set'"
	@echo ""
	@echo "$(COLOR_YELLOW)After creating the bucket and setting quota, continue with:$(COLOR_RESET)"
	@echo "  make tracing-create-s3-secret"
	@echo ""

.PHONY: tracing-create-bucket
tracing-create-bucket: tracing-create-buckets

.PHONY: tracing-enable-monitoring
tracing-enable-monitoring:
	$(call print_info,Enabling monitoring for tracing stack...)
	$(call print_info,Applying ServiceMonitors...)
	@$(KUBECTL) apply -f monitoring/manifests/monitors/tempo-servicemonitor.yaml
	@$(KUBECTL) apply -f monitoring/manifests/monitors/otel-servicemonitor.yaml
	$(call print_info,Applying PrometheusRules...)
	@$(KUBECTL) apply -f monitoring/alerts/tempo-alerts.yaml
	@$(KUBECTL) apply -f monitoring/alerts/otel-collector-alerts.yaml
	$(call print_success,Monitoring enabled for tracing stack!)

.PHONY: tracing-update-otel-operator
tracing-update-otel-operator:
	$(call print_info,Updating OpenTelemetry Operator...)
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$helmfilePath = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, 'tracing', 'helmfile-otel-operator.yaml')); $$env:KUBECONFIG = $$kubeconfig; helmfile -f \"$$helmfilePath\" sync"
else
	$(call run_helmfile,$(HELMFILE_OTEL_OPERATOR),sync)
endif
	$(call print_success,OpenTelemetry Operator updated!)

.PHONY: tracing-update-tempo-operator
tracing-update-tempo-operator:
	$(call print_info,Updating Tempo Operator...)
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$env:KUBECONFIG = $$kubeconfig; kubectl apply -f https://github.com/grafana/tempo-operator/releases/latest/download/tempo-operator.yaml"
else
	@export KUBECONFIG=$(KUBECONFIG); $(KUBECTL) apply -f https://github.com/grafana/tempo-operator/releases/latest/download/tempo-operator.yaml
endif
	$(call print_success,Tempo Operator updated!)

.PHONY: tracing-update-otel-collector
tracing-update-otel-collector:
	$(call print_info,Updating OpenTelemetry Collector CR...)
	@$(KUBECTL) apply -f tracing/manifests/otel-collector.yaml
	$(call print_success,OTel Collector CR updated!)

.PHONY: tracing-update-tempo-stack
tracing-update-tempo-stack:
	$(call print_info,Updating Tempo Stack CR...)
	@$(KUBECTL) apply -f tracing/manifests/tempo-stack.yaml
	$(call print_success,Tempo Stack CR updated!)

.PHONY: tracing-status
tracing-status:
	@echo ""
	@echo "$(COLOR_BOLD)=== Tracing Stack Status ===$(COLOR_RESET)"
	@echo ""
	@echo "$(COLOR_BOLD)Namespace:$(COLOR_RESET)"
	@$(KUBECTL) get namespace tech-tracing 2>/dev/null || echo "Namespace not found"
	@echo ""
	@echo "$(COLOR_BOLD)Operators:$(COLOR_RESET)"
	@$(KUBECTL) -n tech-tracing get pods -l app.kubernetes.io/name=opentelemetry-operator 2>/dev/null || echo "OTel Operator not found"
	@$(KUBECTL) -n tech-tracing get pods -l app.kubernetes.io/name=tempo-operator 2>/dev/null || echo "Tempo Operator not found"
	@echo ""
	@echo "$(COLOR_BOLD)Tempo Stack:$(COLOR_RESET)"
	@$(KUBECTL) -n tech-tracing get tempostack 2>/dev/null || echo "No TempoStack found"
	@$(KUBECTL) -n tech-tracing get pods -l app.kubernetes.io/instance=tempo-stack 2>/dev/null || echo "No Tempo pods found"
	@echo ""
	@echo "$(COLOR_BOLD)OTel Collector:$(COLOR_RESET)"
	@$(KUBECTL) -n tech-tracing get otelcol 2>/dev/null || echo "No OTel Collector found"
	@$(KUBECTL) -n tech-tracing get pods -l app.kubernetes.io/name=otel-gateway-collector 2>/dev/null || echo "No OTel Collector pods found"
	@echo ""
	@echo "$(COLOR_BOLD)Secrets:$(COLOR_RESET)"
	@$(KUBECTL) -n tech-tracing get secret tempo-s3-secret 2>/dev/null || echo "S3 secret not found"
	@echo ""
	@echo "$(COLOR_BOLD)Services:$(COLOR_RESET)"
	@$(KUBECTL) -n tech-tracing get svc 2>/dev/null || echo "No services found"
	@echo ""

.PHONY: tracing-logs-otel
tracing-logs-otel:
	$(call print_info,Showing OTel Collector logs...)
	@$(KUBECTL) -n tech-tracing logs -l app.kubernetes.io/name=otel-gateway-collector --tail=100 -f

.PHONY: tracing-logs-tempo
tracing-logs-tempo:
	$(call print_info,Showing Tempo logs...)
	@echo "$(COLOR_YELLOW)Select component:$(COLOR_RESET)"
	@echo "  1. Distributor"
	@echo "  2. Ingester"
	@echo "  3. Querier"
	@echo "  4. Query Frontend"
	@echo "  5. Compactor"
	@echo "  6. Gateway"
	@echo ""
	@echo "$(COLOR_YELLOW)Enter choice (1-6):$(COLOR_RESET)"
	@read choice; \
	case $$choice in \
		1) $(KUBECTL) -n tech-tracing logs -l app.kubernetes.io/component=distributor --tail=100 -f ;; \
		2) $(KUBECTL) -n tech-tracing logs -l app.kubernetes.io/component=ingester --tail=100 -f ;; \
		3) $(KUBECTL) -n tech-tracing logs -l app.kubernetes.io/component=querier --tail=100 -f ;; \
		4) $(KUBECTL) -n tech-tracing logs -l app.kubernetes.io/component=query-frontend --tail=100 -f ;; \
		5) $(KUBECTL) -n tech-tracing logs -l app.kubernetes.io/component=compactor --tail=100 -f ;; \
		6) $(KUBECTL) -n tech-tracing logs -l app.kubernetes.io/component=gateway --tail=100 -f ;; \
		*) echo "Invalid choice" ;; \
	esac

.PHONY: tracing-port-forward-tempo
tracing-port-forward-tempo:
	$(call print_info,Port-forwarding Tempo query frontend to localhost:3200...)
	@echo "$(COLOR_YELLOW)Access Tempo at: http://localhost:3200$(COLOR_RESET)"
	@$(KUBECTL) -n tech-tracing port-forward svc/tempo-tempo-stack-query-frontend 3200:3200

.PHONY: tracing-test-query
tracing-test-query:
	$(call print_info,Testing Tempo query API...)
	@$(KUBECTL) -n tech-tracing run tempo-test --rm -i --restart=Never --image=curlimages/curl:latest -- \
		curl -s http://tempo-tempo-stack-query-frontend:3200/api/search?tags=%7B%7D

.PHONY: tracing-uninstall
tracing-uninstall:
	$(call print_warning,This will DELETE all tracing data!)
	@echo "$(COLOR_YELLOW)Press Ctrl+C to cancel, or wait 10 seconds to continue...$(COLOR_RESET)"
	$(call sleep_seconds,10)
	$(call print_info,Uninstalling tracing stack...)
	-@$(KUBECTL) delete -f tracing/manifests/otel-collector.yaml 2>/dev/null || true
	-@$(KUBECTL) delete -f tracing/manifests/tempo-stack.yaml 2>/dev/null || true
ifeq ($(DETECTED_OS),Windows)
	-@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$env:KUBECONFIG = $$kubeconfig; kubectl delete -f https://github.com/grafana/tempo-operator/releases/latest/download/tempo-operator.yaml" 2>nul || echo "Tempo Operator may already be uninstalled"
	-@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$helmfilePath = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, 'tracing', 'helmfile-otel-operator.yaml')); $$env:KUBECONFIG = $$kubeconfig; helmfile -f \"$$helmfilePath\" destroy" 2>nul || echo "OTel Operator may already be uninstalled"
else
	-@$(KUBECTL) delete -f https://github.com/grafana/tempo-operator/releases/latest/download/tempo-operator.yaml || true
	-@$(call run_helmfile,$(HELMFILE_OTEL_OPERATOR),destroy) || true
endif
	-@$(KUBECTL) delete namespace tech-tracing 2>/dev/null || true
	$(call print_success,Tracing stack uninstalled!)