## Monitoring Stack Management
## Metrics monitoring with VictoriaMetrics, Prometheus, and Grafana

# Include common variables and functions
include common/Makefile

# Monitoring configuration
HELMFILE_VM := monitoring/helmfile-victoria-metrics.yaml
HELMFILE_PROMETHEUS := monitoring/helmfile-prometheus.yaml
HELMFILE_GRAFANA := monitoring/helmfile-grafana.yaml

.PHONY: monitoring-help
monitoring-help:
	@echo ""
	@echo "$(COLOR_BOLD)Monitoring Commands:$(COLOR_RESET)"
	@echo "  make monitoring-install-all            - Install complete monitoring stack"
	@echo "  make monitoring-install-victoria-metrics - Install VictoriaMetrics only"
	@echo "  make monitoring-install-prometheus     - Install Prometheus stack only"
	@echo "  make monitoring-install-grafana        - Install Grafana only"
	@echo "  make monitoring-update-victoria-metrics - Update VictoriaMetrics configuration"
	@echo "  make monitoring-update-prometheus      - Update Prometheus configuration"
	@echo "  make monitoring-update-grafana         - Update Grafana configuration"
	@echo "  make monitoring-sync-dashboards        - Sync dashboards from repo to Grafana (full, with delete)"
	@echo "  make monitoring-sync-dashboards-incremental - Sync only changed dashboards (recommended)"
	@echo "  make monitoring-create-telegram-secret - Create Telegram secret (interactive)"
	@echo "  make monitoring-create-nexus-metrics-secret - Create Nexus Prometheus metrics auth secret (interactive)"
	@echo "  make monitoring-import-dashboards      - Import Grafana dashboards"
	@echo "  make monitoring-status                 - Check monitoring stack status"
	@echo "  make monitoring-apply-monitors         - Apply all ServiceMonitor and PodMonitor manifests"
	@echo "  make monitoring-apply-alerts           - Apply all PrometheusRule alerts"
	@echo "  make monitoring-fix-kube-proxy-metrics - Fix kube-proxy metrics endpoint for Prometheus"
	@echo "  make monitoring-uninstall-victoria-metrics - Uninstall VictoriaMetrics only (DELETES DATA!)"
	@echo "  make monitoring-uninstall-prometheus   - Uninstall Prometheus stack only (DELETES DATA!)"
	@echo "  make monitoring-uninstall-grafana      - Uninstall Grafana only (DELETES DATA!)"
	@echo "  make monitoring-uninstall              - Uninstall complete monitoring stack (DELETES DATA!)"
	@echo ""
	@echo "$(COLOR_YELLOW)Note:$(COLOR_RESET) Before installing, ensure:"
	@echo "  1. Longhorn is installed: make longhorn-install-all"
	@echo "  2. Nodes are prepared: cd ../../.. && make k8s-prepare-nodes"

.PHONY: monitoring-check-longhorn
monitoring-check-longhorn:
	$(call check_longhorn)

.PHONY: monitoring-install-all
monitoring-install-all: monitoring-check-longhorn
	$(call print_info,Installing complete monitoring stack...)
	@$(MAKE) monitoring-install-prometheus
	@$(MAKE) monitoring-install-victoria-metrics
	@$(MAKE) monitoring-install-grafana
	$(call print_success,Monitoring stack installed successfully!)
	@echo ""
	@echo "$(COLOR_YELLOW)Next steps:$(COLOR_RESET)"
	@echo "  1. Create Telegram secret: make monitoring-create-telegram-secret"
	@echo "  2. Apply ingresses: kubectl apply -f monitoring/manifests/ingress/"
	@echo "  3. Check status: make monitoring-status"
	@echo "  4. Open Grafana: https://grafana.internal.ai-ops.tech"

.PHONY: monitoring-install-victoria-metrics
monitoring-install-victoria-metrics: monitoring-check-longhorn
	$(call print_info,Creating monitoring namespace...)
	@$(KUBECTL) apply -f monitoring/manifests/base/namespace.yaml
	@$(KUBECTL) apply -f monitoring/manifests/base/resourcequota.yaml
	@$(KUBECTL) apply -f monitoring/manifests/base/storageclass.yaml
	@$(KUBECTL) apply -f monitoring/manifests/base/limitrange.yaml
	$(call print_info,Installing VictoriaMetrics via helmfile...)
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$helmfilePath = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, 'monitoring', 'helmfile-victoria-metrics.yaml')); Write-Host 'Using helmfile: ' $$helmfilePath; if (-not (Test-Path $$helmfilePath)) { Write-Host 'Error: Helmfile not found at: ' $$helmfilePath; exit 1 }; $$env:KUBECONFIG = $$kubeconfig; helmfile -f \"$$helmfilePath\" sync"
else
	$(call run_helmfile,$(HELMFILE_VM),sync)
endif
	$(call print_success,VictoriaMetrics installed successfully!)

.PHONY: monitoring-install-prometheus
monitoring-install-prometheus: monitoring-check-longhorn
	$(call print_info,Creating monitoring namespace...)
	@$(KUBECTL) apply -f monitoring/manifests/base/namespace.yaml
	@$(KUBECTL) apply -f monitoring/manifests/base/resourcequota.yaml
	@$(KUBECTL) apply -f monitoring/manifests/base/storageclass.yaml
	@$(KUBECTL) apply -f monitoring/manifests/base/limitrange.yaml
	$(call print_info,Checking for AlertManager Telegram secret...)
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "\
		$$curDir = [System.IO.Directory]::GetCurrentDirectory(); \
		$$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); \
		$$env:KUBECONFIG = $$kubeconfig; \
		$$secretExists = kubectl -n tech-monitoring get secret alertmanager-telegram-secret 2>$$null; \
		if ($$LASTEXITCODE -ne 0) { \
			Write-Host 'Error: Secret alertmanager-telegram-secret not found in namespace tech-monitoring' -ForegroundColor Red; \
			Write-Host ''; \
			Write-Host 'Please create the secret first:' -ForegroundColor Yellow; \
			Write-Host '  make monitoring-create-telegram-secret' -ForegroundColor Cyan; \
			Write-Host ''; \
			exit 1; \
		} else { \
			Write-Host 'Secret found, checking required keys...' -ForegroundColor Green; \
			$$botToken = kubectl -n tech-monitoring get secret alertmanager-telegram-secret -o jsonpath='{.data.bot_token}' 2>$$null; \
			$$chatId = kubectl -n tech-monitoring get secret alertmanager-telegram-secret -o jsonpath='{.data.chat_id}' 2>$$null; \
			if (-not $$botToken -or -not $$chatId) { \
				Write-Host 'Error: Secret exists but missing required keys (bot_token or chat_id)' -ForegroundColor Red; \
				Write-Host 'Please recreate the secret:' -ForegroundColor Yellow; \
				Write-Host '  make monitoring-create-telegram-secret' -ForegroundColor Cyan; \
				exit 1; \
			} \
		}"
else
	@if ! kubectl -n tech-monitoring get secret alertmanager-telegram-secret >/dev/null 2>&1; then \
		echo "$(COLOR_RED)Error: Secret alertmanager-telegram-secret not found in namespace tech-monitoring$(COLOR_RESET)"; \
		echo ""; \
		echo "$(COLOR_YELLOW)Please create the secret first:$(COLOR_RESET)"; \
		echo "  make monitoring-create-telegram-secret"; \
		echo ""; \
		exit 1; \
	fi; \
	if ! kubectl -n tech-monitoring get secret alertmanager-telegram-secret -o jsonpath='{.data.bot_token}' >/dev/null 2>&1 || \
	   ! kubectl -n tech-monitoring get secret alertmanager-telegram-secret -o jsonpath='{.data.chat_id}' >/dev/null 2>&1; then \
		echo "$(COLOR_RED)Error: Secret exists but missing required keys (bot_token or chat_id)$(COLOR_RESET)"; \
		echo "$(COLOR_YELLOW)Please recreate the secret:$(COLOR_RESET)"; \
		echo "  make monitoring-create-telegram-secret"; \
		exit 1; \
	fi; \
	echo "$(COLOR_GREEN)Secret found, checking required keys...$(COLOR_RESET)"
endif
	$(call print_info,Installing Prometheus stack via helmfile...)
	$(call print_info,Extracting chat_id from secret...)
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "\
		$$curDir = [System.IO.Directory]::GetCurrentDirectory(); \
		$$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); \
		$$valuesFile = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, 'monitoring', 'charts', 'kube-prometheus-stack', 'values.yaml')); \
		$$helmfilePath = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, 'monitoring', 'helmfile-prometheus.yaml')); \
		Write-Host \"Using helmfile: $$helmfilePath\"; \
		if (-not (Test-Path $$helmfilePath)) { Write-Host \"Error: Helmfile not found at: $$helmfilePath\" -ForegroundColor Red; exit 1 }; \
		$$env:KUBECONFIG = $$kubeconfig; \
		try { \
			$$chatId = kubectl -n tech-monitoring get secret alertmanager-telegram-secret -o jsonpath='{.data.chat_id}' 2>$$null | ForEach-Object { [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$_)) }; \
			if (-not $$chatId) { \
				Write-Host 'Error: chat_id not found in secret' -ForegroundColor Red; \
				exit 1; \
			}; \
			Write-Host \"Using chat_id from secret: $$chatId\" -ForegroundColor Green; \
			$$content = Get-Content $$valuesFile -Raw; \
			$$content = $$content -replace 'chat_id: <PROVIDE_CHAT_ID>', \"chat_id: $$chatId\"; \
			Set-Content $$valuesFile $$content; \
			Write-Host 'Applying configuration...' -ForegroundColor Cyan; \
			helmfile -f \"$$helmfilePath\" sync; \
			$$exitCode = $$LASTEXITCODE; \
			Write-Host 'Restoring placeholder...' -ForegroundColor Cyan; \
			$$content = Get-Content $$valuesFile -Raw; \
			$$content = $$content -replace \"chat_id: $$chatId\", 'chat_id: <PROVIDE_CHAT_ID>'; \
			Set-Content $$valuesFile $$content; \
			if ($$exitCode -ne 0) { exit $$exitCode }; \
		} catch { \
			Write-Host 'Error occurred, restoring placeholder...' -ForegroundColor Red; \
			$$content = Get-Content $$valuesFile -Raw -ErrorAction SilentlyContinue; \
			if ($$content) { \
				$$content = $$content -replace 'chat_id: \d+', 'chat_id: <PROVIDE_CHAT_ID>'; \
				Set-Content $$valuesFile $$content; \
			}; \
			throw; \
		}"
else
	@CHAT_ID=$$(kubectl -n tech-monitoring get secret alertmanager-telegram-secret -o jsonpath='{.data.chat_id}' 2>/dev/null | base64 -d) || { \
		echo "$(COLOR_RED)Error: chat_id not found in secret$(COLOR_RESET)"; \
		exit 1; \
	}; \
	echo "Using chat_id: $$CHAT_ID"; \
	VALUES_FILE="monitoring/charts/kube-prometheus-stack/values.yaml"; \
	sed -i.bak "s/chat_id: <PROVIDE_CHAT_ID>/chat_id: $$CHAT_ID/" $$VALUES_FILE; \
	trap "sed -i.bak 's/chat_id: $$CHAT_ID/chat_id: <PROVIDE_CHAT_ID>/' $$VALUES_FILE; rm -f $$VALUES_FILE.bak" EXIT; \
	echo "Applying configuration..."; \
	helmfile -f monitoring/helmfile-prometheus.yaml sync; \
	EXIT_CODE=$$?; \
	sed -i.bak "s/chat_id: $$CHAT_ID/chat_id: <PROVIDE_CHAT_ID>/" $$VALUES_FILE; \
	rm -f $$VALUES_FILE.bak; \
	exit $$EXIT_CODE
endif
	$(call print_info,Applying PrometheusRule for alerts...)
	@$(KUBECTL) apply -f monitoring/alerts/
	$(call print_success,Prometheus stack installed successfully!)

.PHONY: monitoring-install-grafana
monitoring-install-grafana:
	$(call print_info,Creating monitoring namespace...)
	@$(KUBECTL) apply -f monitoring/manifests/base/namespace.yaml
	$(call print_info,Installing Grafana via helmfile...)
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$helmfilePath = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, 'monitoring', 'helmfile-grafana.yaml')); Write-Host 'Using helmfile: ' $$helmfilePath; if (-not (Test-Path $$helmfilePath)) { Write-Host 'Error: Helmfile not found at: ' $$helmfilePath; exit 1 }; $$env:KUBECONFIG = $$kubeconfig; helmfile -f \"$$helmfilePath\" sync"
else
	$(call run_helmfile,$(HELMFILE_GRAFANA),sync)
endif
	$(call print_success,Grafana installed successfully!)
	@echo ""
	@echo "$(COLOR_YELLOW)Grafana credentials:$(COLOR_RESET)"
	@echo "  Username: admin"
	@echo "  Password: admin (change on first login)"

.PHONY: monitoring-update-victoria-metrics
monitoring-update-victoria-metrics:
	$(call print_info,Updating VictoriaMetrics configuration...)
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$helmfilePath = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, 'monitoring', 'helmfile-victoria-metrics.yaml')); $$env:KUBECONFIG = $$kubeconfig; helmfile -f \"$$helmfilePath\" sync"
else
	$(call run_helmfile,$(HELMFILE_VM),sync)
endif
	$(call print_success,VictoriaMetrics updated successfully!)

.PHONY: monitoring-update-prometheus
monitoring-update-prometheus:
	$(call print_info,Updating Prometheus configuration...)
	$(call print_info,Extracting chat_id from secret...)
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "\
		$$curDir = [System.IO.Directory]::GetCurrentDirectory(); \
		$$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); \
		$$valuesFile = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, 'monitoring', 'charts', 'kube-prometheus-stack', 'values.yaml')); \
		$$helmfilePath = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, 'monitoring', 'helmfile-prometheus.yaml')); \
		$$env:KUBECONFIG = $$kubeconfig; \
		try { \
			$$chatId = kubectl -n tech-monitoring get secret alertmanager-telegram-secret -o jsonpath='{.data.chat_id}' 2>$$null | ForEach-Object { [System.Text.Encoding]::UTF8.GetString([System.Convert]::FromBase64String($$_)) }; \
			if (-not $$chatId) { Write-Host 'Error: Could not read chat_id from secret' -ForegroundColor Red; exit 1 }; \
			Write-Host \"Using chat_id from secret: $$chatId\" -ForegroundColor Green; \
			$$content = Get-Content $$valuesFile -Raw; \
			$$content = $$content -replace 'chat_id: <PROVIDE_CHAT_ID>', \"chat_id: $$chatId\"; \
			Set-Content $$valuesFile $$content; \
			Write-Host 'Applying configuration...' -ForegroundColor Cyan; \
			Write-Host 'Note: If you see network errors, check your internet connection and try again' -ForegroundColor Yellow; \
			helmfile -f \"$$helmfilePath\" sync; \
			$$exitCode = $$LASTEXITCODE; \
			Write-Host 'Restoring placeholder...' -ForegroundColor Cyan; \
			$$content = Get-Content $$valuesFile -Raw; \
			$$content = $$content -replace \"chat_id: $$chatId\", 'chat_id: <PROVIDE_CHAT_ID>'; \
			Set-Content $$valuesFile $$content; \
			if ($$exitCode -ne 0) { \
				Write-Host 'Deployment failed, but placeholder was restored.' -ForegroundColor Yellow; \
				Write-Host 'If the error was network-related, try again when connection is stable.' -ForegroundColor Yellow; \
				exit $$exitCode \
			}; \
		} catch { \
			Write-Host 'Error occurred, restoring placeholder...' -ForegroundColor Red; \
			$$content = Get-Content $$valuesFile -Raw -ErrorAction SilentlyContinue; \
			if ($$content) { \
				$$content = $$content -replace 'chat_id: \d+', 'chat_id: <PROVIDE_CHAT_ID>'; \
				Set-Content $$valuesFile $$content; \
			}; \
			throw; \
		}"
else
	@CHAT_ID=$$(kubectl -n tech-monitoring get secret alertmanager-telegram-secret -o jsonpath='{.data.chat_id}' 2>/dev/null | base64 -d) || { echo "Error: Could not read chat_id from secret"; exit 1; }; \
	echo "Using chat_id: $$CHAT_ID"; \
	VALUES_FILE="monitoring/charts/kube-prometheus-stack/values.yaml"; \
	sed -i.bak "s/chat_id: <PROVIDE_CHAT_ID>/chat_id: $$CHAT_ID/" $$VALUES_FILE; \
	trap "sed -i.bak 's/chat_id: $$CHAT_ID/chat_id: <PROVIDE_CHAT_ID>/' $$VALUES_FILE; rm -f $$VALUES_FILE.bak" EXIT; \
	echo "Applying configuration..."; \
	echo "Note: If you see network errors, check your internet connection and try again"; \
	helmfile -f monitoring/helmfile-prometheus.yaml sync; \
	EXIT_CODE=$$?; \
	sed -i.bak "s/chat_id: $$CHAT_ID/chat_id: <PROVIDE_CHAT_ID>/" $$VALUES_FILE; \
	rm -f $$VALUES_FILE.bak; \
	if [ $$EXIT_CODE -ne 0 ]; then \
		echo "Deployment failed, but placeholder was restored."; \
		echo "If the error was network-related, try again when connection is stable."; \
	fi; \
	exit $$EXIT_CODE
endif
	$(call print_success,Prometheus updated successfully!)

.PHONY: monitoring-update-grafana
monitoring-update-grafana:
	$(call print_info,Updating Grafana configuration...)
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$helmfilePath = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, 'monitoring', 'helmfile-grafana.yaml')); $$env:KUBECONFIG = $$kubeconfig; helmfile -f \"$$helmfilePath\" sync"
else
	$(call run_helmfile,$(HELMFILE_GRAFANA),sync)
endif
	$(call print_success,Grafana updated successfully!)

.PHONY: monitoring-sync-dashboards
monitoring-sync-dashboards:
	$(call print_info,Syncing Grafana dashboards from repository via API...)
	@echo "$(COLOR_YELLOW)Note:$(COLOR_RESET) This will import all JSON dashboards from monitoring/dashboards/ to Grafana"
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -ExecutionPolicy Bypass -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$scriptPath = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, 'monitoring', 'scripts', 'sync-dashboards.py')); python \"$$scriptPath\" --kubeconfig \"$$kubeconfig\""
else
	@python3 monitoring/scripts/sync-dashboards.py --kubeconfig "$(KUBECONFIG_REL)"
endif
	$(call print_success,Dashboards synced successfully!)

.PHONY: monitoring-sync-dashboards-incremental
monitoring-sync-dashboards-incremental:
	$(call print_info,Syncing Grafana dashboards incrementally (only changed)...)
	@echo "$(COLOR_YELLOW)Note:$(COLOR_RESET) This will update only changed dashboards based on version/hash"
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -ExecutionPolicy Bypass -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$scriptPath = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, 'monitoring', 'scripts', 'sync-dashboards.py')); python \"$$scriptPath\" --no-delete --kubeconfig \"$$kubeconfig\""
else
	@python3 monitoring/scripts/sync-dashboards.py --no-delete --kubeconfig "$(KUBECONFIG_REL)"
endif
	$(call print_success,Dashboards synced successfully!)


.PHONY: monitoring-create-telegram-secret
monitoring-create-telegram-secret:
	$(call print_info,Creating Telegram credentials secret...)
	@$(call print_info,Ensuring namespace exists...)
	@$(KUBECTL) apply -f monitoring/manifests/base/namespace.yaml
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$namespace = 'tech-monitoring'; $$secretName = 'alertmanager-telegram-secret'; $$env:KUBECONFIG = $$kubeconfig; $$secretExists = kubectl -n $$namespace get secret $$secretName 2>&1; if ($$secretExists -notmatch 'NotFound|not found') { Write-Host 'Secret already exists. Do you want to recreate it? (y/N):' -NoNewline; $$response = Read-Host; if ($$response -ne 'y' -and $$response -ne 'Y') { Write-Host 'Aborted.'; exit 0 }; kubectl -n $$namespace delete secret $$secretName }; Write-Host ''; Write-Host 'To create a Telegram bot:' -ForegroundColor Cyan; Write-Host '1. Open Telegram and search for @BotFather'; Write-Host '2. Send /newbot and follow instructions'; Write-Host '3. Copy the bot token'; Write-Host '4. Add bot to a group and get chat ID'; Write-Host '5. Get chat ID from: https://api.telegram.org/bot<TOKEN>/getUpdates'; Write-Host ''; Write-Host 'Enter Telegram Bot Token:' -NoNewline -ForegroundColor Yellow; $$botToken = Read-Host; Write-Host 'Enter Telegram Chat ID (e.g., -1001234567890):' -NoNewline -ForegroundColor Yellow; $$chatId = Read-Host; if ([string]::IsNullOrWhiteSpace($$botToken) -or [string]::IsNullOrWhiteSpace($$chatId)) { Write-Host 'Error: Bot token and chat ID cannot be empty' -ForegroundColor Red; exit 1 }; kubectl -n $$namespace create secret generic $$secretName --from-literal=bot_token=$$botToken --from-literal=chat_id=$$chatId; Write-Host 'Secret created successfully!' -ForegroundColor Green; Write-Host ''; Write-Host 'Restart Alertmanager:' -ForegroundColor Yellow; Write-Host '  kubectl rollout restart statefulset -n tech-monitoring alertmanager-kube-prometheus-stack-alertmanager'"
else
	@bash monitoring/scripts/create-telegram-secret.sh
endif

.PHONY: monitoring-create-nexus-metrics-secret
monitoring-create-nexus-metrics-secret:
	$(call print_info,Creating Nexus Prometheus metrics authentication secret...)
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$namespace = 'tech-monitoring'; $$secretName = 'nexus-prom-metrics-auth'; $$env:KUBECONFIG = $$kubeconfig; $$secretExists = kubectl -n $$namespace get secret $$secretName 2>&1; if ($$secretExists -notmatch 'NotFound|not found') { Write-Host 'Secret already exists. Do you want to recreate it? (y/N):' -NoNewline; $$response = Read-Host; if ($$response -ne 'y' -and $$response -ne 'Y') { Write-Host 'Aborted.'; exit 0 }; kubectl -n $$namespace delete secret $$secretName }; Write-Host ''; Write-Host 'To create a Nexus user for metrics:' -ForegroundColor Cyan; Write-Host '1. Log in to Nexus UI (http://localhost:8081 via port-forward)'; Write-Host '2. Go to Administration → Users → Create local user'; Write-Host '3. Username: prometheus (or your choice)'; Write-Host '4. Grant role: nx-metrics-all (or create custom role with nx-metrics-read permission)'; Write-Host '5. Save the password'; Write-Host ''; Write-Host 'Enter Nexus username for metrics:' -NoNewline -ForegroundColor Yellow; $$username = Read-Host; Write-Host 'Enter Nexus password:' -NoNewline -ForegroundColor Yellow; $$passwordSecure = Read-Host -AsSecureString; $$passwordPlain = [Runtime.InteropServices.Marshal]::PtrToStringAuto([Runtime.InteropServices.Marshal]::SecureStringToBSTR($$passwordSecure)); if ([string]::IsNullOrWhiteSpace($$username) -or [string]::IsNullOrWhiteSpace($$passwordPlain)) { Write-Host 'Error: Username and password cannot be empty' -ForegroundColor Red; exit 1 }; kubectl -n $$namespace create secret generic $$secretName --from-literal=username=$$username --from-literal=password=$$passwordPlain; Write-Host 'Secret created successfully!' -ForegroundColor Green; Write-Host ''; Write-Host 'Next steps:' -ForegroundColor Yellow; Write-Host '1. Uncomment basicAuth section in monitoring/manifests/monitors/nexus-servicemonitor.yaml'; Write-Host '2. Apply ServiceMonitor: make monitoring-apply-monitors'"
else
	@echo "$(COLOR_CYAN)To create a Nexus user for metrics:$(COLOR_RESET)"
	@echo "1. Log in to Nexus UI (http://localhost:8081 via port-forward)"
	@echo "2. Go to Administration → Users → Create local user"
	@echo "3. Username: prometheus (or your choice)"
	@echo "4. Grant role: nx-metrics-all (or create custom role with nx-metrics-read permission)"
	@echo "5. Save the password"
	@echo ""
	@read -p "Enter Nexus username for metrics: " username; \
	read -sp "Enter Nexus password: " password; \
	echo ""; \
	if [ -z "$$username" ] || [ -z "$$password" ]; then \
		echo "$(COLOR_RED)Error: Username and password cannot be empty$(COLOR_RESET)"; \
		exit 1; \
	fi; \
	SECRET_EXISTS=$$(kubectl -n tech-monitoring get secret nexus-prom-metrics-auth 2>/dev/null); \
	if [ -n "$$SECRET_EXISTS" ]; then \
		read -p "Secret already exists. Do you want to recreate it? (y/N): " response; \
		if [ "$$response" != "y" ] && [ "$$response" != "Y" ]; then \
			echo "Aborted."; \
			exit 0; \
		fi; \
		kubectl -n tech-monitoring delete secret nexus-prom-metrics-auth; \
	fi; \
	kubectl -n tech-monitoring create secret generic nexus-prom-metrics-auth \
		--from-literal=username=$$username --from-literal=password=$$password; \
	echo "$(COLOR_GREEN)Secret created successfully!$(COLOR_RESET)"; \
	echo ""; \
	echo "$(COLOR_YELLOW)Next steps:$(COLOR_RESET)"; \
	echo "1. Uncomment basicAuth section in monitoring/manifests/monitors/nexus-servicemonitor.yaml"; \
	echo "2. Apply ServiceMonitor: make monitoring-apply-monitors"
endif

.PHONY: monitoring-import-dashboards
monitoring-import-dashboards:
	@echo ""
	@echo "$(COLOR_BOLD)=== Import Grafana Dashboards ===$(COLOR_RESET)"
	@echo ""
	@echo "$(COLOR_YELLOW)Usage:$(COLOR_RESET) bash monitoring/scripts/import-dashboards.sh <dashboard_id>"
	@echo ""
	@echo "Popular dashboards:"
	@echo "  7249  - Kubernetes Cluster Monitoring"
	@echo "  1860  - Node Exporter Full"
	@echo "  10229 - VictoriaMetrics"
	@echo "  2     - Prometheus Stats"
	@echo ""
	@echo "Example: bash monitoring/scripts/import-dashboards.sh 7249"
	@echo ""

.PHONY: monitoring-status
monitoring-status:
	@echo ""
	@echo "$(COLOR_BOLD)=== Monitoring Stack Status ===$(COLOR_RESET)"
	@echo ""
	@echo "$(COLOR_BOLD)Namespace:$(COLOR_RESET)"
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$env:KUBECONFIG = $$kubeconfig; $$result = kubectl get namespace tech-monitoring 2>&1; if ($$LASTEXITCODE -eq 0) { Write-Output $$result } else { Write-Output 'Namespace not found' }"
else
	@$(KUBECTL) get namespace tech-monitoring 2>/dev/null || echo "Namespace not found"
endif
	@echo ""
	@echo "$(COLOR_BOLD)VictoriaMetrics:$(COLOR_RESET)"
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$env:KUBECONFIG = $$kubeconfig; $$result = kubectl -n tech-monitoring get pods -l app.kubernetes.io/name=victoria-metrics-single 2>&1; if ($$LASTEXITCODE -eq 0) { Write-Output $$result } else { Write-Output 'VictoriaMetrics not found' }"
else
	@$(KUBECTL) -n tech-monitoring get pods -l app.kubernetes.io/name=victoria-metrics-single 2>/dev/null || echo "VictoriaMetrics not found"
endif
	@echo ""
	@echo "$(COLOR_BOLD)Prometheus:$(COLOR_RESET)"
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$env:KUBECONFIG = $$kubeconfig; $$result = kubectl -n tech-monitoring get pods -l app.kubernetes.io/name=prometheus 2>&1; if ($$LASTEXITCODE -eq 0) { Write-Output $$result } else { Write-Output 'Prometheus not found' }"
else
	@$(KUBECTL) -n tech-monitoring get pods -l app.kubernetes.io/name=prometheus 2>/dev/null || echo "Prometheus not found"
endif
	@echo ""
	@echo "$(COLOR_BOLD)Alertmanager:$(COLOR_RESET)"
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$env:KUBECONFIG = $$kubeconfig; $$result = kubectl -n tech-monitoring get pods -l app.kubernetes.io/name=alertmanager 2>&1; if ($$LASTEXITCODE -eq 0) { Write-Output $$result } else { Write-Output 'Alertmanager not found' }"
else
	@$(KUBECTL) -n tech-monitoring get pods -l app.kubernetes.io/name=alertmanager 2>/dev/null || echo "Alertmanager not found"
endif
	@echo ""
	@echo "$(COLOR_BOLD)Grafana:$(COLOR_RESET)"
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$env:KUBECONFIG = $$kubeconfig; $$result = kubectl -n tech-monitoring get pods -l app.kubernetes.io/name=grafana 2>&1; if ($$LASTEXITCODE -eq 0) { Write-Output $$result } else { Write-Output 'Grafana not found' }"
else
	@$(KUBECTL) -n tech-monitoring get pods -l app.kubernetes.io/name=grafana 2>/dev/null || echo "Grafana not found"
endif
	@echo ""
	@echo "$(COLOR_BOLD)PVCs:$(COLOR_RESET)"
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$env:KUBECONFIG = $$kubeconfig; $$result = kubectl -n tech-monitoring get pvc 2>&1; if ($$LASTEXITCODE -eq 0) { Write-Output $$result } else { Write-Output 'No PVCs found' }"
else
	@$(KUBECTL) -n tech-monitoring get pvc 2>/dev/null || echo "No PVCs found"
endif

.PHONY: monitoring-apply-monitors
monitoring-apply-monitors:
	$(call print_info,Applying all ServiceMonitor and PodMonitor manifests...)
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$monitorsDir = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, 'monitoring', 'manifests', 'monitors')); $$env:KUBECONFIG = $$kubeconfig; if (Test-Path $$monitorsDir) { Get-ChildItem -Path $$monitorsDir -Filter '*.yaml' -File | Where-Object { $$_.Name -ne 'examples.yaml' } | ForEach-Object { $$file = $$_; $$fileName = $$($$file.Name); $$filePath = $$($$file.FullName); Write-Host \"Applying $$fileName...\" -ForegroundColor Cyan; kubectl apply -f \"$$filePath\" } } else { Write-Host 'Directory not found: $$monitorsDir' -ForegroundColor Red; exit 1 }"
else
	@if [ -d "monitoring/manifests/monitors" ]; then \
		for file in monitoring/manifests/monitors/*.yaml; do \
			if [ -f "$$file" ] && [ "$$(basename $$file)" != "examples.yaml" ]; then \
				echo "Applying $$(basename $$file)..."; \
				$(KUBECTL) apply -f "$$file"; \
			fi; \
		done; \
	else \
		echo "$(COLOR_RED)Directory not found: monitoring/manifests/monitors$(COLOR_RESET)"; \
		exit 1; \
	fi
endif
	$(call print_success,All ServiceMonitor and PodMonitor manifests applied successfully!)

.PHONY: monitoring-apply-alerts
monitoring-apply-alerts:
	$(call print_info,Applying all PrometheusRule alerts...)
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$alertsDir = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, 'monitoring', 'alerts')); $$env:KUBECONFIG = $$kubeconfig; if (Test-Path $$alertsDir) { Get-ChildItem -Path $$alertsDir -Filter '*.yaml' -File | Where-Object { $$_.Name -ne 'README.md' } | ForEach-Object { $$file = $$_; $$fileName = $$($$file.Name); $$filePath = $$($$file.FullName); Write-Host \"Applying $$fileName...\" -ForegroundColor Cyan; kubectl apply -f \"$$filePath\" } } else { Write-Host 'Directory not found: $$alertsDir' -ForegroundColor Red; exit 1 }"
else
	@if [ -d "monitoring/alerts" ]; then \
		for file in monitoring/alerts/*.yaml; do \
			if [ -f "$$file" ]; then \
				echo "Applying $$(basename $$file)..."; \
				$(KUBECTL) apply -f "$$file"; \
			fi; \
		done; \
	else \
		echo "$(COLOR_RED)Directory not found: monitoring/alerts$(COLOR_RESET)"; \
		exit 1; \
	fi
endif
	$(call print_success,All PrometheusRule alerts applied successfully!)

.PHONY: monitoring-fix-kube-proxy-metrics
monitoring-fix-kube-proxy-metrics:
	$(call print_info,Fixing kube-proxy metrics endpoint for Prometheus...)
	@echo "This will update metricsBindAddress from empty/127.0.0.1:10249 to 0.0.0.0:10249"
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$env:KUBECONFIG = $$kubeconfig; $$config = kubectl get configmap kube-proxy -n kube-system -o jsonpath='{.data.config\.conf}'; if ($$config -match 'metricsBindAddress:\s*\"0\.0\.0\.0:10249\"') { Write-Output 'kube-proxy metricsBindAddress already configured correctly' } else { $$updated = $$config -replace 'metricsBindAddress:\s*[^\r\n]*', 'metricsBindAddress: \"0.0.0.0:10249\"'; $$tempFile = [System.IO.Path]::GetTempFileName(); [System.IO.File]::WriteAllText($$tempFile, $$updated); kubectl create configmap kube-proxy --from-file=config.conf=$$tempFile -n kube-system --dry-run=client -o yaml | kubectl apply -f -; Remove-Item $$tempFile; kubectl rollout restart daemonset kube-proxy -n kube-system; Write-Output 'kube-proxy ConfigMap updated and DaemonSet restarted' }"
else
	@$(KUBECTL) get configmap kube-proxy -n kube-system -o jsonpath='{.data.config\.conf}' > /tmp/kube-proxy-config.conf && \
		sed -i 's/metricsBindAddress:[[:space:]]*[^[:space:]]*/metricsBindAddress: "0.0.0.0:10249"/' /tmp/kube-proxy-config.conf && \
		$(KUBECTL) create configmap kube-proxy --from-file=config.conf=/tmp/kube-proxy-config.conf -n kube-system --dry-run=client -o yaml | \
		$(KUBECTL) apply -f - && \
		rm -f /tmp/kube-proxy-config.conf && \
		$(KUBECTL) rollout restart daemonset kube-proxy -n kube-system && \
		echo "kube-proxy ConfigMap updated and DaemonSet restarted" || \
		echo "kube-proxy metricsBindAddress already configured correctly"
endif
	$(call print_success,kube-proxy metrics endpoint fixed!)
	@echo "Wait a few seconds for pods to restart, then check Prometheus targets"

.PHONY: monitoring-uninstall-victoria-metrics
monitoring-uninstall-victoria-metrics:
	$(call print_warning,This will DELETE VictoriaMetrics data!)
	@echo "$(COLOR_YELLOW)Press Ctrl+C to cancel, or wait 5 seconds to continue...$(COLOR_RESET)"
	$(call sleep_seconds,5)
	$(call print_info,Uninstalling VictoriaMetrics...)
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$helmfilePath = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, 'monitoring', 'helmfile-victoria-metrics.yaml')); $$env:KUBECONFIG = $$kubeconfig; helmfile -f \"$$helmfilePath\" destroy" || true
else
	-$(call run_helmfile,$(HELMFILE_VM),destroy)
endif
	$(call print_success,VictoriaMetrics uninstalled!)

.PHONY: monitoring-uninstall-prometheus
monitoring-uninstall-prometheus:
	$(call print_warning,This will DELETE Prometheus and Alertmanager data!)
	@echo "$(COLOR_YELLOW)Press Ctrl+C to cancel, or wait 5 seconds to continue...$(COLOR_RESET)"
	$(call sleep_seconds,5)
	$(call print_info,Uninstalling Prometheus stack...)
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$helmfilePath = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, 'monitoring', 'helmfile-prometheus.yaml')); $$env:KUBECONFIG = $$kubeconfig; helmfile -f \"$$helmfilePath\" destroy" || true
else
	-$(call run_helmfile,$(HELMFILE_PROMETHEUS),destroy)
endif
	$(call print_info,Deleting PrometheusRule resources...)
	-@$(KUBECTL) delete prometheusrule -n tech-monitoring --all 2>/dev/null || true
	$(call print_success,Prometheus stack uninstalled!)

.PHONY: monitoring-uninstall-grafana
monitoring-uninstall-grafana:
	$(call print_warning,This will DELETE Grafana data!)
	@echo "$(COLOR_YELLOW)Press Ctrl+C to cancel, or wait 5 seconds to continue...$(COLOR_RESET)"
	$(call sleep_seconds,5)
	$(call print_info,Uninstalling Grafana...)
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$helmfilePath = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, 'monitoring', 'helmfile-grafana.yaml')); $$env:KUBECONFIG = $$kubeconfig; helmfile -f \"$$helmfilePath\" destroy" || true
else
	-$(call run_helmfile,$(HELMFILE_GRAFANA),destroy)
endif
	$(call print_success,Grafana uninstalled!)

.PHONY: monitoring-uninstall
monitoring-uninstall:
	$(call print_warning,This will DELETE all monitoring data!)
	@echo "$(COLOR_YELLOW)Press Ctrl+C to cancel, or wait 10 seconds to continue...$(COLOR_RESET)"
	$(call sleep_seconds,10)
	$(call print_info,Uninstalling Grafana...)
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$helmfilePath = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, 'monitoring', 'helmfile-grafana.yaml')); $$env:KUBECONFIG = $$kubeconfig; helmfile -f \"$$helmfilePath\" destroy" || true
else
	-$(call run_helmfile,$(HELMFILE_GRAFANA),destroy)
endif
	$(call print_info,Uninstalling Prometheus stack...)
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$helmfilePath = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, 'monitoring', 'helmfile-prometheus.yaml')); $$env:KUBECONFIG = $$kubeconfig; helmfile -f \"$$helmfilePath\" destroy" || true
else
	-$(call run_helmfile,$(HELMFILE_PROMETHEUS),destroy)
endif
	$(call print_info,Uninstalling VictoriaMetrics...)
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$helmfilePath = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, 'monitoring', 'helmfile-victoria-metrics.yaml')); $$env:KUBECONFIG = $$kubeconfig; helmfile -f \"$$helmfilePath\" destroy" || true
else
	-$(call run_helmfile,$(HELMFILE_VM),destroy)
endif
	$(call print_info,Deleting namespace and resources...)
	-@$(KUBECTL) delete namespace tech-monitoring --wait=false
	$(call print_success,Monitoring stack uninstalled!)