## Logging Stack Management
## Centralized log collection with Loki and Fluent Bit

# Include common variables and functions
include common/Makefile

# Logging configuration
HELMFILE_LOKI := logging/helmfile-loki.yaml
HELMFILE_FLUENT_BIT := logging/helmfile-fluent-bit.yaml

.PHONY: logging-help
logging-help:
	@echo ""
	@echo "$(COLOR_BOLD)Logging Commands:$(COLOR_RESET)"
	@echo "  make logging-install-all      - Install complete logging stack (Loki + Fluent Bit)"
	@echo "  make logging-install-loki     - Install Loki only"
	@echo "  make logging-install-fluent   - Install Fluent Bit only"
	@echo "  make logging-update-loki      - Update Loki configuration"
	@echo "  make logging-update-fluent    - Update Fluent Bit configuration"
	@echo "  make logging-create-s3-secret - Create S3 credentials secret (interactive)"
	@echo "  make logging-create-buckets   - Create S3 buckets in MinIO (interactive)"
	@echo "  make logging-status           - Check logging stack status"
	@echo "  make logging-logs-loki        - Show Loki logs"
	@echo "  make logging-logs-fluent      - Show Fluent Bit logs"
	@echo "  make logging-port-forward     - Port-forward Loki (http://localhost:3100)"
	@echo "  make logging-test-query       - Test Loki query API"
	@echo "  make logging-verify-collection - Verify log collection and S3 storage"
	@echo "  make logging-uninstall-loki   - Uninstall Loki only (keeps Fluent Bit + namespace)"
	@echo "  make logging-uninstall-fluent - Uninstall Fluent Bit only (keeps Loki + namespace)"
	@echo "  make logging-uninstall        - Uninstall logging stack (DELETES DATA!)"
	@echo ""
	@echo "$(COLOR_YELLOW)Note:$(COLOR_RESET) Before installing, ensure:"
	@echo "  1. Longhorn is installed: make longhorn-install-all"
	@echo "  2. MinIO S3 is installed: make minio-install-all"
	@echo "  3. Nodes are prepared: cd ../.. && make k8s-prepare-nodes"

.PHONY: logging-check-longhorn
logging-check-longhorn:
	$(call check_longhorn)

.PHONY: logging-check-minio
logging-check-minio:
	@echo ""
	@echo "$(COLOR_BOLD)=== Checking MinIO availability ===$(COLOR_RESET)"
ifeq ($(DETECTED_OS),Windows)
	@$(KUBECTL) get namespace tech-minio-tenants >nul 2>&1 || \
		($(call print_warning,MinIO namespace not found. MinIO may not be installed.) && \
		 echo "$(COLOR_YELLOW)Please install MinIO first:$(COLOR_RESET)" && \
		 echo "  make minio-install-all" && \
		 exit 1)
	@$(KUBECTL) -n tech-minio-tenants get tenant s3-public >nul 2>&1 || \
		($(call print_warning,MinIO tenant s3-public not found.) && \
		 echo "$(COLOR_YELLOW)Please create MinIO tenant first:$(COLOR_RESET)" && \
		 echo "  make minio-create-tenant" && \
		 exit 1)
else
	@$(KUBECTL) get namespace tech-minio-tenants >/dev/null 2>&1 || \
		($(call print_warning,MinIO namespace not found. MinIO may not be installed.) && \
		 echo "$(COLOR_YELLOW)Please install MinIO first:$(COLOR_RESET)" && \
		 echo "  make minio-install-all" && \
		 exit 1)
	@$(KUBECTL) -n tech-minio-tenants get tenant s3-public >/dev/null 2>&1 || \
		($(call print_warning,MinIO tenant s3-public not found.) && \
		 echo "$(COLOR_YELLOW)Please create MinIO tenant first:$(COLOR_RESET)" && \
		 echo "  make minio-create-tenant" && \
		 exit 1)
endif
	$(call print_success,MinIO is available)

.PHONY: logging-install-all
logging-install-all: logging-check-longhorn logging-check-minio
	$(call print_info,Installing complete logging stack...)
	@$(MAKE) logging-install-loki
	@$(MAKE) logging-install-fluent
	$(call print_success,Logging stack installed successfully!)
	@echo ""
	@echo "$(COLOR_YELLOW)Next steps:$(COLOR_RESET)"
	@echo "  1. Create S3 secret: make logging-create-s3-secret"
	@echo "  2. Create S3 buckets: make logging-create-buckets"
	@echo "  3. Check status: make logging-status"
	@echo "  4. Test query: make logging-test-query"

.PHONY: logging-install-loki
logging-install-loki: logging-check-longhorn logging-check-minio
	$(call print_info,Creating logging namespace...)
	@$(KUBECTL) apply -f logging/manifests/tech-logging.yaml
	$(call print_info,Applying logging resource policies...)
	@$(KUBECTL) apply -f logging/manifests/limitrange.yaml
	@$(KUBECTL) apply -f logging/manifests/resourcequota.yaml
	@$(KUBECTL) apply -f logging/manifests/networkpolicy.yaml
	@$(KUBECTL) apply -f logging/manifests/storageclass.yaml
	$(call print_info,Installing Loki via helmfile...)
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$helmfilePath = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, 'logging', 'helmfile-loki.yaml')); Write-Host 'Using helmfile: ' $$helmfilePath; if (-not (Test-Path $$helmfilePath)) { Write-Host 'Error: Helmfile not found at: ' $$helmfilePath; exit 1 }; $$content = Get-Content $$helmfilePath -Raw; if ($$content -notmatch 'loki') { Write-Host 'Error: Wrong helmfile content!'; exit 1 }; $$env:KUBECONFIG = $$kubeconfig; helmfile -f \"$$helmfilePath\" sync"
else
	$(call run_helmfile,$(HELMFILE_LOKI),sync)
endif
	$(call print_success,Loki installed successfully!)
	@echo ""
	@echo "$(COLOR_YELLOW)Important:$(COLOR_RESET) Create S3 secret before Loki can start:"
	@echo "  make logging-create-s3-secret"

.PHONY: logging-install-fluent
logging-install-fluent:
	$(call print_info,Installing Fluent Bit via helmfile...)
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$helmfilePath = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, 'logging', 'helmfile-fluent-bit.yaml')); Write-Host 'Using helmfile: ' $$helmfilePath; if (-not (Test-Path $$helmfilePath)) { Write-Host 'Error: Helmfile not found at: ' $$helmfilePath; exit 1 }; $$content = Get-Content $$helmfilePath -Raw; if ($$content -notmatch 'fluent-bit') { Write-Host 'Error: Wrong helmfile content!'; exit 1 }; $$env:KUBECONFIG = $$kubeconfig; helmfile -f \"$$helmfilePath\" sync"
else
	$(call run_helmfile,$(HELMFILE_FLUENT_BIT),sync)
endif
	$(call print_success,Fluent Bit installed successfully!)

.PHONY: logging-update-loki
logging-update-loki:
	$(call print_info,Updating Loki configuration...)
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$helmfilePath = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, 'logging', 'helmfile-loki.yaml')); $$env:KUBECONFIG = $$kubeconfig; helmfile -f \"$$helmfilePath\" sync"
else
	$(call run_helmfile,$(HELMFILE_LOKI),sync)
endif
	$(call print_success,Loki updated successfully!)

.PHONY: logging-update-fluent
logging-update-fluent:
	$(call print_info,Updating Fluent Bit configuration...)
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$helmfilePath = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, 'logging', 'helmfile-fluent-bit.yaml')); $$env:KUBECONFIG = $$kubeconfig; helmfile -f \"$$helmfilePath\" sync"
else
	$(call run_helmfile,$(HELMFILE_FLUENT_BIT),sync)
endif
	$(call print_success,Fluent Bit updated successfully!)

.PHONY: logging-create-s3-secret
logging-create-s3-secret:
	$(call print_info,Creating S3 credentials secret...)
	@$(call print_info,Ensuring namespace exists...)
	@$(KUBECTL) apply -f logging/manifests/tech-logging.yaml
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$namespace = 'tech-logging'; $$secretName = 'loki-s3-secret'; $$env:KUBECONFIG = $$kubeconfig; $$secretExists = kubectl -n $$namespace get secret $$secretName 2>&1; if ($$secretExists -notmatch 'NotFound|not found') { Write-Host 'Secret already exists. Do you want to recreate it? (y/N):' -NoNewline; $$response = Read-Host; if ($$response -ne 'y' -and $$response -ne 'Y') { Write-Host 'Aborted.'; exit 0 }; kubectl -n $$namespace delete secret $$secretName }; Write-Host 'Enter MinIO S3 Access Key ID:' -NoNewline; $$accessKey = Read-Host; Write-Host 'Enter MinIO S3 Secret Access Key:' -NoNewline; $$secretKey = Read-Host -AsSecureString; $$secretKeyPlain = [Runtime.InteropServices.Marshal]::PtrToStringAuto([Runtime.InteropServices.Marshal]::SecureStringToBSTR($$secretKey)); if ([string]::IsNullOrWhiteSpace($$accessKey) -or [string]::IsNullOrWhiteSpace($$secretKeyPlain)) { Write-Host 'Error: Access Key ID and Secret Access Key cannot be empty' -ForegroundColor Red; exit 1 }; kubectl -n $$namespace create secret generic $$secretName --from-literal=accessKeyId=$$accessKey --from-literal=secretAccessKey=$$secretKeyPlain; Write-Host 'Secret created successfully!' -ForegroundColor Green; kubectl -n $$namespace get secret $$secretName"
else
	@bash logging/scripts/create-s3-secret.sh
endif

.PHONY: logging-create-buckets
logging-create-buckets:
	@echo ""
	@echo "$(COLOR_BOLD)=== Creating Loki S3 Buckets in MinIO ===$(COLOR_RESET)"
	@echo ""
	@echo "$(COLOR_YELLOW)Please create the following buckets manually through MinIO Console:$(COLOR_RESET)"
	@echo ""
	@echo "$(COLOR_BOLD)Required buckets:$(COLOR_RESET)"
	@echo "  - loki-chunks"
	@echo "  - loki-ruler"
	@echo "  - loki-admin"
	@echo ""
	@echo "$(COLOR_BOLD)Instructions:$(COLOR_RESET)"
	@echo "  1. Open MinIO Console: https://s3.internal.ai-ops.tech"
	@echo "  2. Login with your MinIO credentials"
	@echo "  3. Go to 'Buckets' section (left sidebar)"
	@echo "  4. Click 'Create Bucket' button"
	@echo "  5. Create each bucket with the following names:"
	@echo "     - loki-chunks"
	@echo "     - loki-ruler"
	@echo "     - loki-admin"
	@echo "  6. Leave all bucket settings as default (no versioning, no encryption)"
	@echo ""
	@echo "$(COLOR_BOLD)Recommended: Set Quotas (100 GB total for logging)$(COLOR_RESET)"
	@echo "  After creating buckets, set size quotas to limit storage usage:"
	@echo ""
	@echo "  For 'loki-chunks' bucket (main log storage):"
	@echo "    - Recommended: 95 GB"
	@echo "    - This is the main storage for log chunks (compressed)"
	@echo ""
	@echo "  For 'loki-ruler' bucket (alerting rules):"
	@echo "    - Recommended: 4 GB"
	@echo "    - Alerting rules are small files, but quota prevents accidental growth"
	@echo ""
	@echo "  For 'loki-admin' bucket (admin data):"
	@echo "    - Recommended: 1 GB"
	@echo "    - Admin data is minimal, but quota prevents accidental growth"
	@echo ""
	@echo "  To set quota in MinIO Console:"
	@echo "    1. Select the bucket (e.g., loki-chunks)"
	@echo "    2. Go to 'Summary' tab"
	@echo "    3. Click 'Set Quota' button"
	@echo "    4. Select 'Size' quota type"
	@echo "    5. Enter quota value (e.g., 95GB for loki-chunks)"
	@echo "    6. Click 'Set'"
	@echo "    7. Repeat for other buckets"
	@echo ""
	@echo "$(COLOR_YELLOW)Total allocated: 100 GB (95 + 4 + 1 GB)$(COLOR_RESET)"
	@echo ""
	@echo "$(COLOR_YELLOW)After creating all buckets and setting quotas, continue with:$(COLOR_RESET)"
	@echo "  make logging-create-s3-secret"
	@echo ""

.PHONY: logging-status
logging-status:
	@echo ""
	@echo "$(COLOR_BOLD)=== Logging Stack Status ===$(COLOR_RESET)"
	@echo ""
	@echo "$(COLOR_BOLD)Namespace:$(COLOR_RESET)"
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$env:KUBECONFIG = $$kubeconfig; $$result = kubectl get namespace tech-logging 2>&1; if ($$LASTEXITCODE -eq 0) { Write-Output $$result } else { Write-Output 'Namespace not found' }"
else
	@$(KUBECTL) get namespace tech-logging 2>/dev/null || echo "Namespace not found"
endif
	@echo ""
	@echo "$(COLOR_BOLD)Loki Pods:$(COLOR_RESET)"
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$env:KUBECONFIG = $$kubeconfig; $$result = kubectl -n tech-logging get pods -l app.kubernetes.io/name=loki 2>&1; if ($$LASTEXITCODE -eq 0) { Write-Output $$result } else { Write-Output 'No Loki pods found' }"
else
	@$(KUBECTL) -n tech-logging get pods -l app.kubernetes.io/name=loki 2>/dev/null || echo "No Loki pods found"
endif
	@echo ""
	@echo "$(COLOR_BOLD)Fluent Bit Pods:$(COLOR_RESET)"
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$env:KUBECONFIG = $$kubeconfig; $$result = kubectl -n tech-logging get pods -l app=fluent-bit 2>&1; if ($$LASTEXITCODE -eq 0) { Write-Output $$result } else { Write-Output 'No Fluent Bit pods found' }"
else
	@$(KUBECTL) -n tech-logging get pods -l app=fluent-bit 2>/dev/null || echo "No Fluent Bit pods found"
endif
	@echo ""
	@echo "$(COLOR_BOLD)Services:$(COLOR_RESET)"
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$env:KUBECONFIG = $$kubeconfig; $$result = kubectl -n tech-logging get svc 2>&1; if ($$LASTEXITCODE -eq 0) { Write-Output $$result } else { Write-Output 'No services found' }"
else
	@$(KUBECTL) -n tech-logging get svc 2>/dev/null || echo "No services found"
endif
	@echo ""
	@echo "$(COLOR_BOLD)PVCs:$(COLOR_RESET)"
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$env:KUBECONFIG = $$kubeconfig; $$result = kubectl -n tech-logging get pvc 2>&1; if ($$LASTEXITCODE -eq 0) { Write-Output $$result } else { Write-Output 'No PVCs found' }"
else
	@$(KUBECTL) -n tech-logging get pvc 2>/dev/null || echo "No PVCs found"
endif
	@echo ""
	@echo "$(COLOR_BOLD)Secrets:$(COLOR_RESET)"
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$env:KUBECONFIG = $$kubeconfig; $$result = kubectl -n tech-logging get secrets 2>&1; if ($$LASTEXITCODE -eq 0) { $$result | Select-String -Pattern 'NAME|loki-s3-secret' } else { Write-Output 'S3 secret not found' }"
else
	@$(KUBECTL) -n tech-logging get secrets 2>/dev/null | grep -E "NAME|loki-s3-secret" || echo "S3 secret not found"
endif

.PHONY: logging-logs-loki
logging-logs-loki:
	$(call print_info,Showing Loki logs...)
	@$(KUBECTL) -n tech-logging logs -l app.kubernetes.io/name=loki --tail=100 -f

.PHONY: logging-logs-fluent
logging-logs-fluent:
	$(call print_info,Showing Fluent Bit logs...)
	@$(KUBECTL) -n tech-logging logs -l app=fluent-bit --tail=100 --max-log-requests=10 -f

.PHONY: logging-port-forward
logging-port-forward:
	$(call print_info,Port-forwarding Loki to localhost:3100...)
	@echo "$(COLOR_YELLOW)Access Loki at: http://localhost:3100$(COLOR_RESET)"
	@echo "$(COLOR_YELLOW)Press Ctrl+C to stop$(COLOR_RESET)"
	@$(KUBECTL) -n tech-logging port-forward svc/loki-gateway 3100:80

.PHONY: logging-test-query
logging-test-query:
	$(call print_info,Testing Loki query API...)
	@echo ""
	@echo "$(COLOR_BOLD)Labels:$(COLOR_RESET)"
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$env:KUBECONFIG = $$kubeconfig; $$pod = kubectl -n tech-logging get pods -l app.kubernetes.io/component=gateway -o jsonpath='{.items[0].metadata.name}' 2>&1; if ($$LASTEXITCODE -eq 0 -and $$pod) { $$result = kubectl -n tech-logging exec $$pod -- wget -qO- http://loki-gateway.tech-logging.svc.cluster.local/loki/api/v1/labels 2>&1; if ($$LASTEXITCODE -eq 0) { Write-Output $$result } else { Write-Output 'Failed to query labels' } } else { Write-Output 'Loki gateway pod not found' }"
else
	@POD=$$($(KUBECTL) -n tech-logging get pods -l app.kubernetes.io/component=gateway -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) && \
	if [ -n "$$POD" ]; then \
		$(KUBECTL) -n tech-logging exec $$POD -- wget -qO- http://loki-gateway.tech-logging.svc.cluster.local/loki/api/v1/labels 2>/dev/null || echo "Failed to query labels"; \
	else \
		echo "Loki gateway pod not found"; \
	fi
endif
	@echo ""
	@echo "$(COLOR_BOLD)Recent logs (last 10 minutes):$(COLOR_RESET)"
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$env:KUBECONFIG = $$kubeconfig; $$pod = kubectl -n tech-logging get pods -l app.kubernetes.io/component=gateway -o jsonpath='{.items[0].metadata.name}' 2>&1; if ($$LASTEXITCODE -eq 0 -and $$pod) { $$query = 'http://loki-gateway.tech-logging.svc.cluster.local/loki/api/v1/query_range?query={namespace=\"kube-system\"}&limit=10'; $$result = kubectl -n tech-logging exec $$pod -- wget -qO- $$query 2>&1; if ($$LASTEXITCODE -eq 0) { Write-Output $$result } else { Write-Output 'Failed to query logs' } } else { Write-Output 'Loki gateway pod not found' }"
else
	@POD=$$($(KUBECTL) -n tech-logging get pods -l app.kubernetes.io/component=gateway -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) && \
	if [ -n "$$POD" ]; then \
		$(KUBECTL) -n tech-logging exec $$POD -- wget -qO- 'http://loki-gateway.tech-logging.svc.cluster.local/loki/api/v1/query_range?query={namespace="kube-system"}&limit=10' 2>/dev/null || echo "Failed to query logs"; \
	else \
		echo "Loki gateway pod not found"; \
	fi
endif

.PHONY: logging-verify-collection
logging-verify-collection:
	$(call print_info,Verifying log collection and S3 storage...)
	@echo ""
	@echo "$(COLOR_BOLD)1. Checking Fluent Bit status:$(COLOR_RESET)"
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$env:KUBECONFIG = $$kubeconfig; $$result = kubectl get daemonset -n tech-logging fluent-bit -o jsonpath='{.status.numberReady}/{.status.desiredNumberScheduled} pods ready' 2>&1; if ($$LASTEXITCODE -eq 0 -and $$result) { Write-Output $$result } else { Write-Output 'Fluent Bit not found' }"
else
	@$(KUBECTL) get daemonset -n tech-logging fluent-bit -o jsonpath='{.status.numberReady}/{.status.desiredNumberScheduled} pods ready' 2>/dev/null || echo "Fluent Bit not found"
endif
	@echo ""
	@echo "$(COLOR_BOLD)2. Checking Loki status:$(COLOR_RESET)"
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$env:KUBECONFIG = $$kubeconfig; $$result = kubectl get deployment -n tech-logging loki-gateway -o jsonpath='{.status.readyReplicas}/{.status.replicas} replicas ready' 2>&1; if ($$LASTEXITCODE -eq 0 -and $$result) { Write-Output $$result } else { Write-Output 'Loki gateway not found' }"
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$env:KUBECONFIG = $$kubeconfig; $$result = kubectl get statefulset -n tech-logging loki-backend -o jsonpath='{.status.readyReplicas}/{.status.replicas} replicas ready' 2>&1; if ($$LASTEXITCODE -eq 0 -and $$result) { Write-Output $$result } else { Write-Output 'Loki backend (single-binary) not found or not running' }"
else
	@$(KUBECTL) get deployment -n tech-logging loki-gateway -o jsonpath='{.status.readyReplicas}/{.status.replicas} replicas ready' 2>/dev/null || echo "Loki gateway not found"
	@$(KUBECTL) get statefulset -n tech-logging loki-backend -o jsonpath='{.status.readyReplicas}/{.status.replicas} replicas ready' 2>/dev/null || echo "Loki backend (single-binary) not found or not running"
endif
	@echo ""
	@echo "$(COLOR_BOLD)3. Checking Fluent Bit metrics (logs collected):$(COLOR_RESET)"
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$env:KUBECONFIG = $$kubeconfig; $$pod = kubectl -n tech-logging get pods -l app.kubernetes.io/name=fluent-bit -o jsonpath='{.items[0].metadata.name}' 2>&1; if ($$LASTEXITCODE -eq 0 -and $$pod) { $$result = kubectl -n tech-logging exec $$pod -- wget -qO- http://localhost:2020/api/v1/metrics/prometheus 2>&1 | Select-String -Pattern 'fluentbit_input_bytes_total|fluentbit_output_proc_records_total' | Select-Object -First 5; if ($$result) { Write-Output $$result } else { Write-Output 'No metrics found (Fluent Bit may still be starting)' } } else { Write-Output 'Fluent Bit pod not found' }"
else
	@POD=$$($(KUBECTL) -n tech-logging get pods -l app.kubernetes.io/name=fluent-bit -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) && \
	if [ -n "$$POD" ]; then \
		$(KUBECTL) -n tech-logging exec $$POD -- wget -qO- http://localhost:2020/api/v1/metrics/prometheus 2>/dev/null | grep -E 'fluentbit_input_bytes_total|fluentbit_output_proc_records_total' | head -5 || echo "No metrics found (Fluent Bit may still be starting)"; \
	else \
		echo "Fluent Bit pod not found"; \
	fi
endif
	@echo ""
	@echo "$(COLOR_BOLD)4. Checking Loki metrics (logs ingested):$(COLOR_RESET)"
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$env:KUBECONFIG = $$kubeconfig; $$pod = kubectl -n tech-logging get pods -l app.kubernetes.io/component=backend -o jsonpath='{.items[0].metadata.name}' 2>&1; if ($$LASTEXITCODE -eq 0 -and $$pod -and $$pod -notmatch 'No resources found') { $$result = kubectl -n tech-logging exec $$pod -- wget -qO- http://localhost:3100/metrics 2>&1 | Select-String -Pattern 'loki_ingester_chunks_created_total|loki_distributor_lines_received_total|loki_ingester_chunk_stored_bytes_total' | Select-Object -First 5; if ($$result) { Write-Output $$result } else { Write-Output 'No metrics found (Loki may still be starting)' } } else { Write-Output 'Loki backend pod not found (check if Single Binary is running)' }"
else
	@POD=$$($(KUBECTL) -n tech-logging get pods -l app.kubernetes.io/component=backend -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) && \
	if [ -n "$$POD" ]; then \
		$(KUBECTL) -n tech-logging exec $$POD -- wget -qO- http://localhost:3100/metrics 2>/dev/null | grep -E 'loki_ingester_chunks_created_total|loki_distributor_lines_received_total|loki_ingester_chunk_stored_bytes_total' | head -5 || echo "No metrics found (Loki may still be starting)"; \
	else \
		echo "Loki backend pod not found (check if Single Binary is running)"; \
	fi
endif
	@echo ""
	@echo "$(COLOR_BOLD)5. Checking Loki S3 storage metrics:$(COLOR_RESET)"
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$env:KUBECONFIG = $$kubeconfig; $$pod = kubectl -n tech-logging get pods -l app.kubernetes.io/component=backend -o jsonpath='{.items[0].metadata.name}' 2>&1; if ($$LASTEXITCODE -eq 0 -and $$pod -and $$pod -notmatch 'No resources found') { $$result = kubectl -n tech-logging exec $$pod -- wget -qO- http://localhost:3100/metrics 2>&1 | Select-String -Pattern 'loki_boltdb_shipper_operations_total|loki_compactor_compaction_operations_total|loki_compactor_compaction_deleted_chunks_total' | Select-Object -First 5; if ($$result) { Write-Output $$result } else { Write-Output 'No S3 metrics found (compactor may not have run yet)' } } else { Write-Output 'Loki backend pod not found (check if Single Binary is running)' }"
else
	@POD=$$($(KUBECTL) -n tech-logging get pods -l app.kubernetes.io/component=backend -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) && \
	if [ -n "$$POD" ]; then \
		$(KUBECTL) -n tech-logging exec $$POD -- wget -qO- http://localhost:3100/metrics 2>/dev/null | grep -E 'loki_boltdb_shipper_operations_total|loki_compactor_compaction_operations_total|loki_compactor_compaction_deleted_chunks_total' | head -5 || echo "No S3 metrics found (compactor may not have run yet)"; \
	else \
		echo "Loki backend pod not found (check if Single Binary is running)"; \
	fi
endif
	@echo ""
	@echo "$(COLOR_BOLD)6. Testing Loki query (checking for logs):$(COLOR_RESET)"
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$env:KUBECONFIG = $$kubeconfig; $$pod = kubectl -n tech-logging get pods -l app.kubernetes.io/component=gateway -o jsonpath='{.items[0].metadata.name}' 2>&1; if ($$LASTEXITCODE -eq 0 -and $$pod) { $$result = kubectl -n tech-logging exec $$pod -- wget -qO- 'http://loki-gateway.tech-logging.svc.cluster.local/loki/api/v1/labels' 2>&1; if ($$LASTEXITCODE -eq 0) { $$json = $$result | ConvertFrom-Json; if ($$json.data) { Write-Output 'Available labels:'; $$json.data | ForEach-Object { Write-Output \"  - $$_\" } } else { Write-Output 'No labels found (no logs collected yet)' } } else { Write-Output 'Failed to query labels' } } else { Write-Output 'Loki gateway pod not found' }"
else
	@POD=$$($(KUBECTL) -n tech-logging get pods -l app.kubernetes.io/component=gateway -o jsonpath='{.items[0].metadata.name}' 2>/dev/null) && \
	if [ -n "$$POD" ]; then \
		$(KUBECTL) -n tech-logging exec $$POD -- wget -qO- 'http://loki-gateway.tech-logging.svc.cluster.local/loki/api/v1/labels' 2>/dev/null | python3 -m json.tool 2>/dev/null | grep -A 100 '"data"' || echo "No labels found (no logs collected yet)"; \
	else \
		echo "Loki gateway pod not found"; \
	fi
endif
	@echo ""
	@echo "$(COLOR_BOLD)7. Checking S3 buckets (via MinIO):$(COLOR_RESET)"
	@echo "$(COLOR_YELLOW)Note:$(COLOR_RESET) To check S3 buckets manually:"
	@echo "  1. Access MinIO Console: make minio-console"
	@echo "  2. Check buckets: loki-chunks, loki-ruler, loki-admin"
	@echo "  3. Verify files are being created in loki-chunks bucket"
	@echo ""
	@echo "$(COLOR_BOLD)8. Summary:$(COLOR_RESET)"
	@echo "  - If Fluent Bit metrics show input_bytes_total > 0: logs are being collected"
	@echo "  - If Loki metrics show lines_received_total > 0: logs are being ingested"
	@echo "  - If Loki query returns labels: logs are queryable"
	@echo "  - If S3 buckets contain files: logs are being stored in S3"
	@echo ""
	@echo "$(COLOR_GREEN)For detailed logs, run:$(COLOR_RESET)"
	@echo "  make logging-logs-fluent  # Fluent Bit logs"
	@echo "  make logging-logs-loki    # Loki logs"

.PHONY: logging-uninstall-loki
logging-uninstall-loki:
	$(call print_warning,This will uninstall Loki only!)
	@echo "$(COLOR_YELLOW)Note: Loki data (PVCs/S3 objects) may remain depending on storage settings.$(COLOR_RESET)"
	@echo "$(COLOR_YELLOW)Press Ctrl+C to cancel, or wait 10 seconds to continue...$(COLOR_RESET)"
	$(call sleep_seconds,10)
	$(call print_info,Uninstalling Loki...)
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$helmfilePath = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, 'logging', 'helmfile-loki.yaml')); $$env:KUBECONFIG = $$kubeconfig; if (Test-Path $$helmfilePath) { helmfile -f \"$$helmfilePath\" destroy } else { Write-Host 'Helmfile not found:' $$helmfilePath; exit 1 }" || true
else
	-$(call run_helmfile,$(HELMFILE_LOKI),destroy)
endif
	$(call print_success,Loki uninstalled. Fluent Bit (if installed) remains.)

.PHONY: logging-uninstall-fluent
logging-uninstall-fluent:
	$(call print_warning,This will uninstall Fluent Bit only!)
	@echo "$(COLOR_YELLOW)Note: Loki will remain running (if installed).$(COLOR_RESET)"
	@echo "$(COLOR_YELLOW)Press Ctrl+C to cancel, or wait 10 seconds to continue...$(COLOR_RESET)"
	$(call sleep_seconds,10)
	$(call print_info,Uninstalling Fluent Bit...)
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$helmfilePath = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, 'logging', 'helmfile-fluent-bit.yaml')); $$env:KUBECONFIG = $$kubeconfig; if (Test-Path $$helmfilePath) { helmfile -f \"$$helmfilePath\" destroy } else { Write-Host 'Helmfile not found:' $$helmfilePath; exit 1 }" || true
else
	-$(call run_helmfile,$(HELMFILE_FLUENT_BIT),destroy)
endif
	$(call print_success,Fluent Bit uninstalled. Loki (if installed) remains.)

.PHONY: logging-uninstall
logging-uninstall:
	$(call print_warning,This will DELETE all logging data!)
	@echo "$(COLOR_YELLOW)Press Ctrl+C to cancel, or wait 10 seconds to continue...$(COLOR_RESET)"
	$(call sleep_seconds,10)
	$(call print_info,Uninstalling Fluent Bit...)
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$helmfilePath = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, 'logging', 'helmfile-fluent-bit.yaml')); $$env:KUBECONFIG = $$kubeconfig; helmfile -f \"$$helmfilePath\" destroy" || true
else
	-$(call run_helmfile,$(HELMFILE_FLUENT_BIT),destroy)
endif
	$(call print_info,Uninstalling Loki...)
ifeq ($(DETECTED_OS),Windows)
	@powershell -NoProfile -Command "$$curDir = [System.IO.Directory]::GetCurrentDirectory(); $$kubeconfig = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, '$(KUBECONFIG_REL)')); $$helmfilePath = [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($$curDir, 'logging', 'helmfile-loki.yaml')); $$env:KUBECONFIG = $$kubeconfig; helmfile -f \"$$helmfilePath\" destroy" || true
else
	-$(call run_helmfile,$(HELMFILE_LOKI),destroy)
endif
	$(call print_info,Deleting namespace and resources...)
	-@$(KUBECTL) delete namespace tech-logging --wait=false
	$(call print_success,Logging stack uninstalled!)