Commit 86747c
2025-07-06 22:18:21 Tebby Dog: Init| /dev/null .. 2-code/k8s/backup-pushover-notification.md | |
| @@ 0,0 1,88 @@ | |
| + | # Backup-Pushover-Notification |
| + | |
| + | This is a deployment for Kubernetes that will send a push notification through Pushover when a velero backup completes with stats of what happened |
| + | |
| + | I know this page sucks, its a **WIP** |
| + | |
| + | ```yaml |
| + | # velero-pushover-notification.yaml |
| + | # ConfigMap with enhanced notification script |
| + | apiVersion: v1 |
| + | kind: ConfigMap |
| + | metadata: |
| + | name: velero-pushover-script |
| + | namespace: velero |
| + | data: |
| + | check-and-notify.sh: "#!/bin/bash\nset -e\n\n# Get Pushover credentials\nPUSHOVER_TOKEN=\"${PUSHOVER_TOKEN}\"\nPUSHOVER_USER=\"${PUSHOVER_USER}\"\n\n# Function to convert bytes to human readable\nhuman_readable() {\n local bytes=$1\n local units=(\"B\" \"KB\" \"MB\" \"GB\" \"TB\")\n local unit=0\n local size=$bytes\n \n while (( $(echo \"$size > 1024\" | bc -l) )) && (( unit < 4 )); do\n size=$(echo \"scale=2; $size / 1024\" | bc)\n ((unit++))\n done\n \n echo \"${size}${units[$unit]}\"\n}\n\n# Function to format duration\nformat_duration() {\n local start=$1\n local end=$2\n local duration=$(( $(date -d \"$end\" +%s) - $(date -d \"$start\" +%s) ))\n local hours=$(( duration / 3600 ))\n local minutes=$(( (duration % 3600) / 60 ))\n local seconds=$(( duration % 60 ))\n \n if [ $hours -gt 0 ]; then\n echo \"${hours}h ${minutes}m ${seconds}s\"\n elif [ $minutes -gt 0 ]; then\n echo \"${minutes}m ${seconds}s\"\n else\n echo \"${seconds}s\"\n fi\n}\n\n# Check for recent backups (last 30 minutes)\nBACKUPS=$(velero backup get -o json)\nCURRENT_TIME=$(date -u +%s)\n\necho \"$BACKUPS\" | jq -c '.items[]' | while read -r backup; do\n # Extract backup details\n NAME=$(echo \"$backup\" | jq -r '.metadata.name')\n PHASE=$(echo \"$backup\" | jq -r '.status.phase // \"Unknown\"')\n ERRORS=$(echo \"$backup\" | jq -r '.status.errors // 0')\n WARNINGS=$(echo \"$backup\" | jq -r '.status.warnings // 0')\n STARTED=$(echo \"$backup\" | jq -r '.status.startTimestamp // \"\"')\n COMPLETED=$(echo \"$backup\" | jq -r '.status.completionTimestamp // \"\"')\n CREATED=$(echo \"$backup\" | jq -r '.metadata.creationTimestamp')\n \n # Check if backup is recent (within last 30 minutes)\n CREATED_EPOCH=$(date -d \"$CREATED\" +%s 2>/dev/null || echo 0)\n TIME_DIFF=$(( CURRENT_TIME - CREATED_EPOCH ))\n \n if [ $TIME_DIFF -gt 1800 ]; then\n continue # Skip backups older than 30 minutes\n fi\n \n # Skip if already notified\n NOTIFIED=$(echo \"$backup\" | jq -r '.metadata.annotations.notified // \"false\"')\n if [ \"$NOTIFIED\" = \"true\" ]; then\n continue\n fi\n \n # Skip in-progress backups\n if [ \"$PHASE\" = \"InProgress\" ] || [ \"$PHASE\" = \"New\" ]; then\n continue\n fi\n \n # Calculate duration\n if [ -n \"$STARTED\" ] && [ -n \"$COMPLETED\" ]; then\n DURATION=$(format_duration \"$STARTED\" \"$COMPLETED\")\n else\n DURATION=\"Unknown\"\n fi\n \n # Get backup details\n BACKUP_DETAILS=$(velero backup describe \"$NAME\" --details -o json 2>/dev/null || echo \"{}\")\n \n # Extract metrics\n TOTAL_ITEMS=$(echo \"$BACKUP_DETAILS\" | jq -r '.status.progress.totalItems // 0')\n ITEMS_BACKED_UP=$(echo \"$BACKUP_DETAILS\" | jq -r '.status.progress.itemsBackedUp // 0')\n \n # Get resource counts\n NAMESPACES=$(echo \"$BACKUP_DETAILS\" | jq -r '.spec.includedNamespaces // [] | length')\n \n # Try to get backup size from logs or storage\n BACKUP_SIZE=\"N/A\"\n \n # Check S3 for backup size (if using AWS CLI)\n if command -v aws &> /dev/null; then\n SIZE_BYTES=$(aws s3 ls \"s3://${S3_BUCKET}/backups/${NAME}/\" --recursive 2>/dev/null | awk '{sum+=$3} END {print sum}')\n if [ -n \"$SIZE_BYTES\" ] && [ \"$SIZE_BYTES\" -gt 0 ]; then\n BACKUP_SIZE=$(human_readable \"$SIZE_BYTES\")\n fi\n fi\n \n # Determine status emoji and priority\n if [ \"$PHASE\" = \"Completed\" ] && [ \"$ERRORS\" = \"0\" ]; then\n STATUS_EMOJI=\"✅\"\n TITLE=\"Backup Completed Successfully\"\n PRIORITY=\"-1\" # Low priority for successful backups\n HTML_COLOR=\"3CB371\" # Green\n elif [ \"$PHASE\" = \"PartiallyFailed\" ]; then\n STATUS_EMOJI=\"⚠️\"\n TITLE=\"Backup Partially Failed\"\n PRIORITY=\"0\" # Normal priority for partial failures\n HTML_COLOR=\"FFA500\" # Orange\n elif [ \"$PHASE\" = \"Failed\" ]; then\n STATUS_EMOJI=\"❌\"\n TITLE=\"Backup Failed\"\n PRIORITY=\"1\" # High priority for failures\n HTML_COLOR=\"DC143C\" # Red\n else\n STATUS_EMOJI=\"❓\"\n TITLE=\"Backup Status Unknown\"\n PRIORITY=\"0\" # Normal priority for unknown status\n HTML_COLOR=\"808080\" # Gray\n fi\n \n # Get backup type from name or labels\n BACKUP_TYPE=\"Manual\"\n if [[ \"$NAME\" == *\"daily\"* ]]; then\n BACKUP_TYPE=\"Daily\"\n elif [[ \"$NAME\" == *\"weekly\"* ]]; then\n BACKUP_TYPE=\"Weekly\"\n elif [[ \"$NAME\" == *\"monthly\"* ]]; then\n BACKUP_TYPE=\"Monthly\"\n elif [[ \"$NAME\" == *\"yearly\"* ]]; then\n BACKUP_TYPE=\"Yearly\"\n fi\n \n # Build HTML message\n HTML_MESSAGE=\"<b>Backup Name:</b> ${NAME}<br>\"\n HTML_MESSAGE+=\"<b>Type:</b> ${BACKUP_TYPE}<br>\"\n HTML_MESSAGE+=\"<b>Status:</b> <font color='#${HTML_COLOR}'>${PHASE}</font><br>\"\n HTML_MESSAGE+=\"<br>\"\n \n # Add metrics section\n HTML_MESSAGE+=\"<b>📊 Metrics:</b><br>\"\n HTML_MESSAGE+=\"• Duration: <b>${DURATION}</b><br>\"\n HTML_MESSAGE+=\"• Size: <b>${BACKUP_SIZE}</b><br>\"\n HTML_MESSAGE+=\"• Items: ${ITEMS_BACKED_UP}/${TOTAL_ITEMS}<br>\"\n \n if [ \"$NAMESPACES\" -gt 0 ]; then\n HTML_MESSAGE+=\"• Namespaces: ${NAMESPACES}<br>\"\n fi\n \n # Add errors/warnings if any\n if [ \"$ERRORS\" -gt 0 ] || [ \"$WARNINGS\" -gt 0 ]; then\n HTML_MESSAGE+=\"<br><b>⚠️ Issues:</b><br>\"\n [ \"$ERRORS\" -gt 0 ] && HTML_MESSAGE+=\"• Errors: <font color='#DC143C'><b>${ERRORS}</b></font><br>\"\n [ \"$WARNINGS\" -gt 0 ] && HTML_MESSAGE+=\"• Warnings: <font color='#FFA500'>${WARNINGS}</font><br>\"\n fi\n \n # Add timestamp\n HTML_MESSAGE+=\"<br><b>🕒 Time:</b><br>\"\n HTML_MESSAGE+=\"• Started: $(date -d \"$STARTED\" '+%Y-%m-%d %H:%M:%S %Z' 2>/dev/null || echo 'N/A')<br>\"\n HTML_MESSAGE+=\"• Completed: $(date -d \"$COMPLETED\" '+%Y-%m-%d %H:%M:%S %Z' 2>/dev/null || echo 'N/A')\"\n \n # Add performance indicators\n if [ -n \"$DURATION\" ] && [ \"$DURATION\" != \"Unknown\" ]; then\n DURATION_SECONDS=$(( $(date -d \"$COMPLETED\" +%s) - $(date -d \"$STARTED\" +%s) ))\n if [ $DURATION_SECONDS -lt 300 ]; then\n PERF_INDICATOR=\"🚀 Fast\"\n elif [ $DURATION_SECONDS -lt 900 ]; then\n PERF_INDICATOR=\"✓ Normal\"\n else\n PERF_INDICATOR=\"🐌 Slow\"\n fi\n HTML_MESSAGE+=\"<br><br><b>Performance:</b> ${PERF_INDICATOR}\"\n fi\n \n # Send notification\n RESPONSE=$(curl -s \\\n --form-string \"token=$PUSHOVER_TOKEN\" \\\n --form-string \"user=$PUSHOVER_USER\" \\\n --form-string \"priority=$PRIORITY\" \\\n --form-string \"title=${STATUS_EMOJI} ${TITLE}\" \\\n --form-string \"message=${HTML_MESSAGE}\" \\\n --form-string \"html=1\" \\\n --form-string \"timestamp=$(date +%s)\" \\\n https://api.pushover.net/1/messages.json)\n \n # Mark as notified\n kubectl annotate backup.velero.io \"$NAME\" notified=true --overwrite -n velero\n \n echo \"Sent notification for backup: $NAME\"\ndone\n" |
| + | --- |
| + | # Secret template with placeholders for your credentials |
| + | # REPLACE WITH YOUR ACTUAL VALUES BEFORE APPLYING |
| + | apiVersion: v1 |
| + | kind: Secret |
| + | metadata: |
| + | name: pushover-credentials |
| + | namespace: velero |
| + | type: Opaque |
| + | stringData: |
| + | token: "arg4ei5fh7cp6ehfr7ndpp37mb6us1" # <-- REPLACE THIS! |
| + | user: "g66f9x5s5uybr7a8qg9i287yqx4nu9" # <-- REPLACE THIS! |
| + | --- |
| + | # CronJob to run the notification script |
| + | apiVersion: batch/v1 |
| + | kind: CronJob |
| + | metadata: |
| + | name: velero-pushover-notification |
| + | namespace: velero |
| + | spec: |
| + | schedule: "*/10 * * * *" # Check every 10 minutes |
| + | successfulJobsHistoryLimit: 1 |
| + | failedJobsHistoryLimit: 3 |
| + | jobTemplate: |
| + | spec: |
| + | template: |
| + | spec: |
| + | serviceAccountName: velero |
| + | containers: |
| + | - name: notifier |
| + | image: nicolaka/netshoot:latest # Has more tools including bc, jq, curl |
| + | env: |
| + | - name: PUSHOVER_TOKEN |
| + | valueFrom: |
| + | secretKeyRef: |
| + | name: pushover-credentials |
| + | key: token |
| + | - name: PUSHOVER_USER |
| + | valueFrom: |
| + | secretKeyRef: |
| + | name: pushover-credentials |
| + | key: user |
| + | - name: S3_BUCKET |
| + | value: "k8sbackup" # Your S3 bucket name |
| + | command: |
| + | - /bin/bash |
| + | - -c |
| + | - | |
| + | # Install kubectl |
| + | curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" |
| + | chmod +x kubectl |
| + | mv kubectl /usr/local/bin/ |
| + | |
| + | # Install velero CLI |
| + | wget -q https://github.com/vmware-tanzu/velero/releases/download/v1.15.0/velero-v1.15.0-linux-amd64.tar.gz |
| + | tar -xzf velero-v1.15.0-linux-amd64.tar.gz |
| + | mv velero-v1.15.0-linux-amd64/velero /usr/local/bin/ |
| + | |
| + | # Run the notification script |
| + | /scripts/check-and-notify.sh |
| + | volumeMounts: |
| + | - name: script |
| + | mountPath: /scripts |
| + | readOnly: true |
| + | volumes: |
| + | - name: script |
| + | configMap: |
| + | name: velero-pushover-script |
| + | defaultMode: 0755 |
| + | restartPolicy: OnFailure |
| + | |
| + | ``` |