Backup-Pushover-Notification
This is a deployment for Kubernetes that will send a push notification through Pushover when a velero backup completes with stats of what happened
I know this page sucks, its a WIP
# velero-pushover-notification.yaml # ConfigMap with enhanced notification script apiVersion: v1 kind: ConfigMap metadata: name: velero-pushover-script namespace: velero data: check-and-notify.sh: "#!/bin/bash\nset -e\n\n# Get Pushover credentials\nPUSHOVER_TOKEN=\"${PUSHOVER_TOKEN}\"\nPUSHOVER_USER=\"${PUSHOVER_USER}\"\n\n# Function to convert bytes to human readable\nhuman_readable() {\n local bytes=$1\n local units=(\"B\" \"KB\" \"MB\" \"GB\" \"TB\")\n local unit=0\n local size=$bytes\n \n while (( $(echo \"$size > 1024\" | bc -l) )) && (( unit < 4 )); do\n size=$(echo \"scale=2; $size / 1024\" | bc)\n ((unit++))\n done\n \n echo \"${size}${units[$unit]}\"\n}\n\n# Function to format duration\nformat_duration() {\n local start=$1\n local end=$2\n local duration=$(( $(date -d \"$end\" +%s) - $(date -d \"$start\" +%s) ))\n local hours=$(( duration / 3600 ))\n local minutes=$(( (duration % 3600) / 60 ))\n local seconds=$(( duration % 60 ))\n \n if [ $hours -gt 0 ]; then\n echo \"${hours}h ${minutes}m ${seconds}s\"\n elif [ $minutes -gt 0 ]; then\n echo \"${minutes}m ${seconds}s\"\n else\n echo \"${seconds}s\"\n fi\n}\n\n# Check for recent backups (last 30 minutes)\nBACKUPS=$(velero backup get -o json)\nCURRENT_TIME=$(date -u +%s)\n\necho \"$BACKUPS\" | jq -c '.items[]' | while read -r backup; do\n # Extract backup details\n NAME=$(echo \"$backup\" | jq -r '.metadata.name')\n PHASE=$(echo \"$backup\" | jq -r '.status.phase // \"Unknown\"')\n ERRORS=$(echo \"$backup\" | jq -r '.status.errors // 0')\n WARNINGS=$(echo \"$backup\" | jq -r '.status.warnings // 0')\n STARTED=$(echo \"$backup\" | jq -r '.status.startTimestamp // \"\"')\n COMPLETED=$(echo \"$backup\" | jq -r '.status.completionTimestamp // \"\"')\n CREATED=$(echo \"$backup\" | jq -r '.metadata.creationTimestamp')\n \n # Check if backup is recent (within last 30 minutes)\n CREATED_EPOCH=$(date -d \"$CREATED\" +%s 2>/dev/null || echo 0)\n TIME_DIFF=$(( CURRENT_TIME - CREATED_EPOCH ))\n \n if [ $TIME_DIFF -gt 1800 ]; then\n continue # Skip backups older than 30 minutes\n fi\n \n # Skip if already notified\n NOTIFIED=$(echo \"$backup\" | jq -r '.metadata.annotations.notified // \"false\"')\n if [ \"$NOTIFIED\" = \"true\" ]; then\n continue\n fi\n \n # Skip in-progress backups\n if [ \"$PHASE\" = \"InProgress\" ] || [ \"$PHASE\" = \"New\" ]; then\n continue\n fi\n \n # Calculate duration\n if [ -n \"$STARTED\" ] && [ -n \"$COMPLETED\" ]; then\n DURATION=$(format_duration \"$STARTED\" \"$COMPLETED\")\n else\n DURATION=\"Unknown\"\n fi\n \n # Get backup details\n BACKUP_DETAILS=$(velero backup describe \"$NAME\" --details -o json 2>/dev/null || echo \"{}\")\n \n # Extract metrics\n TOTAL_ITEMS=$(echo \"$BACKUP_DETAILS\" | jq -r '.status.progress.totalItems // 0')\n ITEMS_BACKED_UP=$(echo \"$BACKUP_DETAILS\" | jq -r '.status.progress.itemsBackedUp // 0')\n \n # Get resource counts\n NAMESPACES=$(echo \"$BACKUP_DETAILS\" | jq -r '.spec.includedNamespaces // [] | length')\n \n # Try to get backup size from logs or storage\n BACKUP_SIZE=\"N/A\"\n \n # Check S3 for backup size (if using AWS CLI)\n if command -v aws &> /dev/null; then\n SIZE_BYTES=$(aws s3 ls \"s3://${S3_BUCKET}/backups/${NAME}/\" --recursive 2>/dev/null | awk '{sum+=$3} END {print sum}')\n if [ -n \"$SIZE_BYTES\" ] && [ \"$SIZE_BYTES\" -gt 0 ]; then\n BACKUP_SIZE=$(human_readable \"$SIZE_BYTES\")\n fi\n fi\n \n # Determine status emoji and priority\n if [ \"$PHASE\" = \"Completed\" ] && [ \"$ERRORS\" = \"0\" ]; then\n STATUS_EMOJI=\"✅\"\n TITLE=\"Backup Completed Successfully\"\n PRIORITY=\"-1\" # Low priority for successful backups\n HTML_COLOR=\"3CB371\" # Green\n elif [ \"$PHASE\" = \"PartiallyFailed\" ]; then\n STATUS_EMOJI=\"⚠️\"\n TITLE=\"Backup Partially Failed\"\n PRIORITY=\"0\" # Normal priority for partial failures\n HTML_COLOR=\"FFA500\" # Orange\n elif [ \"$PHASE\" = \"Failed\" ]; then\n STATUS_EMOJI=\"❌\"\n TITLE=\"Backup Failed\"\n PRIORITY=\"1\" # High priority for failures\n HTML_COLOR=\"DC143C\" # Red\n else\n STATUS_EMOJI=\"❓\"\n TITLE=\"Backup Status Unknown\"\n PRIORITY=\"0\" # Normal priority for unknown status\n HTML_COLOR=\"808080\" # Gray\n fi\n \n # Get backup type from name or labels\n BACKUP_TYPE=\"Manual\"\n if [[ \"$NAME\" == *\"daily\"* ]]; then\n BACKUP_TYPE=\"Daily\"\n elif [[ \"$NAME\" == *\"weekly\"* ]]; then\n BACKUP_TYPE=\"Weekly\"\n elif [[ \"$NAME\" == *\"monthly\"* ]]; then\n BACKUP_TYPE=\"Monthly\"\n elif [[ \"$NAME\" == *\"yearly\"* ]]; then\n BACKUP_TYPE=\"Yearly\"\n fi\n \n # Build HTML message\n HTML_MESSAGE=\"<b>Backup Name:</b> ${NAME}<br>\"\n HTML_MESSAGE+=\"<b>Type:</b> ${BACKUP_TYPE}<br>\"\n HTML_MESSAGE+=\"<b>Status:</b> <font color='#${HTML_COLOR}'>${PHASE}</font><br>\"\n HTML_MESSAGE+=\"<br>\"\n \n # Add metrics section\n HTML_MESSAGE+=\"<b>📊 Metrics:</b><br>\"\n HTML_MESSAGE+=\"• Duration: <b>${DURATION}</b><br>\"\n HTML_MESSAGE+=\"• Size: <b>${BACKUP_SIZE}</b><br>\"\n HTML_MESSAGE+=\"• Items: ${ITEMS_BACKED_UP}/${TOTAL_ITEMS}<br>\"\n \n if [ \"$NAMESPACES\" -gt 0 ]; then\n HTML_MESSAGE+=\"• Namespaces: ${NAMESPACES}<br>\"\n fi\n \n # Add errors/warnings if any\n if [ \"$ERRORS\" -gt 0 ] || [ \"$WARNINGS\" -gt 0 ]; then\n HTML_MESSAGE+=\"<br><b>⚠️ Issues:</b><br>\"\n [ \"$ERRORS\" -gt 0 ] && HTML_MESSAGE+=\"• Errors: <font color='#DC143C'><b>${ERRORS}</b></font><br>\"\n [ \"$WARNINGS\" -gt 0 ] && HTML_MESSAGE+=\"• Warnings: <font color='#FFA500'>${WARNINGS}</font><br>\"\n fi\n \n # Add timestamp\n HTML_MESSAGE+=\"<br><b>🕒 Time:</b><br>\"\n HTML_MESSAGE+=\"• Started: $(date -d \"$STARTED\" '+%Y-%m-%d %H:%M:%S %Z' 2>/dev/null || echo 'N/A')<br>\"\n HTML_MESSAGE+=\"• Completed: $(date -d \"$COMPLETED\" '+%Y-%m-%d %H:%M:%S %Z' 2>/dev/null || echo 'N/A')\"\n \n # Add performance indicators\n if [ -n \"$DURATION\" ] && [ \"$DURATION\" != \"Unknown\" ]; then\n DURATION_SECONDS=$(( $(date -d \"$COMPLETED\" +%s) - $(date -d \"$STARTED\" +%s) ))\n if [ $DURATION_SECONDS -lt 300 ]; then\n PERF_INDICATOR=\"🚀 Fast\"\n elif [ $DURATION_SECONDS -lt 900 ]; then\n PERF_INDICATOR=\"✓ Normal\"\n else\n PERF_INDICATOR=\"🐌 Slow\"\n fi\n HTML_MESSAGE+=\"<br><br><b>Performance:</b> ${PERF_INDICATOR}\"\n fi\n \n # Send notification\n RESPONSE=$(curl -s \\\n --form-string \"token=$PUSHOVER_TOKEN\" \\\n --form-string \"user=$PUSHOVER_USER\" \\\n --form-string \"priority=$PRIORITY\" \\\n --form-string \"title=${STATUS_EMOJI} ${TITLE}\" \\\n --form-string \"message=${HTML_MESSAGE}\" \\\n --form-string \"html=1\" \\\n --form-string \"timestamp=$(date +%s)\" \\\n https://api.pushover.net/1/messages.json)\n \n # Mark as notified\n kubectl annotate backup.velero.io \"$NAME\" notified=true --overwrite -n velero\n \n echo \"Sent notification for backup: $NAME\"\ndone\n" --- # Secret template with placeholders for your credentials # REPLACE WITH YOUR ACTUAL VALUES BEFORE APPLYING apiVersion: v1 kind: Secret metadata: name: pushover-credentials namespace: velero type: Opaque stringData: token: "DOPEFUCKNTOKN" # <-- REPLACE THIS! user: "PLZTOUCHMYAPI" # <-- REPLACE THIS! --- # CronJob to run the notification script apiVersion: batch/v1 kind: CronJob metadata: name: velero-pushover-notification namespace: velero spec: schedule: "*/10 * * * *" # Check every 10 minutes successfulJobsHistoryLimit: 1 failedJobsHistoryLimit: 3 jobTemplate: spec: template: spec: serviceAccountName: velero containers: - name: notifier image: nicolaka/netshoot:latest # Has more tools including bc, jq, curl env: - name: PUSHOVER_TOKEN valueFrom: secretKeyRef: name: pushover-credentials key: token - name: PUSHOVER_USER valueFrom: secretKeyRef: name: pushover-credentials key: user - name: S3_BUCKET value: "k8sbackup" # Your S3 bucket name command: - /bin/bash - -c - | # Install kubectl curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" chmod +x kubectl mv kubectl /usr/local/bin/ # Install velero CLI wget -q https://github.com/vmware-tanzu/velero/releases/download/v1.15.0/velero-v1.15.0-linux-amd64.tar.gz tar -xzf velero-v1.15.0-linux-amd64.tar.gz mv velero-v1.15.0-linux-amd64/velero /usr/local/bin/ # Run the notification script /scripts/check-and-notify.sh volumeMounts: - name: script mountPath: /scripts readOnly: true volumes: - name: script configMap: name: velero-pushover-script defaultMode: 0755 restartPolicy: OnFailure