# Self-Hosted Push Notifications Part-6

[MyrinNew]

Self-Hosted Push Notifications Specification

Part 6: Monitoring, Debugging & Troubleshooting

Version: 1.0

Last Updated: October 2025

Prerequisites: Part 5: Advanced Features & Production Deployment

Author: Bunty9

License: MIT (Free to use and adapt)



Table of Contents

1. Metrics & Monitoring
2. Logging Best Practices
3. Performance Profiling
4. Health Checks
5. Error Tracking
6. Alerting Strategies
7. Debugging Tools
8. Common Issues & Solutions
9. Incident Response
10. Analytics & Reporting

Metrics & Monitoring

Key Metrics to Track

1. Delivery Metrics




-- Overall delivery rate (last 24 hours)
SELECT
status,
COUNT(*) as count,
ROUND(COUNT(*) * 100.0 / SUM(COUNT(*)) OVER (), 2) as percentage
FROM push_notification_logs
WHERE sent_at > NOW() - INTERVAL '24 hours'
GROUP BY status;

-- Expected result:
-- status | count | percentage
-- --------+-------+-----------
-- sent | 8,543 | 95.23
-- failed | 234 | 2.61
-- expired | 193 | 2.16






Target: >90% success rate

2. Active Subscriptions




-- Active subscriptions by user type
SELECT
CASE
WHEN user_id IS NOT NULL THEN 'users'
WHEN admin_id IS NOT NULL THEN 'admins'
END as type,
COUNT(*) as active_subscriptions,
COUNT(DISTINCT COALESCE(user_id, admin_id)) as unique_accounts,
ROUND(AVG(failure_count), 2) as avg_failure_count
FROM push_subscriptions
WHERE is_active = TRUE
GROUP BY type;






Target: Failure count

3. Device Distribution




-- Browser and OS distribution
SELECT
browser,
os,
device_type,
COUNT(*) as count
FROM push_subscriptions
WHERE is_active = TRUE
GROUP BY browser, os, device_type
ORDER BY count DESC;





4. Notification Volume




-- Notifications sent per hour (last 24 hours)
SELECT
DATE_TRUNC('hour', sent_at) as hour,
COUNT(*) as notifications_sent,
COUNT(DISTINCT subscription_id) as unique_devices,
COUNT(*) FILTER (WHERE status = 'sent') as successful,
COUNT(*) FILTER (WHERE status = 'failed') as failed
FROM push_notification_logs
WHERE sent_at > NOW() - INTERVAL '24 hours'
GROUP BY hour
ORDER BY hour DESC;







Prometheus Metrics

Go Backend Metrics




// metrics/push_metrics.go

package metrics

import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)

var (
// Notification counters
NotificationsSent = promauto.NewCounterVec(
prometheus.CounterOpts{
Name: "push_notifications_sent_total",
Help: "Total number of push notifications sent",
},
[]string{"status", "type"},
)

NotificationDuration = promauto.NewHistogramVec(
prometheus.HistogramOpts{
Name: "push_notification_duration_seconds",
Help: "Duration of push notification sending",
Buckets: prometheus.DefBuckets,
},
[]string{"status"},
)

// Subscription metrics
ActiveSubscriptions = promauto.NewGaugeVec(
prometheus.GaugeOpts{
Name: "push_subscriptions_active",
Help: "Number of active push subscriptions",
},
[]string{"user_type"},
)

SubscriptionFailures = promauto.NewCounterVec(
prometheus.CounterOpts{
Name: "push_subscription_failures_total",
Help: "Total number of subscription failures",
},
[]string{"reason"},
)

// Worker metrics
WorkerRuns = promauto.NewCounterVec(
prometheus.CounterOpts{
Name: "push_worker_runs_total",
Help: "Total number of worker executions",
},
[]string{"worker", "status"},
)

WorkerDuration = promauto.NewHistogramVec(
prometheus.HistogramOpts{
Name: "push_worker_duration_seconds",
Help: "Duration of worker execution",
Buckets: prometheus.DefBuckets,
},
[]string{"worker"},
)
)

// Usage in PushService
func (ps *PushService) sendToSubscription(subscription models.PushSubscription, payload types.PushPayload) error {
start := time.Now()

err := ps.doSend(subscription, payload)

status := "success"
if err != nil {
status = "error"
}

// Record metrics
NotificationsSent.WithLabelValues(status, payload.Type).Inc()
NotificationDuration.WithLabelValues(status).Obser ve(time.Since(start).Seconds())

return err
}





Prometheus Configuration




# prometheus.yml
global:
scrape_interval: 15s
evaluation_interval: 15s

scrape_configs:
- job_name: 'user-service'
static_configs:
- targets: ['user-service:8081']
metrics_path: /metrics

- job_name: 'admin-service'
static_configs:
- targets: ['admin-service:8082']
metrics_path: /metrics







Grafana Dashboards

Push Notifications Dashboard




{
"dashboard": {
"title": "Push Notifications Monitoring",
"panels": [
{
"title": "Delivery Rate",
"type": "graph",
"targets": [
{
"expr": "rate(push_notifications_sent_total{status="succes s"}[5m])",
"legendFormat": "Successful"
},
{
"expr": "rate(push_notifications_sent_total{status="error" }[5m])",
"legendFormat": "Failed"
}
]
},
{
"title": "Active Subscriptions",
"type": "stat",
"targets": [
{
"expr": "sum(push_subscriptions_active)"
}
]
},
{
"title": "Notification Latency (p50, p95, p99)",
"type": "graph",
"targets": [
{
"expr": "histogram_quantile(0.50, rate(push_notification_duration_seconds_bucket[5m]))",
"legendFormat": "p50"
},
{
"expr": "histogram_quantile(0.95, rate(push_notification_duration_seconds_bucket[5m]))",
"legendFormat": "p95"
},
{
"expr": "histogram_quantile(0.99, rate(push_notification_duration_seconds_bucket[5m]))",
"legendFormat": "p99"
}
]
},
{
"title": "Worker Execution Duration",
"type": "graph",
"targets": [
{
"expr": "rate(push_worker_duration_seconds_sum[5m]) / rate(push_worker_duration_seconds_count[5m])",
"legendFormat": "{{worker}}"
}
]
}
]
}
}







Logging Best Practices

Structured Logging




// Use structured logging with logrus
import (
log "github.com/sirupsen/logrus"
)

func init() {
// Set JSON formatter for production
log.SetFormatter(&log.JSONFormatter{})

// Set log level from environment
level, err := log.ParseLevel(os.Getenv("LOG_LEVEL"))
if err != nil {
level = log.InfoLevel
}
log.SetLevel(level)
}

// Structured log entries
func (ps *PushService) sendToSubscription(subscription models.PushSubscription, payload types.PushPayload) error {
log.WithFields(log.Fields{
"subscription_id": subscription.ID,
"user_id": subscription.UserID,
"device_name": subscription.DeviceName,
"notification_type": payload.Type,
}).Info("Sending push notification")

err := ps.doSend(subscription, payload)

if err != nil {
log.WithFields(log.Fields{
"subscription_id": subscription.ID,
"error": err.Error(),
"notification_type": payload.Type,
}).Error("Failed to send push notification")
} else {
log.WithFields(log.Fields{
"subscription_id": subscription.ID,
"notification_type": payload.Type,
"duration_ms": time.Since(start).Milliseconds(),
}).Info("Push notification sent successfully")
}

return err
}





Log Aggregation

Fluentd Configuration:






# fluentd.conf

@type tail
path /var/log/user-service/*.log
pos_file /var/log/td-agent/user-service.log.pos
tag user-service


@type json
time_key time
time_format %Y-%m-%dT%H:%M:%S.%NZ




@type record_transformer

service ${tag}
environment production




@type elasticsearch
host elasticsearch
port 9200
logstash_format true
logstash_prefix user-service
include_tag_key true
type_name _doc











Performance Profiling

Go pprof Integration





// Enable pprof endpoints
import (
_ "net/http/pprof"
)

func main() {
// Start pprof server on separate port
go func() {
log.Println("pprof server listening on :6060")
log.Println(http.ListenAndServe(":6060", nil))
}()

// Start main application...
}







Profiling Commands





# CPU profiling
go tool pprof http://localhost:6060/debug/pprof/profile?seconds=30

# Memory profiling
go tool pprof http://localhost:6060/debug/pprof/heap

# Goroutine profiling
go tool pprof http://localhost:6060/debug/pprof/goroutine

# View profile in web browser
go tool pprof -http=:8080 profile.pb.gz







Profiling Analysis





// Add custom profiling markers
import "runtime/trace"

func (ps *PushService) SendBatch(subscriptions []models.PushSubscription, payload types.PushPayload) error {
ctx, task := trace.NewTask(context.Background(), "SendBatch")
defer task.End()

trace.Logf(ctx, "info", "Processing %d subscriptions", len(subscriptions))

// Process batch...

return nil
}










Health Checks

Comprehensive Health Endpoint





// controllers/health_controller.go

type HealthResponse struct {
Status string `json:"status"`
Timestamp time.Time `json:"timestamp"`
Checks map[string]HealthCheck `json:"checks"`
}

type HealthCheck struct {
Status string `json:"status"`
Message string `json:"message,omitempty"`
}

func (hc *HealthController) CheckHealth(c *gin.Context) {
response := HealthResponse{
Timestamp: time.Now(),
Checks: make(map[string]HealthCheck),
}

// Check database connection
sqlDB, err := hc.db.DB()
if err != nil {
response.Checks["database"] = HealthCheck{
Status: "unhealthy",
Message: err.Error(),
}
} else {
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()

if err := sqlDB.PingContext(ctx); err != nil {
response.Checks["database"] = HealthCheck{
Status: "unhealthy",
Message: err.Error(),
}
} else {
response.Checks["database"] = HealthCheck{Status: "healthy"}
}
}

// Check VAPID configuration
if os.Getenv("VAPID_PRIVATE_KEY") == "" || os.Getenv("VAPID_PUBLIC_KEY") == "" {
response.Checks["vapid"] = HealthCheck{
Status: "unhealthy",
Message: "VAPID keys not configured",
}
} else {
response.Checks["vapid"] = HealthCheck{Status: "healthy"}
}

// Check active subscriptions
var activeCount int64
hc.db.Model(&models.PushSubscription{}).Where("is_ active = ?", true).Count(&activeCount)

response.Checks["subscriptions"] = HealthCheck{
Status: "healthy",
Message: fmt.Sprintf("%d active subscriptions", activeCount),
}

// Check worker status (optional)
if hc.workerStatus != nil && !hc.workerStatus.IsRunning() {
response.Checks["worker"] = HealthCheck{
Status: "degraded",
Message: "BookingReminderWorker not running",
}
} else {
response.Checks["worker"] = HealthCheck{Status: "healthy"}
}

// Determine overall status
allHealthy := true
for _, check := range response.Checks {
if check.Status != "healthy" {
allHealthy = false
break
}
}

if allHealthy {
response.Status = "healthy"
c.JSON(http.StatusOK, response)
} else {
response.Status = "unhealthy"
c.JSON(http.StatusServiceUnavailable, response)
}
}







Kubernetes Liveness & Readiness





apiVersion: v1
kind: Pod
spec:
containers:
- name: user-service
livenessProbe:
httpGet:
path: /health
port: 8081
initialDelaySeconds: 30
periodSeconds: 10
timeoutSeconds: 5
failureThreshold: 3

readinessProbe:
httpGet:
path: /health
port: 8081
initialDelaySeconds: 5
periodSeconds: 5
timeoutSeconds: 3
failureThreshold: 3










Error Tracking

Sentry Integration





// Initialize Sentry
import "github.com/getsentry/sentry-go"

func initSentry() {
err := sentry.Init(sentry.ClientOptions{
Dsn: os.Getenv("SENTRY_DSN"),
Environment: os.Getenv("ENV"),
Release: "user-service@1.0.0",
})
if err != nil {
log.Fatalf("Sentry init failed: %v", err)
}
}

// Capture errors
func (ps *PushService) sendToSubscription(subscription models.PushSubscription, payload types.PushPayload) error {
err := ps.doSend(subscription, payload)

if err != nil {
// Report to Sentry
sentry.WithScope(func(scope *sentry.Scope) {
scope.SetContext("subscription", map[string]interface{}{
"id": subscription.ID,
"device_name": subscription.DeviceName,
"browser": subscription.Browser,
})
scope.SetContext("notification", map[string]interface{}{
"type": payload.Type,
"title": payload.Title,
})
sentry.CaptureException(err)
})
}

return err
}










Alerting Strategies

Alert Rules





# prometheus-alerts.yml
groups:
- name: push_notifications
interval: 30s
rules:
# High failure rate
- alert: PushNotificationFailureRateHigh
expr: |
(
rate(push_notifications_sent_total{status="error"}[5m])
/
rate(push_notifications_sent_total[5m])
) > 0.10
for: 5m
labels:
severity: critical
annotations:
summary: "Push notification failure rate > 10%"
description: "{{ $value | humanizePercentage }} of push notifications are failing"

# No notifications sent
- alert: PushNotificationsNotSending
expr: |
rate(push_notifications_sent_total[10m]) == 0
for: 10m
labels:
severity: warning
annotations:
summary: "No push notifications sent in last 10 minutes"

# Subscription count dropping
- alert: ActiveSubscriptionsDropping
expr: |
(
push_subscriptions_active - push_subscriptions_active offset 1h
)
for: 5m
labels:
severity: warning
annotations:
summary: "Active subscriptions dropped by {{ $value }}"

# Worker not running
- alert: BookingReminderWorkerDown
expr: |
push_worker_runs_total{worker="booking_reminder"} offset 10m
==
push_worker_runs_total{worker="booking_reminder"}
for: 10m
labels:
severity: critical
annotations:
summary: "BookingReminderWorker hasn't run in 10 minutes"

# High latency
- alert: PushNotificationLatencyHigh
expr: |
histogram_quantile(0.95, rate(push_notification_duration_seconds_bucket[5m])) > 5
for: 5m
labels:
severity: warning
annotations:
summary: "Push notification p95 latency > 5s"







Alertmanager Configuration





# alertmanager.yml
global:
resolve_timeout: 5m

route:
group_by: ['alertname', 'severity']
group_wait: 10s
group_interval: 10s
repeat_interval: 1h
receiver: 'default'

routes:
- match:
severity: critical
receiver: 'pagerduty'
continue: true

- match:
severity: warning
receiver: 'slack'

receivers:
- name: 'default'
email_configs:
- to: 'alerts@yourdomain.com'

- name: 'slack'
slack_configs:
- api_url: 'https://hooks.slack.com/services/YOUR/WEBHOOK/URL'
channel: '#alerts'
title: '{{ .GroupLabels.alertname }}'
text: '{{ range .Alerts }}{{ .Annotations.description }}{{ end }}'

- name: 'pagerduty'
pagerduty_configs:
- service_key: 'YOUR_PAGERDUTY_KEY'










Debugging Tools

Database Query Debugging





-- Find subscriptions with high failure counts
SELECT
id,
COALESCE(user_id::TEXT, admin_id::TEXT) as owner_id,
device_name,
browser,
failure_count,
last_used_at,
created_at
FROM push_subscriptions
WHERE failure_count >= 2
ORDER BY failure_count DESC, last_used_at DESC
LIMIT 20;

-- Check recent failed notifications
SELECT
pnl.id,
pnl.title,
pnl.status,
pnl.error_message,
pnl.http_status_code,
pnl.sent_at,
ps.device_name,
ps.browser
FROM push_notification_logs pnl
LEFT JOIN push_subscriptions ps ON pnl.subscription_id = ps.id
WHERE pnl.status = 'failed'
AND pnl.sent_at > NOW() - INTERVAL '1 hour'
ORDER BY pnl.sent_at DESC
LIMIT 50;

-- Analyze notification volume by hour
SELECT
DATE_TRUNC('hour', sent_at) as hour,
COUNT(*) as total,
COUNT(*) FILTER (WHERE status = 'sent') as sent,
COUNT(*) FILTER (WHERE status = 'failed') as failed,
ROUND(COUNT(*) FILTER (WHERE status = 'sent')::NUMERIC * 100 / COUNT(*), 2) as success_rate
FROM push_notification_logs
WHERE sent_at > NOW() - INTERVAL '24 hours'
GROUP BY hour
ORDER BY hour DESC;







Frontend Debugging





// Enable verbose logging
localStorage.setItem('debug_push_notifications', 'true');

// Check subscription status
navigator.serviceWorker.ready.then(async reg => {
const subscription = await reg.pushManager.getSubscription();
console.log('Current subscription:', subscription);

if (subscription) {
console.log('Endpoint:', subscription.endpoint);
console.log('Keys:', {
p256dh: subscription.toJSON().keys.p256dh,
auth: subscription.toJSON().keys.auth
});
}
});

// Monitor service worker messages
navigator.serviceWorker.addEventListener('message' , event => {
console.log('[SW Message]', event.data);
});

// Test push notification
async function testPushNotification() {
const registration = await navigator.serviceWorker.ready;
await registration.showNotification('Test Notification', {
body: 'This is a test',
icon: '/icon-192x192.png',
data: { url: '/' }
});
}










Common Issues & Solutions

Issue 1: Notifications Not Received

Symptoms:

• Backend logs show "sent" status
• User doesn't receive notification

Debugging:






# Check browser notification permission
# Browser Console:
Notification.permission

# Check service worker status
navigator.serviceWorker.ready.then(reg => console.log(reg.active));

# Check subscription exists in backend
curl -X GET http://localhost:8081/api/push/devices \
-H "Cookie: auth-token=YOUR_TOKEN"







Solutions:

1. Permission denied: User must grant permission in browser settings
2. Service worker not registered: Ensure sw.js exists at /public/sw.js
3. Subscription expired: User needs to resubscribe

Issue 2: High Failure Rate

Symptoms:

• >10% of notifications failing
• failure_count increasing for subscriptions

Debugging:






-- Check error messages
SELECT
error_message,
http_status_code,
COUNT(*) as occurrences
FROM push_notification_logs
WHERE status = 'failed'
AND sent_at > NOW() - INTERVAL '24 hours'
GROUP BY error_message, http_status_code
ORDER BY occurrences DESC;







Common Causes & Solutions:

410 Gone	Subscription expired	Deactivate subscription (auto-handled)
403 Forbidden	Invalid VAPID keys	Verify VAPID configuration
400 Bad Request	Malformed payload	Check payload format
429 Too Many Requests	Rate limit hit	Implement backoff
500 Server Error	Push service issue	Retry with exponential backoff

Issue 3: Worker Not Running

Symptoms:

• Reminder notifications not sent
• Worker metrics flat

Debugging:






# Check service logs
kubectl logs -f deployment/user-service | grep WorkerGoroutine

# Expected:
# [BookingReminderWorker] Started successfully
# [BookingReminderWorker] Processing 3 bookings

# Check if multiple instances are running
kubectl get pods -l app=user-service







Solutions:

1. Restart service: kubectl rollout restart deployment/user-service
2. Check leader election: Ensure only one worker instance has leadership
3. Verify database connection: Worker requires database access

Issue 4: Memory Leak

Symptoms:

• Memory usage increasing over time
• OOM kills in Kubernetes

Debugging:






# Memory profiling
go tool pprof http://localhost:6060/debug/pprof/heap

# Check goroutine leaks
go tool pprof http://localhost:6060/debug/pprof/goroutine

# Kubernetes memory usage
kubectl top pod -l app=user-service







Common Causes:

1. Goroutine leaks: Not closing database connections
2. Unbounded caches: No cache eviction policy
3. Large payloads: Storing too much in memory

Solutions:






// Use context with timeout
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()

// Close database connections
defer db.Close()

// Use bounded channels
queue := make(chan *PushNotification, 1000)










Incident Response

Incident Response Playbook

1. Detection

• Alert fires in Prometheus/Alertmanager
• User reports issue
• Monitoring dashboard shows anomaly

2. Assessment





# Check service health
curl http://api.yourdomain.com/health

# Check recent errors
kubectl logs -f deployment/user-service --tail=100 | grep ERROR

# Check metrics
# Visit Grafana dashboard: http://grafana.yourdomain.com/d/push-notifications







3. Mitigation

High Failure Rate:






# Check VAPID keys
kubectl get secret vapid-keys -o jsonpath='{.data.public-key}' | base64 -d

# Restart services
kubectl rollout restart deployment/user-service
kubectl rollout restart deployment/admin-service







Database Connection Issues:






# Check database health
kubectl exec -it deployment/postgres -- psql -U push_service -c "SELECT 1;"

# Scale down workers temporarily
kubectl scale deployment/user-service --replicas=1







High Latency:






# Check active goroutines
curl http://localhost:6060/debug/pprof/goroutine?debug=1

# Scale up if needed
kubectl scale deployment/user-service --replicas=5







4. Communication

• Update status page
• Notify stakeholders
• Document in incident log

5. Resolution

• Apply fix
• Verify metrics returned to normal
• Write postmortem

Analytics & Reporting

Weekly Report Query





-- Weekly push notification report
WITH weekly_stats AS (
SELECT
DATE_TRUNC('week', sent_at) as week,
COUNT(*) as total_sent,
COUNT(*) FILTER (WHERE status = 'sent') as successful,
COUNT(*) FILTER (WHERE status = 'failed') as failed,
COUNT(DISTINCT subscription_id) as unique_devices,
COUNT(DISTINCT COALESCE(user_id, admin_id)) as unique_users
FROM push_notification_logs
WHERE sent_at > NOW() - INTERVAL '4 weeks'
GROUP BY week
)
SELECT
week,
total_sent,
successful,
failed,
ROUND(successful::NUMERIC * 100 / total_sent, 2) as success_rate,
unique_devices,
unique_users
FROM weekly_stats
ORDER BY week DESC;







Export to CSV





# Export last 7 days of notification data
psql -h localhost -U postgres -d ohhspaces_prod -c "
COPY (
SELECT
pnl.sent_at,
pnl.title,
pnl.status,
ps.device_name,
ps.browser,
ps.os
FROM push_notification_logs pnl
LEFT JOIN push_subscriptions ps ON pnl.subscription_id = ps.id
WHERE pnl.sent_at > NOW() - INTERVAL '7 days'
ORDER BY pnl.sent_at DESC
) TO STDOUT WITH CSV HEADER" > push_notifications_export.csv










Summary

You now have:


✅ Comprehensive metrics and monitoring

✅ Structured logging with aggregation

✅ Performance profiling tools

✅ Health check endpoints

✅ Error tracking integration

✅ Alerting rules and strategies

✅ Debugging tools and techniques

✅ Common issues with solutions

✅ Incident response playbook

✅ Analytics and reporting queries





Next Steps

➡️ Part 7: Best Practices, Security & Optimization


Part 7 will cover:

1. Security best practices
2. VAPID key management
3. Input validation
4. SQL injection prevention
5. Performance optimization
6. Code organization
7. Testing strategies
8. Documentation standards

More...