Staging Rollback Procedures

# Verify rollback tools availability
ls -la scripts/rollback_*.sh
ls -la scripts/verify_staging_deployment.sh
ls -la scripts/backup_staging.sh

# Health check monitoring (runs every 30 seconds)
while true; do
    ./scripts/health_check.sh
    if [ $? -ne 0 ]; then
        echo "Health check failed, triggering automatic rollback"
        ./scripts/rollback_staging.sh
        break
    fi
    sleep 30
done

# 1. Verify current deployment health
./scripts/verify_staging_deployment.sh

# 2. Create backup before rollback
./scripts/backup_staging.sh

# 3. Execute rollback
./scripts/rollback_staging.sh

# 4. Verify rollback success
./scripts/verify_staging_deployment.sh

# Force immediate rollback (bypasses health checks)
./scripts/emergency_rollback.sh

# Verify critical services are running
./scripts/verify_critical_services.sh

# Rollback application to previous version
git checkout HEAD~1
docker-compose build app
docker-compose up -d app

# Verify application health
curl http://localhost:8001/health

# Restore database from backup
./scripts/restore_staging.sh backup_$(date +%Y-%m-%d_%H-%M-%S).sql

# Verify database integrity
docker-compose exec app python -c "
from services.database import get_db
db = get_db()
result = db.execute('SELECT COUNT(*) FROM information_schema.tables')
print(f'Database tables: {result.fetchone()[0]}')
"

# Restore previous configuration
git checkout HEAD~1 -- .env
git checkout HEAD~1 -- docker-compose.yml

# Restart services with old config
docker-compose down
docker-compose up -d

# Rollback monitoring services
docker-compose down prometheus grafana
docker-compose up -d prometheus grafana

# Verify monitoring is working
curl http://localhost:9090/-/healthy
curl http://localhost:3001/api/health

#!/bin/bash
# scripts/rollback_staging.sh

set -e

echo "🚨 Initiating staging rollback..."

# 1. Stop current deployment
echo "Stopping current deployment..."
docker-compose down

# 2. Checkout previous version
echo "Checking out previous version..."
git checkout HEAD~1

# 3. Restore from backup if available
if [ -f "backup_latest.sql" ]; then
    echo "Restoring database from backup..."
    ./scripts/restore_staging.sh backup_latest.sql
fi

# 4. Rebuild and restart
echo "Rebuilding and restarting services..."
docker-compose build
docker-compose up -d

# 5. Wait for services to be ready
echo "Waiting for services to be ready..."
sleep 30

# 6. Verify rollback success
echo "Verifying rollback success..."
./scripts/verify_staging_deployment.sh

echo "✅ Rollback completed successfully"

#!/bin/bash
# scripts/emergency_rollback.sh

set -e

echo "🚨 EMERGENCY ROLLBACK - Bypassing health checks"

# 1. Force stop all services
docker-compose down --remove-orphans

# 2. Checkout last known good version
git checkout HEAD~5  # Go back 5 commits for safety

# 3. Restart with minimal verification
docker-compose up -d

# 4. Basic health check only
curl -f http://localhost:8001/health || echo "Health check failed but services are running"

echo "⚠️  Emergency rollback completed - manual verification required"

#!/bin/bash
# scripts/verify_staging_deployment.sh

set -e

echo "🔍 Verifying staging deployment..."

# Test 1: Application health
echo "Test 1: Application health..."
curl -f http://localhost:8001/health

# Test 2: Database connection
echo "Test 2: Database connection..."
docker-compose exec app python -c "from services.database import get_db; print('Database OK')"

# Test 3: Redis connection
echo "Test 3: Redis connection..."
docker-compose exec app python -c "import redis; r = redis.Redis(host='redis'); r.ping(); print('Redis OK')"

# Test 4: MCP health
echo "Test 4: MCP health..."
curl -f http://localhost:8001/health/mcp

# Test 5: Performance test
echo "Test 5: Performance test..."
start_time=$(date +%s%N)
curl -s http://localhost:8001/health > /dev/null
end_time=$(date +%s%N)
response_time=$(( (end_time - start_time) / 1000000 ))

if [ $response_time -lt 500 ]; then
    echo "✅ Performance OK: ${response_time}ms"
else
    echo "⚠️  Performance degraded: ${response_time}ms"
fi

echo "✅ All verification tests passed"

# Monitor rollback progress
watch -n 5 'docker-compose ps && echo "---" && curl -s http://localhost:8001/health | jq'

# Check service logs during rollback
docker-compose logs -f app postgres redis

# Create rollback status dashboard
cat > rollback_status.md << EOF
# Rollback Status: $(date)

## Services Status
$(docker-compose ps)

## Health Checks
$(curl -s http://localhost:8001/health | jq)

## Performance Metrics
$(curl -s http://localhost:8001/metrics | grep -E "(response_time|error_rate)")
EOF

# 1. Create backup
./scripts/backup_staging.sh

# 2. Tag current version
git tag "pre-deploy-$(date +%Y%m%d-%H%M%S)"

# 3. Verify rollback tools
./scripts/test_rollback_tools.sh

# 1. Monitor health checks
./scripts/monitor_deployment.sh

# 2. Watch for warning signs
- Response time increases
- Error rate spikes
- Service health degradation

# 1. Verify system stability
./scripts/verify_staging_deployment.sh

# 2. Document rollback details
./scripts/document_rollback.sh

# 3. Plan next deployment
./scripts/plan_next_deployment.sh

# Check script permissions
chmod +x scripts/rollback_staging.sh

# Verify script dependencies
./scripts/check_rollback_dependencies.sh

# Run with debug output
bash -x scripts/rollback_staging.sh

# Check Docker resources
docker system df
docker system prune -f

# Verify port availability
netstat -tulpn | grep -E "(8001|8081|3001|9090)"

# Check service logs
docker-compose logs [service_name]

# Verify backup file integrity
./scripts/verify_backup.sh backup_file.sql

# Check database permissions
docker-compose exec postgres psql -U piper_user -l

# Manual database restore
docker-compose exec -T postgres psql -U piper_user -d piper_morgan_staging < backup_file.sql

# Track rollback success rate
./scripts/rollback_metrics.sh

# Expected metrics:
# - Rollback success rate: >95%
# - Average rollback time: <5 minutes
# - Recovery time: <10 minutes