Chapter 11.2: Incident Management

Incident Management is a critical component of Site Reliability Engineering that ensures rapid detection, response, and resolution of service disruptions.

Quick Decision

Situation	Action	Watch Out
User impact is large	Declare incident	Declaring early is better than late
Cause is unknown	Commander + communication channel	Everyone should not debug at once
Mitigation is found	Reduce impact first	Root cause can go deeper later
Incident is over	Post-mortem	Action owner must be assigned

Production Checklist

• Problem: Which user impact and which SLO violation exist?
• Solution: Are severity, incident commander, communication, mitigation, and status-update cadence clear?
• Trade-off: Process slows slightly; uncoordinated response causes more damage.
• Failure mode: Wrong severity, silent incidents, delayed rollback, and actionless post-mortems should be handled.
• Measurement: Track MTTD, MTTA, MTTR, customer impact duration, and repeat incident count.
• Security/cost: Do not share secrets/PII in incident channels; long outages create direct revenue and trust cost.

1. Incident Classification and Severity Levels

Severity Level Framework

incident_severity_levels:
  sev_1_critical:
    description: "Complete service outage affecting all users"
    response_time: "5 minutes"
    escalation_time: "15 minutes"
    communication_frequency: "Every 15 minutes"
    stakeholders:
      - on_call_engineer
      - incident_commander
      - engineering_director
      - customer_support
    
  sev_2_high:
    description: "Major functionality impacted, affecting significant user base"
    response_time: "15 minutes"
    escalation_time: "30 minutes"
    communication_frequency: "Every 30 minutes"
    stakeholders:
      - on_call_engineer
      - team_lead
      - product_manager
    
  sev_3_medium:
    description: "Minor functionality impacted, workaround available"
    response_time: "1 hour"
    escalation_time: "2 hours"
    communication_frequency: "Every 2 hours"
    stakeholders:
      - on_call_engineer
      - team_lead
    
  sev_4_low:
    description: "Minimal impact, can be resolved during business hours"
    response_time: "4 hours"
    escalation_time: "Next business day"
    communication_frequency: "Daily updates"
    stakeholders:
      - assigned_engineer

Automated Incident Detection

@Component
@Slf4j
public class IncidentDetectionEngine {
    
    private final MetricsCollector metricsCollector;
    private final AlertManager alertManager;
    private final IncidentService incidentService;
    private final Map<String, IncidentRule> detectionRules;
    
    @EventListener
    public void handleMetricAlert(MetricAlertEvent event) {
        IncidentRule rule = detectionRules.get(event.getMetricName());
        if (rule != null && rule.matches(event)) {
            processIncident(event, rule);
        }
    }
    
    private void processIncident(MetricAlertEvent event, IncidentRule rule) {
        // Check if incident already exists
        Optional<Incident> existingIncident = incidentService
            .findActiveIncidentByService(event.getServiceName());
        
        if (existingIncident.isPresent()) {
            updateExistingIncident(existingIncident.get(), event);
        } else {
            createNewIncident(event, rule);
        }
    }
    
    private void createNewIncident(MetricAlertEvent event, IncidentRule rule) {
        IncidentSeverity severity = determineSeverity(event, rule);
        
        Incident incident = Incident.builder()
            .id(generateIncidentId())
            .title(generateIncidentTitle(event))
            .description(generateIncidentDescription(event))
            .severity(severity)
            .status(IncidentStatus.INVESTIGATING)
            .affectedService(event.getServiceName())
            .detectedAt(Instant.now())
            .detectionSource(DetectionSource.AUTOMATED)
            .metadata(buildIncidentMetadata(event))
            .build();
        
        incidentService.createIncident(incident);
        
        // Trigger incident response
        triggerIncidentResponse(incident);
        
        log.info("New incident created: {} - Severity: {} - Service: {}", 
            incident.getId(), severity, event.getServiceName());
    }
    
    private void triggerIncidentResponse(Incident incident) {
        // Page on-call engineer
        pagerDutyService.triggerIncident(incident);
        
        // Create incident war room
        slackService.createIncidentChannel(incident);
        
        // Initialize incident commander if SEV-1 or SEV-2
        if (incident.getSeverity().ordinal() <= IncidentSeverity.SEV_2.ordinal()) {
            incidentCommanderService.assignCommander(incident);
        }
        
        // Start automated diagnostics
        diagnosticsService.startAutomatedDiagnostics(incident);
    }
}

2. Incident Response Process

Incident Response Workflow

incident_response_workflow:
  detection:
    automated_alerts: true
    manual_reporting: true
    customer_reports: true
    
  initial_response:
    acknowledge_alert: "< 5 minutes"
    assess_severity: "< 10 minutes"
    establish_communication: "< 15 minutes"
    
  investigation:
    gather_context: true
    review_recent_changes: true
    check_dependencies: true
    analyze_metrics: true
    
  mitigation:
    implement_workaround: true
    rollback_changes: true
    scale_resources: true
    enable_circuit_breakers: true
    
  resolution:
    verify_fix: true
    monitor_stability: true
    close_incident: true
    
  post_incident:
    conduct_postmortem: true
    implement_improvements: true
    update_documentation: true

Incident Management Service

@Service
@Slf4j
public class IncidentManagementService {
    
    private final IncidentRepository incidentRepository;
    private final TimelineService timelineService;
    private final CommunicationService communicationService;
    private final EscalationService escalationService;
    
    public Incident updateIncidentStatus(String incidentId, 
                                        IncidentStatus newStatus, 
                                        String updateMessage) {
        Incident incident = incidentRepository.findById(incidentId)
            .orElseThrow(() -> new IncidentNotFoundException(incidentId));
        
        IncidentStatus previousStatus = incident.getStatus();
        incident.setStatus(newStatus);
        incident.setLastUpdated(Instant.now());
        
        // Add timeline entry
        timelineService.addEntry(incidentId, TimelineEntry.builder()
            .timestamp(Instant.now())
            .type(TimelineEntryType.STATUS_CHANGE)
            .message(String.format("Status changed from %s to %s: %s", 
                previousStatus, newStatus, updateMessage))
            .author(getCurrentUser())
            .build());
        
        // Handle status-specific actions
        handleStatusChange(incident, previousStatus, newStatus);
        
        return incidentRepository.save(incident);
    }
    
    private void handleStatusChange(Incident incident, 
                                   IncidentStatus from, 
                                   IncidentStatus to) {
        switch (to) {
            case INVESTIGATING:
                startInvestigation(incident);
                break;
            case MITIGATED:
                handleMitigation(incident);
                break;
            case RESOLVED:
                handleResolution(incident);
                break;
            case CLOSED:
                handleClosure(incident);
                break;
        }
    }
    
    private void startInvestigation(Incident incident) {
        // Assign primary responder if not already assigned
        if (incident.getAssignedTo() == null) {
            String onCallEngineer = getOnCallEngineer(incident.getAffectedService());
            assignIncident(incident.getId(), onCallEngineer);
        }
        
        // Start automated diagnostics
        diagnosticsService.startDiagnostics(incident);
        
        // Check escalation criteria
        scheduleEscalationCheck(incident);
    }
    
    @Scheduled(fixedDelay = 300000) // Check every 5 minutes
    public void checkEscalations() {
        List<Incident> activeIncidents = incidentRepository
            .findByStatusIn(List.of(
                IncidentStatus.INVESTIGATING, 
                IncidentStatus.MITIGATED
            ));
        
        activeIncidents.forEach(this::checkEscalationCriteria);
    }
    
    private void checkEscalationCriteria(Incident incident) {
        Duration timeElapsed = Duration.between(incident.getDetectedAt(), Instant.now());
        EscalationPolicy policy = getEscalationPolicy(incident.getSeverity());
        
        if (shouldEscalate(incident, timeElapsed, policy)) {
            escalateIncident(incident, policy);
        }
    }
    
    private void escalateIncident(Incident incident, EscalationPolicy policy) {
        EscalationLevel nextLevel = policy.getNextLevel(incident.getCurrentEscalationLevel());
        
        if (nextLevel != null) {
            incident.setCurrentEscalationLevel(nextLevel);
            
            // Notify escalation contacts
            escalationService.notifyEscalation(incident, nextLevel);
            
            // Add timeline entry
            timelineService.addEntry(incident.getId(), TimelineEntry.builder()
                .timestamp(Instant.now())
                .type(TimelineEntryType.ESCALATION)
                .message("Incident escalated to " + nextLevel.getName())
                .automated(true)
                .build());
            
            log.warn("Incident {} escalated to level {}", 
                incident.getId(), nextLevel.getName());
        }
    }
}

Incident Communication System

@Component
public class IncidentCommunicationService {
    
    private final SlackService slackService;
    private final EmailService emailService;
    private final StatusPageService statusPageService;
    private final CustomerNotificationService customerNotificationService;
    
    @EventListener
    public void handleIncidentStatusChange(IncidentStatusChangeEvent event) {
        Incident incident = event.getIncident();
        
        // Internal communications
        sendInternalUpdates(incident, event.getUpdateMessage());
        
        // External communications for customer-facing incidents
        if (isCustomerFacing(incident)) {
            sendExternalUpdates(incident, event.getUpdateMessage());
        }
    }
    
    private void sendInternalUpdates(Incident incident, String updateMessage) {
        // Slack notifications
        String channelId = incident.getSlackChannelId();
        if (channelId != null) {
            SlackMessage message = SlackMessage.builder()
                .channel(channelId)
                .text(formatInternalUpdate(incident, updateMessage))
                .attachments(buildIncidentAttachments(incident))
                .build();
            
            slackService.sendMessage(message);
        }
        
        // Email to stakeholders
        List<String> stakeholders = getStakeholders(incident.getSeverity());
        stakeholders.forEach(email -> {
            emailService.sendIncidentUpdate(email, incident, updateMessage);
        });
    }
    
    private void sendExternalUpdates(Incident incident, String updateMessage) {
        // Update status page
        statusPageService.updateIncident(incident.getId(), 
            mapToStatusPageUpdate(incident, updateMessage));
        
        // Notify affected customers
        if (shouldNotifyCustomers(incident)) {
            customerNotificationService.sendIncidentNotification(
                incident, updateMessage);
        }
    }
    
    public void sendScheduledUpdate(String incidentId) {
        Incident incident = getIncident(incidentId);
        
        if (incident.getStatus() == IncidentStatus.INVESTIGATING || 
            incident.getStatus() == IncidentStatus.MITIGATED) {
            
            String updateMessage = generateStatusUpdate(incident);
            sendInternalUpdates(incident, updateMessage);
            
            if (isCustomerFacing(incident)) {
                sendExternalUpdates(incident, updateMessage);
            }
            
            // Schedule next update
            scheduleNextUpdate(incident);
        }
    }
    
    private String generateStatusUpdate(Incident incident) {
        StringBuilder update = new StringBuilder();
        update.append("Incident Update - ").append(incident.getTitle()).append("\n\n");
        update.append("Status: ").append(incident.getStatus()).append("\n");
        update.append("Duration: ").append(getIncidentDuration(incident)).append("\n");
        
        if (incident.getCurrentAction() != null) {
            update.append("Current Action: ").append(incident.getCurrentAction()).append("\n");
        }
        
        if (incident.getEstimatedResolutionTime() != null) {
            update.append("ETA: ").append(incident.getEstimatedResolutionTime()).append("\n");
        }
        
        return update.toString();
    }
}

3. Root Cause Analysis (RCA)

RCA Framework Implementation

@Service
public class RootCauseAnalysisService {
    
    private final IncidentRepository incidentRepository;
    private final ChangeLogService changeLogService;
    private final MetricsAnalysisService metricsAnalysisService;
    private final DependencyService dependencyService;
    
    public RCAnalysisReport conductRCA(String incidentId) {
        Incident incident = incidentRepository.findById(incidentId)
            .orElseThrow(() -> new IncidentNotFoundException(incidentId));
        
        RCAnalysisReport.Builder reportBuilder = RCAnalysisReport.builder()
            .incidentId(incidentId)
            .analysisDate(LocalDateTime.now())
            .analyst(getCurrentUser());
        
        // Analyze timeline
        Timeline timeline = analyzeIncidentTimeline(incident);
        reportBuilder.timeline(timeline);
        
        // Analyze recent changes
        List<Change> recentChanges = analyzeRecentChanges(incident);
        reportBuilder.recentChanges(recentChanges);
        
        // Analyze metrics and patterns
        MetricsAnalysis metricsAnalysis = analyzeMetricsPatterns(incident);
        reportBuilder.metricsAnalysis(metricsAnalysis);
        
        // Analyze dependencies
        DependencyAnalysis dependencyAnalysis = analyzeDependencies(incident);
        reportBuilder.dependencyAnalysis(dependencyAnalysis);
        
        // Apply RCA techniques
        FiveWhysAnalysis fiveWhys = conductFiveWhysAnalysis(incident);
        reportBuilder.fiveWhysAnalysis(fiveWhys);
        
        FishboneAnalysis fishbone = conductFishboneAnalysis(incident);
        reportBuilder.fishboneAnalysis(fishbone);
        
        // Generate recommendations
        List<Recommendation> recommendations = generateRecommendations(
            timeline, recentChanges, metricsAnalysis, dependencyAnalysis);
        reportBuilder.recommendations(recommendations);
        
        return reportBuilder.build();
    }
    
    private FiveWhysAnalysis conductFiveWhysAnalysis(Incident incident) {
        FiveWhysAnalysis.Builder builder = FiveWhysAnalysis.builder()
            .problem(incident.getTitle());
        
        String currentWhy = incident.getTitle();
        for (int i = 1; i <= 5; i++) {
            String why = analyzeWhy(currentWhy, incident);
            builder.addWhy(i, why);
            currentWhy = why;
            
            if (isRootCause(why)) {
                break;
            }
        }
        
        return builder.build();
    }
    
    private FishboneAnalysis conductFishboneAnalysis(Incident incident) {
        return FishboneAnalysis.builder()
            .problem(incident.getTitle())
            .addCause(CauseCategory.METHODS, analyzeMethodCauses(incident))
            .addCause(CauseCategory.MACHINES, analyzeMachineCauses(incident))
            .addCause(CauseCategory.MATERIALS, analyzeMaterialCauses(incident))
            .addCause(CauseCategory.PEOPLE, analyzePeopleCauses(incident))
            .addCause(CauseCategory.ENVIRONMENT, analyzeEnvironmentCauses(incident))
            .addCause(CauseCategory.MANAGEMENT, analyzeManagementCauses(incident))
            .build();
    }
    
    private List<Recommendation> generateRecommendations(Timeline timeline,
                                                        List<Change> recentChanges,
                                                        MetricsAnalysis metricsAnalysis,
                                                        DependencyAnalysis dependencyAnalysis) {
        List<Recommendation> recommendations = new ArrayList<>();
        
        // Analyze change-related recommendations
        if (!recentChanges.isEmpty()) {
            recommendations.addAll(generateChangeRecommendations(recentChanges));
        }
        
        // Analyze monitoring recommendations
        if (metricsAnalysis.hasGaps()) {
            recommendations.addAll(generateMonitoringRecommendations(metricsAnalysis));
        }
        
        // Analyze process recommendations
        if (timeline.hasDelays()) {
            recommendations.addAll(generateProcessRecommendations(timeline));
        }
        
        // Analyze dependency recommendations
        if (dependencyAnalysis.hasIssues()) {
            recommendations.addAll(generateDependencyRecommendations(dependencyAnalysis));
        }
        
        return recommendations.stream()
            .sorted(Comparator.comparing(Recommendation::getPriority))
            .collect(Collectors.toList());
    }
}

Automated RCA Tools

@Component
public class AutomatedRCATools {
    
    private final LogAnalysisService logAnalysisService;
    private final AnomalyDetectionService anomalyDetectionService;
    private final CorrelationAnalysisService correlationService;
    
    public AutomatedRCAResults analyzeIncident(Incident incident) {
        AutomatedRCAResults.Builder builder = AutomatedRCAResults.builder();
        
        // Analyze logs for anomalies
        LogAnalysisResults logResults = logAnalysisService.analyzeLogsAroundIncident(
            incident.getAffectedService(),
            incident.getDetectedAt().minus(Duration.ofHours(1)),
            incident.getDetectedAt().plus(Duration.ofHours(1))
        );
        builder.logAnalysis(logResults);
        
        // Detect metric anomalies
        List<MetricAnomaly> anomalies = anomalyDetectionService.detectAnomalies(
            incident.getAffectedService(),
            incident.getDetectedAt().minus(Duration.ofHours(2)),
            incident.getDetectedAt()
        );
        builder.metricAnomalies(anomalies);
        
        // Correlate with external events
        List<CorrelatedEvent> correlatedEvents = correlationService.findCorrelatedEvents(
            incident.getDetectedAt(),
            Duration.ofHours(2)
        );
        builder.correlatedEvents(correlatedEvents);
        
        // Generate automated insights
        List<RCAInsight> insights = generateAutomatedInsights(
            logResults, anomalies, correlatedEvents);
        builder.insights(insights);
        
        return builder.build();
    }
    
    private List<RCAInsight> generateAutomatedInsights(LogAnalysisResults logResults,
                                                      List<MetricAnomaly> anomalies,
                                                      List<CorrelatedEvent> correlatedEvents) {
        List<RCAInsight> insights = new ArrayList<>();
        
        // Analyze error patterns in logs
        if (logResults.hasErrorSpikes()) {
            insights.add(RCAInsight.builder()
                .type(InsightType.ERROR_PATTERN)
                .confidence(0.8)
                .description("Error spike detected in logs starting " + 
                    logResults.getErrorSpikeStart())
                .evidence(logResults.getTopErrors())
                .build());
        }
        
        // Analyze metric correlations
        for (MetricAnomaly anomaly : anomalies) {
            if (anomaly.getConfidence() > 0.7) {
                insights.add(RCAInsight.builder()
                    .type(InsightType.METRIC_ANOMALY)
                    .confidence(anomaly.getConfidence())
                    .description(String.format("Anomaly detected in %s: %s", 
                        anomaly.getMetricName(), anomaly.getDescription()))
                    .evidence(List.of(anomaly.getMetricName()))
                    .build());
            }
        }
        
        // Analyze external correlations
        for (CorrelatedEvent event : correlatedEvents) {
            if (event.getCorrelationStrength() > 0.6) {
                insights.add(RCAInsight.builder()
                    .type(InsightType.EXTERNAL_CORRELATION)
                    .confidence(event.getCorrelationStrength())
                    .description(String.format("Correlated with %s at %s", 
                        event.getEventType(), event.getTimestamp()))
                    .evidence(List.of(event.getEventDescription()))
                    .build());
            }
        }
        
        return insights;
    }
}

4. Post-Incident Review and Improvement

Post-Mortem Process

@Entity
@Table(name = "post_mortems")
public class PostMortem {
    
    @Id
    private String incidentId;
    
    @Column(nullable = false)
    private String title;
    
    @Column(columnDefinition = "TEXT")
    private String summary;
    
    @Column(columnDefinition = "TEXT")
    private String timeline;
    
    @Column(columnDefinition = "TEXT")
    private String rootCause;
    
    @Column(columnDefinition = "TEXT")
    private String impact;
    
    @ElementCollection
    @CollectionTable(name = "postmortem_action_items")
    private List<ActionItem> actionItems;
    
    @Column(nullable = false)
    private LocalDateTime createdAt;
    
    @Column
    private LocalDateTime reviewedAt;
    
    @Enumerated(EnumType.STRING)
    private PostMortemStatus status;
    
    @Column
    private String facilitator;
    
    @ElementCollection
    private List<String> participants;
    
    // Constructors, getters, setters
}

@Embeddable
public class ActionItem {
    
    @Column(nullable = false)
    private String description;
    
    @Column
    private String assignee;
    
    @Column
    private LocalDateTime dueDate;
    
    @Enumerated(EnumType.STRING)
    private ActionItemPriority priority;
    
    @Enumerated(EnumType.STRING)
    private ActionItemStatus status;
    
    @Column
    private String jiraTicket;
    
    // Constructors, getters, setters
}

Post-Mortem Management Service

@Service
@Slf4j
public class PostMortemService {
    
    private final PostMortemRepository postMortemRepository;
    private final IncidentRepository incidentRepository;
    private final ActionItemTrackingService actionItemService;
    private final NotificationService notificationService;
    
    public PostMortem createPostMortem(String incidentId, String facilitator) {
        Incident incident = incidentRepository.findById(incidentId)
            .orElseThrow(() -> new IncidentNotFoundException(incidentId));
        
        PostMortem postMortem = PostMortem.builder()
            .incidentId(incidentId)
            .title("Post-mortem: " + incident.getTitle())
            .createdAt(LocalDateTime.now())
            .status(PostMortemStatus.DRAFT)
            .facilitator(facilitator)
            .build();
        
        // Pre-populate with incident data
        populateFromIncident(postMortem, incident);
        
        return postMortemRepository.save(postMortem);
    }
    
    private void populateFromIncident(PostMortem postMortem, Incident incident) {
        // Generate initial summary
        String summary = generateInitialSummary(incident);
        postMortem.setSummary(summary);
        
        // Generate timeline from incident timeline
        String timeline = generateTimelineFromIncident(incident);
        postMortem.setTimeline(timeline);
        
        // Generate impact assessment
        String impact = generateImpactAssessment(incident);
        postMortem.setImpact(impact);
    }
    
    public PostMortem conductPostMortemReview(String incidentId, 
                                             PostMortemReviewData reviewData) {
        PostMortem postMortem = postMortemRepository.findById(incidentId)
            .orElseThrow(() -> new PostMortemNotFoundException(incidentId));
        
        // Update with review data
        postMortem.setRootCause(reviewData.getRootCause());
        postMortem.setParticipants(reviewData.getParticipants());
        postMortem.setReviewedAt(LocalDateTime.now());
        postMortem.setStatus(PostMortemStatus.REVIEWED);
        
        // Create action items
        List<ActionItem> actionItems = reviewData.getActionItems().stream()
            .map(this::createActionItem)
            .collect(Collectors.toList());
        postMortem.setActionItems(actionItems);
        
        // Schedule follow-up tracking
        scheduleActionItemTracking(postMortem);
        
        // Notify stakeholders
        notificationService.notifyPostMortemCompleted(postMortem);
        
        return postMortemRepository.save(postMortem);
    }
    
    @Scheduled(cron = "0 0 9 * * MON") // Every Monday at 9 AM
    public void trackActionItemProgress() {
        List<PostMortem> activePostMortems = postMortemRepository
            .findByStatusAndActionItemsStatusNot(
                PostMortemStatus.REVIEWED, 
                ActionItemStatus.COMPLETED
            );
        
        activePostMortems.forEach(this::checkActionItemProgress);
    }
    
    private void checkActionItemProgress(PostMortem postMortem) {
        boolean hasOverdueItems = false;
        
        for (ActionItem item : postMortem.getActionItems()) {
            if (item.getStatus() != ActionItemStatus.COMPLETED &&
                item.getDueDate().isBefore(LocalDateTime.now())) {
                
                hasOverdueItems = true;
                
                // Notify assignee
                notificationService.notifyOverdueActionItem(item);
                
                log.warn("Overdue action item: {} - Assignee: {} - Due: {}", 
                    item.getDescription(), item.getAssignee(), item.getDueDate());
            }
        }
        
        if (hasOverdueItems) {
            // Escalate if needed
            escalateOverdueActionItems(postMortem);
        }
    }
    
    public PostMortemMetrics generateMetrics(Duration period) {
        LocalDateTime startDate = LocalDateTime.now().minus(period);
        
        List<PostMortem> postMortems = postMortemRepository
            .findByCreatedAtAfter(startDate);
        
        return PostMortemMetrics.builder()
            .totalIncidents(postMortems.size())
            .averageTimeToPostMortem(calculateAverageTimeToPostMortem(postMortems))
            .actionItemCompletionRate(calculateActionItemCompletionRate(postMortems))
            .topRootCauses(analyzeTopRootCauses(postMortems))
            .improvementTrends(analyzeImprovementTrends(postMortems))
            .build();
    }
}

Integration with External Tools

# PagerDuty Integration
pagerduty:
  api_key: "${PAGERDUTY_API_KEY}"
  service_id: "${PAGERDUTY_SERVICE_ID}"
  escalation_policy: "${PAGERDUTY_ESCALATION_POLICY}"
  integration_key: "${PAGERDUTY_INTEGRATION_KEY}"

# Jira Integration
jira:
  url: "${JIRA_URL}"
  username: "${JIRA_USERNAME}"
  api_token: "${JIRA_API_TOKEN}"
  project_key: "INC"
  issue_type: "Incident"

# Slack Integration
slack:
  bot_token: "${SLACK_BOT_TOKEN}"
  incident_channel: "#incidents"
  escalation_channel: "#on-call-escalation"
  postmortem_channel: "#postmortems"

# StatusPage Integration
statuspage:
  api_key: "${STATUSPAGE_API_KEY}"
  page_id: "${STATUSPAGE_PAGE_ID}"

@Configuration
@EnableConfigurationProperties({
    PagerDutyProperties.class,
    JiraProperties.class,
    SlackProperties.class,
    StatusPageProperties.class
})
public class IncidentToolsConfiguration {
    
    @Bean
    public PagerDutyClient pagerDutyClient(PagerDutyProperties properties) {
        return PagerDutyClient.builder()
            .apiKey(properties.getApiKey())
            .serviceId(properties.getServiceId())
            .build();
    }
    
    @Bean
    public JiraClient jiraClient(JiraProperties properties) {
        return JiraClient.builder()
            .url(properties.getUrl())
            .username(properties.getUsername())
            .apiToken(properties.getApiToken())
            .build();
    }
    
    @Bean
    public SlackClient slackClient(SlackProperties properties) {
        return SlackClient.builder()
            .botToken(properties.getBotToken())
            .build();
    }
}

This comprehensive incident management system provides the foundation for effective incident response, from automated detection through post-incident improvement. The integration with external tools ensures seamless workflow and communication throughout the incident lifecycle.