Marat's Notes

Effective monitoring and alerting are crucial for maintaining system reliability and performance. This guide explores how to set up and optimize DataDog for comprehensive system monitoring, with a focus on practical implementation and best practices, particularly for Node.js/TypeScript applications.

Prerequisites

Before getting started, ensure you have:

DataDog Account:
- Active DataDog account
- Appropriate permissions
- API and application keys
System Access:
- Access to target systems
- Required credentials
- Network access to DataDog endpoints

Initial Setup

1. DataDog Agent Installation

When setting up DataDog monitoring, consider:

Choosing the right agent version
Selecting appropriate installation method
Configuring system requirements
Setting up agent authentication

Example configuration for a Node.js application:

// datadog.config.ts
import { StatsD } from 'hot-shots';

export const statsd = new StatsD({
  host: 'localhost',
  port: 8125,
  errorHandler: (error) => {
    console.error('StatsD error:', error);
  },
  globalTags: {
    env: process.env.NODE_ENV,
    service: 'my-node-app'
  }
});

// Example usage in your application
import { statsd } from './datadog.config';

// Track API response time
app.use((req, res, next) => {
  const start = Date.now();
  res.on('finish', () => {
    const duration = Date.now() - start;
    statsd.timing('http.request.duration', duration, {
      method: req.method,
      route: req.route?.path || 'unknown',
      status: res.statusCode
    });
  });
  next();
});

2. Basic Monitoring Setup

Configure essential monitoring components:

// monitoring.ts
import { statsd } from './datadog.config';

export class MonitoringService {
  // Track custom business metrics
  static trackUserSignup(userId: string) {
    statsd.increment('user.signup', {
      userId,
      source: 'web'
    });
  }

  // Track error rates
  static trackError(error: Error, context: any) {
    statsd.increment('app.error', {
      errorType: error.name,
      errorMessage: error.message,
      ...context
    });
  }

  // Track performance metrics
  static trackDatabaseQuery(duration: number, query: string) {
    statsd.timing('db.query.duration', duration, {
      query: query.substring(0, 50) // Truncate long queries
    });
  }
}

// Usage example
try {
  const start = Date.now();
  await db.query('SELECT * FROM users');
  MonitoringService.trackDatabaseQuery(Date.now() - start, 'SELECT * FROM users');
} catch (error) {
  MonitoringService.trackError(error, { query: 'SELECT * FROM users' });
}

Advanced Monitoring Configuration

1. Custom Metrics

Set up custom metrics for your Node.js application:

// metrics.ts
import { statsd } from './datadog.config';

export class CustomMetrics {
  // Track business KPIs
  static trackOrderValue(orderId: string, value: number) {
    statsd.gauge('order.value', value, {
      orderId,
      currency: 'USD'
    });
  }

  // Track user behavior
  static trackUserAction(userId: string, action: string) {
    statsd.increment('user.action', {
      userId,
      action,
      timestamp: new Date().toISOString()
    });
  }

  // Track system health
  static trackMemoryUsage() {
    const memoryUsage = process.memoryUsage();
    statsd.gauge('system.memory.heapUsed', memoryUsage.heapUsed);
    statsd.gauge('system.memory.heapTotal', memoryUsage.heapTotal);
    statsd.gauge('system.memory.rss', memoryUsage.rss);
  }
}

// Usage in your application
setInterval(() => {
  CustomMetrics.trackMemoryUsage();
}, 60000); // Every minute

2. Service Level Objectives (SLOs)

Define and monitor SLOs for your application:

// slos.ts
import { statsd } from './datadog.config';

export class SLOMonitoring {
  // Track API availability
  static trackAPIAvailability(endpoint: string, success: boolean) {
    statsd.increment('api.availability', {
      endpoint,
      success: success.toString()
    });
  }

  // Track response time percentiles
  static trackResponseTime(endpoint: string, duration: number) {
    statsd.histogram('api.response_time', duration, {
      endpoint,
      percentile: 'p95'
    });
  }

  // Track error rates
  static trackErrorRate(endpoint: string, errorCount: number) {
    statsd.gauge('api.error_rate', errorCount, {
      endpoint
    });
  }
}

// Usage example
app.use((req, res, next) => {
  const start = Date.now();
  res.on('finish', () => {
    const duration = Date.now() - start;
    SLOMonitoring.trackResponseTime(req.path, duration);
    SLOMonitoring.trackAPIAvailability(req.path, res.statusCode < 500);
  });
  next();
});

Alerting Strategy

1. Alert Configuration

Set up alerts for your Node.js application:

// alerts.ts
import { statsd } from './datadog.config';

export class AlertMonitoring {
  // Track critical errors
  static trackCriticalError(error: Error, context: any) {
    statsd.increment('alert.critical_error', {
      errorType: error.name,
      errorMessage: error.message,
      ...context
    });
  }

  // Track resource utilization
  static trackResourceUtilization(cpu: number, memory: number) {
    statsd.gauge('system.cpu.usage', cpu);
    statsd.gauge('system.memory.usage', memory);
  }

  // Track business metrics
  static trackBusinessMetric(metric: string, value: number, tags: any) {
    statsd.gauge(`business.${metric}`, value, tags);
  }
}

// Usage example
process.on('uncaughtException', (error) => {
  AlertMonitoring.trackCriticalError(error, {
    process: process.pid,
    timestamp: new Date().toISOString()
  });
});

2. Notification Channels

Configure notification methods in your application:

// notifications.ts
import { statsd } from './datadog.config';

export class NotificationService {
  // Track alert notifications
  static trackAlertNotification(alert: string, channel: string) {
    statsd.increment('alert.notification', {
      alert,
      channel,
      timestamp: new Date().toISOString()
    });
  }

  // Track notification delivery
  static trackNotificationDelivery(notificationId: string, success: boolean) {
    statsd.increment('notification.delivery', {
      notificationId,
      success: success.toString()
    });
  }
}

// Usage example
async function sendAlert(alert: string) {
  try {
    await sendSlackNotification(alert);
    NotificationService.trackAlertNotification(alert, 'slack');
    NotificationService.trackNotificationDelivery(alert, true);
  } catch (error) {
    NotificationService.trackNotificationDelivery(alert, false);
    throw error;
  }
}

Dashboard Creation

1. System Overview

Create dashboards for your Node.js application:

// dashboard.ts
import { statsd } from './datadog.config';

export class DashboardMetrics {
  // Track application health
  static trackApplicationHealth() {
    const health = {
      status: 'healthy',
      uptime: process.uptime(),
      memory: process.memoryUsage(),
      cpu: process.cpuUsage()
    };

    statsd.gauge('app.health.status', health.status === 'healthy' ? 1 : 0);
    statsd.gauge('app.health.uptime', health.uptime);
    statsd.gauge('app.health.memory', health.memory.heapUsed);
    statsd.gauge('app.health.cpu', health.cpu.user);
  }

  // Track API performance
  static trackAPIPerformance(endpoint: string, duration: number) {
    statsd.histogram('api.performance', duration, {
      endpoint,
      percentile: 'p95'
    });
  }
}

// Usage example
setInterval(() => {
  DashboardMetrics.trackApplicationHealth();
}, 30000); // Every 30 seconds

2. Custom Visualizations

Design visualizations for your application:

// visualizations.ts
import { statsd } from './datadog.config';

export class VisualizationMetrics {
  // Track user engagement
  static trackUserEngagement(userId: string, action: string) {
    statsd.increment('user.engagement', {
      userId,
      action,
      timestamp: new Date().toISOString()
    });
  }

  // Track feature usage
  static trackFeatureUsage(feature: string, userId: string) {
    statsd.increment('feature.usage', {
      feature,
      userId,
      timestamp: new Date().toISOString()
    });
  }
}

// Usage example
app.post('/api/feature', (req, res) => {
  const { feature, userId } = req.body;
  VisualizationMetrics.trackFeatureUsage(feature, userId);
  res.json({ success: true });
});

Best Practices

1. Monitoring Strategy

Follow these best practices for your Node.js application:

// best-practices.ts
import { statsd } from './datadog.config';

export class MonitoringBestPractices {
  // Use consistent naming conventions
  static trackMetric(name: string, value: number, tags: any) {
    const metricName = `app.${name}`; // Consistent prefix
    statsd.gauge(metricName, value, {
      ...tags,
      env: process.env.NODE_ENV,
      version: process.env.APP_VERSION
    });
  }

  // Implement proper error handling
  static trackError(error: Error, context: any) {
    statsd.increment('app.error', {
      errorType: error.name,
      errorMessage: error.message,
      ...context
    });
  }

  // Use appropriate metric types
  static trackMetrics() {
    // Counters for events
    statsd.increment('app.event');

    // Gauges for current values
    statsd.gauge('app.memory', process.memoryUsage().heapUsed);

    // Histograms for distributions
    statsd.histogram('app.response_time', 100);

    // Sets for unique values
    statsd.set('app.unique_users', 'user123');
  }
}

2. Cost Management

Optimize costs in your monitoring setup:

// cost-management.ts
import { statsd } from './datadog.config';

export class CostManagement {
  // Batch metrics to reduce API calls
  static batchMetrics(metrics: any[]) {
    const batch = new Map();
    
    metrics.forEach(metric => {
      const key = `${metric.name}:${JSON.stringify(metric.tags)}`;
      if (!batch.has(key)) {
        batch.set(key, []);
      }
      batch.get(key).push(metric.value);
    });

    batch.forEach((values, key) => {
      const [name, tags] = key.split(':');
      statsd.gauge(name, values[values.length - 1], JSON.parse(tags));
    });
  }

  // Sample metrics to reduce volume
  static sampleMetric(name: string, value: number, sampleRate: number) {
    statsd.gauge(name, value, { sampleRate });
  }
}

// Usage example
const metrics = [
  { name: 'app.metric1', value: 100, tags: { tag1: 'value1' } },
  { name: 'app.metric1', value: 200, tags: { tag1: 'value1' } }
];
CostManagement.batchMetrics(metrics);

Conclusion

Setting up effective monitoring with DataDog in your Node.js/TypeScript application requires careful planning and implementation. By following this guide, you can:

Set up comprehensive system monitoring
Configure effective alerting
Create informative dashboards
Implement best practices
Optimize costs and performance

Remember to:

Regularly review and update monitoring
Optimize alert thresholds
Maintain dashboard relevance
Monitor costs
Follow security best practices

With proper implementation and maintenance, DataDog can provide valuable insights into your system’s performance and help ensure reliable operation.

Setting up Monitoring and Alerting with DataDog