Module 9: Final Project - Real-Time Analytics Platform

#!/bin/bash
# setup.sh - Complete project setup script

echo "🚀 Setting up Real-Time Analytics Platform..."

# Create project structure
mkdir -p analytics-platform/{config,producers,consumers,models,dashboard/{templates,static/{css,js,images}},monitoring/{grafana/{dashboards,datasources}},tests}

# Create Docker Compose file
cat > docker-compose.yml << 'EOF'
version: '3.8'

services:
  zookeeper:
    image: confluentinc/cp-zookeeper:latest
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000

  kafka:
    image: confluentinc/cp-kafka:latest
    depends_on:
      - zookeeper
    ports:
      - "9092:9092"
      - "9999:9999"
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092
      KAFKA_JMX_PORT: 9999
      KAFKA_JMX_HOSTNAME: localhost
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1

  kafka-ui:
    image: provectuslabs/kafka-ui:latest
    depends_on:
      - kafka
    ports:
      - "8080:8080"
    environment:
      KAFKA_CLUSTERS_0_NAME: local
      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafka:29092

  postgres:
    image: postgres:13
    environment:
      POSTGRES_DB: analytics
      POSTGRES_USER: admin
      POSTGRES_PASSWORD: password
    ports:
      - "5432:5432"

  redis:
    image: redis:6-alpine
    ports:
      - "6379:6379"

  prometheus:
    image: prom/prometheus:latest
    ports:
      - "9090:9090"
    volumes:
      - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml

  grafana:
    image: grafana/grafana:latest
    ports:
      - "3000:3000"
    environment:
      GF_SECURITY_ADMIN_PASSWORD: admin
EOF

# Create requirements.txt
cat > requirements.txt << 'EOF'
kafka-python==2.0.2
psycopg2-binary==2.9.5
redis==4.5.4
flask==2.3.2
flask-sqlalchemy==3.0.5
prometheus-client==0.17.1
pandas==2.0.3
numpy==1.24.3
python-dotenv==1.0.0
requests==2.31.0
EOF

# Create startup script
cat > start_platform.sh << 'EOF'
#!/bin/bash
echo "Starting Real-Time Analytics Platform..."

# Start infrastructure
docker-compose up -d

# Wait for services to be ready
echo "Waiting for services to start..."
sleep 30

# Create Kafka topics
docker exec kafka kafka-topics.sh --create --topic user-events --bootstrap-server localhost:9092 --partitions 3 --replication-factor 1
docker exec kafka kafka-topics.sh --create --topic page-views --bootstrap-server localhost:9092 --partitions 3 --replication-factor 1
docker exec kafka kafka-topics.sh --create --topic purchases --bootstrap-server localhost:9092 --partitions 3 --replication-factor 1
docker exec kafka kafka-topics.sh --create --topic user-actions --bootstrap-server localhost:9092 --partitions 3 --replication-factor 1

echo "✅ Platform started successfully!"
echo ""
echo "Access points:"
echo "  Dashboard: http://localhost:5000"
echo "  Kafka UI: http://localhost:8080"
echo "  Prometheus: http://localhost:9090"
echo "  Grafana: http://localhost:3000 (admin/admin)"
echo ""
echo "To start producers and consumers:"
echo "  python producers/web_producer.py"
echo "  python consumers/analytics_consumer.py"
echo "  python dashboard/app.py"
EOF

chmod +x start_platform.sh

# Create monitoring configuration
cat > monitoring/prometheus.yml << 'EOF'
global:
  scrape_interval: 15s

scrape_configs:
  - job_name: 'kafka'
    static_configs:
      - targets: ['kafka:9999']
    scrape_interval: 5s
EOF

echo "✅ Project setup complete!"
echo ""
echo "Next steps:"
echo "1. Run: ./start_platform.sh"
echo "2. Install Python dependencies: pip install -r requirements.txt"
echo "3. Start the components as shown in the tutorial"
echo ""
echo "Happy coding! 🎉"