filebeat in progress

2025-12-16 14:07:49 +01:00 · 2024-08-07 14:12:59 +02:00 · 2024-08-07 14:12:59 +02:00 · f2b558abe5
commit f2b558abe5
parent b3b2bdf18c
7 changed files with 219 additions and 57 deletions
--- a/.env
+++ b/.env
@ -24,7 +24,3 @@ ELASTIC_PASSWORD=qwerty42
 # Kibana settings
 KIBANA_PASSWORD=qwerty42
 KIBANA_PORT=5601
-
-ES_MEM_LIMIT=1073741824
-KB_MEM_LIMIT=1073741824
-LS_MEM_LIMIT=1073741824
--- a/.gitignore
+++ b/.gitignore
@ -1,3 +1,4 @@
 venv/
 __pycache__/
 data/
+.env
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@ -1,7 +1,5 @@
 services:
  backend:
-    depends_on:
-      - db
    build:
      context: .
      dockerfile: Dockerfile
@ -25,6 +23,8 @@ services:
      DB_NAME: ${POSTGRES_DB}
      DB_USER: ${POSTGRES_USER}
      DB_PASSWORD: ${POSTGRES_PASSWORD}
+    depends_on:
+      - db

  db:
    image: postgres:latest
@ -40,11 +40,11 @@ services:
      POSTGRES_DB: ${POSTGRES_DB}
      POSTGRES_USER: ${POSTGRES_USER}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
+    #healthcheck:
+      #test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
+      #interval: 10s
+      #timeout: 5s
+      #retries: 5

  es01:
    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
@ -62,20 +62,14 @@ services:
      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
      - xpack.security.enabled=false 
      - xpack.license.self_generated.type=trial
-    mem_limit: ${ES_MEM_LIMIT}
-    ulimits:
-      memlock:
-        soft: -1
-        hard: -1
+    depends_on:
+      - logstash01

  logstash01:
-    depends_on:
-      - es01
-      - kibana
    image: docker.elastic.co/logstash/logstash:${STACK_VERSION}
    container_name: logstash01
    volumes:
-      - ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf
+      - ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro
    ports:
      - "5044:5044"
    networks:
@ -88,8 +82,6 @@ services:

  kibana:
    image: docker.elastic.co/kibana/kibana:${STACK_VERSION}
-    depends_on:
-      - es01
    container_name: kibana
    volumes:
      - pong_kibana:/usr/share/kibana/data
@ -103,13 +95,32 @@ services:
      - ELASTICSEARCH_HOSTS=http://es01:9200
      - ELASTICSEARCH_USERNAME=${ELASTIC_USERNAME}
      - ELASTICSEARCH_PASSWORD=${ELASTIC_PASSWORD}
-    mem_limit: ${KB_MEM_LIMIT}
+    depends_on:
+      - es01
    #healthcheck:
      #test: ["CMD-SHELL", "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'"]
      #interval: 20s
      #timeout: 10s
      #retries: 120

+  filebeat01:
+    depends_on:
+      - es01
+    image: docker.elastic.co/beats/filebeat:${STACK_VERSION}
+    volumes:
+      - pong_filebeat_data_01:/usr/share/filebeat/data
+      - ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    environment:
+      - ELASTIC_USER=elastic
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+      - ELASTIC_HOSTS=https://es01:9200
+      - KIBANA_HOSTS=http://kibana:5601
+      - LOGSTASH_HOSTS=http://logstash01:9600
+    networks:
+      - app-network
+
 volumes:
  pong:
    driver: local
@ -125,6 +136,8 @@ volumes:
    driver: local
  pong_logstash_data_01:
    driver: local
+  pong_filebeat_data_01:
+    driver: local

 networks:
  app-network:
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -0,0 +1,144 @@
+services:
+  backend:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    image: backend
+    container_name: backend
+    restart: always
+    command: /bin/sh -c "sleep 5 &&
+      venv/bin/python manage.py makemigrations --noinput &&
+      venv/bin/python manage.py migrate --noinput && 
+      venv/bin/python manage.py collectstatic --noinput &&
+      venv/bin/daphne -b 0.0.0.0 -p 80 pong.asgi:application"
+    volumes:
+      - pong:/transcendence/pong
+    ports:
+      - "8080:8080"
+    networks:
+      - app-network
+    environment:
+      DB_HOST: db
+      DB_PORT: 5432
+      DB_NAME: ${POSTGRES_DB}
+      DB_USER: ${POSTGRES_USER}
+      DB_PASSWORD: ${POSTGRES_PASSWORD}
+    depends_on:
+      - db
+
+  db:
+    image: postgres:latest
+    container_name: postgres
+    restart: always
+    volumes:
+      - pong_pg_data:/var/lib/postgresql/data
+    ports:
+      - "5432:5432"
+    networks:
+      - app-network
+    environment:
+      POSTGRES_DB: ${POSTGRES_DB}
+      POSTGRES_USER: ${POSTGRES_USER}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+    #healthcheck:
+      #test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
+      #interval: 10s
+      #timeout: 5s
+      #retries: 5
+
+  es01:
+    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
+    container_name: es01
+    volumes:
+      - pong_logstash_data_01:/usr/share/elasticsearch/data
+    ports:
+      - "127.0.0.1:9200:9200"
+    networks:
+      - app-network
+    environment:
+      - node.name=es01
+      - cluster.name=${CLUSTER_NAME}
+      - discovery.type=single-node
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+      - xpack.security.enabled=false 
+      - xpack.license.self_generated.type=trial
+    depends_on:
+      - logstash01
+
+  logstash01:
+    image: docker.elastic.co/logstash/logstash:${STACK_VERSION}
+    container_name: logstash01
+    volumes:
+      - ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro
+    ports:
+      - "5044:5044"
+    networks:
+      - app-network
+    environment:
+      - ELASTIC_HOSTS=http://es01:9200
+      - ELASTIC_USER=${ELASTIC_USERNAME}
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+      - xpack.monitoring.enabled=false
+
+  kibana:
+    image: docker.elastic.co/kibana/kibana:${STACK_VERSION}
+    container_name: kibana
+    volumes:
+      - pong_kibana:/usr/share/kibana/data
+    user: "1000:1000"
+    ports:
+      - 5601:5601
+    networks:
+      - app-network
+    environment:
+      - SERVERNAME=pong.kibana.org
+      - ELASTICSEARCH_HOSTS=http://es01:9200
+      - ELASTICSEARCH_USERNAME=${ELASTIC_USERNAME}
+      - ELASTICSEARCH_PASSWORD=${ELASTIC_PASSWORD}
+    depends_on:
+      - es01
+    #healthcheck:
+      #test: ["CMD-SHELL", "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'"]
+      #interval: 20s
+      #timeout: 10s
+      #retries: 120
+
+  filebeat01:
+    depends_on:
+      - es01
+    image: docker.elastic.co/beats/filebeat:${STACK_VERSION}
+    volumes:
+      - pong_filebeat_data_01:/usr/share/filebeat/data
+      - ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    environment:
+      - ELASTIC_USER=elastic
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+      - ELASTIC_HOSTS=https://es01:9200
+      - KIBANA_HOSTS=http://kibana:5601
+      - LOGSTASH_HOSTS=http://logstash01:9600
+    networks:
+      - app-network
+
+volumes:
+  pong:
+    driver: local
+    driver_opts:
+      type: none
+      device: ${PROJECT_PATH}
+      o: bind
+  pong_pg_data:
+    driver: local
+  pong_es_data_01:
+    driver: local
+  pong_kibana:
+    driver: local
+  pong_logstash_data_01:
+    driver: local
+  pong_filebeat_data_01:
+    driver: local
+
+networks:
+  app-network:
+    driver: bridge
--- a/filebeat.yml
+++ b/filebeat.yml
@ -0,0 +1,32 @@
+filebeat.inputs:
+#- type: docker
+  #containers.ids:
+  #  - "*"
+- type: filestream
+  id: default-filestream
+  paths:
+    - ingest_data/*.log
+
+filebeat.autodiscover:
+ providers:
+   - type: docker
+     hints.enabled: true
+
+processors:
+- add_docker_metadata: ~
+
+
+setup.kibana:
+ host: ${KIBANA_HOSTS}
+ username: ${ELASTIC_USER}
+ password: ${ELASTIC_PASSWORD}
+
+
+output.elasticsearch:
+ hosts: ${ELASTIC_HOSTS}
+ username: ${ELASTIC_USER}
+ password: ${ELASTIC_PASSWORD}
+ ssl.enabled: false
+
+#output.logstash:
+  #hosts: ["http://logstash01:9600"]
--- a/logstash.conf
+++ b/logstash.conf
@ -1,40 +1,17 @@
 input {
-  # Input from Docker container's stdout
-  	beats {
+	beat {
 		port => 5044
 	}
-  }
-
-filter {
-  # Grok filter to parse the log lines
-  grok {
-    match => {
-      "message" => [
-        "%{DATA:container_name}\s*\|\s*%{IP:client_ip}:%{NUMBER:client_port} - - \[%{HTTPDATE:timestamp}\] \"%{WORD:method} %{URIPATHPARAM:request} HTTP/%{NUMBER:http_version}\" %{NUMBER:status} %{NUMBER:bytes}"
-      ]
-    }
-  }
-
-  # Date filter to convert timestamp to a proper date format
-  date {
-    match => ["timestamp", "dd/MMM/yyyy:HH:mm:ss Z"]
-    target => "@timestamp"
-    remove_field => ["timestamp"]
-  }
 }

-output {
-  # Output to stdout for testing purposes
-  stdout {
-    codec => rubydebug
-  }
+filter {
+}

-  # Optionally, output to Elasticsearch
-elasticsearch {
-	hosts => ["localhost:9200"]
+output {
+	elasticsearch {
+		hosts => ["http://es01:9200"]
 		index => "docker-logs-%{+YYYY.MM.dd}"
 		user=> "${ELASTIC_USER}"
 		password=> "${ELASTIC_PASSWORD}"
 	}
-
 }
--- a/3
+++ b/3
@ -1,4 +1,4 @@
-COMPOSE_FILE=docker-compose.yaml
+COMPOSE_FILE=docker-compose.yml
 COMPOSE=docker compose -f $(COMPOSE_FILE)
 CONTAINER=$(c)

@ -44,4 +44,3 @@ help:
 	@echo "  make logs [c=service]         # Tail logs of containers"
 	@echo "  make ps                       # List containers"
 	@echo "  make help                     # Show this help"
-