filebeat in progress

.env

@@ -24,7 +24,3 @@ ELASTIC_PASSWORD=qwerty42
 # Kibana settings
 KIBANA_PASSWORD=qwerty42
 KIBANA_PORT=5601
-
-ES_MEM_LIMIT=1073741824
-KB_MEM_LIMIT=1073741824
-LS_MEM_LIMIT=1073741824

.gitignore

@@ -1,3 +1,4 @@
 venv/
 __pycache__/
 data/
+.env

docker-compose.yaml

@@ -1,7 +1,5 @@
 services:
   backend:
-    depends_on:
-      - db
     build:
       context: .
       dockerfile: Dockerfile
@@ -25,6 +23,8 @@ services:
       DB_NAME: ${POSTGRES_DB}
       DB_USER: ${POSTGRES_USER}
       DB_PASSWORD: ${POSTGRES_PASSWORD}
+    depends_on:
+      - db
 
   db:
     image: postgres:latest
@@ -40,11 +40,11 @@ services:
       POSTGRES_DB: ${POSTGRES_DB}
       POSTGRES_USER: ${POSTGRES_USER}
       POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
-    healthcheck:
+    #healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
+      #test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
-      interval: 10s
+      #interval: 10s
-      timeout: 5s
+      #timeout: 5s
-      retries: 5
+      #retries: 5
 
   es01:
     image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
@@ -62,20 +62,14 @@ services:
       - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
       - xpack.security.enabled=false 
       - xpack.license.self_generated.type=trial
-    mem_limit: ${ES_MEM_LIMIT}
+    depends_on:
-    ulimits:
+      - logstash01
-      memlock:
-        soft: -1
-        hard: -1
 
   logstash01:
-    depends_on:
-      - es01
-      - kibana
     image: docker.elastic.co/logstash/logstash:${STACK_VERSION}
     container_name: logstash01
     volumes:
-      - ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf
+      - ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro
     ports:
       - "5044:5044"
     networks:
@@ -88,8 +82,6 @@ services:
 
   kibana:
     image: docker.elastic.co/kibana/kibana:${STACK_VERSION}
-    depends_on:
-      - es01
     container_name: kibana
     volumes:
       - pong_kibana:/usr/share/kibana/data
@@ -103,13 +95,32 @@ services:
       - ELASTICSEARCH_HOSTS=http://es01:9200
       - ELASTICSEARCH_USERNAME=${ELASTIC_USERNAME}
       - ELASTICSEARCH_PASSWORD=${ELASTIC_PASSWORD}
-    mem_limit: ${KB_MEM_LIMIT}
+    depends_on:
+      - es01
     #healthcheck:
       #test: ["CMD-SHELL", "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'"]
       #interval: 20s
       #timeout: 10s
       #retries: 120
 
+  filebeat01:
+    depends_on:
+      - es01
+    image: docker.elastic.co/beats/filebeat:${STACK_VERSION}
+    volumes:
+      - pong_filebeat_data_01:/usr/share/filebeat/data
+      - ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    environment:
+      - ELASTIC_USER=elastic
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+      - ELASTIC_HOSTS=https://es01:9200
+      - KIBANA_HOSTS=http://kibana:5601
+      - LOGSTASH_HOSTS=http://logstash01:9600
+    networks:
+      - app-network
+
 volumes:
   pong:
     driver: local
@@ -125,6 +136,8 @@ volumes:
     driver: local
   pong_logstash_data_01:
     driver: local
+  pong_filebeat_data_01:
+    driver: local
 
 networks:
   app-network:

docker-compose.yml

@@ -0,0 +1,144 @@
+services:
+  backend:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    image: backend
+    container_name: backend
+    restart: always
+    command: /bin/sh -c "sleep 5 &&
+      venv/bin/python manage.py makemigrations --noinput &&
+      venv/bin/python manage.py migrate --noinput && 
+      venv/bin/python manage.py collectstatic --noinput &&
+      venv/bin/daphne -b 0.0.0.0 -p 80 pong.asgi:application"
+    volumes:
+      - pong:/transcendence/pong
+    ports:
+      - "8080:8080"
+    networks:
+      - app-network
+    environment:
+      DB_HOST: db
+      DB_PORT: 5432
+      DB_NAME: ${POSTGRES_DB}
+      DB_USER: ${POSTGRES_USER}
+      DB_PASSWORD: ${POSTGRES_PASSWORD}
+    depends_on:
+      - db
+
+  db:
+    image: postgres:latest
+    container_name: postgres
+    restart: always
+    volumes:
+      - pong_pg_data:/var/lib/postgresql/data
+    ports:
+      - "5432:5432"
+    networks:
+      - app-network
+    environment:
+      POSTGRES_DB: ${POSTGRES_DB}
+      POSTGRES_USER: ${POSTGRES_USER}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+    #healthcheck:
+      #test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
+      #interval: 10s
+      #timeout: 5s
+      #retries: 5
+
+  es01:
+    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
+    container_name: es01
+    volumes:
+      - pong_logstash_data_01:/usr/share/elasticsearch/data
+    ports:
+      - "127.0.0.1:9200:9200"
+    networks:
+      - app-network
+    environment:
+      - node.name=es01
+      - cluster.name=${CLUSTER_NAME}
+      - discovery.type=single-node
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+      - xpack.security.enabled=false 
+      - xpack.license.self_generated.type=trial
+    depends_on:
+      - logstash01
+
+  logstash01:
+    image: docker.elastic.co/logstash/logstash:${STACK_VERSION}
+    container_name: logstash01
+    volumes:
+      - ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro
+    ports:
+      - "5044:5044"
+    networks:
+      - app-network
+    environment:
+      - ELASTIC_HOSTS=http://es01:9200
+      - ELASTIC_USER=${ELASTIC_USERNAME}
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+      - xpack.monitoring.enabled=false
+
+  kibana:
+    image: docker.elastic.co/kibana/kibana:${STACK_VERSION}
+    container_name: kibana
+    volumes:
+      - pong_kibana:/usr/share/kibana/data
+    user: "1000:1000"
+    ports:
+      - 5601:5601
+    networks:
+      - app-network
+    environment:
+      - SERVERNAME=pong.kibana.org
+      - ELASTICSEARCH_HOSTS=http://es01:9200
+      - ELASTICSEARCH_USERNAME=${ELASTIC_USERNAME}
+      - ELASTICSEARCH_PASSWORD=${ELASTIC_PASSWORD}
+    depends_on:
+      - es01
+    #healthcheck:
+      #test: ["CMD-SHELL", "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'"]
+      #interval: 20s
+      #timeout: 10s
+      #retries: 120
+
+  filebeat01:
+    depends_on:
+      - es01
+    image: docker.elastic.co/beats/filebeat:${STACK_VERSION}
+    volumes:
+      - pong_filebeat_data_01:/usr/share/filebeat/data
+      - ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+    environment:
+      - ELASTIC_USER=elastic
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+      - ELASTIC_HOSTS=https://es01:9200
+      - KIBANA_HOSTS=http://kibana:5601
+      - LOGSTASH_HOSTS=http://logstash01:9600
+    networks:
+      - app-network
+
+volumes:
+  pong:
+    driver: local
+    driver_opts:
+      type: none
+      device: ${PROJECT_PATH}
+      o: bind
+  pong_pg_data:
+    driver: local
+  pong_es_data_01:
+    driver: local
+  pong_kibana:
+    driver: local
+  pong_logstash_data_01:
+    driver: local
+  pong_filebeat_data_01:
+    driver: local
+
+networks:
+  app-network:
+    driver: bridge

filebeat.yml

@@ -0,0 +1,32 @@
+filebeat.inputs:
+#- type: docker
+  #containers.ids:
+  #  - "*"
+- type: filestream
+  id: default-filestream
+  paths:
+    - ingest_data/*.log
+
+filebeat.autodiscover:
+ providers:
+   - type: docker
+     hints.enabled: true
+
+processors:
+- add_docker_metadata: ~
+
+
+setup.kibana:
+ host: ${KIBANA_HOSTS}
+ username: ${ELASTIC_USER}
+ password: ${ELASTIC_PASSWORD}
+
+
+output.elasticsearch:
+ hosts: ${ELASTIC_HOSTS}
+ username: ${ELASTIC_USER}
+ password: ${ELASTIC_PASSWORD}
+ ssl.enabled: false
+
+#output.logstash:
+  #hosts: ["http://logstash01:9600"]

logstash.conf

@@ -1,40 +1,17 @@
 input {
-  # Input from Docker container's stdout
+	beat {
-  	beats {
 		port => 5044
 	}
 }
 
 filter {
-  # Grok filter to parse the log lines
-  grok {
-    match => {
-      "message" => [
-        "%{DATA:container_name}\s*\|\s*%{IP:client_ip}:%{NUMBER:client_port} - - \[%{HTTPDATE:timestamp}\] \"%{WORD:method} %{URIPATHPARAM:request} HTTP/%{NUMBER:http_version}\" %{NUMBER:status} %{NUMBER:bytes}"
-      ]
-    }
-  }
-
-  # Date filter to convert timestamp to a proper date format
-  date {
-    match => ["timestamp", "dd/MMM/yyyy:HH:mm:ss Z"]
-    target => "@timestamp"
-    remove_field => ["timestamp"]
-  }
 }
 
 output {
-  # Output to stdout for testing purposes
-  stdout {
-    codec => rubydebug
-  }
-
-  # Optionally, output to Elasticsearch
 	elasticsearch {
-	hosts => ["localhost:9200"]
+		hosts => ["http://es01:9200"]
 		index => "docker-logs-%{+YYYY.MM.dd}"
 		user=> "${ELASTIC_USER}"
 		password=> "${ELASTIC_PASSWORD}"
 	}
-
 }

makefile

@@ -1,4 +1,4 @@
-COMPOSE_FILE=docker-compose.yaml
+COMPOSE_FILE=docker-compose.yml
 COMPOSE=docker compose -f $(COMPOSE_FILE)
 CONTAINER=$(c)
 
@@ -44,4 +44,3 @@ help:
 	@echo "  make logs [c=service]         # Tail logs of containers"
 	@echo "  make ps                       # List containers"
 	@echo "  make help                     # Show this help"
-