elk in progress

.env

@@ -11,14 +11,24 @@ POSTGRES_PASSWORD=qwerty
 DB_HOST=db
 DB_PORT=5432
 
-PROJECT_PATH=${PWD}/pong
+#PROJECT_PATH=${PWD}/pong
-POSTGRES_DATA_PATH=${PWD}/data/db
+#POSTGRES_DATA_PATH=${PWD}/data/db
+#ES_DATA_PATH=${PWD}/data/es
+#KIBA_DATA_PATH=${PWD}/data/kiba
+#LSTASH_DATA_PATH=${PWD}/data/lstash
 
-ELASTIC_PASSWORD=qwerty42
+# ElasticSearch settings
 STACK_VERSION=8.14.3
-ES_PORT=127.0.0.1:9200
 CLUSTER_NAME=docker-cluster
 LICENSE=basic
 
+ELASTIC_PASSWORD=qwerty42
+ES_PORT=127.0.0.1:9200
+
+# Kibana settings
 KIBANA_PASSWORD=qwerty42
 KIBANA_PORT=5601
+
+ES_MEM_LIMIT=1073741824
+KB_MEM_LIMIT=1073741824
+LS_MEM_LIMIT=1073741824

docker-compose.yaml

@@ -1,25 +1,7 @@
 services:
-  db:
-    image: postgres:latest
-    container_name: postgres
-    restart: always
-    volumes:
-      - postgres_data:/var/lib/postgresql/data
-    ports:
-        - "5432:5432"
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-    networks:
-      - app-network
-    environment:
-      POSTGRES_DB: ${POSTGRES_DB}
-      POSTGRES_USER: ${POSTGRES_USER}
-      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
-
   backend:
+    depends_on:
+      - db
     build:
       context: .
       dockerfile: Dockerfile
@@ -27,16 +9,14 @@ services:
     container_name: backend
     restart: always
     command: /bin/sh -c "sleep 5 &&
-                         venv/bin/python manage.py makemigrations --noinput &&
+      venv/bin/python manage.py makemigrations --noinput &&
-                         venv/bin/python manage.py migrate --noinput && 
+      venv/bin/python manage.py migrate --noinput && 
-                         venv/bin/python manage.py collectstatic --noinput &&
+      venv/bin/python manage.py collectstatic --noinput &&
-                         venv/bin/daphne -b 0.0.0.0 -p 80 pong.asgi:application"
+      venv/bin/daphne -b 0.0.0.0 -p 80 pong.asgi:application"
     volumes:
       - pong:/transcendence/pong
     ports:
       - "80:80"
-    depends_on:
-      - db
     networks:
       - app-network
     environment:
@@ -45,25 +25,92 @@ services:
       DB_NAME: ${POSTGRES_DB}
       DB_USER: ${POSTGRES_USER}
       DB_PASSWORD: ${POSTGRES_PASSWORD}
+        #healthcheck:
+        #test: ["CMD-SHELL", "curl -f http://localhost:80 || exit 1"]
+        #interval: 20s
+        #timeout: 5s
+        #retries: 5
+
+  db:
+    image: postgres:latest
+    container_name: postgres
+    restart: always
+    volumes:
+      - pong_pg_data:/var/lib/postgresql/data
+    ports:
+      - "5432:5432"
+    networks:
+      - app-network
+    environment:
+      POSTGRES_DB: ${POSTGRES_DB}
+      POSTGRES_USER: ${POSTGRES_USER}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
     healthcheck:
-      test: ["CMD-SHELL", "curl -f http://localhost:80 || exit 1"]
+      test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
       interval: 10s
       timeout: 5s
       retries: 5
 
+  es01:
+    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
+    volumes:
+      - pong_logstash_data_01:/usr/share/elasticsearch/data
+    ports:
+      - ${ES_PORT}:9200
+    environment:
+      - node.name=es01
+      - cluster.name=${CLUSTER_NAME}
+      - discovery.type=single-node
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+    mem_limit: ${ES_MEM_LIMIT}
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+
+  logstash01:
+    depends_on:
+      - es01
+      - kibana
+    image: docker.elastic.co/logstash/logstash:${STACK_VERSION}
+    container_name: logstash
+    volumes:
+      - pong_logstash_data_01:/usr/share/logstash/data/logstash.conf
+    ports:
+      - "5044:5044"
+
+  kibana:
+    image: docker.elastic.co/kibana/kibana:${STACK_VERSION}
+    container_name: kibana
+    volumes:
+      - pong_kibana:/usr/share/kibana/data
+      - pong_logstash_data_01:/usr/share/logstash/data
+    user: root
+    ports:
+      - 5601:5601
+    environment:
+      - SERVERNAME=kibana
+      - ELASTICSEARCH_HOSTS=https://es01:9200
+      - ELASTICSEARCH_USERNAME=kibana_system
+      - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD}
+    mem_limit: ${KB_MEM_LIMIT}
+      #healthcheck:
+      #test: ["CMD-SHELL", "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'"]
+      #interval: 20s
+      #timeout: 10s
+      #retries: 120
+
 volumes:
-  postgres_data:
-    driver: local
-    driver_opts:
-      type: none
-      device: ${POSTGRES_DATA_PATH}
-      o: bind
   pong:
     driver: local
-    driver_opts:
+  pong_pg_data:
-      type: none
+    driver: local
-      device: ${PROJECT_PATH}
+  pong_es_data_01:
-      o: bind
+    driver: local
+  pong_kibana:
+    driver: local
+  pong_logstash_data_01:
+    driver: local
 
 networks:
   app-network:

logstash.conf

@@ -0,0 +1,23 @@
+input {
+  stdin { }
+}
+
+filter {
+  # Adjust the grok pattern according to the PostgreSQL log format
+  # Example log format: "2024-07-30 10:20:30 UTC LOG:  statement: SELECT * FROM table"
+  grok {
+    match => { "message" => "%{TIMESTAMP_ISO8601:timestamp} %{GREEDYDATA:log_message}" }
+  }
+
+  # Optionally, parse and format the extracted timestamp field
+  date {
+    match => [ "timestamp", "ISO8601" ]
+    target => "@timestamp"
+    # Optional: specify timezone if needed
+    # timezone => "UTC"
+  }
+}
+
+output {
+  stdout { codec => rubydebug }
+}

makefile

@@ -22,8 +22,8 @@ down:
 destroy:
 	$(COMPOSE) down -v --rmi all
 	sudo rm -rf $$PWD/data/db
-	#sudo lsof -i :5432 | awk 'NR>1 {print $$2}' | xargs sudo kill -9 || true
+	sudo lsof -i :5432 | awk 'NR>1 {print $$2}' | xargs sudo kill -9 || true
-	#sudo lsof -i :80 | awk 'NR>1 {print $$2}' | xargs sudo kill -9 || true
+	sudo lsof -i :80 | awk 'NR>1 {print $$2}' | xargs sudo kill -9 || true
 
 logs:
 	$(COMPOSE) logs -f $(CONTAINER)