first logs from logstash

.env

@@ -17,11 +17,15 @@ PROJECT_PATH=${PWD_PATH}/pong
 # ElasticSearch settings
 STACK_VERSION=8.14.3
 CLUSTER_NAME=docker-cluster
-LICENSE=trial
+LICENSE=basic
 
-ELASTIC_USERNAME=adrien
+ELASTIC_USERNAME=elastic
-ELASTIC_PASSWORD=qwerty42
+ELASTIC_PASSWORD=elastic_pass
 
 # Kibana settings
-KIBANA_PASSWORD=qwerty42
 KIBANA_PORT=5601
+KIBANA_USERNAME=kibana_system
+KIBANA_PASSWORD=kibana_pass
+
+ENCRYPTION_KEY=c34d38b3a14956121ff2170e5030b471551370178f43e5626eec58b04a30fae2
+

config/filebeat.yml

@@ -1,10 +0,0 @@
-filebeat.inputs:
-- type: docker
-  containers.ids:
-    - "*"
-
-processors:
-- add_docker_metadata: ~
-
-output.logstash:
-  hosts: ["http://logstash01:5044"]

config/kibana.yml

@@ -1,54 +0,0 @@
-# Kibana's Elasticsearch URL
-elasticsearch.hosts: ["https://es01:9200"]
-
-# The name of the Kibana instance
-server.name: "kibana"
-
-# The base path for Kibana
-#server.basePath: ""
-
-# Serve the Kibana instance at the root path
-#server.rewriteBasePath: false
-
-# Enable or disable the Kibana server
-server.enabled: true
-
-# The port Kibana will listen on
-server.port: 5601
-
-# Enable or disable the xpack features
-xpack.enabled: true
-
-# Set the encryption key for encrypted saved objects
-#xpack.encryptedSavedObjects.encryptionKey: ""
-
-# Set the key for the kibana security features
-#xpack.security.encryptionKey: ""
-
-# Enable or disable the monitoring feature
-xpack.monitoring.enabled: true
-
-# Set the URL of the Elasticsearch instance to which Kibana should connect
-#xpack.reporting.enabled: true
-
-  #xpack.reporting.roles.enabled: false
-
-# Enable the Kibana alerting feature
-#xpack.alerting.enabled: false
-
-# Enable or disable the usage collection
-#xpack.usageCollection.enabled: true
-
-# Configure the log level for Kibana
-#logging.level: info
-
-# Configure the directory where logs will be stored
-logging.dest: /var/log/kibana/kibana.log
-
-# Configure security and authentication settings
-elasticsearch.username: test
-elasticsearch.password: test 
-
-# Disable or enable the Kibana plugin
-#xpack.license.management.enabled: false
-

config/logstash.conf

@@ -1,30 +1,21 @@
 input {
-	file {
+  udp {
-		path => "/var/lib/docker/containers/*/*.log"
+    host => "0.0.0.0"
-		start_position => "beginning"
+    port => 5044
-		sincedb_path => "/dev/null"
+  }
-		type => "docker"
-		codec => "json"
-	}
 }
 
-filter {
+filter {}
-	if [log_message] =~ /GET/ {
-		grok {
-			match => { "log_message" => "%{IP:client_ip} - - \[%{HTTPDATE:timestamp}\] \"%{WORD:method} %{URIPATH:request_path} HTTP/%{NUMBER:http_version}\" %{NUMBER:response_code} %{NUMBER:response_size}" }
-		}
-		date {
-			match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
-				target => "@timestamp"
-		}
-	}
-}
 
 output {
-	elasticsearch {
+  elasticsearch {
-		hosts => ["http://es01:9200"]
+    index => "logstash-%{+YYYY.MM.dd}"
-		index => "docker-logs-%{+YYYY.MM.dd}"
+    hosts => ["https://es01:9200"]
-		user=> "${ELASTIC_USER}"
+    user => "elastic"
-		password=> "${ELASTIC_PASSWORD}"
+    password => elastic_pass
-	}
+	ssl_enabled => true
+    ssl_certificate_authorities => "/usr/share/logstash/certs/ca/ca.crt"
+    ssl_verification_mode => "full"
+  }
+  #stdout {}
 }

docker-compose.yaml

@@ -1,134 +0,0 @@
-services:
-  backend:
-    build:
-      context: .
-      dockerfile: Dockerfile
-    image: backend
-    container_name: backend
-    restart: always
-    command: /bin/sh -c "sleep 5 &&
-      venv/bin/python manage.py makemigrations --noinput &&
-      venv/bin/python manage.py migrate --noinput && 
-      venv/bin/python manage.py collectstatic --noinput &&
-      venv/bin/daphne -b 0.0.0.0 -p 8080 pong.asgi:application"
-    volumes:
-      - pong:/transcendence/pong
-    ports:
-      - "8080:8080"
-    networks:
-      - app-network
-    environment:
-      DB_HOST: db
-      DB_PORT: 5432
-      DB_NAME: ${POSTGRES_DB}
-      DB_USER: ${POSTGRES_USER}
-      DB_PASSWORD: ${POSTGRES_PASSWORD}
-    depends_on:
-      - db
-    healthcheck:
-      test: ["CMD-SHELL", "curl", "http://localhost:8080"]
-      interval: 30s
-      timeout: 10s
-      retries: 3
-      start_period: 10s
-
-  db:
-    image: postgres:latest
-    container_name: postgres
-    restart: always
-    volumes:
-      - pong_pg_data:/var/lib/postgresql/data
-    ports:
-      - "5432:5432"
-    networks:
-      - app-network
-    environment:
-      POSTGRES_DB: ${POSTGRES_DB}
-      POSTGRES_USER: ${POSTGRES_USER}
-      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-
-  es01:
-    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
-    container_name: es01
-    volumes:
-      - pong_es_data_01:/usr/share/elasticsearch/data
-    ports:
-     - "9200:9200"
-    networks:
-     - app-network
-    environment:
-     - node.name=es01
-     - cluster.name=${CLUSTER_NAME}
-     - discovery.type=single-node
-     - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
-     - bootstrap.memory_lock=true
-     - xpack.security.enabled=false
-    healthcheck:
-      test: ["CMD-SHELL", "curl -s http://localhost:9200/_cluster/health | grep -q '\"status\":\"green\"'"]
-      interval: 10s
-      timeout: 10s
-      retries: 5
-
-  kibana:
-    image: docker.elastic.co/kibana/kibana:${STACK_VERSION}
-    container_name: kibana
-    volumes:
-      - pong_kibana:/usr/share/kibana/data
-    user: "1000:1000"
-    ports:
-      - 5601:5601
-    networks:
-      - app-network
-    environment:
-      - SERVERNAME=pong.kibana.org
-      - ELASTICSEARCH_HOSTS=http://es01:9200
-      - ELASTICSEARCH_USERNAME=${ELASTIC_USERNAME}
-      - ELASTICSEARCH_PASSWORD=${ELASTIC_PASSWORD}
-    depends_on:
-      - es01
-    healthcheck:
-      test: ["CMD-SHELL", "curl -s -I http://localhost:5601 | grep -q '200 OK'"]
-      interval: 10s
-      timeout: 20s
-      retries: 200
-
-    
-  logstash01:
-    image: docker.elastic.co/logstash/logstash:${STACK_VERSION}
-    container_name: logstash01
-    volumes:
-      - ./config/logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro
-    ports:
-      - "5044:5044"
-    networks:
-      - app-network
-    environment:
-      - ELASTIC_HOSTS=http://es01:9200
-      - ELASTIC_USER=${ELASTIC_USERNAME}
-      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
-      - xpack.monitoring.enabled=false
-
-volumes:
-  pong:
-    driver: local
-    driver_opts:
-      type: none
-      device: ${PROJECT_PATH}
-      o: bind
-  pong_pg_data:
-    driver: local
-  pong_es_data_01:
-    driver: local
-  pong_kibana:
-    driver: local
-
-  
-
-networks:
-  app-network:
-    driver: bridge

docker-compose.yml

@@ -6,38 +6,59 @@ services:
     volumes:
       - certs:/usr/share/elasticsearch/config/certs
     command: >
-      sh -c '
+      bash -c '
-      if [ ! -f /usr/share/elasticsearch/config/certs/elastic-certificate.p12 ]; then
+       if [ x${ELASTIC_PASSWORD} == x ]; then
-        echo "Creating elastic-certificate.p12...";
+         echo "Set the ELASTIC_PASSWORD environment variable in the .env file";
-        bin/elasticsearch-certutil cert --name elastic-certificate --days 365 --self-signed --out /usr/share/elasticsearch/config/certs/elastic-certificate.p12 --pass "";
+         exit 1;
-        else
+       elif [ x${KIBANA_PASSWORD} == x ]; then
-          echo "elastic-certificate.p12 already exists, skipping certificate creation.";
+         echo "Set the KIBANA_PASSWORD environment variable in the .env file";
-      fi;
+         exit 1;
-
+       fi;
-      if [ ! -f /usr/share/elasticsearch/config/certs/instances.yml ]; then
+       if [ ! -f config/certs/ca.zip ]; then
-        echo "Creating certs";
+         echo "Creating CA";
-        echo -ne \
+         bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
-        "instances:\n"\
+         unzip config/certs/ca.zip -d config/certs;
-        "  - name: es01\n"\
+       fi;
-        "    dns:\n"\
+       if [ ! -f config/certs/certs.zip ]; then
-        "      - es01\n"\
+         echo "Creating certs";
-        "      - localhost\n"\
+         echo -ne \
-        "    ip:\n"\
+         "instances:\n"\
-        "      - 127.0.0.1\n"\
+         "  - name: es01\n"\
-        "  - name: kibana\n"\
+         "    dns:\n"\
-        "    dns:\n"\
+         "      - es01\n"\
-        "      - kibana\n"\
+         "      - localhost\n"\
-        "      - localhost\n"\
+         "    ip:\n"\
-        "    ip:\n"\
+         "      - 127.0.0.1\n"\
-        "      - 127.0.0.1\n"\
+         "  - name: kibana\n"\
+         "    dns:\n"\
+         "      - kibana\n"\
+         "      - localhost\n"\
+         "    ip:\n"\
+         "      - 127.0.0.1\n"\
          > config/certs/instances.yml;
-      fi;
 
-      echo "Setting file permissions";
+         bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
-      find /usr/share/elasticsearch/config/certs -type d -exec chmod 750 \{\} \;;
+         unzip config/certs/certs.zip -d config/certs;
-      find /usr/share/elasticsearch/config/certs -type f -exec chmod 640 \{\} \;;
+       fi;
-      tail -f /dev/null;
+
+       echo "Setting file permissions"
+       chown -R root:root config/certs;
+       find . -type d -exec chmod 750 \{\} \;;
+       find . -type f -exec chmod 640 \{\} \;;
+
+       echo "Waiting for Elasticsearch availability";
+       until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
+       echo "Setting kibana_system password";
+       until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done;
+       echo "All done!";      
       '
+    healthcheck:
+     test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"]
+     interval: 1s
+     timeout: 5s
+     retries: 120
+
+
 
   backend:
     build:
@@ -54,7 +75,7 @@ services:
     volumes:
       - pong:/transcendence/pong
     ports:
-      - "8080:8080"
+      - 8080:8080
     networks:
       - app-network
     environment:
@@ -95,12 +116,16 @@ services:
   es01:
     image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
     container_name: es01
+    depends_on:
+      setup:
+        condition: service_healthy
     volumes:
       - certs:/usr/share/elasticsearch/config/certs:ro
+      - pong_es_data_01:/usr/share/elasticsearch/data
+    labels:
+      co.elastic.logs/module: elasticsearch
     ports:
-      - "9200:9200"
+      - 9200:9200
-    networks:
-      - app-network
     environment:
       - node.name=es01
       - cluster.name=${CLUSTER_NAME}
@@ -108,55 +133,79 @@ services:
       - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
       - bootstrap.memory_lock=true
       - xpack.security.enabled=true
+      - xpack.security.http.ssl.enabled=true
+      - xpack.security.http.ssl.key=certs/es01/es01.key
+      - xpack.security.http.ssl.certificate=certs/es01/es01.crt
+      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
       - xpack.security.transport.ssl.enabled=true
-      - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/certs/elastic-certificate.p12
+      - xpack.security.transport.ssl.key=certs/es01/es01.key
-      - xpack.security.transport.ssl.keystore.type=PKCS12
+      - xpack.security.transport.ssl.certificate=certs/es01/es01.crt
-      - xpack.security.transport.ssl.keystore.password=""
+      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
+      - xpack.security.transport.ssl.verification_mode=certificate
       - xpack.license.self_generated.type=${LICENSE}
-    depends_on:
-      - setup
-      - logstash01
     healthcheck:
-      test: ["CMD-SHELL", "curl -s http://localhost:9200/_cluster/health | grep -q '\"status\":\"green\"'"]
+      test:
+        [
+         "CMD-SHELL",
+         "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
+        ]
       interval: 10s
       timeout: 10s
-      retries: 5
+      retries: 120
 
   kibana:
     image: docker.elastic.co/kibana/kibana:${STACK_VERSION}
     container_name: kibana
+    labels:
+      co.elastic.logs/module: kibana
+    depends_on:
+      es01:
+        condition: service_healthy
     volumes:
       - certs:/usr/share/kibana/config/certs:ro
       - pong_kibana:/usr/share/kibana/data
-    user: "1000:1000"
     ports:
       - 5601:5601
-    networks:
-      - app-network
     environment:
-      - SERVERNAME=pong.kibana.org
+      - SERVERNAME=kibana
-      - ELASTICSEARCH_HOSTS=http://es01:9200
+      - ELASTICSEARCH_HOSTS=https://es01:9200
-      - ELASTICSEARCH_USERNAME=${ELASTIC_USERNAME}
+      - ELASTICSEARCH_USERNAME=${KIBANA_USERNAME}
-      - ELASTICSEARCH_PASSWORD=${ELASTIC_PASSWORD}
+      - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD}
-    depends_on:
+      - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt
-      - es01
+      - XPACK_SECURITY_ENCRYPTIONKEY=${ENCRYPTION_KEY}
+      - XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=${ENCRYPTION_KEY}
+      - XPACK_REPORTING_ENCRYPTIONKEY=${ENCRYPTION_KEY}
     healthcheck:
-      test: ["CMD-SHELL", "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'"]
+      test: 
+        [
+        "CMD-SHELL", 
+        "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'"
+        ]
       interval: 10s
-      timeout: 20s
+      timeout: 10s
-      retries: 200
+      retries: 120
  
   logstash01:
     image: docker.elastic.co/logstash/logstash:${STACK_VERSION}
     container_name: logstash01
+    labels:
+      co.elastic.logs/module: logstash 
+    user: root 
+    depends_on:
+      es01:
+        condition: service_healthy
+      kibana:
+        condition: service_healthy
     volumes:
+      - certs:/usr/share/logstash/certs
+      - pong_logstash_data01:/usr/share/logstash/data
       - ./config/logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro
     ports:
-      - "5044:5044"
+      - "5044:5044/udp"
-    networks:
+    command: logstash -f /usr/share/logstash/pipeline/logstash.conf
-      - app-network
     environment:
-      - ELASTIC_HOSTS=http://es01:9200
+      - NODE_NAME="logstash"
+      - ELASTIC_HOSTS=https://es01:9200
       - ELASTIC_USER=${ELASTIC_USERNAME}
       - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
       - xpack.monitoring.enabled=false
@@ -176,32 +225,10 @@ volumes:
     driver: local
   pong_kibana:
     driver: local
+  pong_logstash_data01:
+    driver: local
 
 networks:
   app-network:
     name: app-network
     driver: bridge
-
-
-
-  #filebeat01:
-  #depends_on:
-  #- es01
-  #  image: docker.elastic.co/beats/filebeat:${STACK_VERSION}
-  #  volumes:
-  #    - certs:/usr/share/logstash/certs
-  #    - pong_filebeat_data_01:/usr/share/filebeat/data
-  #    - ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
-  #    - /var/lib/docker/containers:/var/lib/docker/containers:ro
-  #    - /var/run/docker.sock:/var/run/docker.sock:ro
-  #  environment:
-  #    - ELASTIC_USER=elastic
-  #    - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
-  #    - ELASTIC_HOSTS=https://es01:9200
-  #    - KIBANA_HOSTS=http://kibana:5601
-  #    - LOGSTASH_HOSTS=http://logstash01:9600
-  #    - xpack.monitoring.enabled=false
-  #  networks:
-  #    - app-network
-
-

env_template

@@ -4,32 +4,27 @@ DEBUG=True
 DJANGO_ALLOWED_HOSTS=['*']
 
 # PostgreSQL settings
-POSTGRES_DB= 
+POSTGRES_DB=players_db 
-POSTGRES_USER=
+POSTGRES_USER=42student
 POSTGRES_PASSWORD=
 
+# Django settings
 DB_HOST=db
 DB_PORT=5432
-
+PWD_PATH=${PWD}
-PROJECT_PATH=${PWD}/pong
+PROJECT_PATH=${PWD_PATH}/pong
-POSTGRES_DATA_PATH=${PWD}/data/db
-ES_DATA_PATH=${PWD}/data/es
-KIBA_DATA_PATH=${PWD}/data/kiba
-LSTASH_DATA_PATH=${PWD}/data/lstash
 
 # ElasticSearch settings
 STACK_VERSION=8.14.3
 CLUSTER_NAME=docker-cluster
 LICENSE=basic
 
+ELASTIC_USERNAME=elastic
 ELASTIC_PASSWORD=
-ES_PORT=9200
 
 # Kibana settings
-KIBANA_PASSWORD=
 KIBANA_PORT=5601
+KIBANA_USERNAME=
+KIBANA_PASSWORD=
 
-ES_MEM_LIMIT=1073741824
+ENCRYPTION_KEY=c34d38b3a14956121ff2170e5030b471551370178f43e5626eec58b04a30fae2
-KB_MEM_LIMIT=1073741824
-LS_MEM_LIMIT=1073741824
-

makefile

@@ -2,9 +2,11 @@ COMPOSE_FILE=docker-compose.yml
 COMPOSE=docker compose -f $(COMPOSE_FILE)
 CONTAINER=$(c)
 
+.PHONY: up build start stop down destroy logs ps db-shell help
+
 up:
 	$(COMPOSE) build 
-	$(COMPOSE) up $(CONTAINER)
+	$(COMPOSE) up $(CONTAINER) || true
 
 build:
 	$(COMPOSE) build $(CONTAINER)
@@ -40,7 +42,7 @@ help:
 	@echo "  make down [c=service]         # Stop and remove containers"
 	@echo "  make destroy				   # Stop and remove containers and volumes"
 	@echo "  make stop [c=service]         # Stop containers"
-	@echo "  make restart [c=service]      # Restart containers"
 	@echo "  make logs [c=service]         # Tail logs of containers"
 	@echo "  make ps                       # List containers"
 	@echo "  make help                     # Show this help"
+