first logs from logstash

2025-12-16 05:57:48 +01:00 · 2024-08-17 20:11:18 +02:00 · 2024-08-17 20:11:18 +02:00 · b0706f1c36
commit b0706f1c36
parent c871a71135
8 changed files with 144 additions and 323 deletions
--- a/.env
+++ b/.env
@ -17,11 +17,15 @@ PROJECT_PATH=${PWD_PATH}/pong
 # ElasticSearch settings
 STACK_VERSION=8.14.3
 CLUSTER_NAME=docker-cluster
-LICENSE=trial
+LICENSE=basic

-ELASTIC_USERNAME=adrien
-ELASTIC_PASSWORD=qwerty42
+ELASTIC_USERNAME=elastic
+ELASTIC_PASSWORD=elastic_pass

 # Kibana settings
-KIBANA_PASSWORD=qwerty42
 KIBANA_PORT=5601
+KIBANA_USERNAME=kibana_system
+KIBANA_PASSWORD=kibana_pass
+
+ENCRYPTION_KEY=c34d38b3a14956121ff2170e5030b471551370178f43e5626eec58b04a30fae2
+
--- a/config/filebeat.yml
+++ b/config/filebeat.yml
@ -1,10 +0,0 @@
-filebeat.inputs:
- type: docker
-  containers.ids:
-    - "*"
-
-processors:
- add_docker_metadata: ~
-
-output.logstash:
-  hosts: ["http://logstash01:5044"]
--- a/config/kibana.yml
+++ b/config/kibana.yml
@ -1,54 +0,0 @@
-# Kibana's Elasticsearch URL
-elasticsearch.hosts: ["https://es01:9200"]
-
-# The name of the Kibana instance
-server.name: "kibana"
-
-# The base path for Kibana
-#server.basePath: ""
-
-# Serve the Kibana instance at the root path
-#server.rewriteBasePath: false
-
-# Enable or disable the Kibana server
-server.enabled: true
-
-# The port Kibana will listen on
-server.port: 5601
-
-# Enable or disable the xpack features
-xpack.enabled: true
-
-# Set the encryption key for encrypted saved objects
-#xpack.encryptedSavedObjects.encryptionKey: ""
-
-# Set the key for the kibana security features
-#xpack.security.encryptionKey: ""
-
-# Enable or disable the monitoring feature
-xpack.monitoring.enabled: true
-
-# Set the URL of the Elasticsearch instance to which Kibana should connect
-#xpack.reporting.enabled: true
-
-  #xpack.reporting.roles.enabled: false
-
-# Enable the Kibana alerting feature
-#xpack.alerting.enabled: false
-
-# Enable or disable the usage collection
-#xpack.usageCollection.enabled: true
-
-# Configure the log level for Kibana
-#logging.level: info
-
-# Configure the directory where logs will be stored
-logging.dest: /var/log/kibana/kibana.log
-
-# Configure security and authentication settings
-elasticsearch.username: test
-elasticsearch.password: test 
-
-# Disable or enable the Kibana plugin
-#xpack.license.management.enabled: false
-
--- a/config/logstash.conf
+++ b/config/logstash.conf
@ -1,30 +1,21 @@
 input {
-	file {
-		path => "/var/lib/docker/containers/*/*.log"
-		start_position => "beginning"
-		sincedb_path => "/dev/null"
-		type => "docker"
-		codec => "json"
-	}
+  udp {
+    host => "0.0.0.0"
+    port => 5044
+  }
 }

-filter {
-	if [log_message] =~ /GET/ {
-		grok {
-			match => { "log_message" => "%{IP:client_ip} - - \[%{HTTPDATE:timestamp}\] \"%{WORD:method} %{URIPATH:request_path} HTTP/%{NUMBER:http_version}\" %{NUMBER:response_code} %{NUMBER:response_size}" }
-		}
-		date {
-			match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
-				target => "@timestamp"
-		}
-	}
-}
+filter {}

 output {
-	elasticsearch {
-		hosts => ["http://es01:9200"]
-		index => "docker-logs-%{+YYYY.MM.dd}"
-		user=> "${ELASTIC_USER}"
-		password=> "${ELASTIC_PASSWORD}"
-	}
+  elasticsearch {
+    index => "logstash-%{+YYYY.MM.dd}"
+    hosts => ["https://es01:9200"]
+    user => "elastic"
+    password => elastic_pass
+	ssl_enabled => true
+    ssl_certificate_authorities => "/usr/share/logstash/certs/ca/ca.crt"
+    ssl_verification_mode => "full"
+  }
+  #stdout {}
 }
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@ -1,134 +0,0 @@
-services:
-  backend:
-    build:
-      context: .
-      dockerfile: Dockerfile
-    image: backend
-    container_name: backend
-    restart: always
-    command: /bin/sh -c "sleep 5 &&
-      venv/bin/python manage.py makemigrations --noinput &&
-      venv/bin/python manage.py migrate --noinput && 
-      venv/bin/python manage.py collectstatic --noinput &&
-      venv/bin/daphne -b 0.0.0.0 -p 8080 pong.asgi:application"
-    volumes:
-      - pong:/transcendence/pong
-    ports:
-      - "8080:8080"
-    networks:
-      - app-network
-    environment:
-      DB_HOST: db
-      DB_PORT: 5432
-      DB_NAME: ${POSTGRES_DB}
-      DB_USER: ${POSTGRES_USER}
-      DB_PASSWORD: ${POSTGRES_PASSWORD}
-    depends_on:
-      - db
-    healthcheck:
-      test: ["CMD-SHELL", "curl", "http://localhost:8080"]
-      interval: 30s
-      timeout: 10s
-      retries: 3
-      start_period: 10s
-
-  db:
-    image: postgres:latest
-    container_name: postgres
-    restart: always
-    volumes:
-      - pong_pg_data:/var/lib/postgresql/data
-    ports:
-      - "5432:5432"
-    networks:
-      - app-network
-    environment:
-      POSTGRES_DB: ${POSTGRES_DB}
-      POSTGRES_USER: ${POSTGRES_USER}
-      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-
-  es01:
-    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
-    container_name: es01
-    volumes:
-      - pong_es_data_01:/usr/share/elasticsearch/data
-    ports:
-     - "9200:9200"
-    networks:
-     - app-network
-    environment:
-     - node.name=es01
-     - cluster.name=${CLUSTER_NAME}
-     - discovery.type=single-node
-     - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
-     - bootstrap.memory_lock=true
-     - xpack.security.enabled=false
-    healthcheck:
-      test: ["CMD-SHELL", "curl -s http://localhost:9200/_cluster/health | grep -q '\"status\":\"green\"'"]
-      interval: 10s
-      timeout: 10s
-      retries: 5
-
-  kibana:
-    image: docker.elastic.co/kibana/kibana:${STACK_VERSION}
-    container_name: kibana
-    volumes:
-      - pong_kibana:/usr/share/kibana/data
-    user: "1000:1000"
-    ports:
-      - 5601:5601
-    networks:
-      - app-network
-    environment:
-      - SERVERNAME=pong.kibana.org
-      - ELASTICSEARCH_HOSTS=http://es01:9200
-      - ELASTICSEARCH_USERNAME=${ELASTIC_USERNAME}
-      - ELASTICSEARCH_PASSWORD=${ELASTIC_PASSWORD}
-    depends_on:
-      - es01
-    healthcheck:
-      test: ["CMD-SHELL", "curl -s -I http://localhost:5601 | grep -q '200 OK'"]
-      interval: 10s
-      timeout: 20s
-      retries: 200
-
-    
-  logstash01:
-    image: docker.elastic.co/logstash/logstash:${STACK_VERSION}
-    container_name: logstash01
-    volumes:
-      - ./config/logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro
-    ports:
-      - "5044:5044"
-    networks:
-      - app-network
-    environment:
-      - ELASTIC_HOSTS=http://es01:9200
-      - ELASTIC_USER=${ELASTIC_USERNAME}
-      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
-      - xpack.monitoring.enabled=false
-
-volumes:
-  pong:
-    driver: local
-    driver_opts:
-      type: none
-      device: ${PROJECT_PATH}
-      o: bind
-  pong_pg_data:
-    driver: local
-  pong_es_data_01:
-    driver: local
-  pong_kibana:
-    driver: local
-
-  
-
-networks:
-  app-network:
-    driver: bridge
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -6,38 +6,59 @@ services:
    volumes:
      - certs:/usr/share/elasticsearch/config/certs
    command: >
-      sh -c '
-      if [ ! -f /usr/share/elasticsearch/config/certs/elastic-certificate.p12 ]; then
-        echo "Creating elastic-certificate.p12...";
-        bin/elasticsearch-certutil cert --name elastic-certificate --days 365 --self-signed --out /usr/share/elasticsearch/config/certs/elastic-certificate.p12 --pass "";
-        else
-          echo "elastic-certificate.p12 already exists, skipping certificate creation.";
-      fi;
-
-      if [ ! -f /usr/share/elasticsearch/config/certs/instances.yml ]; then
-        echo "Creating certs";
-        echo -ne \
-        "instances:\n"\
-        "  - name: es01\n"\
-        "    dns:\n"\
-        "      - es01\n"\
-        "      - localhost\n"\
-        "    ip:\n"\
-        "      - 127.0.0.1\n"\
-        "  - name: kibana\n"\
-        "    dns:\n"\
-        "      - kibana\n"\
-        "      - localhost\n"\
-        "    ip:\n"\
-        "      - 127.0.0.1\n"\
+      bash -c '
+       if [ x${ELASTIC_PASSWORD} == x ]; then
+         echo "Set the ELASTIC_PASSWORD environment variable in the .env file";
+         exit 1;
+       elif [ x${KIBANA_PASSWORD} == x ]; then
+         echo "Set the KIBANA_PASSWORD environment variable in the .env file";
+         exit 1;
+       fi;
+       if [ ! -f config/certs/ca.zip ]; then
+         echo "Creating CA";
+         bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
+         unzip config/certs/ca.zip -d config/certs;
+       fi;
+       if [ ! -f config/certs/certs.zip ]; then
+         echo "Creating certs";
+         echo -ne \
+         "instances:\n"\
+         "  - name: es01\n"\
+         "    dns:\n"\
+         "      - es01\n"\
+         "      - localhost\n"\
+         "    ip:\n"\
+         "      - 127.0.0.1\n"\
+         "  - name: kibana\n"\
+         "    dns:\n"\
+         "      - kibana\n"\
+         "      - localhost\n"\
+         "    ip:\n"\
+         "      - 127.0.0.1\n"\
         > config/certs/instances.yml;
-      fi;

-      echo "Setting file permissions";
-      find /usr/share/elasticsearch/config/certs -type d -exec chmod 750 \{\} \;;
-      find /usr/share/elasticsearch/config/certs -type f -exec chmod 640 \{\} \;;
-      tail -f /dev/null;
+         bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
+         unzip config/certs/certs.zip -d config/certs;
+       fi;
+
+       echo "Setting file permissions"
+       chown -R root:root config/certs;
+       find . -type d -exec chmod 750 \{\} \;;
+       find . -type f -exec chmod 640 \{\} \;;
+
+       echo "Waiting for Elasticsearch availability";
+       until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
+       echo "Setting kibana_system password";
+       until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done;
+       echo "All done!";      
      '
+    healthcheck:
+     test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"]
+     interval: 1s
+     timeout: 5s
+     retries: 120
+
+

  backend:
    build:
@ -54,7 +75,7 @@ services:
    volumes:
      - pong:/transcendence/pong
    ports:
-      - "8080:8080"
+      - 8080:8080
    networks:
      - app-network
    environment:
@ -95,12 +116,16 @@ services:
  es01:
    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
    container_name: es01
+    depends_on:
+      setup:
+        condition: service_healthy
    volumes:
      - certs:/usr/share/elasticsearch/config/certs:ro
+      - pong_es_data_01:/usr/share/elasticsearch/data
+    labels:
+      co.elastic.logs/module: elasticsearch
    ports:
-      - "9200:9200"
-    networks:
-      - app-network
+      - 9200:9200
    environment:
      - node.name=es01
      - cluster.name=${CLUSTER_NAME}
@ -108,55 +133,79 @@ services:
      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
+      - xpack.security.http.ssl.enabled=true
+      - xpack.security.http.ssl.key=certs/es01/es01.key
+      - xpack.security.http.ssl.certificate=certs/es01/es01.crt
+      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.enabled=true
-      - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/certs/elastic-certificate.p12
-      - xpack.security.transport.ssl.keystore.type=PKCS12
-      - xpack.security.transport.ssl.keystore.password=""
+      - xpack.security.transport.ssl.key=certs/es01/es01.key
+      - xpack.security.transport.ssl.certificate=certs/es01/es01.crt
+      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
+      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.license.self_generated.type=${LICENSE}
-    depends_on:
-      - setup
-      - logstash01
    healthcheck:
-      test: ["CMD-SHELL", "curl -s http://localhost:9200/_cluster/health | grep -q '\"status\":\"green\"'"]
+      test:
+        [
+         "CMD-SHELL",
+         "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
+        ]
      interval: 10s
      timeout: 10s
-      retries: 5
+      retries: 120

  kibana:
    image: docker.elastic.co/kibana/kibana:${STACK_VERSION}
    container_name: kibana
+    labels:
+      co.elastic.logs/module: kibana
+    depends_on:
+      es01:
+        condition: service_healthy
    volumes:
      - certs:/usr/share/kibana/config/certs:ro
      - pong_kibana:/usr/share/kibana/data
-    user: "1000:1000"
    ports:
      - 5601:5601
-    networks:
-      - app-network
    environment:
-      - SERVERNAME=pong.kibana.org
-      - ELASTICSEARCH_HOSTS=http://es01:9200
-      - ELASTICSEARCH_USERNAME=${ELASTIC_USERNAME}
-      - ELASTICSEARCH_PASSWORD=${ELASTIC_PASSWORD}
-    depends_on:
-      - es01
+      - SERVERNAME=kibana
+      - ELASTICSEARCH_HOSTS=https://es01:9200
+      - ELASTICSEARCH_USERNAME=${KIBANA_USERNAME}
+      - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD}
+      - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt
+      - XPACK_SECURITY_ENCRYPTIONKEY=${ENCRYPTION_KEY}
+      - XPACK_ENCRYPTEDSAVEDOBJECTS_ENCRYPTIONKEY=${ENCRYPTION_KEY}
+      - XPACK_REPORTING_ENCRYPTIONKEY=${ENCRYPTION_KEY}
    healthcheck:
-      test: ["CMD-SHELL", "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'"]
+      test: 
+        [
+        "CMD-SHELL", 
+        "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'"
+        ]
      interval: 10s
-      timeout: 20s
-      retries: 200
+      timeout: 10s
+      retries: 120
 
  logstash01:
    image: docker.elastic.co/logstash/logstash:${STACK_VERSION}
    container_name: logstash01
+    labels:
+      co.elastic.logs/module: logstash 
+    user: root 
+    depends_on:
+      es01:
+        condition: service_healthy
+      kibana:
+        condition: service_healthy
    volumes:
+      - certs:/usr/share/logstash/certs
+      - pong_logstash_data01:/usr/share/logstash/data
      - ./config/logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro
    ports:
-      - "5044:5044"
-    networks:
-      - app-network
+      - "5044:5044/udp"
+    command: logstash -f /usr/share/logstash/pipeline/logstash.conf
    environment:
-      - ELASTIC_HOSTS=http://es01:9200
+      - NODE_NAME="logstash"
+      - ELASTIC_HOSTS=https://es01:9200
      - ELASTIC_USER=${ELASTIC_USERNAME}
      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
      - xpack.monitoring.enabled=false
@ -176,32 +225,10 @@ volumes:
    driver: local
  pong_kibana:
    driver: local
+  pong_logstash_data01:
+    driver: local

 networks:
  app-network:
    name: app-network
    driver: bridge
-
-
-
-  #filebeat01:
-  #depends_on:
-  #- es01
-  #  image: docker.elastic.co/beats/filebeat:${STACK_VERSION}
-  #  volumes:
-  #    - certs:/usr/share/logstash/certs
-  #    - pong_filebeat_data_01:/usr/share/filebeat/data
-  #    - ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
-  #    - /var/lib/docker/containers:/var/lib/docker/containers:ro
-  #    - /var/run/docker.sock:/var/run/docker.sock:ro
-  #  environment:
-  #    - ELASTIC_USER=elastic
-  #    - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
-  #    - ELASTIC_HOSTS=https://es01:9200
-  #    - KIBANA_HOSTS=http://kibana:5601
-  #    - LOGSTASH_HOSTS=http://logstash01:9600
-  #    - xpack.monitoring.enabled=false
-  #  networks:
-  #    - app-network
-
-
--- a/23
+++ b/23
@ -4,32 +4,27 @@ DEBUG=True
 DJANGO_ALLOWED_HOSTS=['*']

 # PostgreSQL settings
-POSTGRES_DB= 
-POSTGRES_USER=
+POSTGRES_DB=players_db 
+POSTGRES_USER=42student
 POSTGRES_PASSWORD=

+# Django settings
 DB_HOST=db
 DB_PORT=5432
-
-PROJECT_PATH=${PWD}/pong
-POSTGRES_DATA_PATH=${PWD}/data/db
-ES_DATA_PATH=${PWD}/data/es
-KIBA_DATA_PATH=${PWD}/data/kiba
-LSTASH_DATA_PATH=${PWD}/data/lstash
+PWD_PATH=${PWD}
+PROJECT_PATH=${PWD_PATH}/pong

 # ElasticSearch settings
 STACK_VERSION=8.14.3
 CLUSTER_NAME=docker-cluster
 LICENSE=basic

+ELASTIC_USERNAME=elastic
 ELASTIC_PASSWORD=
-ES_PORT=9200

 # Kibana settings
-KIBANA_PASSWORD=
 KIBANA_PORT=5601
+KIBANA_USERNAME=
+KIBANA_PASSWORD=

-ES_MEM_LIMIT=1073741824
-KB_MEM_LIMIT=1073741824
-LS_MEM_LIMIT=1073741824
-
+ENCRYPTION_KEY=c34d38b3a14956121ff2170e5030b471551370178f43e5626eec58b04a30fae2
--- a/6
+++ b/6
@ -2,9 +2,11 @@ COMPOSE_FILE=docker-compose.yml
 COMPOSE=docker compose -f $(COMPOSE_FILE)
 CONTAINER=$(c)

+.PHONY: up build start stop down destroy logs ps db-shell help
+
 up:
 	$(COMPOSE) build 
-	$(COMPOSE) up $(CONTAINER)
+	$(COMPOSE) up $(CONTAINER) || true

 build:
 	$(COMPOSE) build $(CONTAINER)
@ -40,7 +42,7 @@ help:
 	@echo "  make down [c=service]         # Stop and remove containers"
 	@echo "  make destroy				   # Stop and remove containers and volumes"
 	@echo "  make stop [c=service]         # Stop containers"
-	@echo "  make restart [c=service]      # Restart containers"
 	@echo "  make logs [c=service]         # Tail logs of containers"
 	@echo "  make ps                       # List containers"
 	@echo "  make help                     # Show this help"
+