wip

.env

@@ -8,10 +8,11 @@ POSTGRES_DB=players_db
 POSTGRES_USER=42student
 POSTGRES_PASSWORD=qwerty
 
+# Django settings
 DB_HOST=db
 DB_PORT=5432
-
+PWD_PATH=${PWD}
-PROJECT_PATH=${PWD}/pong
+PROJECT_PATH=${PWD_PATH}/pong
 
 # ElasticSearch settings
 STACK_VERSION=8.14.3
@@ -24,5 +25,3 @@ ELASTIC_PASSWORD=qwerty42
 # Kibana settings
 KIBANA_PASSWORD=qwerty42
 KIBANA_PORT=5601
-
-ENCRYPTION_KEY=c34d38b3a14956121ff2170e5030b471551370178f43e5626eec58b04a30fae2

config/kibana.yml

@@ -0,0 +1,54 @@
+# Kibana's Elasticsearch URL
+elasticsearch.hosts: ["https://es01:9200"]
+
+# The name of the Kibana instance
+server.name: "kibana"
+
+# The base path for Kibana
+#server.basePath: ""
+
+# Serve the Kibana instance at the root path
+#server.rewriteBasePath: false
+
+# Enable or disable the Kibana server
+server.enabled: true
+
+# The port Kibana will listen on
+server.port: 5601
+
+# Enable or disable the xpack features
+xpack.enabled: true
+
+# Set the encryption key for encrypted saved objects
+#xpack.encryptedSavedObjects.encryptionKey: ""
+
+# Set the key for the kibana security features
+#xpack.security.encryptionKey: ""
+
+# Enable or disable the monitoring feature
+xpack.monitoring.enabled: true
+
+# Set the URL of the Elasticsearch instance to which Kibana should connect
+#xpack.reporting.enabled: true
+
+  #xpack.reporting.roles.enabled: false
+
+# Enable the Kibana alerting feature
+#xpack.alerting.enabled: false
+
+# Enable or disable the usage collection
+#xpack.usageCollection.enabled: true
+
+# Configure the log level for Kibana
+#logging.level: info
+
+# Configure the directory where logs will be stored
+logging.dest: /var/log/kibana/kibana.log
+
+# Configure security and authentication settings
+elasticsearch.username: test
+elasticsearch.password: test 
+
+# Disable or enable the Kibana plugin
+#xpack.license.management.enabled: false
+

config/logstash.conf

@@ -0,0 +1,30 @@
+input {
+	file {
+		path => "/var/lib/docker/containers/*/*.log"
+		start_position => "beginning"
+		sincedb_path => "/dev/null"
+		type => "docker"
+		codec => "json"
+	}
+}
+
+filter {
+	if [log_message] =~ /GET/ {
+		grok {
+			match => { "log_message" => "%{IP:client_ip} - - \[%{HTTPDATE:timestamp}\] \"%{WORD:method} %{URIPATH:request_path} HTTP/%{NUMBER:http_version}\" %{NUMBER:response_code} %{NUMBER:response_size}" }
+		}
+		date {
+			match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
+				target => "@timestamp"
+		}
+	}
+}
+
+output {
+	elasticsearch {
+		hosts => ["http://es01:9200"]
+		index => "docker-logs-%{+YYYY.MM.dd}"
+		user=> "${ELASTIC_USER}"
+		password=> "${ELASTIC_PASSWORD}"
+	}
+}

docker-compose.yaml

@@ -25,6 +25,12 @@ services:
       DB_PASSWORD: ${POSTGRES_PASSWORD}
     depends_on:
       - db
+    healthcheck:
+      test: ["CMD-SHELL", "curl", "http://localhost:8080"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 10s
 
   db:
     image: postgres:latest
@@ -40,47 +46,33 @@ services:
       POSTGRES_DB: ${POSTGRES_DB}
       POSTGRES_USER: ${POSTGRES_USER}
       POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
-    #healthcheck:
+    healthcheck:
-      #test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
+      test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
-      #interval: 10s
+      interval: 10s
-      #timeout: 5s
+      timeout: 5s
-      #retries: 5
+      retries: 5
 
   es01:
     image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
     container_name: es01
     volumes:
-      - pong_logstash_data_01:/usr/share/elasticsearch/data
+      - pong_es_data_01:/usr/share/elasticsearch/data
     ports:
-      - "127.0.0.1:9200:9200"
+     - "9200:9200"
     networks:
-      - app-network
+     - app-network
     environment:
-      - node.name=es01
+     - node.name=es01
-      - cluster.name=${CLUSTER_NAME}
+     - cluster.name=${CLUSTER_NAME}
-      - discovery.type=single-node
+     - discovery.type=single-node
-      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+     - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
-      - xpack.security.enabled=false 
+     - bootstrap.memory_lock=true
-      - xpack.license.self_generated.type=trial
+     - xpack.security.enabled=false
-    depends_on:
+    healthcheck:
-      - logstash01
+      test: ["CMD-SHELL", "curl -s http://localhost:9200/_cluster/health | grep -q '\"status\":\"green\"'"]
-
+      interval: 10s
-  logstash01:
+      timeout: 10s
-    image: docker.elastic.co/logstash/logstash:${STACK_VERSION}
+      retries: 5
-    container_name: logstash01
-    volumes:
-      - ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro
-      - /var/lib/docker/containers:/var/lib/docker/containers:ro
-      - /var/run/docker.sock:/var/run/docker.sock:ro
-    ports:
-      - "5044:5044"
-    networks:
-      - app-network
-    environment:
-      - ELASTIC_HOSTS=http://es01:9200
-      - ELASTIC_USER=${ELASTIC_USERNAME}
-      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
-      - xpack.monitoring.enabled=false
 
   kibana:
     image: docker.elastic.co/kibana/kibana:${STACK_VERSION}
@@ -99,11 +91,27 @@ services:
       - ELASTICSEARCH_PASSWORD=${ELASTIC_PASSWORD}
     depends_on:
       - es01
-    #healthcheck:
+    healthcheck:
-      #test: ["CMD-SHELL", "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'"]
+      test: ["CMD-SHELL", "curl -s -I http://localhost:5601 | grep -q '200 OK'"]
-      #interval: 20s
+      interval: 10s
-      #timeout: 10s
+      timeout: 20s
-      #retries: 120
+      retries: 200
+
+    
+  logstash01:
+    image: docker.elastic.co/logstash/logstash:${STACK_VERSION}
+    container_name: logstash01
+    volumes:
+      - ./config/logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro
+    ports:
+      - "5044:5044"
+    networks:
+      - app-network
+    environment:
+      - ELASTIC_HOSTS=http://es01:9200
+      - ELASTIC_USER=${ELASTIC_USERNAME}
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+      - xpack.monitoring.enabled=false
 
 volumes:
   pong:
@@ -118,10 +126,8 @@ volumes:
     driver: local
   pong_kibana:
     driver: local
-  pong_logstash_data_01:
+
-    driver: local
+  
-  pong_filebeat_data_01:
-    driver: local
 
 networks:
   app-network:

docker-compose.yaml-simle

@@ -1,61 +0,0 @@
-services:
-  backend:
-    build:
-      context: .
-      dockerfile: Dockerfile
-    image: backend
-    container_name: backend
-    restart: always
-    command: /bin/sh -c "sleep 5 &&
-      venv/bin/python manage.py makemigrations --noinput &&
-      venv/bin/python manage.py migrate --noinput && 
-      venv/bin/python manage.py collectstatic --noinput &&
-      venv/bin/daphne -b 0.0.0.0 -p 8080 pong.asgi:application"
-    volumes:
-      - pong:/transcendence/pong
-    ports:
-      - "8080:8080"
-    networks:
-      - app-network
-    environment:
-      DB_HOST: db
-      DB_PORT: 5432
-      DB_NAME: ${POSTGRES_DB}
-      DB_USER: ${POSTGRES_USER}
-      DB_PASSWORD: ${POSTGRES_PASSWORD}
-    depends_on:
-      - db
-
-  db:
-    image: postgres:latest
-    container_name: postgres
-    restart: always
-    volumes:
-      - pong_pg_data:/var/lib/postgresql/data
-    ports:
-      - "5432:5432"
-    networks:
-      - app-network
-    environment:
-      POSTGRES_DB: ${POSTGRES_DB}
-      POSTGRES_USER: ${POSTGRES_USER}
-      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
-      interval: 10s
-      timeout: 5s
-      retries: 5
-
-volumes:
-  pong:
-    driver: local
-    driver_opts:
-      type: none
-      device: ${PROJECT_PATH}
-      o: bind
-  pong_pg_data:
-    driver: local
-
-networks:
-  app-network:
-    driver: bridge

docker-compose.yml

@@ -1,58 +1,43 @@
 services:
   setup:
-   image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
+    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
-   volumes:
+    container_name: setup
-     - certs:/usr/share/elasticsearch/config/certs
+    user: "0"
-   user: "0"
+    volumes:
-   command: >
+      - certs:/usr/share/elasticsearch/config/certs
-     bash -c '
+    command: >
-       if [ x${ELASTIC_PASSWORD} == x ]; then
+      sh -c '
-         echo "Set the ELASTIC_PASSWORD environment variable in the .env file";
+      if [ ! -f /usr/share/elasticsearch/config/certs/elastic-certificate.p12 ]; then
-         exit 1;
+        echo "Creating elastic-certificate.p12...";
-       elif [ x${KIBANA_PASSWORD} == x ]; then
+        bin/elasticsearch-certutil cert --name elastic-certificate --days 365 --self-signed --out /usr/share/elasticsearch/config/certs/elastic-certificate.p12 --pass "";
-         echo "Set the KIBANA_PASSWORD environment variable in the .env file";
+        else
-         exit 1;
+          echo "elastic-certificate.p12 already exists, skipping certificate creation.";
-       fi;
+      fi;
-       if [ ! -f config/certs/ca.zip ]; then
+
-         echo "Creating CA";
+      if [ ! -f /usr/share/elasticsearch/config/certs/instances.yml ]; then
-         bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
+        echo "Creating certs";
-         unzip config/certs/ca.zip -d config/certs;
+        echo -ne \
-       fi;
+        "instances:\n"\
-       if [ ! -f config/certs/certs.zip ]; then
+        "  - name: es01\n"\
-         echo "Creating certs";
+        "    dns:\n"\
-         echo -ne \
+        "      - es01\n"\
-         "instances:\n"\
+        "      - localhost\n"\
-         "  - name: es01\n"\
+        "    ip:\n"\
-         "    dns:\n"\
+        "      - 127.0.0.1\n"\
-         "      - es01\n"\
+        "  - name: kibana\n"\
-         "      - localhost\n"\
+        "    dns:\n"\
-         "    ip:\n"\
+        "      - kibana\n"\
-         "      - 127.0.0.1\n"\
+        "      - localhost\n"\
-         "  - name: kibana\n"\
+        "    ip:\n"\
-         "    dns:\n"\
+        "      - 127.0.0.1\n"\
-         "      - kibana\n"\
-         "      - localhost\n"\
-         "    ip:\n"\
-         "      - 127.0.0.1\n"\
          > config/certs/instances.yml;
-         bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
+      fi;
-         unzip config/certs/certs.zip -d config/certs;
+
-       fi;
+      echo "Setting file permissions";
-       echo "Setting file permissions"
+      find /usr/share/elasticsearch/config/certs -type d -exec chmod 750 \{\} \;;
-       chown -R root:root config/certs;
+      find /usr/share/elasticsearch/config/certs -type f -exec chmod 640 \{\} \;;
-       find . -type d -exec chmod 750 \{\} \;;
+      tail -f /dev/null;
-       find . -type f -exec chmod 640 \{\} \;;
+      '
-       echo "Waiting for Elasticsearch availability";
-       until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
-       echo "Setting kibana_system password";
-       until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done;
-       echo "All done!";
-     '
-   healthcheck:
-     test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"]
-     interval: 1s
-     timeout: 5s
-     retries: 120 
 
   backend:
     build:
@@ -80,6 +65,12 @@ services:
       DB_PASSWORD: ${POSTGRES_PASSWORD}
     depends_on:
       - db
+    healthcheck:
+      test: ["CMD-SHELL", "curl", "http://localhost:8080"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 10s
 
   db:
     image: postgres:latest
@@ -95,20 +86,19 @@ services:
       POSTGRES_DB: ${POSTGRES_DB}
       POSTGRES_USER: ${POSTGRES_USER}
       POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
-    #healthcheck:
+    healthcheck:
-      #test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
+      test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
-      #interval: 10s
+      interval: 10s
-      #timeout: 5s
+      timeout: 5s
-      #retries: 5
+      retries: 5
 
   es01:
     image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
     container_name: es01
     volumes:
-      - certs:/usr/share/elasticsearch/config/certs
+      - certs:/usr/share/elasticsearch/config/certs:ro
-      - pong_logstash_data_01:/usr/share/elasticsearch/data
     ports:
-      - "127.0.0.1:9200:9200"
+      - "9200:9200"
     networks:
       - app-network
     environment:
@@ -118,38 +108,25 @@ services:
       - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
       - bootstrap.memory_lock=true
       - xpack.security.enabled=true
-      - xpack.security.http.ssl.enabled=true
-      - xpack.security.http.ssl.key=certs/es01/es01.key
-      - xpack.security.http.ssl.certificate=certs/es01/es01.crt
-      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
       - xpack.security.transport.ssl.enabled=true
-      - xpack.security.transport.ssl.key=certs/es01/es01.key
+      - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/certs/elastic-certificate.p12
-      - xpack.security.transport.ssl.certificate=certs/es01/es01.crt
+      - xpack.security.transport.ssl.keystore.type=PKCS12
-      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
+      - xpack.security.transport.ssl.keystore.password=""
-      - xpack.security.transport.ssl.verification_mode=certificate
       - xpack.license.self_generated.type=${LICENSE}
     depends_on:
+      - setup
       - logstash01
-
+    healthcheck:
-  logstash01:
+      test: ["CMD-SHELL", "curl -s http://localhost:9200/_cluster/health | grep -q '\"status\":\"green\"'"]
-    image: docker.elastic.co/logstash/logstash:${STACK_VERSION}
+      interval: 10s
-    container_name: logstash01
+      timeout: 10s
-    volumes:
+      retries: 5
-      - ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro
-    ports:
-      - "5044:5044"
-    networks:
-      - app-network
-    environment:
-      - ELASTIC_HOSTS=http://es01:9200
-      - ELASTIC_USER=${ELASTIC_USERNAME}
-      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
-      - xpack.monitoring.enabled=false
 
   kibana:
     image: docker.elastic.co/kibana/kibana:${STACK_VERSION}
     container_name: kibana
     volumes:
+      - certs:/usr/share/kibana/config/certs:ro
       - pong_kibana:/usr/share/kibana/data
     user: "1000:1000"
     ports:
@@ -163,30 +140,27 @@ services:
       - ELASTICSEARCH_PASSWORD=${ELASTIC_PASSWORD}
     depends_on:
       - es01
-    #healthcheck:
+    healthcheck:
-      #test: ["CMD-SHELL", "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'"]
+      test: ["CMD-SHELL", "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'"]
-      #interval: 20s
+      interval: 10s
-      #timeout: 10s
+      timeout: 20s
-      #retries: 120
+      retries: 200
-
+ 
-  filebeat01:
+  logstash01:
-    depends_on:
+    image: docker.elastic.co/logstash/logstash:${STACK_VERSION}
-      - es01
+    container_name: logstash01
-    image: docker.elastic.co/beats/filebeat:${STACK_VERSION}
     volumes:
-      - pong_filebeat_data_01:/usr/share/filebeat/data
+      - ./config/logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro
-      - ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
+    ports:
-      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - "5044:5044"
-      - /var/run/docker.sock:/var/run/docker.sock:ro
-    environment:
-      - ELASTIC_USER=elastic
-      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
-      - ELASTIC_HOSTS=https://es01:9200
-      - KIBANA_HOSTS=http://kibana:5601
-      - LOGSTASH_HOSTS=http://logstash01:9600
     networks:
       - app-network
-
+    environment:
+      - ELASTIC_HOSTS=http://es01:9200
+      - ELASTIC_USER=${ELASTIC_USERNAME}
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+      - xpack.monitoring.enabled=false
+      
 volumes:
   certs:
     driver: local
@@ -202,11 +176,32 @@ volumes:
     driver: local
   pong_kibana:
     driver: local
-  pong_logstash_data_01:
-    driver: local
-  pong_filebeat_data_01:
-    driver: local
 
 networks:
   app-network:
+    name: app-network
     driver: bridge
+
+
+
+  #filebeat01:
+  #depends_on:
+  #- es01
+  #  image: docker.elastic.co/beats/filebeat:${STACK_VERSION}
+  #  volumes:
+  #    - certs:/usr/share/logstash/certs
+  #    - pong_filebeat_data_01:/usr/share/filebeat/data
+  #    - ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
+  #    - /var/lib/docker/containers:/var/lib/docker/containers:ro
+  #    - /var/run/docker.sock:/var/run/docker.sock:ro
+  #  environment:
+  #    - ELASTIC_USER=elastic
+  #    - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+  #    - ELASTIC_HOSTS=https://es01:9200
+  #    - KIBANA_HOSTS=http://kibana:5601
+  #    - LOGSTASH_HOSTS=http://logstash01:9600
+  #    - xpack.monitoring.enabled=false
+  #  networks:
+  #    - app-network
+
+

logstash.conf

@@ -1,21 +0,0 @@
-input {
-	file {
-		path => "/var/lib/docker/containers/*/*.log"
-		start_position => "beginning"
-		sincedb_path => "/usr/share/logstash/data/sincedb"
-		type => "docker"
-		codec => "json"
-	}
-}
-
-filter {
-}
-
-output {
-	elasticsearch {
-		hosts => ["http://es01:9200"]
-		index => "docker-logs-%{+YYYY.MM.dd}"
-		user=> "${ELASTIC_USER}"
-		password=> "${ELASTIC_PASSWORD}"
-	}
-}

makefile

@@ -1,10 +1,10 @@
-COMPOSE_FILE=docker-compose.yaml
+COMPOSE_FILE=docker-compose.yml
 COMPOSE=docker compose -f $(COMPOSE_FILE)
 CONTAINER=$(c)
 
 up:
 	$(COMPOSE) build 
-	$(COMPOSE) up
+	$(COMPOSE) up $(CONTAINER)
 
 build:
 	$(COMPOSE) build $(CONTAINER)