in progress

.env

@@ -16,7 +16,7 @@ PROJECT_PATH=${PWD}/pong
 # ElasticSearch settings
 STACK_VERSION=8.14.3
 CLUSTER_NAME=docker-cluster
-LICENSE=basic
+LICENSE=trial
 
 ELASTIC_USERNAME=adrien
 ELASTIC_PASSWORD=qwerty42
@@ -24,3 +24,5 @@ ELASTIC_PASSWORD=qwerty42
 # Kibana settings
 KIBANA_PASSWORD=qwerty42
 KIBANA_PORT=5601
+
+ENCRYPTION_KEY=c34d38b3a14956121ff2170e5030b471551370178f43e5626eec58b04a30fae2

docker-compose.yaml

@@ -10,11 +10,11 @@ services:
       venv/bin/python manage.py makemigrations --noinput &&
       venv/bin/python manage.py migrate --noinput && 
       venv/bin/python manage.py collectstatic --noinput &&
-      venv/bin/daphne -b 0.0.0.0 -p 80 pong.asgi:application"
+      venv/bin/daphne -b 0.0.0.0 -p 8080 pong.asgi:application"
     volumes:
       - pong:/transcendence/pong
     ports:
-      - "80:80"
+      - "8080:8080"
     networks:
       - app-network
     environment:
@@ -70,6 +70,8 @@ services:
     container_name: logstash01
     volumes:
       - ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro
+      - /var/lib/docker/containers:/var/lib/docker/containers:ro
+      - /var/run/docker.sock:/var/run/docker.sock:ro
     ports:
       - "5044:5044"
     networks:
@@ -103,24 +105,6 @@ services:
       #timeout: 10s
       #retries: 120
 
-  filebeat01:
-    depends_on:
-      - es01
-    image: docker.elastic.co/beats/filebeat:${STACK_VERSION}
-    volumes:
-      - pong_filebeat_data_01:/usr/share/filebeat/data
-      - ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
-      - /var/lib/docker/containers:/var/lib/docker/containers:ro
-      - /var/run/docker.sock:/var/run/docker.sock:ro
-    environment:
-      - ELASTIC_USER=elastic
-      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
-      - ELASTIC_HOSTS=https://es01:9200
-      - KIBANA_HOSTS=http://kibana:5601
-      - LOGSTASH_HOSTS=http://logstash01:9600
-    networks:
-      - app-network
-
 volumes:
   pong:
     driver: local

docker-compose.yaml-simle

@@ -0,0 +1,61 @@
+services:
+  backend:
+    build:
+      context: .
+      dockerfile: Dockerfile
+    image: backend
+    container_name: backend
+    restart: always
+    command: /bin/sh -c "sleep 5 &&
+      venv/bin/python manage.py makemigrations --noinput &&
+      venv/bin/python manage.py migrate --noinput && 
+      venv/bin/python manage.py collectstatic --noinput &&
+      venv/bin/daphne -b 0.0.0.0 -p 8080 pong.asgi:application"
+    volumes:
+      - pong:/transcendence/pong
+    ports:
+      - "8080:8080"
+    networks:
+      - app-network
+    environment:
+      DB_HOST: db
+      DB_PORT: 5432
+      DB_NAME: ${POSTGRES_DB}
+      DB_USER: ${POSTGRES_USER}
+      DB_PASSWORD: ${POSTGRES_PASSWORD}
+    depends_on:
+      - db
+
+  db:
+    image: postgres:latest
+    container_name: postgres
+    restart: always
+    volumes:
+      - pong_pg_data:/var/lib/postgresql/data
+    ports:
+      - "5432:5432"
+    networks:
+      - app-network
+    environment:
+      POSTGRES_DB: ${POSTGRES_DB}
+      POSTGRES_USER: ${POSTGRES_USER}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD}
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+volumes:
+  pong:
+    driver: local
+    driver_opts:
+      type: none
+      device: ${PROJECT_PATH}
+      o: bind
+  pong_pg_data:
+    driver: local
+
+networks:
+  app-network:
+    driver: bridge

docker-compose.yml

@@ -1,4 +1,59 @@
 services:
+  setup:
+   image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
+   volumes:
+     - certs:/usr/share/elasticsearch/config/certs
+   user: "0"
+   command: >
+     bash -c '
+       if [ x${ELASTIC_PASSWORD} == x ]; then
+         echo "Set the ELASTIC_PASSWORD environment variable in the .env file";
+         exit 1;
+       elif [ x${KIBANA_PASSWORD} == x ]; then
+         echo "Set the KIBANA_PASSWORD environment variable in the .env file";
+         exit 1;
+       fi;
+       if [ ! -f config/certs/ca.zip ]; then
+         echo "Creating CA";
+         bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
+         unzip config/certs/ca.zip -d config/certs;
+       fi;
+       if [ ! -f config/certs/certs.zip ]; then
+         echo "Creating certs";
+         echo -ne \
+         "instances:\n"\
+         "  - name: es01\n"\
+         "    dns:\n"\
+         "      - es01\n"\
+         "      - localhost\n"\
+         "    ip:\n"\
+         "      - 127.0.0.1\n"\
+         "  - name: kibana\n"\
+         "    dns:\n"\
+         "      - kibana\n"\
+         "      - localhost\n"\
+         "    ip:\n"\
+         "      - 127.0.0.1\n"\
+         > config/certs/instances.yml;
+         bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
+         unzip config/certs/certs.zip -d config/certs;
+       fi;
+       echo "Setting file permissions"
+       chown -R root:root config/certs;
+       find . -type d -exec chmod 750 \{\} \;;
+       find . -type f -exec chmod 640 \{\} \;;
+       echo "Waiting for Elasticsearch availability";
+       until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
+       echo "Setting kibana_system password";
+       until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done;
+       echo "All done!";
+     '
+   healthcheck:
+     test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"]
+     interval: 1s
+     timeout: 5s
+     retries: 120 
+
   backend:
     build:
       context: .
@@ -10,7 +65,7 @@ services:
       venv/bin/python manage.py makemigrations --noinput &&
       venv/bin/python manage.py migrate --noinput && 
       venv/bin/python manage.py collectstatic --noinput &&
-      venv/bin/daphne -b 0.0.0.0 -p 80 pong.asgi:application"
+      venv/bin/daphne -b 0.0.0.0 -p 8080 pong.asgi:application"
     volumes:
       - pong:/transcendence/pong
     ports:
@@ -50,6 +105,7 @@ services:
     image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
     container_name: es01
     volumes:
+      - certs:/usr/share/elasticsearch/config/certs
       - pong_logstash_data_01:/usr/share/elasticsearch/data
     ports:
       - "127.0.0.1:9200:9200"
@@ -60,8 +116,18 @@ services:
       - cluster.name=${CLUSTER_NAME}
       - discovery.type=single-node
       - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
-      - xpack.security.enabled=false 
+      - bootstrap.memory_lock=true
-      - xpack.license.self_generated.type=trial
+      - xpack.security.enabled=true
+      - xpack.security.http.ssl.enabled=true
+      - xpack.security.http.ssl.key=certs/es01/es01.key
+      - xpack.security.http.ssl.certificate=certs/es01/es01.crt
+      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
+      - xpack.security.transport.ssl.enabled=true
+      - xpack.security.transport.ssl.key=certs/es01/es01.key
+      - xpack.security.transport.ssl.certificate=certs/es01/es01.crt
+      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
+      - xpack.security.transport.ssl.verification_mode=certificate
+      - xpack.license.self_generated.type=${LICENSE}
     depends_on:
       - logstash01
 
@@ -122,6 +188,8 @@ services:
       - app-network
 
 volumes:
+  certs:
+    driver: local
   pong:
     driver: local
     driver_opts:

filebeat.yml

@@ -1,32 +1,10 @@
 filebeat.inputs:
-#- type: docker
+- type: docker
-  #containers.ids:
+  containers.ids:
-  #  - "*"
+    - "*"
-- type: filestream
-  id: default-filestream
-  paths:
-    - ingest_data/*.log
-
-filebeat.autodiscover:
- providers:
-   - type: docker
-     hints.enabled: true
 
 processors:
 - add_docker_metadata: ~
 
-
+output.logstash:
-setup.kibana:
+  hosts: ["http://logstash01:5044"]
- host: ${KIBANA_HOSTS}
- username: ${ELASTIC_USER}
- password: ${ELASTIC_PASSWORD}
-
-
-output.elasticsearch:
- hosts: ${ELASTIC_HOSTS}
- username: ${ELASTIC_USER}
- password: ${ELASTIC_PASSWORD}
- ssl.enabled: false
-
-#output.logstash:
-  #hosts: ["http://logstash01:9600"]

logstash.conf

@@ -1,6 +1,10 @@
 input {
-	beat {
+	file {
-		port => 5044
+		path => "/var/lib/docker/containers/*/*.log"
+		start_position => "beginning"
+		sincedb_path => "/usr/share/logstash/data/sincedb"
+		type => "docker"
+		codec => "json"
 	}
 }
 

makefile

@@ -1,10 +1,10 @@
-COMPOSE_FILE=docker-compose.yml
+COMPOSE_FILE=docker-compose.yaml
 COMPOSE=docker compose -f $(COMPOSE_FILE)
 CONTAINER=$(c)
 
 up:
 	$(COMPOSE) build 
-	$(COMPOSE) up $(CONTAINER)
+	$(COMPOSE) up
 
 build:
 	$(COMPOSE) build $(CONTAINER)