From 4b82bbbfd185ab7fad2aedb8eb510e1a8a7e127c Mon Sep 17 00:00:00 2001 From: Adrien Audebert Date: Wed, 7 Aug 2024 15:30:11 +0200 Subject: [PATCH] in progress --- .env | 4 ++- docker-compose.yaml | 24 +++---------- docker-compose.yaml-simle | 61 ++++++++++++++++++++++++++++++++ docker-compose.yml | 74 +++++++++++++++++++++++++++++++++++++-- filebeat.yml | 32 +++-------------- logstash.conf | 8 +++-- makefile | 4 +-- 7 files changed, 152 insertions(+), 55 deletions(-) create mode 100644 docker-compose.yaml-simle diff --git a/.env b/.env index d87bad8..78f2dda 100644 --- a/.env +++ b/.env @@ -16,7 +16,7 @@ PROJECT_PATH=${PWD}/pong # ElasticSearch settings STACK_VERSION=8.14.3 CLUSTER_NAME=docker-cluster -LICENSE=basic +LICENSE=trial ELASTIC_USERNAME=adrien ELASTIC_PASSWORD=qwerty42 @@ -24,3 +24,5 @@ ELASTIC_PASSWORD=qwerty42 # Kibana settings KIBANA_PASSWORD=qwerty42 KIBANA_PORT=5601 + +ENCRYPTION_KEY=c34d38b3a14956121ff2170e5030b471551370178f43e5626eec58b04a30fae2 diff --git a/docker-compose.yaml b/docker-compose.yaml index 6eb4002..8b2b3dd 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -10,11 +10,11 @@ services: venv/bin/python manage.py makemigrations --noinput && venv/bin/python manage.py migrate --noinput && venv/bin/python manage.py collectstatic --noinput && - venv/bin/daphne -b 0.0.0.0 -p 80 pong.asgi:application" + venv/bin/daphne -b 0.0.0.0 -p 8080 pong.asgi:application" volumes: - pong:/transcendence/pong ports: - - "80:80" + - "8080:8080" networks: - app-network environment: @@ -70,6 +70,8 @@ services: container_name: logstash01 volumes: - ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - /var/run/docker.sock:/var/run/docker.sock:ro ports: - "5044:5044" networks: @@ -103,24 +105,6 @@ services: #timeout: 10s #retries: 120 - filebeat01: - depends_on: - - es01 - image: docker.elastic.co/beats/filebeat:${STACK_VERSION} - volumes: - - pong_filebeat_data_01:/usr/share/filebeat/data - - ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro - - /var/lib/docker/containers:/var/lib/docker/containers:ro - - /var/run/docker.sock:/var/run/docker.sock:ro - environment: - - ELASTIC_USER=elastic - - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} - - ELASTIC_HOSTS=https://es01:9200 - - KIBANA_HOSTS=http://kibana:5601 - - LOGSTASH_HOSTS=http://logstash01:9600 - networks: - - app-network - volumes: pong: driver: local diff --git a/docker-compose.yaml-simle b/docker-compose.yaml-simle new file mode 100644 index 0000000..d11b2d0 --- /dev/null +++ b/docker-compose.yaml-simle @@ -0,0 +1,61 @@ +services: + backend: + build: + context: . + dockerfile: Dockerfile + image: backend + container_name: backend + restart: always + command: /bin/sh -c "sleep 5 && + venv/bin/python manage.py makemigrations --noinput && + venv/bin/python manage.py migrate --noinput && + venv/bin/python manage.py collectstatic --noinput && + venv/bin/daphne -b 0.0.0.0 -p 8080 pong.asgi:application" + volumes: + - pong:/transcendence/pong + ports: + - "8080:8080" + networks: + - app-network + environment: + DB_HOST: db + DB_PORT: 5432 + DB_NAME: ${POSTGRES_DB} + DB_USER: ${POSTGRES_USER} + DB_PASSWORD: ${POSTGRES_PASSWORD} + depends_on: + - db + + db: + image: postgres:latest + container_name: postgres + restart: always + volumes: + - pong_pg_data:/var/lib/postgresql/data + ports: + - "5432:5432" + networks: + - app-network + environment: + POSTGRES_DB: ${POSTGRES_DB} + POSTGRES_USER: ${POSTGRES_USER} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] + interval: 10s + timeout: 5s + retries: 5 + +volumes: + pong: + driver: local + driver_opts: + type: none + device: ${PROJECT_PATH} + o: bind + pong_pg_data: + driver: local + +networks: + app-network: + driver: bridge diff --git a/docker-compose.yml b/docker-compose.yml index 5f5beee..944e595 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,59 @@ services: + setup: + image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} + volumes: + - certs:/usr/share/elasticsearch/config/certs + user: "0" + command: > + bash -c ' + if [ x${ELASTIC_PASSWORD} == x ]; then + echo "Set the ELASTIC_PASSWORD environment variable in the .env file"; + exit 1; + elif [ x${KIBANA_PASSWORD} == x ]; then + echo "Set the KIBANA_PASSWORD environment variable in the .env file"; + exit 1; + fi; + if [ ! -f config/certs/ca.zip ]; then + echo "Creating CA"; + bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip; + unzip config/certs/ca.zip -d config/certs; + fi; + if [ ! -f config/certs/certs.zip ]; then + echo "Creating certs"; + echo -ne \ + "instances:\n"\ + " - name: es01\n"\ + " dns:\n"\ + " - es01\n"\ + " - localhost\n"\ + " ip:\n"\ + " - 127.0.0.1\n"\ + " - name: kibana\n"\ + " dns:\n"\ + " - kibana\n"\ + " - localhost\n"\ + " ip:\n"\ + " - 127.0.0.1\n"\ + > config/certs/instances.yml; + bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key; + unzip config/certs/certs.zip -d config/certs; + fi; + echo "Setting file permissions" + chown -R root:root config/certs; + find . -type d -exec chmod 750 \{\} \;; + find . -type f -exec chmod 640 \{\} \;; + echo "Waiting for Elasticsearch availability"; + until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done; + echo "Setting kibana_system password"; + until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done; + echo "All done!"; + ' + healthcheck: + test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"] + interval: 1s + timeout: 5s + retries: 120 + backend: build: context: . @@ -10,7 +65,7 @@ services: venv/bin/python manage.py makemigrations --noinput && venv/bin/python manage.py migrate --noinput && venv/bin/python manage.py collectstatic --noinput && - venv/bin/daphne -b 0.0.0.0 -p 80 pong.asgi:application" + venv/bin/daphne -b 0.0.0.0 -p 8080 pong.asgi:application" volumes: - pong:/transcendence/pong ports: @@ -50,6 +105,7 @@ services: image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} container_name: es01 volumes: + - certs:/usr/share/elasticsearch/config/certs - pong_logstash_data_01:/usr/share/elasticsearch/data ports: - "127.0.0.1:9200:9200" @@ -60,8 +116,18 @@ services: - cluster.name=${CLUSTER_NAME} - discovery.type=single-node - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} - - xpack.security.enabled=false - - xpack.license.self_generated.type=trial + - bootstrap.memory_lock=true + - xpack.security.enabled=true + - xpack.security.http.ssl.enabled=true + - xpack.security.http.ssl.key=certs/es01/es01.key + - xpack.security.http.ssl.certificate=certs/es01/es01.crt + - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.security.transport.ssl.enabled=true + - xpack.security.transport.ssl.key=certs/es01/es01.key + - xpack.security.transport.ssl.certificate=certs/es01/es01.crt + - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt + - xpack.security.transport.ssl.verification_mode=certificate + - xpack.license.self_generated.type=${LICENSE} depends_on: - logstash01 @@ -122,6 +188,8 @@ services: - app-network volumes: + certs: + driver: local pong: driver: local driver_opts: diff --git a/filebeat.yml b/filebeat.yml index f5be95b..3082ccb 100644 --- a/filebeat.yml +++ b/filebeat.yml @@ -1,32 +1,10 @@ filebeat.inputs: -#- type: docker - #containers.ids: - # - "*" -- type: filestream - id: default-filestream - paths: - - ingest_data/*.log - -filebeat.autodiscover: - providers: - - type: docker - hints.enabled: true +- type: docker + containers.ids: + - "*" processors: - add_docker_metadata: ~ - -setup.kibana: - host: ${KIBANA_HOSTS} - username: ${ELASTIC_USER} - password: ${ELASTIC_PASSWORD} - - -output.elasticsearch: - hosts: ${ELASTIC_HOSTS} - username: ${ELASTIC_USER} - password: ${ELASTIC_PASSWORD} - ssl.enabled: false - -#output.logstash: - #hosts: ["http://logstash01:9600"] +output.logstash: + hosts: ["http://logstash01:5044"] diff --git a/logstash.conf b/logstash.conf index 9236eb1..4872c61 100644 --- a/logstash.conf +++ b/logstash.conf @@ -1,6 +1,10 @@ input { - beat { - port => 5044 + file { + path => "/var/lib/docker/containers/*/*.log" + start_position => "beginning" + sincedb_path => "/usr/share/logstash/data/sincedb" + type => "docker" + codec => "json" } } diff --git a/makefile b/makefile index 169500b..126ed39 100644 --- a/makefile +++ b/makefile @@ -1,10 +1,10 @@ -COMPOSE_FILE=docker-compose.yml +COMPOSE_FILE=docker-compose.yaml COMPOSE=docker compose -f $(COMPOSE_FILE) CONTAINER=$(c) up: $(COMPOSE) build - $(COMPOSE) up $(CONTAINER) + $(COMPOSE) up build: $(COMPOSE) build $(CONTAINER)