From c871a711355563046434ec6c50442f887ab9152a Mon Sep 17 00:00:00 2001 From: Adrien Audebert Date: Wed, 14 Aug 2024 16:29:20 +0200 Subject: [PATCH] wip --- .env | 7 +- filebeat.yml => config/filebeat.yml | 0 config/kibana.yml | 54 +++++++ config/logstash.conf | 30 ++++ docker-compose.yaml | 90 ++++++------ docker-compose.yaml-simle | 61 -------- docker-compose.yml | 211 ++++++++++++++-------------- logstash.conf | 21 --- makefile | 4 +- manage.py | 0 10 files changed, 240 insertions(+), 238 deletions(-) rename filebeat.yml => config/filebeat.yml (100%) create mode 100644 config/kibana.yml create mode 100644 config/logstash.conf delete mode 100644 docker-compose.yaml-simle delete mode 100644 logstash.conf mode change 100755 => 100644 manage.py diff --git a/.env b/.env index 78f2dda..9dd1c1a 100644 --- a/.env +++ b/.env @@ -8,10 +8,11 @@ POSTGRES_DB=players_db POSTGRES_USER=42student POSTGRES_PASSWORD=qwerty +# Django settings DB_HOST=db DB_PORT=5432 - -PROJECT_PATH=${PWD}/pong +PWD_PATH=${PWD} +PROJECT_PATH=${PWD_PATH}/pong # ElasticSearch settings STACK_VERSION=8.14.3 @@ -24,5 +25,3 @@ ELASTIC_PASSWORD=qwerty42 # Kibana settings KIBANA_PASSWORD=qwerty42 KIBANA_PORT=5601 - -ENCRYPTION_KEY=c34d38b3a14956121ff2170e5030b471551370178f43e5626eec58b04a30fae2 diff --git a/filebeat.yml b/config/filebeat.yml similarity index 100% rename from filebeat.yml rename to config/filebeat.yml diff --git a/config/kibana.yml b/config/kibana.yml new file mode 100644 index 0000000..a1ed606 --- /dev/null +++ b/config/kibana.yml @@ -0,0 +1,54 @@ +# Kibana's Elasticsearch URL +elasticsearch.hosts: ["https://es01:9200"] + +# The name of the Kibana instance +server.name: "kibana" + +# The base path for Kibana +#server.basePath: "" + +# Serve the Kibana instance at the root path +#server.rewriteBasePath: false + +# Enable or disable the Kibana server +server.enabled: true + +# The port Kibana will listen on +server.port: 5601 + +# Enable or disable the xpack features +xpack.enabled: true + +# Set the encryption key for encrypted saved objects +#xpack.encryptedSavedObjects.encryptionKey: "" + +# Set the key for the kibana security features +#xpack.security.encryptionKey: "" + +# Enable or disable the monitoring feature +xpack.monitoring.enabled: true + +# Set the URL of the Elasticsearch instance to which Kibana should connect +#xpack.reporting.enabled: true + + #xpack.reporting.roles.enabled: false + +# Enable the Kibana alerting feature +#xpack.alerting.enabled: false + +# Enable or disable the usage collection +#xpack.usageCollection.enabled: true + +# Configure the log level for Kibana +#logging.level: info + +# Configure the directory where logs will be stored +logging.dest: /var/log/kibana/kibana.log + +# Configure security and authentication settings +elasticsearch.username: test +elasticsearch.password: test + +# Disable or enable the Kibana plugin +#xpack.license.management.enabled: false + diff --git a/config/logstash.conf b/config/logstash.conf new file mode 100644 index 0000000..28710ab --- /dev/null +++ b/config/logstash.conf @@ -0,0 +1,30 @@ +input { + file { + path => "/var/lib/docker/containers/*/*.log" + start_position => "beginning" + sincedb_path => "/dev/null" + type => "docker" + codec => "json" + } +} + +filter { + if [log_message] =~ /GET/ { + grok { + match => { "log_message" => "%{IP:client_ip} - - \[%{HTTPDATE:timestamp}\] \"%{WORD:method} %{URIPATH:request_path} HTTP/%{NUMBER:http_version}\" %{NUMBER:response_code} %{NUMBER:response_size}" } + } + date { + match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ] + target => "@timestamp" + } + } +} + +output { + elasticsearch { + hosts => ["http://es01:9200"] + index => "docker-logs-%{+YYYY.MM.dd}" + user=> "${ELASTIC_USER}" + password=> "${ELASTIC_PASSWORD}" + } +} diff --git a/docker-compose.yaml b/docker-compose.yaml index 8b2b3dd..a046dbe 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -25,6 +25,12 @@ services: DB_PASSWORD: ${POSTGRES_PASSWORD} depends_on: - db + healthcheck: + test: ["CMD-SHELL", "curl", "http://localhost:8080"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 10s db: image: postgres:latest @@ -40,47 +46,33 @@ services: POSTGRES_DB: ${POSTGRES_DB} POSTGRES_USER: ${POSTGRES_USER} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - #healthcheck: - #test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] - #interval: 10s - #timeout: 5s - #retries: 5 + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] + interval: 10s + timeout: 5s + retries: 5 es01: image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} container_name: es01 volumes: - - pong_logstash_data_01:/usr/share/elasticsearch/data + - pong_es_data_01:/usr/share/elasticsearch/data ports: - - "127.0.0.1:9200:9200" + - "9200:9200" networks: - - app-network + - app-network environment: - - node.name=es01 - - cluster.name=${CLUSTER_NAME} - - discovery.type=single-node - - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} - - xpack.security.enabled=false - - xpack.license.self_generated.type=trial - depends_on: - - logstash01 - - logstash01: - image: docker.elastic.co/logstash/logstash:${STACK_VERSION} - container_name: logstash01 - volumes: - - ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro - - /var/lib/docker/containers:/var/lib/docker/containers:ro - - /var/run/docker.sock:/var/run/docker.sock:ro - ports: - - "5044:5044" - networks: - - app-network - environment: - - ELASTIC_HOSTS=http://es01:9200 - - ELASTIC_USER=${ELASTIC_USERNAME} - - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} - - xpack.monitoring.enabled=false + - node.name=es01 + - cluster.name=${CLUSTER_NAME} + - discovery.type=single-node + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - bootstrap.memory_lock=true + - xpack.security.enabled=false + healthcheck: + test: ["CMD-SHELL", "curl -s http://localhost:9200/_cluster/health | grep -q '\"status\":\"green\"'"] + interval: 10s + timeout: 10s + retries: 5 kibana: image: docker.elastic.co/kibana/kibana:${STACK_VERSION} @@ -99,11 +91,27 @@ services: - ELASTICSEARCH_PASSWORD=${ELASTIC_PASSWORD} depends_on: - es01 - #healthcheck: - #test: ["CMD-SHELL", "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'"] - #interval: 20s - #timeout: 10s - #retries: 120 + healthcheck: + test: ["CMD-SHELL", "curl -s -I http://localhost:5601 | grep -q '200 OK'"] + interval: 10s + timeout: 20s + retries: 200 + + + logstash01: + image: docker.elastic.co/logstash/logstash:${STACK_VERSION} + container_name: logstash01 + volumes: + - ./config/logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro + ports: + - "5044:5044" + networks: + - app-network + environment: + - ELASTIC_HOSTS=http://es01:9200 + - ELASTIC_USER=${ELASTIC_USERNAME} + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - xpack.monitoring.enabled=false volumes: pong: @@ -118,10 +126,8 @@ volumes: driver: local pong_kibana: driver: local - pong_logstash_data_01: - driver: local - pong_filebeat_data_01: - driver: local + + networks: app-network: diff --git a/docker-compose.yaml-simle b/docker-compose.yaml-simle deleted file mode 100644 index d11b2d0..0000000 --- a/docker-compose.yaml-simle +++ /dev/null @@ -1,61 +0,0 @@ -services: - backend: - build: - context: . - dockerfile: Dockerfile - image: backend - container_name: backend - restart: always - command: /bin/sh -c "sleep 5 && - venv/bin/python manage.py makemigrations --noinput && - venv/bin/python manage.py migrate --noinput && - venv/bin/python manage.py collectstatic --noinput && - venv/bin/daphne -b 0.0.0.0 -p 8080 pong.asgi:application" - volumes: - - pong:/transcendence/pong - ports: - - "8080:8080" - networks: - - app-network - environment: - DB_HOST: db - DB_PORT: 5432 - DB_NAME: ${POSTGRES_DB} - DB_USER: ${POSTGRES_USER} - DB_PASSWORD: ${POSTGRES_PASSWORD} - depends_on: - - db - - db: - image: postgres:latest - container_name: postgres - restart: always - volumes: - - pong_pg_data:/var/lib/postgresql/data - ports: - - "5432:5432" - networks: - - app-network - environment: - POSTGRES_DB: ${POSTGRES_DB} - POSTGRES_USER: ${POSTGRES_USER} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - healthcheck: - test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] - interval: 10s - timeout: 5s - retries: 5 - -volumes: - pong: - driver: local - driver_opts: - type: none - device: ${PROJECT_PATH} - o: bind - pong_pg_data: - driver: local - -networks: - app-network: - driver: bridge diff --git a/docker-compose.yml b/docker-compose.yml index 944e595..fc73df1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,58 +1,43 @@ services: setup: - image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} - volumes: - - certs:/usr/share/elasticsearch/config/certs - user: "0" - command: > - bash -c ' - if [ x${ELASTIC_PASSWORD} == x ]; then - echo "Set the ELASTIC_PASSWORD environment variable in the .env file"; - exit 1; - elif [ x${KIBANA_PASSWORD} == x ]; then - echo "Set the KIBANA_PASSWORD environment variable in the .env file"; - exit 1; - fi; - if [ ! -f config/certs/ca.zip ]; then - echo "Creating CA"; - bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip; - unzip config/certs/ca.zip -d config/certs; - fi; - if [ ! -f config/certs/certs.zip ]; then - echo "Creating certs"; - echo -ne \ - "instances:\n"\ - " - name: es01\n"\ - " dns:\n"\ - " - es01\n"\ - " - localhost\n"\ - " ip:\n"\ - " - 127.0.0.1\n"\ - " - name: kibana\n"\ - " dns:\n"\ - " - kibana\n"\ - " - localhost\n"\ - " ip:\n"\ - " - 127.0.0.1\n"\ + image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} + container_name: setup + user: "0" + volumes: + - certs:/usr/share/elasticsearch/config/certs + command: > + sh -c ' + if [ ! -f /usr/share/elasticsearch/config/certs/elastic-certificate.p12 ]; then + echo "Creating elastic-certificate.p12..."; + bin/elasticsearch-certutil cert --name elastic-certificate --days 365 --self-signed --out /usr/share/elasticsearch/config/certs/elastic-certificate.p12 --pass ""; + else + echo "elastic-certificate.p12 already exists, skipping certificate creation."; + fi; + + if [ ! -f /usr/share/elasticsearch/config/certs/instances.yml ]; then + echo "Creating certs"; + echo -ne \ + "instances:\n"\ + " - name: es01\n"\ + " dns:\n"\ + " - es01\n"\ + " - localhost\n"\ + " ip:\n"\ + " - 127.0.0.1\n"\ + " - name: kibana\n"\ + " dns:\n"\ + " - kibana\n"\ + " - localhost\n"\ + " ip:\n"\ + " - 127.0.0.1\n"\ > config/certs/instances.yml; - bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key; - unzip config/certs/certs.zip -d config/certs; - fi; - echo "Setting file permissions" - chown -R root:root config/certs; - find . -type d -exec chmod 750 \{\} \;; - find . -type f -exec chmod 640 \{\} \;; - echo "Waiting for Elasticsearch availability"; - until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done; - echo "Setting kibana_system password"; - until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done; - echo "All done!"; - ' - healthcheck: - test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"] - interval: 1s - timeout: 5s - retries: 120 + fi; + + echo "Setting file permissions"; + find /usr/share/elasticsearch/config/certs -type d -exec chmod 750 \{\} \;; + find /usr/share/elasticsearch/config/certs -type f -exec chmod 640 \{\} \;; + tail -f /dev/null; + ' backend: build: @@ -80,6 +65,12 @@ services: DB_PASSWORD: ${POSTGRES_PASSWORD} depends_on: - db + healthcheck: + test: ["CMD-SHELL", "curl", "http://localhost:8080"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 10s db: image: postgres:latest @@ -95,20 +86,19 @@ services: POSTGRES_DB: ${POSTGRES_DB} POSTGRES_USER: ${POSTGRES_USER} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD} - #healthcheck: - #test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] - #interval: 10s - #timeout: 5s - #retries: 5 + healthcheck: + test: ["CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}"] + interval: 10s + timeout: 5s + retries: 5 es01: image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION} container_name: es01 volumes: - - certs:/usr/share/elasticsearch/config/certs - - pong_logstash_data_01:/usr/share/elasticsearch/data + - certs:/usr/share/elasticsearch/config/certs:ro ports: - - "127.0.0.1:9200:9200" + - "9200:9200" networks: - app-network environment: @@ -118,38 +108,25 @@ services: - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} - bootstrap.memory_lock=true - xpack.security.enabled=true - - xpack.security.http.ssl.enabled=true - - xpack.security.http.ssl.key=certs/es01/es01.key - - xpack.security.http.ssl.certificate=certs/es01/es01.crt - - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt - xpack.security.transport.ssl.enabled=true - - xpack.security.transport.ssl.key=certs/es01/es01.key - - xpack.security.transport.ssl.certificate=certs/es01/es01.crt - - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt - - xpack.security.transport.ssl.verification_mode=certificate + - xpack.security.transport.ssl.keystore.path=/usr/share/elasticsearch/config/certs/elastic-certificate.p12 + - xpack.security.transport.ssl.keystore.type=PKCS12 + - xpack.security.transport.ssl.keystore.password="" - xpack.license.self_generated.type=${LICENSE} depends_on: + - setup - logstash01 - - logstash01: - image: docker.elastic.co/logstash/logstash:${STACK_VERSION} - container_name: logstash01 - volumes: - - ./logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro - ports: - - "5044:5044" - networks: - - app-network - environment: - - ELASTIC_HOSTS=http://es01:9200 - - ELASTIC_USER=${ELASTIC_USERNAME} - - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} - - xpack.monitoring.enabled=false + healthcheck: + test: ["CMD-SHELL", "curl -s http://localhost:9200/_cluster/health | grep -q '\"status\":\"green\"'"] + interval: 10s + timeout: 10s + retries: 5 kibana: image: docker.elastic.co/kibana/kibana:${STACK_VERSION} container_name: kibana volumes: + - certs:/usr/share/kibana/config/certs:ro - pong_kibana:/usr/share/kibana/data user: "1000:1000" ports: @@ -163,30 +140,27 @@ services: - ELASTICSEARCH_PASSWORD=${ELASTIC_PASSWORD} depends_on: - es01 - #healthcheck: - #test: ["CMD-SHELL", "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'"] - #interval: 20s - #timeout: 10s - #retries: 120 - - filebeat01: - depends_on: - - es01 - image: docker.elastic.co/beats/filebeat:${STACK_VERSION} + healthcheck: + test: ["CMD-SHELL", "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'"] + interval: 10s + timeout: 20s + retries: 200 + + logstash01: + image: docker.elastic.co/logstash/logstash:${STACK_VERSION} + container_name: logstash01 volumes: - - pong_filebeat_data_01:/usr/share/filebeat/data - - ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro - - /var/lib/docker/containers:/var/lib/docker/containers:ro - - /var/run/docker.sock:/var/run/docker.sock:ro - environment: - - ELASTIC_USER=elastic - - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} - - ELASTIC_HOSTS=https://es01:9200 - - KIBANA_HOSTS=http://kibana:5601 - - LOGSTASH_HOSTS=http://logstash01:9600 + - ./config/logstash.conf:/usr/share/logstash/pipeline/logstash.conf:ro + ports: + - "5044:5044" networks: - app-network - + environment: + - ELASTIC_HOSTS=http://es01:9200 + - ELASTIC_USER=${ELASTIC_USERNAME} + - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + - xpack.monitoring.enabled=false + volumes: certs: driver: local @@ -202,11 +176,32 @@ volumes: driver: local pong_kibana: driver: local - pong_logstash_data_01: - driver: local - pong_filebeat_data_01: - driver: local networks: app-network: + name: app-network driver: bridge + + + + #filebeat01: + #depends_on: + #- es01 + # image: docker.elastic.co/beats/filebeat:${STACK_VERSION} + # volumes: + # - certs:/usr/share/logstash/certs + # - pong_filebeat_data_01:/usr/share/filebeat/data + # - ./filebeat.yml:/usr/share/filebeat/filebeat.yml:ro + # - /var/lib/docker/containers:/var/lib/docker/containers:ro + # - /var/run/docker.sock:/var/run/docker.sock:ro + # environment: + # - ELASTIC_USER=elastic + # - ELASTIC_PASSWORD=${ELASTIC_PASSWORD} + # - ELASTIC_HOSTS=https://es01:9200 + # - KIBANA_HOSTS=http://kibana:5601 + # - LOGSTASH_HOSTS=http://logstash01:9600 + # - xpack.monitoring.enabled=false + # networks: + # - app-network + + diff --git a/logstash.conf b/logstash.conf deleted file mode 100644 index 4872c61..0000000 --- a/logstash.conf +++ /dev/null @@ -1,21 +0,0 @@ -input { - file { - path => "/var/lib/docker/containers/*/*.log" - start_position => "beginning" - sincedb_path => "/usr/share/logstash/data/sincedb" - type => "docker" - codec => "json" - } -} - -filter { -} - -output { - elasticsearch { - hosts => ["http://es01:9200"] - index => "docker-logs-%{+YYYY.MM.dd}" - user=> "${ELASTIC_USER}" - password=> "${ELASTIC_PASSWORD}" - } -} diff --git a/makefile b/makefile index 126ed39..169500b 100644 --- a/makefile +++ b/makefile @@ -1,10 +1,10 @@ -COMPOSE_FILE=docker-compose.yaml +COMPOSE_FILE=docker-compose.yml COMPOSE=docker compose -f $(COMPOSE_FILE) CONTAINER=$(c) up: $(COMPOSE) build - $(COMPOSE) up + $(COMPOSE) up $(CONTAINER) build: $(COMPOSE) build $(CONTAINER) diff --git a/manage.py b/manage.py old mode 100755 new mode 100644