add method to deploy locally

Author: andresionek91

Makefile

@@ -19,3 +19,8 @@ airflow-push-image: push-to-ecr
 
 airflow-destroy:
 	python -c 'from deploy_cloudformation import destroy_stacks;  destroy_stacks()';
+
+airflow-local:
+	pip install cryptography
+	export AIRFLOW_FERNET_KEY=$(python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())")
+	docker-compose up --build

dags/crypto_extract_dag.py

@@ -0,0 +1,109 @@
+version: '3'
+
+services:
+  redis:
+    image: 'redis:5.0.3'
+    command: redis-server
+
+  postgres:
+    image: postgres:10.4
+    environment:
+      - POSTGRES_USER=airflow
+      - POSTGRES_PASSWORD=airflow
+      - POSTGRES_DB=airflow
+    volumes:
+      - ./postgres_data:/var/lib/postgresql/data
+    ports:
+      - 5432:5432
+
+  webserver:
+    image: airflow:latest
+    restart: always
+    build: .
+    depends_on:
+      - postgres
+      - redis
+    environment:
+      - REDIS_HOST=redis
+      - REDIS_PORT=6379
+      - POSTGRES_HOST=postgres
+      - POSTGRES_PORT=5432
+      - POSTGRES_USER=airflow
+      - POSTGRES_PASSWORD=airflow
+      - POSTGRES_DB=airflow
+      - FERNET_KEY=${AIRFLOW_FERNET_KEY}
+      - AIRFLOW_BASE_URL=http://localhost:8080
+      - ENABLE_REMOTE_LOGGING=False
+      - STAGE=dev
+    volumes:
+        - ./dags:/usr/local/airflow/dags
+    ports:
+        - "8080:8080"
+    command: webserver
+    healthcheck:
+      test: ["CMD-SHELL", "[ -f /usr/local/airflow/airflow-webserver.pid ]"]
+      interval: 30s
+      timeout: 30s
+      retries: 3
+
+  flower:
+    image: airflow:latest
+    restart: always
+    depends_on:
+      - redis
+      - webserver
+    environment:
+      - REDIS_HOST=redis
+      - REDIS_PORT=6379
+      - POSTGRES_HOST=postgres
+      - POSTGRES_PORT=5432
+      - POSTGRES_USER=airflow
+      - POSTGRES_PASSWORD=airflow
+      - POSTGRES_DB=airflow
+      - STAGE=dev
+    ports:
+      - "5555:5555"
+    command: flower
+
+  scheduler:
+    image: airflow:latest
+    restart: always
+    depends_on:
+      - webserver
+    volumes:
+      - ./dags:/usr/local/airflow/dags
+    environment:
+      - REDIS_HOST=redis
+      - REDIS_PORT=6379
+      - POSTGRES_HOST=postgres
+      - POSTGRES_PORT=5432
+      - POSTGRES_USER=airflow
+      - POSTGRES_PASSWORD=airflow
+      - POSTGRES_DB=airflow
+      - FERNET_KEY=${AIRFLOW_FERNET_KEY}
+      - AIRFLOW_BASE_URL=http://localhost:8080
+      - ENABLE_REMOTE_LOGGING=False
+      - STAGE=dev
+    command: scheduler
+
+  worker:
+    image: airflow:latest
+    restart: always
+    depends_on:
+      - webserver
+      - scheduler
+    volumes:
+      - ./dags:/usr/local/airflow/dags
+    environment:
+      - REDIS_HOST=redis
+      - REDIS_PORT=6379
+      - POSTGRES_HOST=postgres
+      - POSTGRES_PORT=5432
+      - POSTGRES_USER=airflow
+      - POSTGRES_PASSWORD=airflow
+      - POSTGRES_DB=airflow
+      - FERNET_KEY=${AIRFLOW_FERNET_KEY}
+      - AIRFLOW_BASE_URL=http://localhost:8080
+      - ENABLE_REMOTE_LOGGING=False
+      - STAGE=dev
+    command: worker