Merge pull request #6 from andresionek91/adjust-ip-whitelisting

Adjust ip whitelisting + Create automated tests

Author: andresionek91

.circleci/config.yml

@@ -0,0 +1,41 @@
+name: Run Tests
+
+on:
+  push:
+    branches:
+      - "*"
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements.txt
+          pip install -r tests/requirements.txt
+      - name: Lint with flake8
+        run: |
+          # stop the build if there are Python syntax errors or undefined names
+          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+      - name: Test with pytest
+        run: |
+          python -m pytest
+      - name: Validate Cloudformation Tempates
+        env:
+          AWS_DEFAULT_REGION: ${{ secrets.AWS_REGION }}
+          ENVIRONMENT: dev
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        run: |
+          python -c "from deploy_cloudformation import validate_templates; validate_templates()"

.github/workflows/run_tests.yml

@@ -8,10 +8,12 @@ Resources:
       GroupDescription: Security group for Postgres Metadata DB. Public access
       GroupName: "{{ serviceName }}-{{ ENVIRONMENT }}-metadata-db-public-security-group"
       SecurityGroupEgress:
-        - CidrIp: 0.0.0.0/0
+      {% for ip in whitelistedIPs %}
+        - CidrIp: "{{ ip }}"
           FromPort: "{{ metadataDb.port }}"
-          IpProtocol: -1
+          IpProtocol: tcp
           ToPort: "{{ metadataDb.port }}"
+      {% endfor %}
       SecurityGroupIngress:
         - CidrIp: 0.0.0.0/0
           FromPort: "{{ metadataDb.port }}"

cloudformation/60_databases.yml.j2

@@ -17,10 +17,12 @@ Resources:
       GroupDescription: Security group for Redis
       GroupName: "{{ serviceName }}-{{ ENVIRONMENT }}-redis-security-group"
       SecurityGroupEgress:
-        - CidrIp: 0.0.0.0/0
-          FromPort: 0
-          IpProtocol: -1
-          ToPort: 0
+      {% for ip in whitelistedIPs %}
+        - CidrIp: "{{ ip }}"
+          FromPort: "{{ celeryBackend.port }}"
+          IpProtocol: tcp
+          ToPort: "{{ celeryBackend.port }}"
+      {% endfor %}
       SecurityGroupIngress:
         - CidrIp: "{{ service.cidrBlock }}/16"
           FromPort: "{{ celeryBackend.port }}"

cloudformation/70_redis.yml.j2

@@ -7,7 +7,16 @@ Resources:
       GroupDescription: Security group for Airflow Flower.  Allow all inbound traffic.
       GroupName: "{{ serviceName }}-{{ ENVIRONMENT }}-flower-external-security-group"
       SecurityGroupEgress:
-        {{ securityGroupEgressRules }}
+      {% for ip in whitelistedIPs %}
+        - CidrIp: "{{ ip }}"
+          FromPort: 80
+          IpProtocol: tcp
+          ToPort: 80
+        - CidrIp: "{{ ip }}"
+          FromPort: 443
+          IpProtocol: tcp
+          ToPort: 443
+      {% endfor %}
       SecurityGroupIngress:
         - CidrIp: 0.0.0.0/0
           FromPort: "{{ service.port }}"

cloudformation/83_airflow-flower.yml.j2

@@ -6,8 +6,6 @@ Resources:
     Properties:
       GroupDescription: Security group for Airflow Scheduler
       GroupName: "{{ serviceName }}-{{ ENVIRONMENT }}-scheduler-security-group"
-      SecurityGroupEgress:
-        {{ securityGroupEgressRules }}
       VpcId: !ImportValue network-VpcId
       Tags:
         - Key: Name

cloudformation/84_airflow-scheduler.yml.j2

@@ -7,7 +7,16 @@ Resources:
       GroupDescription: Security group for Airflow webserver. Allow all inbound traffic.
       GroupName: "{{ serviceName }}-{{ ENVIRONMENT }}-webserver-security-group"
       SecurityGroupEgress:
-        {{ securityGroupEgressRules }}
+      {% for ip in whitelistedIPs %}
+        - CidrIp: "{{ ip }}"
+          FromPort: 80
+          IpProtocol: tcp
+          ToPort: 80
+        - CidrIp: "{{ ip }}"
+          FromPort: 443
+          IpProtocol: tcp
+          ToPort: 443
+      {% endfor %}
       SecurityGroupIngress:
         - CidrIp: 0.0.0.0/0
           FromPort: "{{ service.port }}"

cloudformation/85_airflow-webserver.yml.j2

@@ -6,8 +6,6 @@ Resources:
     Properties:
       GroupDescription: Security group for Airflow workers
       GroupName: "{{ serviceName }}-{{ ENVIRONMENT }}-workers-security-group"
-      SecurityGroupEgress:
-        {{ securityGroupEgressRules }}
       SecurityGroupIngress:
         - CidrIp: "{{ service.cidrBlock }}/16"
           FromPort: "{{ service.workers.port }}"

cloudformation/86_airflow-workers.yml.j2

@@ -44,7 +44,6 @@ def get_cloudformation_templates(reverse=False):
 
 def validate_templates():
     cf_templates = get_cloudformation_templates()
-
     for cf_template in cf_templates:
         logging.info('Validating CF template {}'.format(cf_template['filename']))
         cloudformation_client.validate_template(

deploy_cloudformation.py

@@ -1,6 +1,7 @@
 boto3==1.13.19
 cryptography==2.9.2
-pandas==0.24.0
+pandas==1.1.2
+numpy==1.18.4
 toolz==0.9.0
 jinja2==2.11.2
 click==7.1.2

requirements.txt

@@ -83,11 +83,8 @@ publicSubnet:
 
 # List of SecurityGroupEgress.
 # Will be used to whitelist IPs for webserver, flower, workers and scheduler
-securityGroupEgressRules:
-  - CidrIp: 0.0.0.0/0
-    FromPort: 0
-    IpProtocol: -1
-    ToPort: 0
+whitelistedIPs:
+  - 0.0.0.0/0
 
 metadataDb:
   instanceType: db.t3.micro
@@ -103,6 +100,7 @@ metadataDb:
     maxConnections: 100
 
 
+
 celeryBackend:
   port: 6379
   azMode: single-az

service.yml

@@ -0,0 +1,2 @@
+flake8==3.8.3
+pytest==6.0.1

tests/requirements.txt

@@ -0,0 +1,3 @@
+
+def test_dummy():
+    assert True