From 3be43e572129bdc9c77e85f96c2534a3f57a6057 Mon Sep 17 00:00:00 2001 From: bsatoriu Date: Wed, 15 Nov 2023 15:10:46 -0800 Subject: [PATCH 1/6] Merge live patch from OPS --- api/endpoints/cmr.py | 30 ++++++++++---- api/endpoints/members.py | 89 +++++++++++++++++++++++----------------- 2 files changed, 72 insertions(+), 47 deletions(-) diff --git a/api/endpoints/cmr.py b/api/endpoints/cmr.py index 9b4fd97..67df3dc 100755 --- a/api/endpoints/cmr.py +++ b/api/endpoints/cmr.py @@ -130,16 +130,28 @@ class CmrGranuleData(Resource): """ def get(self, file_uri): - response = edl_federated_request(parse.unquote(file_uri), stream=True) + s = requests.Session() + response = s.get(parse.unquote(file_uri), stream=True) - if response.status_code == status.HTTP_200_OK: - return Response( - response=stream_with_context(response.iter_content(chunk_size=1024 * 10)), - content_type=response.headers.get('Content-Type'), - direct_passthrough=True) - else: - # Propagate the error - return response + if response.status_code == 401: + maap_user = get_authorized_user() + + if maap_user is None: + return Response(response.text, status=401) + else: + urs_token = db.session.query(Member).filter_by(id=maap_user.id).first().urs_token + s.headers.update({'Authorization': f'Bearer {urs_token},Basic {settings.MAAP_EDL_CREDS}', + 'Connection': 'close'}) + + response = s.get(url=response.url, stream=True) + + if response.status_code == 401: + return Response(response.text, status=401) + + return Response( + response=stream_with_context(response.iter_content(chunk_size=1024 * 10)), + content_type=response.headers.get('Content-Type'), + direct_passthrough=True) def get_search_headers(): diff --git a/api/endpoints/members.py b/api/endpoints/members.py index c3603ee..73accb7 100755 --- a/api/endpoints/members.py +++ b/api/endpoints/members.py @@ -24,7 +24,6 @@ from urllib import parse from api.utils.url_util import proxied_url - log = logging.getLogger(__name__) ns = api.namespace('members', description='Operations for MAAP members') s3_client = boto3.client('s3', region_name=settings.AWS_REGION) @@ -79,10 +78,10 @@ def get(self, key): Member_db.creation_date ] - member = db.session\ - .query(Member_db)\ - .with_entities(*cols)\ - .filter_by(username=key)\ + member = db.session \ + .query(Member_db) \ + .with_entities(*cols) \ + .filter_by(username=key) \ .first() if member is None: @@ -92,11 +91,11 @@ def get(self, key): result = json.loads(member_schema.dumps(member)) if valid_dps_request(): - pgt_ticket = db.session\ - .query(MemberSession_db)\ - .with_entities(MemberSession_db.session_key)\ - .filter_by(member_id=member.id)\ - .order_by(MemberSession_db.id.desc())\ + pgt_ticket = db.session \ + .query(MemberSession_db) \ + .with_entities(MemberSession_db.session_key) \ + .filter_by(member_id=member.id) \ + .order_by(MemberSession_db.id.desc()) \ .first() member_session_schema = MemberSessionSchema() @@ -413,7 +412,6 @@ def delete(self): @ns.route('/self/presignedUrlS3//') class PresignedUrlS3(Resource): - expiration_param = reqparse.RequestParser() expiration_param.add_argument('exp', type=int, required=False, default=60 * 60 * 12) expiration_param.add_argument('ws', type=str, required=False, default="") @@ -456,7 +454,6 @@ def mount_key_to_bucket(self, key, ws): @ns.route('/self/awsAccess/requesterPaysBucket') class AwsAccessRequesterPaysBucket(Resource): - expiration_param = reqparse.RequestParser() expiration_param.add_argument('exp', type=int, required=False, default=60 * 60 * 12) @@ -464,7 +461,6 @@ class AwsAccessRequesterPaysBucket(Resource): @login_required @api.expect(expiration_param) def get(self): - member = get_authorized_user() expiration = request.args['exp'] @@ -497,26 +493,42 @@ class AwsAccessEdcCredentials(Resource): Example: https://api.maap-project.org/api/self/edcCredentials/https%3A%2F%2Fdata.lpdaac.earthdatacloud.nasa.gov%2Fs3credentials """ + @api.doc(security='ApiKeyAuth') @login_required def get(self, endpoint_uri): + s = requests.Session() maap_user = get_authorized_user() - if not maap_user: + if maap_user is None: return Response('Unauthorized', status=401) + else: + urs_token = db.session.query(Member_db).filter_by(id=maap_user.id).first().urs_token + s.headers.update({'Authorization': f'Bearer {urs_token},Basic {settings.MAAP_EDL_CREDS}', + 'Connection': 'close'}) - creds = get_edc_credentials(endpoint_uri, maap_user) + endpoint = parse.unquote(endpoint_uri) + login_resp = s.get( + endpoint, allow_redirects=False + ) - response = jsonify( - accessKeyId=creds['accessKeyId'], - secretAccessKey=creds['secretAccessKey'], - sessionToken=creds['sessionToken'], - expiration=creds['expiration'] - ) + if login_resp.status_code == 307: + edl_response = s.get(url=login_resp.headers['location']) + else: + edl_response = edl_federated_request(url=endpoint) - response.headers.add('Access-Control-Allow-Origin', '*') + json_response = json.loads(edl_response.content) - return response + response = jsonify( + accessKeyId=json_response['accessKeyId'], + secretAccessKey=json_response['secretAccessKey'], + sessionToken=json_response['sessionToken'], + expiration=json_response['expiration'] + ) + + response.headers.add('Access-Control-Allow-Origin', '*') + + return response @ns.route('/self/awsAccess/workspaceBucket') @@ -529,6 +541,7 @@ class AwsAccessUserBucketCredentials(Resource): Example: https://api.maap-project.org/api/self/awsAccess/workspaceBucket """ + @api.doc(security='ApiKeyAuth') @login_required def get(self): @@ -629,23 +642,24 @@ def get_edc_credentials(endpoint_uri, user): """ urs_token = db.session.query(Member_db).filter_by(id=user.id).first().urs_token - with requests.Session() as s: - s.headers.update( - { - 'Authorization': f'Bearer {urs_token},Basic {settings.MAAP_EDL_CREDS}', - 'Connection': 'close' - } - ) + s = requests.Session() - endpoint = parse.unquote(endpoint_uri) - login_resp = s.get(endpoint, allow_redirects=False) + s.headers.update( + { + 'Authorization': f'Bearer {urs_token},Basic {settings.MAAP_EDL_CREDS}', + 'Connection': 'close' + } + ) - if login_resp.status_code == status.HTTP_307_TEMPORARY_REDIRECT: - edl_response = s.get(url=login_resp.headers['location']) - else: - edl_response = edl_federated_request(url=endpoint) + endpoint = parse.unquote(endpoint_uri) + login_resp = s.get(endpoint, allow_redirects=False) - return json.loads(edl_response.content) + if login_resp.status_code == status.HTTP_307_TEMPORARY_REDIRECT: + edl_response = s.get(url=login_resp.headers['location']) + else: + edl_response = edl_federated_request(url=endpoint) + + return json.loads(edl_response.content) @ns.route('/pre-approved') @@ -714,7 +728,6 @@ class PreApprovedEmails(Resource): @api.doc(security='ApiKeyAuth') @login_required def delete(self, email): - """ Delete pre-approved email """ From e81d0f5bcd89ece4c7306f96a9aa83bc99893cf3 Mon Sep 17 00:00:00 2001 From: bsatoriu Date: Tue, 21 Nov 2023 12:55:19 -0800 Subject: [PATCH 2/6] Restore public_ssh_key to member endpoint --- api/endpoints/members.py | 1 + 1 file changed, 1 insertion(+) diff --git a/api/endpoints/members.py b/api/endpoints/members.py index 73accb7..7c6897d 100755 --- a/api/endpoints/members.py +++ b/api/endpoints/members.py @@ -75,6 +75,7 @@ def get(self, key): Member_db.last_name, Member_db.email, Member_db.status, + Member_db.public_ssh_key, Member_db.creation_date ] From 5176df3941d6088dac2287c12700d4f5ce59105c Mon Sep 17 00:00:00 2001 From: bsatoriu Date: Wed, 29 Nov 2023 15:06:09 -0800 Subject: [PATCH 3/6] Update members.py --- api/endpoints/members.py | 1 + 1 file changed, 1 insertion(+) diff --git a/api/endpoints/members.py b/api/endpoints/members.py index 7c6897d..937a5e9 100755 --- a/api/endpoints/members.py +++ b/api/endpoints/members.py @@ -341,6 +341,7 @@ def get(self): Member_db.last_name, Member_db.email, Member_db.status, + Member_db.public_ssh_key, Member_db.creation_date ] From 43ba3892fa8384fdf1d584f3e4d9bedf3755a319 Mon Sep 17 00:00:00 2001 From: bsatoriu Date: Wed, 20 Dec 2023 11:47:42 -0800 Subject: [PATCH 4/6] User gunicorn to run API service --- docker/docker-compose.yml | 2 +- requirements.txt | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index b1fe117..173201f 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -16,7 +16,7 @@ services: volumes: - ./logs:/maap-api-nasa/logs/ command: > - sh -c "flask run --host=0.0.0.0" + sh -c "gunicorn --bind 0.0.0.0:5000 api.maapapp:app -w 4" environment: FLASK_APP: /maap-api-nasa/api/maapapp.py restart: always diff --git a/requirements.txt b/requirements.txt index 1f06628..681d778 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ cachetools +gunicorn flask flask-cors flask-restx From 803e945854966836f585096ef5aa8382c4c1d61d Mon Sep 17 00:00:00 2001 From: bsatoriu Date: Sun, 7 Jan 2024 19:07:05 -0800 Subject: [PATCH 5/6] Update cmr.py --- api/endpoints/cmr.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/endpoints/cmr.py b/api/endpoints/cmr.py index 67df3dc..0a184ef 100755 --- a/api/endpoints/cmr.py +++ b/api/endpoints/cmr.py @@ -145,8 +145,8 @@ def get(self, file_uri): response = s.get(url=response.url, stream=True) - if response.status_code == 401: - return Response(response.text, status=401) + if response.status_code >= 400: + return Response(response.text, status=response.status_code) return Response( response=stream_with_context(response.iter_content(chunk_size=1024 * 10)), From 2727b7b57a800862c23d001a065446a821136371 Mon Sep 17 00:00:00 2001 From: Sujen Shah Date: Thu, 18 Jan 2024 14:28:58 -0800 Subject: [PATCH 6/6] Add log rotation for api docker (#92) --- docker/docker-compose.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 173201f..87bc9cd 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -20,6 +20,11 @@ services: environment: FLASK_APP: /maap-api-nasa/api/maapapp.py restart: always + logging: + driver: "json-file" + options: + max-size: 250m + max-file: 10 db: image: postgres:14.5