From a2ad67ca818938dc5b6eaef6d3547e9dd4c4fdf1 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Tue, 11 Jan 2022 17:17:42 +1100 Subject: [PATCH 01/14] Yield a disconnect on the second receive call --- azure/functions/_http_asgi.py | 17 ++++++++++++----- tests/test_http_asgi.py | 3 +++ 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/azure/functions/_http_asgi.py b/azure/functions/_http_asgi.py index 55e44fa1..8ba156e0 100644 --- a/azure/functions/_http_asgi.py +++ b/azure/functions/_http_asgi.py @@ -89,11 +89,18 @@ def _handle_http_response_body(self, message: Dict[str, Any]): # https://github.com/Azure/azure-functions-host/issues/4926 async def _receive(self): - return { - "type": "http.request", - "body": self._request_body, - "more_body": False, - } + if self._request_body is not None: + reply = { + "type": "http.request", + "body": self._request_body, + "more_body": False, + } + self._request_body = None + else: + reply = { + "type": "http.disconnect", + } + return reply async def _send(self, message): logging.debug(f"Received {message} from ASGI worker.") diff --git a/tests/test_http_asgi.py b/tests/test_http_asgi.py index f9410d8c..e48d541c 100644 --- a/tests/test_http_asgi.py +++ b/tests/test_http_asgi.py @@ -65,6 +65,9 @@ async def __call__(self, scope, receive, send): assert isinstance(self.received_request['body'], bytes) assert isinstance(self.received_request['more_body'], bool) + self.next_request = await receive() + assert self.next_request['type'] == 'http.disconnect' + await send( { "type": "http.response.start", From 023d9717bde83f1da9cc4d1e6e6462aea5464c86 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Wed, 25 May 2022 12:34:58 +1000 Subject: [PATCH 02/14] Setup an orjson branch --- azure/functions/_cosmosdb.py | 2 +- azure/functions/_http.py | 3 +- azure/functions/_json.py | 59 +++++++++++++++++++++ azure/functions/_queue.py | 3 +- azure/functions/_sql.py | 3 +- azure/functions/cosmosdb.py | 3 +- azure/functions/decorators/utils.py | 16 +----- azure/functions/durable_functions.py | 2 +- azure/functions/eventgrid.py | 3 +- azure/functions/eventhub.py | 3 +- azure/functions/extension/extension_meta.py | 3 +- azure/functions/http.py | 3 +- azure/functions/kafka.py | 3 +- azure/functions/meta.py | 2 +- azure/functions/queue.py | 2 +- azure/functions/servicebus.py | 4 +- azure/functions/sql.py | 2 +- azure/functions/timer.py | 2 +- setup.py | 3 +- tests/decorators/test_function_app.py | 3 +- tests/decorators/test_utils.py | 15 ++++++ tests/decorators/testutils.py | 2 +- tests/test_blob.py | 2 +- tests/test_durable_functions.py | 2 +- tests/test_eventhub.py | 20 ++++--- tests/test_extension.py | 6 ++- tests/test_kafka.py | 2 +- tests/test_queue.py | 2 +- tests/test_servicebus.py | 3 +- tests/test_sql.py | 2 +- tests/test_timer.py | 3 +- 31 files changed, 129 insertions(+), 54 deletions(-) create mode 100644 azure/functions/_json.py diff --git a/azure/functions/_cosmosdb.py b/azure/functions/_cosmosdb.py index 4a6295c6..684a16df 100644 --- a/azure/functions/_cosmosdb.py +++ b/azure/functions/_cosmosdb.py @@ -2,8 +2,8 @@ # Licensed under the MIT License. import collections -import json +from azure.functions import _json as json from . import _abc diff --git a/azure/functions/_http.py b/azure/functions/_http.py index b52fbb5f..f42a5216 100644 --- a/azure/functions/_http.py +++ b/azure/functions/_http.py @@ -3,7 +3,7 @@ import collections.abc import io -import json + import types import typing @@ -12,6 +12,7 @@ from ._thirdparty.werkzeug import formparser as _wk_parser from ._thirdparty.werkzeug import http as _wk_http from ._thirdparty.werkzeug.datastructures import Headers +from azure.functions import _json as json class BaseHeaders(collections.abc.Mapping): diff --git a/azure/functions/_json.py b/azure/functions/_json.py new file mode 100644 index 00000000..92cec11a --- /dev/null +++ b/azure/functions/_json.py @@ -0,0 +1,59 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +from enum import Enum +import os +import warnings + +try: + import orjson + if 'AZUREFUNCTIONS_ORJSON' in os.environ: + HAS_ORJSON = bool(os.environ['AZUREFUNCTIONS_ORJSON']) + else: + HAS_ORJSON = True + import json +except ImportError: + import json + HAS_ORJSON = False + + +class StringifyEnum(Enum): + """This class output name of enum object when printed as string.""" + + def __str__(self): + return str(self.name) + + +class StringifyEnumJsonEncoder(json.JSONEncoder): + def default(self, o): + if isinstance(o, StringifyEnum): + return str(o) + + return super().default(o) + + +JSONDecodeError = json.JSONDecodeError + +if HAS_ORJSON: + def dumps(v, **kwargs): + sort_keys = False + if 'sort_keys' in kwargs: + del kwargs['sort_keys'] + sort_keys = True + if kwargs: # Unsupported arguments + return json.dumps(v, **kwargs) + if sort_keys: + r = orjson.dumps(v, option=orjson.OPT_SORT_KEYS) + else: + r = orjson.dumps(v) + return r.decode(encoding='utf-8') + + def loads(*args, **kwargs): + if kwargs: + return json.loads(*args, **kwargs) + else: # ORjson takes no kwargs + return orjson.loads(*args) + +else: + dumps = json.dumps + loads = json.loads diff --git a/azure/functions/_queue.py b/azure/functions/_queue.py index c6c7d094..de92772a 100644 --- a/azure/functions/_queue.py +++ b/azure/functions/_queue.py @@ -2,9 +2,10 @@ # Licensed under the MIT License. import datetime -import json + import typing +from azure.functions import _json as json from . import _abc diff --git a/azure/functions/_sql.py b/azure/functions/_sql.py index a673c320..a8beed2c 100644 --- a/azure/functions/_sql.py +++ b/azure/functions/_sql.py @@ -2,8 +2,7 @@ # Licensed under the MIT License. import collections -import json - +from azure.functions import _json as json from . import _abc diff --git a/azure/functions/cosmosdb.py b/azure/functions/cosmosdb.py index 1edc78af..36574caf 100644 --- a/azure/functions/cosmosdb.py +++ b/azure/functions/cosmosdb.py @@ -2,9 +2,10 @@ # Licensed under the MIT License. import collections.abc -import json + import typing +from azure.functions import _json as json from azure.functions import _cosmosdb as cdb from . import meta diff --git a/azure/functions/decorators/utils.py b/azure/functions/decorators/utils.py index 4d7bcbdf..c238f792 100644 --- a/azure/functions/decorators/utils.py +++ b/azure/functions/decorators/utils.py @@ -4,7 +4,6 @@ import re from abc import ABCMeta from enum import Enum -from json import JSONEncoder from typing import TypeVar, Optional, Union, Iterable, Type, Callable T = TypeVar("T", bound=Enum) @@ -12,12 +11,7 @@ re.IGNORECASE) WORD_RE = re.compile(r'^([a-z]+\d*)$', re.IGNORECASE) - -class StringifyEnum(Enum): - """This class output name of enum object when printed as string.""" - - def __str__(self): - return str(self.name) +from azure.functions._json import StringifyEnum, StringifyEnumJsonEncoder # NOQA class BuildDictMeta(type): @@ -166,11 +160,3 @@ def is_word(input_string: str) -> bool: :return: True for one word string, false otherwise. """ return WORD_RE.match(input_string) is not None - - -class StringifyEnumJsonEncoder(JSONEncoder): - def default(self, o): - if isinstance(o, StringifyEnum): - return str(o) - - return super().default(o) diff --git a/azure/functions/durable_functions.py b/azure/functions/durable_functions.py index 16c4fc4c..07b148a2 100644 --- a/azure/functions/durable_functions.py +++ b/azure/functions/durable_functions.py @@ -2,8 +2,8 @@ # Licensed under the MIT License. import typing -import json +from azure.functions import _json as json from azure.functions import _durable_functions from . import meta diff --git a/azure/functions/eventgrid.py b/azure/functions/eventgrid.py index e76f5dde..d3c4d47c 100644 --- a/azure/functions/eventgrid.py +++ b/azure/functions/eventgrid.py @@ -3,9 +3,10 @@ import collections import datetime -import json + from typing import Optional, List, Any, Dict, Union +from azure.functions import _json as json from azure.functions import _eventgrid as azf_eventgrid from . import meta diff --git a/azure/functions/eventhub.py b/azure/functions/eventhub.py index 6aab2be5..946dbcdf 100644 --- a/azure/functions/eventhub.py +++ b/azure/functions/eventhub.py @@ -1,9 +1,10 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -import json + from typing import Dict, Any, List, Union, Optional, Mapping +from azure.functions import _json as json from azure.functions import _eventhub from . import meta diff --git a/azure/functions/extension/extension_meta.py b/azure/functions/extension/extension_meta.py index c875722f..1c8df31d 100644 --- a/azure/functions/extension/extension_meta.py +++ b/azure/functions/extension/extension_meta.py @@ -3,7 +3,8 @@ from typing import Optional, Union, Dict, List import abc -import json + +from azure.functions import _json as json from .app_extension_hooks import AppExtensionHooks from .func_extension_hooks import FuncExtensionHooks from .extension_hook_meta import ExtensionHookMeta diff --git a/azure/functions/http.py b/azure/functions/http.py index 61b303ec..05d1604d 100644 --- a/azure/functions/http.py +++ b/azure/functions/http.py @@ -1,13 +1,14 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -import json + import logging import sys import typing from http.cookies import SimpleCookie from azure.functions import _abc as azf_abc +from azure.functions import _json as json from azure.functions import _http as azf_http from . import meta from ._thirdparty.werkzeug.datastructures import Headers diff --git a/azure/functions/kafka.py b/azure/functions/kafka.py index 5080aa7f..c9bf3d41 100644 --- a/azure/functions/kafka.py +++ b/azure/functions/kafka.py @@ -2,12 +2,11 @@ # Licensed under the MIT License. import typing -import json - from typing import Any, List from . import meta +from azure.functions import _json as json from ._kafka import AbstractKafkaEvent diff --git a/azure/functions/meta.py b/azure/functions/meta.py index 2ca92563..3abcb5d5 100644 --- a/azure/functions/meta.py +++ b/azure/functions/meta.py @@ -4,7 +4,6 @@ import abc import collections.abc import datetime -import json import re from typing import Dict, Optional, Union, Tuple, Mapping, Any @@ -13,6 +12,7 @@ try_parse_datetime_with_formats, try_parse_timedelta_with_formats ) +from azure.functions import _json as json def is_iterable_type_annotation(annotation: object, pytype: object) -> bool: diff --git a/azure/functions/queue.py b/azure/functions/queue.py index d39c56e5..cb55fe66 100644 --- a/azure/functions/queue.py +++ b/azure/functions/queue.py @@ -3,10 +3,10 @@ import collections.abc import datetime -import json from typing import List, Dict, Any, Union, Optional from azure.functions import _abc as azf_abc +from azure.functions import _json as json from azure.functions import _queue as azf_queue from . import meta diff --git a/azure/functions/servicebus.py b/azure/functions/servicebus.py index 76f31b59..9bee3dfe 100644 --- a/azure/functions/servicebus.py +++ b/azure/functions/servicebus.py @@ -2,9 +2,9 @@ # Licensed under the MIT License. import datetime -import json -from typing import Dict, Any, List, Union, Optional, Mapping, cast +from typing import Dict, Any, List, Union, Optional, Mapping, cast +from azure.functions import _json as json from azure.functions import _servicebus as azf_sbus from . import meta diff --git a/azure/functions/sql.py b/azure/functions/sql.py index 60919c0e..310a7b45 100644 --- a/azure/functions/sql.py +++ b/azure/functions/sql.py @@ -2,9 +2,9 @@ # Licensed under the MIT License. import collections.abc -import json import typing +from azure.functions import _json as json from azure.functions import _sql as sql from . import meta diff --git a/azure/functions/timer.py b/azure/functions/timer.py index 56bdc8eb..6120f46d 100644 --- a/azure/functions/timer.py +++ b/azure/functions/timer.py @@ -1,10 +1,10 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -import json import typing from azure.functions import _abc as azf_abc +from azure.functions import _json as json from . import meta diff --git a/setup.py b/setup.py index 89a0ff8c..913a9574 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,8 @@ 'pytest-cov', 'requests==2.*', 'coverage' - ] + ], + 'orjson': ['orjson>=3.6.8,<4.0'] } with open("README.md") as readme: diff --git a/tests/decorators/test_function_app.py b/tests/decorators/test_function_app.py index 103c7268..f1894c00 100644 --- a/tests/decorators/test_function_app.py +++ b/tests/decorators/test_function_app.py @@ -1,9 +1,10 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -import json + import unittest from unittest import mock +from azure.functions import _json as json from azure.functions import WsgiMiddleware, AsgiMiddleware from azure.functions.decorators.constants import HTTP_OUTPUT, HTTP_TRIGGER from azure.functions.decorators.core import DataType, AuthLevel, \ diff --git a/tests/decorators/test_utils.py b/tests/decorators/test_utils.py index 4dd87e06..e8380c81 100644 --- a/tests/decorators/test_utils.py +++ b/tests/decorators/test_utils.py @@ -219,3 +219,18 @@ def test_is_not_supported_trigger_type(self): Trigger.is_supported_trigger_type( GenericTrigger(name='req', type="dummy"), HttpTrigger)) + + +def test_clean_nones_nested_benchmark(benchmark): + data = [ + {"direction": "IN", "type": "b", "name": "t1"}, + {"direction": "IN", "type": "b", "name": "t2"}, + {"direction": "IN", "type": "b", "name": "t3"}, + {"direction": "IN", "type": "b", "name": "t4"}, + {"direction": "IN", "type": "b", "name": "t5"}, + {"direction": "IN", "type": "b", "name": "t6"}, + {"direction": "IN", "type": "b", "name": "t7"}, + {"direction": "IN", "type": "b", "name": "t8"}, + {"direction": "IN", "type": "b", "name": "t9"} + ] + benchmark(BuildDictMeta.clean_nones, data) diff --git a/tests/decorators/testutils.py b/tests/decorators/testutils.py index 6e85f9d1..83e2444e 100644 --- a/tests/decorators/testutils.py +++ b/tests/decorators/testutils.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -import json +from azure.functions import _json as json from azure.functions.decorators.utils import StringifyEnumJsonEncoder diff --git a/tests/test_blob.py b/tests/test_blob.py index 4a218228..27a25bff 100644 --- a/tests/test_blob.py +++ b/tests/test_blob.py @@ -1,12 +1,12 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -import json import unittest from typing import Any, Dict import azure.functions as func import azure.functions.blob as afb from azure.functions.blob import InputStream +from azure.functions import _json as json from azure.functions.meta import Datum diff --git a/tests/test_durable_functions.py b/tests/test_durable_functions.py index 1739cdcd..9f3a9176 100644 --- a/tests/test_durable_functions.py +++ b/tests/test_durable_functions.py @@ -2,7 +2,6 @@ # Licensed under the MIT License. import unittest -import json from azure.functions.durable_functions import ( OrchestrationTriggerConverter, @@ -13,6 +12,7 @@ OrchestrationContext, EntityContext ) +from azure.functions import _json as json from azure.functions.meta import Datum CONTEXT_CLASSES = [OrchestrationContext, EntityContext] diff --git a/tests/test_eventhub.py b/tests/test_eventhub.py index 4b47775a..d30257d1 100644 --- a/tests/test_eventhub.py +++ b/tests/test_eventhub.py @@ -3,12 +3,12 @@ from typing import List, Mapping import unittest -import json from unittest.mock import patch from datetime import datetime import azure.functions as func import azure.functions.eventhub as azf_eh +from azure.functions import _json as json import azure.functions.meta as meta from .testutils import CollectionBytes, CollectionString @@ -113,12 +113,14 @@ def test_eventhub_trigger_multiple_events_json(self): self.assertEqual(result[0].enqueued_time, self.MOCKED_ENQUEUE_TIME) self.assertEqual( - result[0].get_body().decode('utf-8'), '{"device-status": "good1"}' + result[0].get_body().decode('utf-8'), + json.dumps({"device-status": "good1"}) ) self.assertEqual(result[1].enqueued_time, self.MOCKED_ENQUEUE_TIME) self.assertEqual( - result[1].get_body().decode('utf-8'), '{"device-status": "good2"}' + result[1].get_body().decode('utf-8'), + json.dumps({"device-status": "good2"}) ) def test_eventhub_trigger_multiple_events_collection_string(self): @@ -131,12 +133,14 @@ def test_eventhub_trigger_multiple_events_collection_string(self): self.assertEqual(result[0].enqueued_time, self.MOCKED_ENQUEUE_TIME) self.assertEqual( - result[0].get_body().decode('utf-8'), '{"device-status": "good1"}' + result[0].get_body().decode('utf-8'), + json.dumps({"device-status": "good1"}) ) self.assertEqual(result[1].enqueued_time, self.MOCKED_ENQUEUE_TIME) self.assertEqual( - result[1].get_body().decode('utf-8'), '{"device-status": "good2"}' + result[1].get_body().decode('utf-8'), + json.dumps({"device-status": "good2"}) ) def test_eventhub_trigger_multiple_events_collection_bytes(self): @@ -149,12 +153,14 @@ def test_eventhub_trigger_multiple_events_collection_bytes(self): self.assertEqual(result[0].enqueued_time, self.MOCKED_ENQUEUE_TIME) self.assertEqual( - result[0].get_body().decode('utf-8'), '{"device-status": "good1"}' + result[0].get_body().decode('utf-8'), + json.dumps({"device-status": "good1"}) ) self.assertEqual(result[1].enqueued_time, self.MOCKED_ENQUEUE_TIME) self.assertEqual( - result[1].get_body().decode('utf-8'), '{"device-status": "good2"}' + result[1].get_body().decode('utf-8'), + json.dumps({"device-status": "good2"}) ) def test_iothub_metadata_events(self): diff --git a/tests/test_extension.py b/tests/test_extension.py index 2b7452f2..45d20176 100644 --- a/tests/test_extension.py +++ b/tests/test_extension.py @@ -7,6 +7,7 @@ from unittest.mock import MagicMock, patch from logging import Logger +from azure.functions import _json as json from azure.functions.extension import FunctionExtensionException from azure.functions.extension.app_extension_base import AppExtensionBase from azure.functions.extension.func_extension_base import FuncExtensionBase @@ -152,7 +153,8 @@ def test_get_registered_extensions_json_function_ext(self): info_json = self._instance.get_registered_extensions_json() self.assertEqual( info_json, - r'{"FuncExtension": {"HttpTrigger": ["NewFuncExtension"]}}' + json.dumps({"FuncExtension": + {"HttpTrigger": ["NewFuncExtension"]}}) ) def test_get_registered_extension_json_application_ext(self): @@ -164,7 +166,7 @@ def test_get_registered_extension_json_application_ext(self): info_json = self._instance.get_registered_extensions_json() self.assertEqual( info_json, - r'{"AppExtension": ["NewAppExtension"]}' + json.dumps({"AppExtension": ["NewAppExtension"]}) ) def test_get_extension_scope(self): diff --git a/tests/test_kafka.py b/tests/test_kafka.py index 7c90e314..52a65a19 100644 --- a/tests/test_kafka.py +++ b/tests/test_kafka.py @@ -3,10 +3,10 @@ from typing import List import unittest -import json from unittest.mock import patch import azure.functions as func +from azure.functions import _json as json import azure.functions.kafka as azf_ka import azure.functions.meta as meta diff --git a/tests/test_queue.py b/tests/test_queue.py index ab83f923..fdf3e528 100644 --- a/tests/test_queue.py +++ b/tests/test_queue.py @@ -1,9 +1,9 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -import json import unittest +from azure.functions import _json as json import azure.functions as func import azure.functions.queue as azf_q diff --git a/tests/test_servicebus.py b/tests/test_servicebus.py index 4f5c2508..aa13cfe1 100644 --- a/tests/test_servicebus.py +++ b/tests/test_servicebus.py @@ -2,11 +2,12 @@ # Licensed under the MIT License. from typing import Dict, List -import json + import unittest from datetime import datetime, timedelta import azure.functions as func +from azure.functions import _json as json import azure.functions.servicebus as azf_sb from azure.functions import meta diff --git a/tests/test_sql.py b/tests/test_sql.py index 3fcae437..3b4522d1 100644 --- a/tests/test_sql.py +++ b/tests/test_sql.py @@ -6,7 +6,7 @@ import azure.functions as func import azure.functions.sql as sql from azure.functions.meta import Datum -import json +from azure.functions import _json as json class TestSql(unittest.TestCase): diff --git a/tests/test_timer.py b/tests/test_timer.py index 4b21022a..6c943377 100644 --- a/tests/test_timer.py +++ b/tests/test_timer.py @@ -1,9 +1,8 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. -import json import unittest - +from azure.functions import _json as json import azure.functions.timer as timer from azure.functions.meta import Datum From 0a290425afd0bdcd38141f5f68437ce8a617ea60 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Wed, 25 May 2022 12:36:59 +1000 Subject: [PATCH 03/14] Pass the sort_keys arg back --- azure/functions/_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure/functions/_json.py b/azure/functions/_json.py index 92cec11a..f10fb3dc 100644 --- a/azure/functions/_json.py +++ b/azure/functions/_json.py @@ -41,7 +41,7 @@ def dumps(v, **kwargs): del kwargs['sort_keys'] sort_keys = True if kwargs: # Unsupported arguments - return json.dumps(v, **kwargs) + return json.dumps(v, sort_keys=sort_keys, **kwargs) if sort_keys: r = orjson.dumps(v, option=orjson.OPT_SORT_KEYS) else: From dcaf878900d6d1af0ed6bc8ca24d90ca8cfd2dba Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Wed, 25 May 2022 12:37:30 +1000 Subject: [PATCH 04/14] Remove unused import --- azure/functions/_json.py | 1 - 1 file changed, 1 deletion(-) diff --git a/azure/functions/_json.py b/azure/functions/_json.py index f10fb3dc..721bce6a 100644 --- a/azure/functions/_json.py +++ b/azure/functions/_json.py @@ -3,7 +3,6 @@ from enum import Enum import os -import warnings try: import orjson From 976e89ee07d79e4f4d6f0ff9d1c51295255d0a6f Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Wed, 25 May 2022 12:41:48 +1000 Subject: [PATCH 05/14] add pytest-benchmark --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 913a9574..7c7ab877 100644 --- a/setup.py +++ b/setup.py @@ -10,6 +10,7 @@ 'mypy', 'pytest', 'pytest-cov', + 'pytest-benchmark', 'requests==2.*', 'coverage' ], From e9220629c79b0db22649575afef268d5672eb74d Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Wed, 25 May 2022 14:32:46 +1000 Subject: [PATCH 06/14] Switch to ujson --- .github/workflows/gh-tests-ci.yml | 4 +++ azure/functions/_durable_functions.py | 2 +- azure/functions/_json.py | 40 ++++++++++++++------------- setup.py | 3 +- 4 files changed, 28 insertions(+), 21 deletions(-) diff --git a/.github/workflows/gh-tests-ci.yml b/.github/workflows/gh-tests-ci.yml index 2d46f823..139c91a7 100644 --- a/.github/workflows/gh-tests-ci.yml +++ b/.github/workflows/gh-tests-ci.yml @@ -26,6 +26,10 @@ jobs: - name: Test with pytest run: | pytest --cache-clear --cov=./azure --cov-report=xml --cov-branch tests + - name: Test with pytest and ujson + run: | + python -m pip install ujson + pytest --cache-clear tests - name: Codecov if: ${{ matrix.python-version }} == 3.9 uses: codecov/codecov-action@v2 diff --git a/azure/functions/_durable_functions.py b/azure/functions/_durable_functions.py index aa533679..a0d9622a 100644 --- a/azure/functions/_durable_functions.py +++ b/azure/functions/_durable_functions.py @@ -12,7 +12,7 @@ def _serialize_custom_object(obj): This function gets called when `json.dumps` cannot serialize an object and returns a serializable dictionary containing enough - metadata to recontrust the original object. + metadata to reconstruct the original object. Parameters ---------- diff --git a/azure/functions/_json.py b/azure/functions/_json.py index 721bce6a..9cc9bad4 100644 --- a/azure/functions/_json.py +++ b/azure/functions/_json.py @@ -4,16 +4,18 @@ from enum import Enum import os +AZUREFUNCTIONS_UJSON_ENV_VAR = 'AZUREFUNCTIONS_UJSON' + try: - import orjson - if 'AZUREFUNCTIONS_ORJSON' in os.environ: - HAS_ORJSON = bool(os.environ['AZUREFUNCTIONS_ORJSON']) + import ujson + if AZUREFUNCTIONS_UJSON_ENV_VAR in os.environ: + HAS_UJSON = bool(os.environ[AZUREFUNCTIONS_UJSON_ENV_VAR]) else: - HAS_ORJSON = True + HAS_UJSON = True import json except ImportError: import json - HAS_ORJSON = False + HAS_UJSON = False class StringifyEnum(Enum): @@ -22,6 +24,10 @@ class StringifyEnum(Enum): def __str__(self): return str(self.name) + def __json__(self): + """For ujson encoding.""" + return f'"{self.name}"' + class StringifyEnumJsonEncoder(json.JSONEncoder): def default(self, o): @@ -33,25 +39,21 @@ def default(self, o): JSONDecodeError = json.JSONDecodeError -if HAS_ORJSON: +if HAS_UJSON: def dumps(v, **kwargs): - sort_keys = False - if 'sort_keys' in kwargs: - del kwargs['sort_keys'] - sort_keys = True - if kwargs: # Unsupported arguments - return json.dumps(v, sort_keys=sort_keys, **kwargs) - if sort_keys: - r = orjson.dumps(v, option=orjson.OPT_SORT_KEYS) - else: - r = orjson.dumps(v) - return r.decode(encoding='utf-8') + if 'default' in kwargs: + return json.dumps(v, **kwargs) + + if 'cls' in kwargs: + del kwargs['cls'] + + return ujson.dumps(v, **kwargs) def loads(*args, **kwargs): if kwargs: return json.loads(*args, **kwargs) - else: # ORjson takes no kwargs - return orjson.loads(*args) + else: # ujson takes no kwargs + return ujson.loads(*args) else: dumps = json.dumps diff --git a/setup.py b/setup.py index 7c7ab877..2e0b1ff1 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ 'requests==2.*', 'coverage' ], - 'orjson': ['orjson>=3.6.8,<4.0'] + 'ujson': ['ujson>=5.3.0,<6.0'] } with open("README.md") as readme: @@ -36,6 +36,7 @@ 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', 'Operating System :: Microsoft :: Windows', 'Operating System :: POSIX', 'Operating System :: MacOS :: MacOS X', From a126b9704591b71a2e8db636457f3c857bfbe6b1 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Wed, 25 May 2022 14:46:29 +1000 Subject: [PATCH 07/14] Add type stubs and improve annotations --- azure/functions/_json.py | 13 +++++++------ setup.py | 3 ++- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/azure/functions/_json.py b/azure/functions/_json.py index 9cc9bad4..c919d3b9 100644 --- a/azure/functions/_json.py +++ b/azure/functions/_json.py @@ -2,6 +2,7 @@ # Licensed under the MIT License. from enum import Enum +from typing import AnyStr, Any import os AZUREFUNCTIONS_UJSON_ENV_VAR = 'AZUREFUNCTIONS_UJSON' @@ -40,21 +41,21 @@ def default(self, o): JSONDecodeError = json.JSONDecodeError if HAS_UJSON: - def dumps(v, **kwargs): + def dumps(v: Any, **kwargs) -> str: if 'default' in kwargs: return json.dumps(v, **kwargs) if 'cls' in kwargs: del kwargs['cls'] - + return ujson.dumps(v, **kwargs) - def loads(*args, **kwargs): + def loads(s: AnyStr, **kwargs): if kwargs: - return json.loads(*args, **kwargs) + return json.loads(s, **kwargs) else: # ujson takes no kwargs - return ujson.loads(*args) + return ujson.loads(s) else: - dumps = json.dumps + dumps = json.dumps # type: ignore loads = json.loads diff --git a/setup.py b/setup.py index 2e0b1ff1..2bc7d5d4 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,8 @@ 'pytest-cov', 'pytest-benchmark', 'requests==2.*', - 'coverage' + 'coverage', + 'types-ujson' ], 'ujson': ['ujson>=5.3.0,<6.0'] } From 73cac973060a512d83bd457b52fce9ec081d8377 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Wed, 25 May 2022 14:46:42 +1000 Subject: [PATCH 08/14] install via extra --- .github/workflows/gh-tests-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gh-tests-ci.yml b/.github/workflows/gh-tests-ci.yml index 139c91a7..308b43e9 100644 --- a/.github/workflows/gh-tests-ci.yml +++ b/.github/workflows/gh-tests-ci.yml @@ -28,7 +28,7 @@ jobs: pytest --cache-clear --cov=./azure --cov-report=xml --cov-branch tests - name: Test with pytest and ujson run: | - python -m pip install ujson + python -m pip install .[ujson] pytest --cache-clear tests - name: Codecov if: ${{ matrix.python-version }} == 3.9 From b554d2cceb22dc1cd2ceb016d86b10496769a56a Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Wed, 25 May 2022 14:51:04 +1000 Subject: [PATCH 09/14] Add missing space in line comment --- azure/functions/_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure/functions/_json.py b/azure/functions/_json.py index c919d3b9..9a990a49 100644 --- a/azure/functions/_json.py +++ b/azure/functions/_json.py @@ -57,5 +57,5 @@ def loads(s: AnyStr, **kwargs): return ujson.loads(s) else: - dumps = json.dumps # type: ignore + dumps = json.dumps # type: ignore loads = json.loads From 0327e54b37ba8d13cb33db6c0a9429ad5409323d Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Wed, 25 May 2022 14:57:31 +1000 Subject: [PATCH 10/14] Only try ujson over python 3.6 --- .github/workflows/gh-tests-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/gh-tests-ci.yml b/.github/workflows/gh-tests-ci.yml index 308b43e9..00e55148 100644 --- a/.github/workflows/gh-tests-ci.yml +++ b/.github/workflows/gh-tests-ci.yml @@ -27,6 +27,7 @@ jobs: run: | pytest --cache-clear --cov=./azure --cov-report=xml --cov-branch tests - name: Test with pytest and ujson + if: ${{ matrix.python-version }} > 3.6 run: | python -m pip install .[ujson] pytest --cache-clear tests From 3b83a35ba59ecca3bfbd26654c4f637afcc8bd10 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Wed, 25 May 2022 15:08:33 +1000 Subject: [PATCH 11/14] Try this syntax instead Another variation Move expression remove braces Change to underscore --- .github/workflows/gh-tests-ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/gh-tests-ci.yml b/.github/workflows/gh-tests-ci.yml index 00e55148..9cd149fd 100644 --- a/.github/workflows/gh-tests-ci.yml +++ b/.github/workflows/gh-tests-ci.yml @@ -26,13 +26,13 @@ jobs: - name: Test with pytest run: | pytest --cache-clear --cov=./azure --cov-report=xml --cov-branch tests - - name: Test with pytest and ujson - if: ${{ matrix.python-version }} > 3.6 + - if: matrix.python_version != 3.6 + name: Test with pytest and ujson run: | python -m pip install .[ujson] pytest --cache-clear tests - - name: Codecov - if: ${{ matrix.python-version }} == 3.9 + - if: matrix.python_version == 3.9 + name: Codecov uses: codecov/codecov-action@v2 with: file: ./coverage.xml From d40b3dc4625f62ba4f54a65205866c6e7fc9fc56 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Wed, 25 May 2022 16:08:17 +1000 Subject: [PATCH 12/14] Merge coverage for both test passes to accurately record coverage --- .github/workflows/gh-tests-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gh-tests-ci.yml b/.github/workflows/gh-tests-ci.yml index 9cd149fd..38d0c870 100644 --- a/.github/workflows/gh-tests-ci.yml +++ b/.github/workflows/gh-tests-ci.yml @@ -30,7 +30,7 @@ jobs: name: Test with pytest and ujson run: | python -m pip install .[ujson] - pytest --cache-clear tests + pytest --cache-clear --cov=./azure --cov-report=xml --cov-branch --cov-append tests - if: matrix.python_version == 3.9 name: Codecov uses: codecov/codecov-action@v2 From e08d35f4bd0321e8f4cabd9e76dea3ddbdc1effb Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Wed, 25 May 2022 16:22:01 +1000 Subject: [PATCH 13/14] Add benchmark test for request.get_json --- tests/test_http.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test_http.py b/tests/test_http.py index a64503ef..33435944 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -166,3 +166,17 @@ def test_http_response_does_not_have_explicit_output(self): self.assertIsNone( getattr(http.HttpResponseConverter, 'has_implicit_output', None) ) + + +def test_http_request_body_json(benchmark): + data: bytes = b'{ "result": "OK", "message": "Everything is ok", "code": 200 }' + request = func.HttpRequest( + method='POST', + url='/foo', + body=data, + headers={ + 'Content-Type': 'application/json; charset=utf-8' + } + ) + + benchmark(request.get_json) From c090330672f838b836370a7bdbb08290d09b83c9 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Wed, 25 May 2022 16:25:14 +1000 Subject: [PATCH 14/14] Shorten line --- tests/test_http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_http.py b/tests/test_http.py index 33435944..7679f398 100644 --- a/tests/test_http.py +++ b/tests/test_http.py @@ -169,7 +169,7 @@ def test_http_response_does_not_have_explicit_output(self): def test_http_request_body_json(benchmark): - data: bytes = b'{ "result": "OK", "message": "Everything is ok", "code": 200 }' + data: bytes = b'{ "result": "OK", "message": "All good!", "code": 200 }' request = func.HttpRequest( method='POST', url='/foo',