Skip to content

Commit 961e914

Browse files
vivekjillavivekjilla
authored
feat: decorator support for Kafka extension (#234)
* initial changes for decorator support for Kafka * Fixing minor issues * fixing tests * some more changes post validations * Added Oauthbearer options support and fixing tests
1 parent a0b8692 commit 961e914

File tree

4 files changed

+577
-0
lines changed

4 files changed

+577
-0
lines changed

‎azure/functions/decorators/constants.py

+2
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
EVENT_HUB = "eventHub"
99
HTTP_TRIGGER = "httpTrigger"
1010
HTTP_OUTPUT = "http"
11+
KAFKA = "kafka"
12+
KAFKA_TRIGGER = "kafkaTrigger"
1113
QUEUE = "queue"
1214
QUEUE_TRIGGER = "queueTrigger"
1315
SERVICE_BUS = "serviceBus"

‎azure/functions/decorators/function_app.py

+316
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,8 @@
2424
from azure.functions.decorators.eventhub import EventHubTrigger, EventHubOutput
2525
from azure.functions.decorators.http import HttpTrigger, HttpOutput, \
2626
HttpMethod
27+
from azure.functions.decorators.kafka import KafkaTrigger, KafkaOutput, \
28+
BrokerAuthenticationMode, BrokerProtocol, OAuthBearerMethod
2729
from azure.functions.decorators.queue import QueueTrigger, QueueOutput
2830
from azure.functions.decorators.servicebus import ServiceBusQueueTrigger, \
2931
ServiceBusQueueOutput, ServiceBusTopicTrigger, \
@@ -1229,6 +1231,155 @@ def decorator():
12291231

12301232
return wrap
12311233

1234+
def kafka_trigger(self,
1235+
arg_name: str,
1236+
topic: str,
1237+
broker_list: str,
1238+
event_hub_connection_string: Optional[str] = None,
1239+
consumer_group: Optional[str] = None,
1240+
avro_schema: Optional[str] = None,
1241+
username: Optional[str] = None,
1242+
password: Optional[str] = None,
1243+
ssl_key_location: Optional[str] = None,
1244+
ssl_ca_location: Optional[str] = None,
1245+
ssl_certificate_location: Optional[str] = None,
1246+
ssl_key_password: Optional[str] = None,
1247+
schema_registry_url: Optional[str] = None,
1248+
schema_registry_username: Optional[str] = None,
1249+
schema_registry_password: Optional[str] = None,
1250+
o_auth_bearer_method: Optional[Union[OAuthBearerMethod, str]] = None, # noqa E501
1251+
o_auth_bearer_client_id: Optional[str] = None,
1252+
o_auth_bearer_client_secret: Optional[str] = None,
1253+
o_auth_bearer_scope: Optional[str] = None,
1254+
o_auth_bearer_token_endpoint_url: Optional[str] = None,
1255+
o_auth_bearer_extensions: Optional[str] = None,
1256+
authentication_mode: Optional[Union[BrokerAuthenticationMode, str]] = "NotSet", # noqa E501
1257+
protocol: Optional[Union[BrokerProtocol, str]] = "NotSet", # noqa E501
1258+
cardinality: Optional[Union[Cardinality, str]] = "One",
1259+
lag_threshold: int = 1000,
1260+
data_type: Optional[Union[DataType, str]] = None,
1261+
**kwargs) -> Callable[..., Any]:
1262+
"""
1263+
The kafka_trigger decorator adds
1264+
:class:`KafkaTrigger`
1265+
to the :class:`FunctionBuilder` object
1266+
for building :class:`Function` object used in worker function
1267+
indexing model. This is equivalent to defining kafka trigger
1268+
in the function.json which enables function to be triggered to
1269+
respond to an event sent to a kafka topic.
1270+
All optional fields will be given default value by function host when
1271+
they are parsed by function host.
1272+
1273+
Ref: https://aka.ms/kafkatrigger
1274+
1275+
:param arg_name: the variable name used in function code for the
1276+
parameter that has the kafka event data.
1277+
:param topic: The topic monitored by the trigger.
1278+
:param broker_list: The list of Kafka brokers monitored by the trigger.
1279+
:param event_hub_connection_string: The name of an app setting that
1280+
contains the connection string for the eventhub when using Kafka
1281+
protocol header feature of Azure EventHubs.
1282+
:param consumer_group: Kafka consumer group used by the trigger.
1283+
:param avro_schema: This should be used only if a generic record
1284+
should be generated.
1285+
:param username: SASL username for use with the PLAIN and SASL-SCRAM-..
1286+
mechanisms. Default is empty string. This is equivalent to
1287+
'sasl.username' in librdkafka.
1288+
:param password: SASL password for use with the PLAIN and SASL-SCRAM-..
1289+
mechanisms. Default is empty string. This is equivalent to
1290+
'sasl.password' in librdkafka.
1291+
:param ssl_key_location: Path to client's private key (PEM) used for
1292+
authentication. Default is empty string. This is equivalent to
1293+
'ssl.key.location' in librdkafka.
1294+
:param ssl_ca_location: Path to CA certificate file for verifying the
1295+
broker's certificate. This is equivalent to 'ssl.ca.location' in
1296+
librdkafka.
1297+
:param ssl_certificate_location: Path to client's certificate. This is
1298+
equivalent to 'ssl.certificate.location' in librdkafka.
1299+
:param ssl_key_password: Password for client's certificate. This is
1300+
equivalent to 'ssl.key.password' in librdkafka.
1301+
:param schema_registry_url: URL for the Avro Schema Registry.
1302+
:param schema_registry_username: Username for the Avro Schema Registry.
1303+
:param schema_registry_password: Password for the Avro Schema Registry.
1304+
:param o_auth_bearer_method: Either 'default' or 'oidc'.
1305+
sasl.oauthbearer in librdkafka.
1306+
:param o_auth_bearer_client_id: Specify only when o_auth_bearer_method
1307+
is 'oidc'. sasl.oauthbearer.client.id in librdkafka.
1308+
:param o_auth_bearer_client_secret: Specify only when
1309+
o_auth_bearer_method is 'oidc'. sasl.oauthbearer.client.secret in
1310+
librdkafka.
1311+
:param o_auth_bearer_scope: Specify only when o_auth_bearer_method
1312+
is 'oidc'. Client use this to specify the scope of the access request
1313+
to the broker. sasl.oauthbearer.scope in librdkafka.
1314+
:param o_auth_bearer_token_endpoint_url: Specify only when
1315+
o_auth_bearer_method is 'oidc'. sasl.oauthbearer.token.endpoint.url
1316+
in librdkafka.
1317+
:param o_auth_bearer_extensions: Allow additional information to be
1318+
provided to the broker. Comma-separated list of key=value pairs. E.g.,
1319+
"supportFeatureX=true,organizationId=sales-emea".
1320+
sasl.oauthbearer.extensions in librdkafka
1321+
:param authentication_mode: SASL mechanism to use for authentication.
1322+
Allowed values: Gssapi, Plain, ScramSha256, ScramSha512. Default is
1323+
Plain. This is equivalent to 'sasl.mechanism' in librdkafka.
1324+
:param protocol: Gets or sets the security protocol used to communicate
1325+
with brokers. Default is plain text. This is equivalent to
1326+
'security.protocol' in librdkafka. TODO
1327+
:param lag_threshold: Maximum number of unprocessed messages a worker
1328+
is expected to have at an instance. When target-based scaling is not
1329+
disabled, this is used to divide total unprocessed event count to
1330+
determine the number of worker instances, which will then be rounded
1331+
up to a worker instance count that creates a balanced partition
1332+
distribution. Default is 1000.
1333+
:param data_type: Defines how Functions runtime should treat the
1334+
parameter value.
1335+
:param kwargs: Keyword arguments for specifying additional binding
1336+
fields to include in the binding json
1337+
:return: Decorator function.
1338+
"""
1339+
1340+
@self._configure_function_builder
1341+
def wrap(fb):
1342+
def decorator():
1343+
fb.add_trigger(
1344+
trigger=KafkaTrigger(
1345+
name=arg_name,
1346+
topic=topic,
1347+
broker_list=broker_list,
1348+
event_hub_connection_string=event_hub_connection_string, # noqa: E501
1349+
consumer_group=consumer_group,
1350+
avro_schema=avro_schema,
1351+
username=username,
1352+
password=password,
1353+
ssl_key_location=ssl_key_location,
1354+
ssl_ca_location=ssl_ca_location,
1355+
ssl_certificate_location=ssl_certificate_location,
1356+
ssl_key_password=ssl_key_password,
1357+
schema_registry_url=schema_registry_url,
1358+
schema_registry_username=schema_registry_username,
1359+
schema_registry_password=schema_registry_password,
1360+
o_auth_bearer_method=parse_singular_param_to_enum(
1361+
o_auth_bearer_method, OAuthBearerMethod),
1362+
o_auth_bearer_client_id=o_auth_bearer_client_id,
1363+
o_auth_bearer_client_secret=o_auth_bearer_client_secret, # noqa: E501
1364+
o_auth_bearer_scope=o_auth_bearer_scope,
1365+
o_auth_bearer_token_endpoint_url=o_auth_bearer_token_endpoint_url, # noqa: E501
1366+
o_auth_bearer_extensions=o_auth_bearer_extensions,
1367+
authentication_mode=parse_singular_param_to_enum(
1368+
authentication_mode, BrokerAuthenticationMode),
1369+
protocol=parse_singular_param_to_enum(protocol,
1370+
BrokerProtocol),
1371+
cardinality=parse_singular_param_to_enum(cardinality,
1372+
Cardinality),
1373+
lag_threshold=lag_threshold,
1374+
data_type=parse_singular_param_to_enum(data_type,
1375+
DataType),
1376+
**kwargs))
1377+
return fb
1378+
1379+
return decorator()
1380+
1381+
return wrap
1382+
12321383
def sql_trigger(self,
12331384
arg_name: str,
12341385
table_name: str,
@@ -2212,6 +2363,171 @@ def decorator():
22122363

22132364
return wrap
22142365

2366+
def kafka_output(self,
2367+
arg_name: str,
2368+
topic: str,
2369+
broker_list: str,
2370+
avro_schema: Optional[str] = None,
2371+
username: Optional[str] = None,
2372+
password: Optional[str] = None,
2373+
ssl_key_location: Optional[str] = None,
2374+
ssl_ca_location: Optional[str] = None,
2375+
ssl_certificate_location: Optional[str] = None,
2376+
ssl_key_password: Optional[str] = None,
2377+
schema_registry_url: Optional[str] = None,
2378+
schema_registry_username: Optional[str] = None,
2379+
schema_registry_password: Optional[str] = None,
2380+
o_auth_bearer_method: Optional[Union[OAuthBearerMethod, str]] = None, # noqa E501
2381+
o_auth_bearer_client_id: Optional[str] = None,
2382+
o_auth_bearer_client_secret: Optional[str] = None,
2383+
o_auth_bearer_scope: Optional[str] = None,
2384+
o_auth_bearer_token_endpoint_url: Optional[str] = None,
2385+
o_auth_bearer_extensions: Optional[str] = None,
2386+
max_message_bytes: int = 1_000_000,
2387+
batch_size: int = 10_000,
2388+
enable_idempotence: bool = False,
2389+
message_timeout_ms: int = 300_000,
2390+
request_timeout_ms: int = 5_000,
2391+
max_retries: int = 2_147_483_647,
2392+
authentication_mode: Optional[Union[BrokerAuthenticationMode, str]] = "NOTSET", # noqa E501
2393+
protocol: Optional[Union[BrokerProtocol, str]] = "NOTSET",
2394+
linger_ms: int = 5,
2395+
data_type: Optional[Union[DataType, str]] = None,
2396+
**kwargs) -> Callable[..., Any]:
2397+
"""
2398+
The kafka_output decorator adds
2399+
:class:`KafkaOutput`
2400+
to the :class:`FunctionBuilder` object
2401+
for building :class:`Function` object used in worker function
2402+
indexing model. This is equivalent to defining output binding
2403+
in the function.json which enables function to
2404+
write events to a kafka topic.
2405+
All optional fields will be given default value by function host when
2406+
they are parsed by function host.
2407+
2408+
Ref: https://aka.ms/kafkaoutput
2409+
2410+
:param arg_name: The variable name used in function code that
2411+
represents the event.
2412+
:param topic: The topic monitored by the trigger.
2413+
:param broker_list: The list of Kafka brokers monitored by the trigger.
2414+
:param avro_schema: This should be used only if a generic record
2415+
should be generated.
2416+
:param username: SASL username for use with the PLAIN and SASL-SCRAM-..
2417+
mechanisms. Default is empty string. This is equivalent to
2418+
'sasl.username' in librdkafka.
2419+
:param password: SASL password for use with the PLAIN and SASL-SCRAM-..
2420+
mechanisms. Default is empty string. This is equivalent to
2421+
'sasl.password' in librdkafka.
2422+
:param ssl_key_location: Path to client's private key (PEM) used for
2423+
authentication. Default is empty string. This is equivalent to
2424+
'ssl.key.location' in librdkafka.
2425+
:param ssl_ca_location: Path to CA certificate file for verifying the
2426+
broker's certificate. This is equivalent to 'ssl.ca.location' in
2427+
librdkafka.
2428+
:param ssl_certificate_location: Path to client's certificate. This is
2429+
equivalent to 'ssl.certificate.location' in librdkafka.
2430+
:param ssl_key_password: Password for client's certificate. This is
2431+
equivalent to 'ssl.key.password' in librdkafka.
2432+
:param schema_registry_url: URL for the Avro Schema Registry.
2433+
:param schema_registry_username: Username for the Avro Schema Registry.
2434+
:param schema_registry_password: Password for the Avro Schema Registry.
2435+
:param o_auth_bearer_method: Either 'default' or 'oidc'.
2436+
sasl.oauthbearer in librdkafka.
2437+
:param o_auth_bearer_client_id: Specify only when o_auth_bearer_method
2438+
is 'oidc'. sasl.oauthbearer.client.id in librdkafka.
2439+
:param o_auth_bearer_client_secret: Specify only when
2440+
o_auth_bearer_method is 'oidc'. sasl.oauthbearer.client.secret in
2441+
librdkafka.
2442+
:param o_auth_bearer_scope: Specify only when o_auth_bearer_method
2443+
is 'oidc'. Client use this to specify the scope of the access request
2444+
to the broker. sasl.oauthbearer.scope in librdkafka.
2445+
:param o_auth_bearer_token_endpoint_url: Specify only when
2446+
o_auth_bearer_method is 'oidc'. sasl.oauthbearer.token.endpoint.url
2447+
in librdkafka.
2448+
:param o_auth_bearer_extensions: Allow additional information to be
2449+
provided to the broker. Comma-separated list of key=value pairs. E.g.,
2450+
"supportFeatureX=true,organizationId=sales-emea".
2451+
sasl.oauthbearer.extensions in librdkafka
2452+
:param max_message_bytes: Maximum transmit message size. Default is 1MB
2453+
:param batch_size: Maximum number of messages batched in one MessageSet
2454+
Default is 10000.
2455+
:param enable_idempotence: When set to `true`, the producer will ensure
2456+
that messages are successfully produced exactly once and in the
2457+
original produce order. Default is false.
2458+
:param message_timeout_ms: Local message timeout. This value is only
2459+
enforced locally and limits the time a produced message waits for
2460+
successful delivery. A time of 0 is infinite. This is the maximum time
2461+
used to deliver a message (including retries). Delivery error occurs
2462+
when either the retry count or the message timeout are exceeded.
2463+
Default is 300000.
2464+
:param request_timeout_ms: The ack timeout of the producer request in
2465+
milliseconds. Default is 5000.
2466+
:param max_retries: How many times to retry sending a failing Message.
2467+
Default is 2147483647. Retrying may cause reordering unless
2468+
'EnableIdempotence' is set to 'True'.
2469+
:param authentication_mode: SASL mechanism to use for authentication.
2470+
Allowed values: Gssapi, Plain, ScramSha256, ScramSha512. Default is
2471+
Plain. This is equivalent to 'sasl.mechanism' in librdkafka.
2472+
:param protocol: Gets or sets the security protocol used to communicate
2473+
with brokers. Default is plain text. This is equivalent to
2474+
'security.protocol' in librdkafka.
2475+
:param linger_ms: Linger.MS property provides the time between batches
2476+
of messages being sent to cluster. Larger value allows more batching
2477+
results in high throughput.
2478+
:param data_type: Defines how Functions runtime should treat the
2479+
parameter value.
2480+
:param kwargs: Keyword arguments for specifying additional binding
2481+
fields to include in the binding json
2482+
2483+
:return: Decorator function.
2484+
"""
2485+
2486+
@self._configure_function_builder
2487+
def wrap(fb):
2488+
def decorator():
2489+
fb.add_binding(
2490+
binding=KafkaOutput(
2491+
name=arg_name,
2492+
topic=topic,
2493+
broker_list=broker_list,
2494+
avro_schema=avro_schema,
2495+
username=username,
2496+
password=password,
2497+
ssl_key_location=ssl_key_location,
2498+
ssl_ca_location=ssl_ca_location,
2499+
ssl_certificate_location=ssl_certificate_location,
2500+
ssl_key_password=ssl_key_password,
2501+
schema_registry_url=schema_registry_url,
2502+
schema_registry_username=schema_registry_username,
2503+
schema_registry_password=schema_registry_password,
2504+
o_auth_bearer_method=parse_singular_param_to_enum(
2505+
o_auth_bearer_method, OAuthBearerMethod),
2506+
o_auth_bearer_client_id=o_auth_bearer_client_id,
2507+
o_auth_bearer_client_secret=o_auth_bearer_client_secret, # noqa: E501
2508+
o_auth_bearer_scope=o_auth_bearer_scope,
2509+
o_auth_bearer_token_endpoint_url=o_auth_bearer_token_endpoint_url, # noqa: E501
2510+
o_auth_bearer_extensions=o_auth_bearer_extensions,
2511+
max_message_bytes=max_message_bytes,
2512+
batch_size=batch_size,
2513+
enable_idempotence=enable_idempotence,
2514+
message_timeout_ms=message_timeout_ms,
2515+
request_timeout_ms=request_timeout_ms,
2516+
max_retries=max_retries,
2517+
authentication_mode=parse_singular_param_to_enum(
2518+
authentication_mode, BrokerAuthenticationMode),
2519+
protocol=parse_singular_param_to_enum(protocol,
2520+
BrokerProtocol),
2521+
linger_ms=linger_ms,
2522+
data_type=parse_singular_param_to_enum(data_type,
2523+
DataType),
2524+
**kwargs))
2525+
return fb
2526+
2527+
return decorator()
2528+
2529+
return wrap
2530+
22152531
def table_input(self,
22162532
arg_name: str,
22172533
connection: str,

0 commit comments

Comments
 (0)