|
1 | 1 | # Default configuration for kinesis-lzo-s3-sink
|
2 | 2 |
|
3 |
| -sink { |
4 |
| - |
5 |
| - # The following are used to authenticate for the Amazon Kinesis sink. |
6 |
| - # |
7 |
| - # If both are set to 'default', the default provider chain is used |
8 |
| - # (see http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html) |
9 |
| - # |
10 |
| - # If both are set to 'iam', use AWS IAM Roles to provision credentials. |
11 |
| - # |
12 |
| - # If both are set to 'env', use environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY |
13 |
| - aws { |
14 |
| - access-key: "iam" |
15 |
| - secret-key: "iam" |
16 |
| - } |
| 3 | +# Sources currently supported are: |
| 4 | +# 'kinesis' for reading records from a Kinesis stream |
| 5 | +# 'nsq' for reading records from a NSQ topic |
| 6 | +source: "{{source}}" |
17 | 7 |
|
18 |
| - kinesis { |
19 |
| - in { |
20 |
| - # Kinesis input stream name |
21 |
| - stream-name: "{{sinkKinesisInStreamName}}" |
| 8 | +# Sinks currently supported are: |
| 9 | +# 'kinesis' for writing records to a Kinesis stream |
| 10 | +# 'nsq' for writing records to a NSQ topic |
| 11 | +sink: "{{sink}}" |
22 | 12 |
|
23 |
| - # LATEST: most recent data. |
24 |
| - # TRIM_HORIZON: oldest available data. |
25 |
| - # Note: This only affects the first run of this application |
26 |
| - # on a stream. |
27 |
| - initial-position: "TRIM_HORIZON" |
| 13 | +# The following are used to authenticate for the Amazon Kinesis sink. |
| 14 | +# |
| 15 | +# If both are set to 'default', the default provider chain is used |
| 16 | +# (see http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/DefaultAWSCredentialsProviderChain.html) |
| 17 | +# |
| 18 | +# If both are set to 'iam', use AWS IAM Roles to provision credentials. |
| 19 | +# |
| 20 | +# If both are set to 'env', use environment variables AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY |
| 21 | +aws { |
| 22 | + access-key: "iam" |
| 23 | + secret-key: "iam" |
| 24 | +} |
28 | 25 |
|
29 |
| - # Maximum number of records to read per GetRecords call |
30 |
| - max-records: {{sinkKinesisMaxRecords}} |
31 |
| - } |
| 26 | +# Config for NSQ |
| 27 | +nsq { |
| 28 | + # Channel name for NSQ source |
| 29 | + channel-name: "{{NsqSourceChannelName}}" |
| 30 | + |
| 31 | + # Host name for NSQ tools |
| 32 | + host: "{{NsqHost}}" |
32 | 33 |
|
33 |
| - out { |
34 |
| - # Stream for events for which the storage process fails |
35 |
| - stream-name: "{{sinkKinesisOutStreamName}}" |
36 |
| - } |
| 34 | + # Port for nsqd |
| 35 | + port: "{{NsqdPort}}" |
37 | 36 |
|
38 |
| - region: "{{sinkKinesisRegion}}" |
| 37 | + # Port for nsqlookupd |
| 38 | + lookup-port: {{NsqlookupdPort}} |
| 39 | +} |
39 | 40 |
|
40 |
| - # "app-name" is used for a DynamoDB table to maintain stream state. |
41 |
| - # You can set it automatically using: "SnowplowLzoS3Sink-$\\{sink.kinesis.in.stream-name\\}" |
42 |
| - app-name: "{{sinkKinesisAppName}}" |
43 |
| - } |
| 41 | +kinesis { |
| 42 | + # LATEST: most recent data. |
| 43 | + # TRIM_HORIZON: oldest available data. |
| 44 | + # Note: This only affects the first run of this application |
| 45 | + # on a stream. |
| 46 | + initial-position: "TRIM_HORIZON" |
44 | 47 |
|
45 |
| - s3 { |
46 |
| - # If using us-east-1, then endpoint should be "http://s3.amazonaws.com". |
47 |
| - # Otherwise "http://s3-<<region>>.s3.amazonaws.com", e.g. |
48 |
| - # http://s3-eu-west-1.amazonaws.com |
49 |
| - region: "{{sinkKinesisS3Region}}" |
50 |
| - bucket: "{{sinkKinesisS3Bucket}}" |
| 48 | + # Maximum number of records to read per GetRecords call |
| 49 | + max-records: {{sinkKinesisMaxRecords}} |
51 | 50 |
|
52 |
| - # Format is one of lzo or gzip |
53 |
| - # Note, that you can use gzip only for enriched data stream. |
54 |
| - format: "{{sinkKinesisFormat}}" |
| 51 | + region: "{{sinkKinesisRegion}}" |
55 | 52 |
|
56 |
| - # Maximum Timeout that the application is allowed to fail for |
57 |
| - max-timeout: {{sinkKinesisMaxTimeout}} |
58 |
| - } |
| 53 | + # "app-name" is used for a DynamoDB table to maintain stream state. |
| 54 | + # You can set it automatically using: "SnowplowLzoS3Sink-$\\{sink.kinesis.in.stream-name\\}" |
| 55 | + app-name: "{{sinkKinesisAppName}}" |
| 56 | +} |
| 57 | + |
| 58 | +streams { |
| 59 | + # Input stream name |
| 60 | + stream-name-in = "{{InStreamName}}" |
| 61 | + |
| 62 | + # Stream for events for which the storage process fails |
| 63 | + stream-name-out = "{{OutStreamName}}" |
59 | 64 |
|
60 | 65 | # Events are accumulated in a buffer before being sent to S3.
|
61 | 66 | # The buffer is emptied whenever:
|
62 | 67 | # - the combined size of the stored records exceeds byte-limit or
|
63 | 68 | # - the number of stored records exceeds record-limit or
|
64 | 69 | # - the time in milliseconds since it was last emptied exceeds time-limit
|
65 | 70 | buffer {
|
66 |
| - byte-limit: {{sinkLzoBufferByteThreshold}} |
| 71 | + byte-limit: {{sinkLzoBufferByteThreshold}} # Not supported by NSQ; will be ignored |
67 | 72 | record-limit: {{sinkLzoBufferRecordThreshold}}
|
68 |
| - time-limit: {{sinkLzoBufferTimeThreshold}} |
| 73 | + time-limit: {{sinkLzoBufferTimeThreshold}} # Not supported by NSQ; will be ignored |
69 | 74 | }
|
| 75 | +} |
70 | 76 |
|
71 |
| - # Set the Logging Level for the S3 Sink |
72 |
| - # Options: ERROR, WARN, INFO, DEBUG, TRACE |
73 |
| - logging { |
74 |
| - level: "{{sinkLzoLogLevel}}" |
75 |
| - } |
| 77 | +s3 { |
| 78 | + # If using us-east-1, then endpoint should be "http://s3.amazonaws.com". |
| 79 | + # Otherwise "http://s3-<<region>>.s3.amazonaws.com", e.g. |
| 80 | + # http://s3-eu-west-1.amazonaws.com |
| 81 | + region: "{{sinkKinesisS3Region}}" |
| 82 | + bucket: "{{sinkKinesisS3Bucket}}" |
| 83 | + |
| 84 | + # Format is one of lzo or gzip |
| 85 | + # Note, that you can use gzip only for enriched data stream. |
| 86 | + format: "{{sinkKinesisFormat}}" |
| 87 | + |
| 88 | + # Maximum Timeout that the application is allowed to fail for |
| 89 | + max-timeout: {{sinkKinesisMaxTimeout}} |
| 90 | +} |
| 91 | + |
| 92 | +# Set the Logging Level for the S3 Sink |
| 93 | +# Options: ERROR, WARN, INFO, DEBUG, TRACE |
| 94 | +logging { |
| 95 | + level: "{{sinkLzoLogLevel}}" |
| 96 | +} |
76 | 97 |
|
77 |
| - # Optional section for tracking endpoints |
78 |
| - monitoring { |
79 |
| - snowplow { |
80 |
| - collector-uri: "{{collectorUri}}" |
81 |
| - collector-port: 80 |
82 |
| - app-id: "{{sinkLzoAppName}}" |
83 |
| - method: "GET" |
84 |
| - } |
| 98 | +# Optional section for tracking endpoints |
| 99 | +monitoring { |
| 100 | + snowplow { |
| 101 | + collector-uri: "{{collectorUri}}" |
| 102 | + collector-port: 80 |
| 103 | + app-id: "{{sinkLzoAppName}}" |
| 104 | + method: "GET" |
85 | 105 | }
|
86 | 106 | }
|
0 commit comments