Skip to content

Commit db04214

Browse files
mirii1994mirii1994
authored
0.3.0 (#14)
* enrich logs - aws lambda name, aws region * add custom fields * change log message * readme * fix typo in readme
1 parent ca79694 commit db04214

4 files changed

Lines changed: 198 additions & 63 deletions

File tree

‎logzio-lambda-extensions-logs/README.md‎

Lines changed: 31 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -163,38 +163,40 @@ message_nested.field2: val2
163163
164164
### Environment Variables
165165
166-
| Name | Description |Required/Default|
167-
| --- | --- | --- |
168-
| `LOGZIO_LOGS_TOKEN` | Your Logz.io log shipping [token](https://app.logz.io/#/dashboard/settings/manage-tokens/data-shipping). | Required |
169-
| `LOGZIO_LISTENER` | Your Logz.io listener address, with port 8070 (http) or 8071 (https). For example, for example: `https://listener.logz.io:8071` | Required |
170-
| `LOGS_EXT_LOG_LEVEL` | Log level of the extension. Can be set to one of the following: `debug`, `info`, `warn`, `error`, `fatal`, `panic`. |Default: `info` |
171-
| `ENABLE_EXTENSION_LOGS` | Set to `true` if you wish the extension logs will be shipped to your Logz.io account. | Default: `false` |
172-
| `ENABLE_PLATFORM_LOGS` | The platform log captures runtime or execution environment errors. Set to `true` if you wish the platform logs will be shipped to your Logz.io account. | Default: `false` |
173-
| `GROK_PATTERNS` | Must be set with `LOGS_FORMAT`. Use this if you want to parse your logs into fields. A minified JSON list that contains the field name and the regex that will match the field. To understand more see the [parsing logs](https://github.com/logzio/logzio-lambda-extensions/tree/main/logzio-lambda-extensions-logs#parsing-logs) section. | - |
174-
| `LOGS_FORMAT` | Must be set with `GROK_PATTERNS`. Use this if you want to parse your logs into fields. The format in which the logs will appear, in accordance to grok conventions. To understand more see the [parsing logs](https://github.com/logzio/logzio-lambda-extensions/tree/main/logzio-lambda-extensions-logs#parsing-logs) section. | - |
166+
| Name | Description | Required/Default |
167+
|-------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------|
168+
| `LOGZIO_LOGS_TOKEN` | Your Logz.io log shipping [token](https://app.logz.io/#/dashboard/settings/manage-tokens/data-shipping). | Required |
169+
| `LOGZIO_LISTENER` | Your Logz.io listener address, with port 8070 (http) or 8071 (https). For example: `https://listener.logz.io:8071` | Required |
170+
| `LOGS_EXT_LOG_LEVEL` | Log level of the extension. Can be set to one of the following: `debug`, `info`, `warn`, `error`, `fatal`, `panic`. | Default: `info` |
171+
| `ENABLE_EXTENSION_LOGS` | Set to `true` if you wish the extension logs will be shipped to your Logz.io account. | Default: `false` |
172+
| `ENABLE_PLATFORM_LOGS` | The platform log captures runtime or execution environment errors. Set to `true` if you wish the platform logs will be shipped to your Logz.io account. | Default: `false` |
173+
| `GROK_PATTERNS` | Must be set with `LOGS_FORMAT`. Use this if you want to parse your logs into fields. A minified JSON list that contains the field name and the regex that will match the field. To understand more see the [parsing logs](https://github.com/logzio/logzio-lambda-extensions/tree/main/logzio-lambda-extensions-logs#parsing-logs) section. | - |
174+
| `LOGS_FORMAT` | Must be set with `GROK_PATTERNS`. Use this if you want to parse your logs into fields. The format in which the logs will appear, in accordance to grok conventions. To understand more see the [parsing logs](https://github.com/logzio/logzio-lambda-extensions/tree/main/logzio-lambda-extensions-logs#parsing-logs) section. | - |
175+
| `CUSTOM_FIELDS` | Include additional fields with every message sent, formatted as `fieldName1=fieldValue1;fieldName2=fieldValue2` (**NO SPACES**). A custom key that clashes with a key from the log itself will be ignored. | - |
175176
176177
### Lambda extension versions
177178
178-
| Version | Supported Runtimes | AWS ARN |
179-
| --- | --- | --- |
180-
| 0.2.0 | `.NET Core 3.1`, `Java 11`, `Java 8`, `Node.js 14.x`, `Node.js 12.x`, `Python 3.9`, `Python 3.8`, `Python 3.7`, `Ruby 2.7`, `Custom runtime` | `arn:aws:lambda:<<YOUR-AWS-REGION-CODE>>:486140753397:layer:LogzioLambdaExtensionLogs:3` |
181-
| 0.1.0| `.NET Core 3.1`, `Java 11`, `Java 8`, `Node.js 14.x`, `Node.js 12.x`, `Node.js 10.x`, `Python 3.8`, `Python 3.7`, `Ruby 2.7`, `Ruby 2.5`, `Custom runtime`| `arn:aws:lambda:<<YOUR-AWS-REGION-CODE>>:486140753397:layer:LogzioLambdaExtensionLogs:2` |
182-
| 0.0.1 | `Python 3.7`, `Python 3.8` | `arn:aws:lambda:<<YOUR-AWS-REGION-CODE>>:486140753397:layer:LogzioLambdaExtensionLogs:1` |
179+
| Version | Supported Runtimes | AWS ARN |
180+
|---------|------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------|
181+
| 0.3.0 | `.NET Core 3.1`, `Java 11`, `Java 8`, `Node.js 14.x`, `Node.js 12.x`, `Python 3.9`, `Python 3.8`, `Python 3.7`, `Ruby 2.7`, `Custom runtime` | `arn:aws:lambda:<<YOUR-AWS-REGION-CODE>>:486140753397:layer:LogzioLambdaExtensionLogs:4` |
182+
| 0.2.0 | `.NET Core 3.1`, `Java 11`, `Java 8`, `Node.js 14.x`, `Node.js 12.x`, `Python 3.9`, `Python 3.8`, `Python 3.7`, `Ruby 2.7`, `Custom runtime` | `arn:aws:lambda:<<YOUR-AWS-REGION-CODE>>:486140753397:layer:LogzioLambdaExtensionLogs:3` |
183+
| 0.1.0 | `.NET Core 3.1`, `Java 11`, `Java 8`, `Node.js 14.x`, `Node.js 12.x`, `Node.js 10.x`, `Python 3.8`, `Python 3.7`, `Ruby 2.7`, `Ruby 2.5`, `Custom runtime` | `arn:aws:lambda:<<YOUR-AWS-REGION-CODE>>:486140753397:layer:LogzioLambdaExtensionLogs:2` |
184+
| 0.0.1 | `Python 3.7`, `Python 3.8` | `arn:aws:lambda:<<YOUR-AWS-REGION-CODE>>:486140753397:layer:LogzioLambdaExtensionLogs:1` |
183185
184186
### Available AWS regions
185187
186-
| Region Name | Region Code | Notes |
187-
| --- | ---- | --- |
188-
| US East (N. Virginia) | `us-east-1` | - |
189-
| US East (Ohio) | `us-east-2` | - |
190-
| US West (N. California) | `us-west-1` | - |
191-
| US West (Oregon) | `us-west-2` | - |
192-
| Europe (Frankfurt) | `eu-central-1`| - |
193-
| Europe (Ireland) | `eu-west-1` | - |
194-
| Europe (Stockholm) | `eu-north-1` | - |
195-
| Asia Pacific (Sydney) | `ap-southeast-2` | Available from v0.1.0 |
196-
| Canada (Central) | `ca-central-1` | Available from v0.1.0 |
197-
| South America (São Paulo) | `sa-east-1` | Available from v0.2.0 |
188+
| Region Name | Region Code | Notes |
189+
|---------------------------|------------------|-----------------------|
190+
| US East (N. Virginia) | `us-east-1` | - |
191+
| US East (Ohio) | `us-east-2` | - |
192+
| US West (N. California) | `us-west-1` | - |
193+
| US West (Oregon) | `us-west-2` | - |
194+
| Europe (Frankfurt) | `eu-central-1` | - |
195+
| Europe (Ireland) | `eu-west-1` | - |
196+
| Europe (Stockholm) | `eu-north-1` | - |
197+
| Asia Pacific (Sydney) | `ap-southeast-2` | Available from v0.1.0 |
198+
| Canada (Central) | `ca-central-1` | Available from v0.1.0 |
199+
| South America (São Paulo) | `sa-east-1` | Available from v0.2.0 |
198200
199201
**NOTE:** If your AWS region is not in the list, please reach out to Logz.io's support or open an issue in this repo.
200202
@@ -206,6 +208,9 @@ Note: the dependencies layer is deprecated.
206208
207209
### Changelog:
208210
211+
- **0.3.0**:
212+
- Enrich logs with the following fields: `lambda_function_name`, `aws_region`.
213+
- Allow adding custom fields with `CUSTOM_FIELDS` env var.
209214
- **0.2.0**:
210215
- Allow parsing log into fields. To learn more see [parsing logs](https://github.com/logzio/logzio-lambda-extensions/tree/main/logzio-lambda-extensions-logs#parsing-logs) section.
211216
- Allow nested JSON within logs. To learn more see [nested fields](https://github.com/logzio/logzio-lambda-extensions/tree/main/logzio-lambda-extensions-logs#nested-fields) section.

‎logzio-lambda-extensions-logs/utils/converter.go‎

Lines changed: 56 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,17 @@ import (
77
)
88

99
const (
10-
FldLogzioTimestamp = "@timestamp"
11-
FldLambdaTime = "time"
12-
FldLogzioType = "type"
13-
FldLambdaType = "type"
14-
FldLogzioLambdaType = "lambda.log.type"
15-
FldLambdaRecord = "record"
10+
FldLogzioTimestamp = "@timestamp"
11+
FldLambdaTime = "time"
12+
FldLogzioType = "type"
13+
FldLambdaType = "type"
14+
FldLogzioLambdaType = "lambda.log.type"
15+
FldLambdaRecord = "record"
1616
FldLogzioMsg = "message"
1717
FldLogzioMsgNested = "message_nested"
1818
FldLogzioLambdaRecord = "lambda.record"
19+
FldLogzioLambdaName = "lambda_function_name"
20+
FldLogzioAwsRegion = "aws_region"
1921

2022
ExtensionType = "lambda-extension-logs"
2123

@@ -30,6 +32,7 @@ func ConvertLambdaLogToLogzioLog(lambdaLog map[string]interface{}) map[string]in
3032
logzioLog[FldLogzioType] = ExtensionType
3133
logzioLog[FldLogzioLambdaType] = lambdaLog[FldLambdaType]
3234
logger.Debugf("working on: %v", lambdaLog[FldLambdaRecord])
35+
addAwsMetadata(logzioLog)
3336

3437
switch lambdaLog[FldLambdaRecord].(type) {
3538
case string:
@@ -56,7 +59,7 @@ func ConvertLambdaLogToLogzioLog(lambdaLog map[string]interface{}) map[string]in
5659
var nested map[string]interface{}
5760
err := json.Unmarshal([]byte(fmt.Sprintf(`%s`, lambdaLog[FldLambdaRecord])), &nested)
5861
if err != nil {
59-
logger.Infof("error occurred while checking if log %s is JSON. ignore if this is not JSON: %s", lambdaLog[FldLambdaRecord], err.Error())
62+
logger.Infof("ignore if this log is not JSON. while checking if log %s is JSON, receive this : %s", lambdaLog[FldLambdaRecord], err.Error())
6063
logzioLog[FldLogzioMsg] = lambdaLog[FldLambdaRecord]
6164
} else {
6265
logger.Debugf("detected JSON: %s", lambdaLog[FldLambdaRecord])
@@ -67,6 +70,7 @@ func ConvertLambdaLogToLogzioLog(lambdaLog map[string]interface{}) map[string]in
6770
logzioLog[FldLogzioLambdaRecord] = lambdaLog[FldLambdaRecord]
6871
}
6972

73+
addCustomFields(logzioLog)
7074
return logzioLog
7175
}
7276

@@ -142,3 +146,48 @@ func addFields(logsMap map[string]interface{}, fields map[string]string) {
142146
}
143147
}
144148
}
149+
150+
func addAwsMetadata(logzioLog map[string]interface{}) {
151+
lambdaName := GetAwsLambdaFunctionName()
152+
if len(lambdaName) > 0 {
153+
logzioLog[FldLogzioLambdaName] = lambdaName
154+
} else {
155+
logger.Warning("could not get AWS Lambda function name. The field will not appear in the log")
156+
}
157+
158+
awsRegion := GetAwsRegion()
159+
if len(awsRegion) > 0 {
160+
logzioLog[FldLogzioAwsRegion] = awsRegion
161+
} else {
162+
logger.Warning("could not get AWS region. The field will not appear in the log")
163+
}
164+
}
165+
166+
func addCustomFields(logzioLog map[string]interface{}) {
167+
customFields := GetCustomFields()
168+
if len(customFields) > 0 {
169+
logKeys := make([]string, len(logzioLog))
170+
for k := range logzioLog {
171+
logKeys = append(logKeys, k)
172+
}
173+
174+
for key, val := range customFields {
175+
// Making sure that the custom fields don't override an existing field.
176+
// If it exists - custom field will be ignored.
177+
if !contains(logKeys, key) {
178+
logzioLog[key] = val
179+
} else {
180+
logger.Warningf("custom field key %s already exist in log. ignoring this custom field", key)
181+
}
182+
}
183+
}
184+
}
185+
186+
func contains(slice []string, s string) bool {
187+
for _, value := range slice {
188+
if value == s {
189+
return true
190+
}
191+
}
192+
return false
193+
}

0 commit comments

Comments
 (0)