amazon-kinesis-agent
amazon-kinesis-agent copied to clipboard
Cannot make regex to deliver a record
// sample log -> [airflowwebserver] [qa] [2019-09-08T19:40:06.087+00:00] {jobs.py:1559} INFO - Harvesting DAG parsing results
agent.json
{
"cloudwatch.emitMetrics": true,
"cloudwatch.endpoint": "monitoring.eu-west-1.amazonaws.com",
"firehose.endpoint": "firehose.eu-west-1.amazonaws.com",
"kinesis.endpoint": "https://kinesis.eu-west-1.amazonaws.com",
"flows": [
{
"filePattern": "/var/log/containers/*.log",
"kinesisStream": "prod-application-logs",
"dataProcessingOptions": [
{
"optionName": "LOGTOJSON",
"logFormat": "SYSLOG",
"matchPattern": "\\[(\\w+)\\]\\s\\[\\w+\\]\\s\\[(.+)\\]\\s\\{.+\\}\\s(.+)\\s-(.+)",
"customFieldNames": ["service", "@timestamp", "severity", "message"]
}
]
}
],
"log.level": "INFO"
}
Code to check the regex:
import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class HelloWorld{
public static void main(String []args){
String regex = "\\[(\\w+)\\]\\s\\[\\w+\\]\\s\\[(.+)\\]\\s\\{.+\\}\\s(.+)\\s-(.+)";
String string = "[pi_airflow_webserver] [qa] [2019-09-08T19:40:06.087+00:00] {jobs.py:1559} INFO - Harvesting DAG parsing results";
Pattern pattern = Pattern.compile(regex, Pattern.MULTILINE);
Matcher matcher = pattern.matcher(string);
while (matcher.find()) {
System.out.println("Full match: " + matcher.group(0));
for (int i = 1; i <= matcher.groupCount(); i++) {
System.out.println("Group " + i + ": " + matcher.group(i));
}
}
}
}