Skip to content

Commit 9881dfc

Browse files
nikhilsinhaparseablenikhilsinhaparseable
authored
add regex for boomi webserver and container logs (#1580)
1 parent 026e95d commit 9881dfc

2 files changed

Lines changed: 378 additions & 0 deletions

File tree

‎resources/formats.json‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1515,5 +1515,41 @@
15151515
]
15161516
}
15171517
]
1518+
},
1519+
{
1520+
"name": "boomi_container_log",
1521+
"regex": [
1522+
{
1523+
"pattern": "^(?P<timestamp>\\w{3} \\d{1,2}, \\d{4} \\d{2}:\\d{2}:\\d{2} [AP]M \\w+) (?P<level>INFO|WARNING|SEVERE) \\[(?P<logger>[^\\]]+)\\] (?P<body>.*)",
1524+
"fields": [
1525+
"timestamp",
1526+
"level",
1527+
"logger",
1528+
"body"
1529+
]
1530+
}
1531+
]
1532+
},
1533+
{
1534+
"name": "boomi_webserver_log",
1535+
"regex": [
1536+
{
1537+
"pattern": "^(?P<c_ip>[\\w\\.:\\-]+)\\s+\\S+\\s+(?:-|(?P<cs_username>\\S+))\\s+\\[(?P<timestamp>[^\\]]+)\\]\\s+\"(?:\\-|(?P<cs_method>\\w+) (?P<cs_uri_stem>[^ \\?]+)(?:\\?(?P<cs_uri_query>[^ ]*))? (?P<cs_version>[\\w/\\.]+))\"\\s+(?P<sc_status>\\d+)\\s+(?P<sc_bytes>\\d+|-)\\s+\"(?:-|(?P<cs_referer>[^\"]*))\"\\s+\"(?:-|(?P<cs_user_agent>[^\"]+))\"\\s+(?P<duration>\\d+)ms",
1538+
"fields": [
1539+
"c_ip",
1540+
"cs_username",
1541+
"timestamp",
1542+
"cs_method",
1543+
"cs_uri_stem",
1544+
"cs_uri_query",
1545+
"cs_version",
1546+
"sc_status",
1547+
"sc_bytes",
1548+
"cs_referer",
1549+
"cs_user_agent",
1550+
"duration"
1551+
]
1552+
}
1553+
]
15181554
}
15191555
]

‎src/event/format/known_schema.rs‎

Lines changed: 342 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -514,6 +514,348 @@ mod tests {
514514
assert!(!obj.contains_key("timestamp"));
515515
}
516516

517+
#[test]
518+
fn test_boomi_container_log() {
519+
let processor = EventProcessor::new(FORMATS_JSON);
520+
let schema = processor
521+
.schema_definitions
522+
.get("boomi_container_log")
523+
.unwrap();
524+
525+
let test_logs = vec![
526+
// INFO - process scheduling
527+
(
528+
"Dec 21, 2023 12:00:00 AM UTC INFO [com.boomi.process.ProcessListener%ProcessRunner run] Skipping execution of process Main:PDM9000ToMarxLogicPartsDataProc [alstomtransport-BFJG449], schedules are paused.",
529+
"Dec 21, 2023 12:00:00 AM UTC",
530+
"INFO",
531+
"com.boomi.process.ProcessListener%ProcessRunner run",
532+
"Skipping execution of process Main:PDM9000ToMarxLogicPartsDataProc [alstomtransport-BFJG449], schedules are paused.",
533+
),
534+
// INFO - connector operation
535+
(
536+
"Jan 15, 2024 01:03:36 AM UTC INFO [com.boomi.connector.databaseconnector.get.StandardGetOperation] Values appended for prepared statement. Parameters: 5.",
537+
"Jan 15, 2024 01:03:36 AM UTC",
538+
"INFO",
539+
"com.boomi.connector.databaseconnector.get.StandardGetOperation",
540+
"Values appended for prepared statement. Parameters: 5.",
541+
),
542+
// WARNING - notify shape
543+
(
544+
"Dec 21, 2023 12:00:14 AM UTC WARNING [com.boomi.process.shape.NotifyShape appendMessage] Notify Shape message size exceeded.",
545+
"Dec 21, 2023 12:00:14 AM UTC",
546+
"WARNING",
547+
"com.boomi.process.shape.NotifyShape appendMessage",
548+
"Notify Shape message size exceeded.",
549+
),
550+
// SEVERE - OOM error
551+
(
552+
"Jan 15, 2024 12:00:00 AM UTC SEVERE [com.boomi.container.ContainerController start] Failed to start container [acme-corp-PROD01]. java.lang.OutOfMemoryError: Java heap space",
553+
"Jan 15, 2024 12:00:00 AM UTC",
554+
"SEVERE",
555+
"com.boomi.container.ContainerController start",
556+
"Failed to start container [acme-corp-PROD01]. java.lang.OutOfMemoryError: Java heap space",
557+
),
558+
// INFO - startup with version details
559+
(
560+
"Jan 15, 2024 12:00:00 AM UTC INFO [com.boomi.container.ContainerController start] Atom [acme-corp-PROD01] starting. Version: 24.01.2. JVM: OpenJDK 11.0.21. OS: Linux 5.15.0-91-generic amd64.",
561+
"Jan 15, 2024 12:00:00 AM UTC",
562+
"INFO",
563+
"com.boomi.container.ContainerController start",
564+
"Atom [acme-corp-PROD01] starting. Version: 24.01.2. JVM: OpenJDK 11.0.21. OS: Linux 5.15.0-91-generic amd64.",
565+
),
566+
// WARNING - TryCatch with special characters in body
567+
(
568+
"Jan 15, 2024 01:08:37 AM UTC WARNING [com.boomi.process.shape.TryCatchShape handleError] Caught exception in Try/Catch for process Main:ErrorNotificationHandler: java.lang.NumberFormatException: For input string: \"N/A\"",
569+
"Jan 15, 2024 01:08:37 AM UTC",
570+
"WARNING",
571+
"com.boomi.process.shape.TryCatchShape handleError",
572+
"Caught exception in Try/Catch for process Main:ErrorNotificationHandler: java.lang.NumberFormatException: For input string: \"N/A\"",
573+
),
574+
];
575+
576+
for (i, (log_text, expected_ts, expected_level, expected_logger, expected_body)) in
577+
test_logs.iter().enumerate()
578+
{
579+
let mut obj = Map::new();
580+
let log_field = "raw_log";
581+
obj.insert(log_field.to_string(), Value::String(log_text.to_string()));
582+
583+
let result = schema.check_or_extract(&mut obj, Some(log_field));
584+
assert!(
585+
result.is_some(),
586+
"Failed to extract fields from boomi container log {}: {}",
587+
i + 1,
588+
log_text
589+
);
590+
591+
assert_eq!(
592+
obj.get("timestamp").unwrap().as_str().unwrap(),
593+
*expected_ts,
594+
"Timestamp mismatch for log {}",
595+
i + 1
596+
);
597+
assert_eq!(
598+
obj.get("level").unwrap().as_str().unwrap(),
599+
*expected_level,
600+
"Level mismatch for log {}",
601+
i + 1
602+
);
603+
assert_eq!(
604+
obj.get("logger").unwrap().as_str().unwrap(),
605+
*expected_logger,
606+
"Logger mismatch for log {}",
607+
i + 1
608+
);
609+
assert_eq!(
610+
obj.get("body").unwrap().as_str().unwrap(),
611+
*expected_body,
612+
"Body mismatch for log {}",
613+
i + 1
614+
);
615+
}
616+
}
617+
618+
#[test]
619+
fn test_boomi_container_log_no_match() {
620+
let processor = EventProcessor::new(FORMATS_JSON);
621+
let schema = processor
622+
.schema_definitions
623+
.get("boomi_container_log")
624+
.unwrap();
625+
626+
let invalid_logs = vec![
627+
"This is not a Boomi log line",
628+
"2024-01-15T01:03:36Z INFO [some.logger] Not Boomi timestamp format",
629+
"Jan 15, 2024 01:03:36 AM UTC DEBUG [com.boomi.test] Debug is not a valid Boomi level",
630+
];
631+
632+
for (i, log_text) in invalid_logs.iter().enumerate() {
633+
let mut obj = Map::new();
634+
obj.insert("raw_log".to_string(), Value::String(log_text.to_string()));
635+
636+
let result = schema.check_or_extract(&mut obj, Some("raw_log"));
637+
assert!(
638+
result.is_none(),
639+
"Should not match invalid log {}: {}",
640+
i + 1,
641+
log_text
642+
);
643+
}
644+
}
645+
646+
#[test]
647+
fn test_boomi_webserver_log() {
648+
let processor = EventProcessor::new(FORMATS_JSON);
649+
let schema = processor
650+
.schema_definitions
651+
.get("boomi_webserver_log")
652+
.unwrap();
653+
654+
let test_logs = vec![
655+
// Standard POST with 200
656+
(
657+
r#"192.168.1.200 - - [15/Jan/2024:00:00:55 +0000] "POST /ws/rest/v1/invoices HTTP/1.1" 200 481480 "-" "python-requests/2.31.0" 10530ms"#,
658+
"192.168.1.200",
659+
"POST",
660+
"/ws/rest/v1/invoices",
661+
"HTTP/1.1",
662+
"200",
663+
"481480",
664+
"python-requests/2.31.0",
665+
"10530",
666+
),
667+
// GET with 400 error
668+
(
669+
r#"10.0.1.45 - - [15/Jan/2024:00:04:36 +0000] "GET /ws/rest/v1/edi/850 HTTP/1.1" 400 840 "-" "curl/8.1.2" 14225ms"#,
670+
"10.0.1.45",
671+
"GET",
672+
"/ws/rest/v1/edi/850",
673+
"HTTP/1.1",
674+
"400",
675+
"840",
676+
"curl/8.1.2",
677+
"14225",
678+
),
679+
// GET with 500 error
680+
(
681+
r#"172.16.0.88 - - [15/Jan/2024:00:59:04 +0000] "GET /ws/rest/v1/edi/850 HTTP/1.1" 500 369 "-" "Apache-HttpClient/4.5.13" 4620ms"#,
682+
"172.16.0.88",
683+
"GET",
684+
"/ws/rest/v1/edi/850",
685+
"HTTP/1.1",
686+
"500",
687+
"369",
688+
"Apache-HttpClient/4.5.13",
689+
"4620",
690+
),
691+
// PUT with SAP user agent
692+
(
693+
r#"10.0.1.102 - - [15/Jan/2024:00:28:40 +0000] "PUT /ws/rest/v1/products HTTP/1.1" 200 77663 "-" "SAP-NetWeaver/7.50" 1003ms"#,
694+
"10.0.1.102",
695+
"PUT",
696+
"/ws/rest/v1/products",
697+
"HTTP/1.1",
698+
"200",
699+
"77663",
700+
"SAP-NetWeaver/7.50",
701+
"1003",
702+
),
703+
// DELETE with Salesforce user agent
704+
(
705+
r#"172.16.0.15 - - [15/Jan/2024:01:08:39 +0000] "DELETE /ws/rest/v1/orders HTTP/1.1" 400 1302 "-" "Salesforce/1.0" 24919ms"#,
706+
"172.16.0.15",
707+
"DELETE",
708+
"/ws/rest/v1/orders",
709+
"HTTP/1.1",
710+
"400",
711+
"1302",
712+
"Salesforce/1.0",
713+
"24919",
714+
),
715+
];
716+
717+
for (
718+
i,
719+
(
720+
log_text,
721+
expected_ip,
722+
expected_method,
723+
expected_path,
724+
expected_version,
725+
expected_status,
726+
expected_bytes,
727+
expected_ua,
728+
expected_duration,
729+
),
730+
) in test_logs.iter().enumerate()
731+
{
732+
let mut obj = Map::new();
733+
let log_field = "raw_log";
734+
obj.insert(log_field.to_string(), Value::String(log_text.to_string()));
735+
736+
let result = schema.check_or_extract(&mut obj, Some(log_field));
737+
assert!(
738+
result.is_some(),
739+
"Failed to extract fields from boomi webserver log {}: {}",
740+
i + 1,
741+
log_text
742+
);
743+
744+
assert_eq!(
745+
obj.get("c_ip").unwrap().as_str().unwrap(),
746+
*expected_ip,
747+
"IP mismatch for log {}",
748+
i + 1
749+
);
750+
assert_eq!(
751+
obj.get("cs_method").unwrap().as_str().unwrap(),
752+
*expected_method,
753+
"Method mismatch for log {}",
754+
i + 1
755+
);
756+
assert_eq!(
757+
obj.get("cs_uri_stem").unwrap().as_str().unwrap(),
758+
*expected_path,
759+
"Path mismatch for log {}",
760+
i + 1
761+
);
762+
assert_eq!(
763+
obj.get("cs_version").unwrap().as_str().unwrap(),
764+
*expected_version,
765+
"HTTP version mismatch for log {}",
766+
i + 1
767+
);
768+
assert_eq!(
769+
obj.get("sc_status").unwrap().as_str().unwrap(),
770+
*expected_status,
771+
"Status mismatch for log {}",
772+
i + 1
773+
);
774+
assert_eq!(
775+
obj.get("sc_bytes").unwrap().as_str().unwrap(),
776+
*expected_bytes,
777+
"Bytes mismatch for log {}",
778+
i + 1
779+
);
780+
assert_eq!(
781+
obj.get("cs_user_agent").unwrap().as_str().unwrap(),
782+
*expected_ua,
783+
"User agent mismatch for log {}",
784+
i + 1
785+
);
786+
assert_eq!(
787+
obj.get("duration").unwrap().as_str().unwrap(),
788+
*expected_duration,
789+
"Duration mismatch for log {}",
790+
i + 1
791+
);
792+
}
793+
}
794+
795+
#[test]
796+
fn test_boomi_webserver_log_no_match() {
797+
let processor = EventProcessor::new(FORMATS_JSON);
798+
let schema = processor
799+
.schema_definitions
800+
.get("boomi_webserver_log")
801+
.unwrap();
802+
803+
let invalid_logs = vec![
804+
"This is not a web server log",
805+
// Missing duration suffix
806+
r#"192.168.1.200 - - [15/Jan/2024:00:00:55 +0000] "POST /ws/rest/v1/invoices HTTP/1.1" 200 481480 "-" "python-requests/2.31.0""#,
807+
];
808+
809+
for (i, log_text) in invalid_logs.iter().enumerate() {
810+
let mut obj = Map::new();
811+
obj.insert("raw_log".to_string(), Value::String(log_text.to_string()));
812+
813+
let result = schema.check_or_extract(&mut obj, Some("raw_log"));
814+
assert!(
815+
result.is_none(),
816+
"Should not match invalid log {}: {}",
817+
i + 1,
818+
log_text
819+
);
820+
}
821+
}
822+
823+
#[test]
824+
fn test_boomi_container_log_fields_already_exist() {
825+
let processor = EventProcessor::new(FORMATS_JSON);
826+
let schema = processor
827+
.schema_definitions
828+
.get("boomi_container_log")
829+
.unwrap();
830+
831+
let mut obj = Map::new();
832+
obj.insert(
833+
"timestamp".to_string(),
834+
Value::String("Jan 15, 2024 01:03:36 AM UTC".to_string()),
835+
);
836+
obj.insert("level".to_string(), Value::String("INFO".to_string()));
837+
obj.insert(
838+
"logger".to_string(),
839+
Value::String("com.boomi.container.ContainerController".to_string()),
840+
);
841+
obj.insert(
842+
"body".to_string(),
843+
Value::String("Atom started successfully.".to_string()),
844+
);
845+
846+
let result = schema.check_or_extract(&mut obj, None);
847+
assert!(
848+
result.is_some(),
849+
"Should return Some when all fields already exist"
850+
);
851+
852+
// Verify original values weren't changed
853+
assert_eq!(
854+
obj.get("body").unwrap().as_str().unwrap(),
855+
"Atom started successfully."
856+
);
857+
}
858+
517859
#[test]
518860
fn test_parseable_server_logs() {
519861
let processor = EventProcessor::new(FORMATS_JSON);

0 commit comments

Comments
 (0)