xm_multiline help

Tags:

#1 Jereme.Powers

I am having trouble getting multiline to work with ClamSentinel DriveAdd logs.

below is the raw log as ClamSentinel outputs the information.

#####################################################################################################################

##### Thursday, June 07, 2018 2:05:09 PM (jereme.powers@POS-LOGFIL1)

-------------------------------------------------------------------------------


----------- SCAN SUMMARY -----------
Known viruses: 6539461
Engine version: 0.99.4
Scanned directories: 9
Scanned files: 13
Infected files: 0
Data scanned: 21.52 MB
Data read: 169.16 MB (ratio 0.13:1)
Time: 17.069 sec (0 m 17 s)

##### Thursday, June 07, 2018 2:10:31 PM (jereme.powers@POS-LOGFIL1)

-------------------------------------------------------------------------------


----------- SCAN SUMMARY -----------
Known viruses: 6539461
Engine version: 0.99.4
Scanned directories: 9
Scanned files: 13
Infected files: 0
Data scanned: 21.52 MB
Data read: 169.16 MB (ratio 0.13:1)
Time: 16.608 sec (0 m 16 s)

##### Thursday, June 07, 2018 2:12:13 PM (jereme.powers@POS-LOGFIL1)

-------------------------------------------------------------------------------


----------- SCAN SUMMARY -----------
Known viruses: 6539461
Engine version: 0.99.4
Scanned directories: 9
Scanned files: 13
Infected files: 0
Data scanned: 21.52 MB
Data read: 169.16 MB (ratio 0.13:1)
Time: 16.234 sec (0 m 16 s)

#####################################################################################################################

I have tried many different regex combinations for the headerline but so far I have not been able to parse everything together into one line.

#####################################################################################################################

Here is my nxlog config...


define ROOT C:\Program Files (x86)\nxlog
define LOGFILE %ROOT%\data\nxlog.log

Moduledir %ROOT%\modules
CacheDir %ROOT%\data
Pidfile %ROOT%\data\nxlog.pid
SpoolDir %ROOT%\data
LogFile %ROOT%\data\nxlog.log

##################################################################################

##################
# Extensions #
##################

<Extension json>
Module xm_json
</Extension>

<Extension _syslog>
Module xm_syslog
</Extension>

<Extension charconv>
Module xm_charconv
</Extension>

<Extension Eventlog_CSV>
Module xm_csv
Fields $EventTime, $EventType, $Severity, $Channel, $Hostname, $EventID, $SourceName, $AccountName, $AccountType, $Domain, $Message, $Task, $Category, $Keywords, $UserID, $SeverityValue, $ProviderGuid, $Version, $OpcodeValue, $Opcode, $ActivityID, $RelatedActivityID, $ProcessID, $ThreadID, $RecordNumber
FieldTypes string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string
Delimiter ;
</Extension>

<Extension multiline_CLAMWIN-ScanLog>
Module xm_multiline
Exec if $raw_event =~ /^\s*$/ drop();

else

{

$raw_event = replace($raw_event, "\r", " ");

}
HeaderLine /^\w{4}\s\w{7}\s\w{0,5}\s\w{0,4}\s\d{2}\s\d{2}:\d{2}:\d{2}\s\d{4}/
</Extension>

<Extension multiline_CLAMWIN-UpdateLog>
Module xm_multiline
Exec if $raw_event =~ /^\s*$/ drop();

else

{

$raw_event = replace($raw_event, "\r", " ");

}
HeaderLine /^--------------------------------------/
</Extension>

<Extension multiline_CLAMSentinel-DriveAddLog>
Module xm_multiline
Exec if $raw_event =~ /^\s*$/ drop();

else

{

$raw_event = replace($raw_event, "\r", " ");

}
HeaderLine /^[#]{5}\s\w{6,9},\s\w{3,9}\s\d{2},\s\d{4}\s\d+:\d+:\d+\s\w{2}\s[(].[)]$/
</Extension>

##################################################################################

#####################
# WINDOWS Events #
#####################

<Processor Filter_XML>
Module pm_pattern
PatternFile %ROOT%\conf\patterndb.xml
</Processor>

<Input IN_Eventlog>
Module im_msvistalog
Query <QueryList>

<Query Id="0">

<Select Path="Security">
</Select>

<Select Path="Application"></Select>

<Select Path="System">
</Select>

<Select Path="Setup">*</Select>

</Query>

</QueryList>


#########################################

# Windows Events - Filter Application #
#########################################

#Exec if ($Application =~ /appdata\roaming\dropbox\bin\dropbox.exe/) drop();
#Exec if ($Application =~ /nxlog\nxlog.exe/) drop();
#Exec if ($Application =~ /windows\system32\spoolsv.exe/) drop();

#########################################
# Filter by Source and Destination IP #
#########################################

#Exec if ($SourceAddress =~ /8.8.8.8/) drop();
#Exec if ($DestAddress =~ /8.8.8.8/) drop();

#########################
# Filter by EventID #
#########################

Exec if ($EventID == 4656 or $EventID == 4663) drop();
</Input>

<Output OUT_Eventlog>
Module om_file
File "C:\ProgramData\.clamwin\log\WINEVENTLOGS.log"
Exec if not defined $PatternID or not defined $Message { drop(); }
Exec $Message = replace($Message, "\t", " "); $Message = replace($Message, "\n", " "); $Message = replace($Message, "\r", " ");
Exec if not defined $AccountName { $AccountName = "-"; }
Exec if not defined $AccountType { $AccountType = "-"; }
Exec if not defined $Domain { $Domain = "-"; }
Exec Eventlog_CSV->to_csv(); $raw_event = $Hostname + ' WIN-NXLOG ' + $raw_event + ' Task: ' + $Task + ' Category: '+ $Category + ' Keywords: ' + $Keywords + ' UserID: ' + $UserID + ' Severity: ' + $SeverityValue + ' ProviderGuid: ' + $ProviderGuid + ' Version: ' + $Version + ' OpcodeValue: ' + $OpcodeValue + ' Opcode: ' + $Opcode + ' ActivityID: ' + $ActivityID + ' RelatedActivityID: ' + $RelatedActivityID + ' ProcessID: ' + $ProcessID + ' ThreadID: ' + $ThreadID + ' RecordNumber: ' + $RecordNumber;
</Output>

<Route route_winevents_logs>
Path IN_Eventlog => Filter_XML => OUT_Eventlog
</Route>

##################################################################################

###############
# CLAMWIN #
###############

##############
# ScanLog #
##############

<Input IN_CLAMWIN-ScanLog>
Module im_file
File "C:\ProgramData\.clamwin\log\ClamScanLog.txt"
InputType multiline_CLAMWIN-ScanLog
SavePos FALSE
</Input>

<Output OUT_CLAMWIN-ScanLog>
Module om_file
File "C:\ProgramData\.clamwin\log\CLAMWIN-ScanLog.log"
Exec $Hostname = hostname_fqdn();
Exec $raw_event = $Hostname + " " + $raw_event + " " + 'CLAM-NXLOG';
</Output>

<Route route_CLAMWIN-ScanLog>
Path IN_CLAMWIN-ScanLog => OUT_CLAMWIN-ScanLog
</Route>

#################
# UpdateLog #
#################

<Input IN_CLAMWIN-UpdateLog>
Module im_file
File "C:\ProgramData\.clamwin\log\ClamUpdateLog.txt"
InputType multiline_CLAMWIN-UpdateLog
SavePos FALSE
</Input>

<Output OUT_CLAMWIN-UpdateLog>
Module om_file
File "C:\ProgramData\.clamwin\log\CLAMWIN-UpdateLog.log"
Exec $Hostname = hostname_fqdn();
Exec $raw_event = $Hostname + " " + $raw_event + " " + 'CLAM-NXLOG';
</Output>

<Route route_CLAMWIN-UpdateLog>
Path IN_CLAMWIN-UpdateLog => OUT_CLAMWIN-UpdateLog
</Route>

##################################################################################

#####################
# CLAMSentinel #
#####################

###################
# DriveAddLog #
###################

<Input IN_CLAMSentinel-DriveAddLog>
Module im_file
File "C:\ProgramData\.clamwin\log\ClamSentinel_DriveAddLog.txt"
InputType multiline_CLAMSentinel-DriveAddLog
SavePos FALSE
</Input>

<Output OUT_CLAMSentinel-DriveAddLog>
Module om_file
File "C:\ProgramData\.clamwin\log\CLAMSentinel-DriveAddLog.log"
Exec $Hostname = hostname_fqdn();
Exec $raw_event = $Hostname + " " + $raw_event + " " + 'CLAM-NXLOG';
</Output>

<Route route_CLAMSentinel-DriveAddLog>
Path IN_CLAMSentinel-DriveAddLog => OUT_CLAMSentinel-DriveAddLog
</Route>

#####################################################################################################################

Everything is working but the DriveAddLog section. Please help!

#2 Jereme.Powers
#1 Jereme.Powers
I am having trouble getting multiline to work with ClamSentinel DriveAdd logs. below is the raw log as ClamSentinel outputs the information. ##################################################################################################################### ##### Thursday, June 07, 2018 2:05:09 PM (jereme.powers@POS-LOGFIL1) ------------------------------------------------------------------------------- ----------- SCAN SUMMARY ----------- Known viruses: 6539461 Engine version: 0.99.4 Scanned directories: 9 Scanned files: 13 Infected files: 0 Data scanned: 21.52 MB Data read: 169.16 MB (ratio 0.13:1) Time: 17.069 sec (0 m 17 s) ##### Thursday, June 07, 2018 2:10:31 PM (jereme.powers@POS-LOGFIL1) ------------------------------------------------------------------------------- ----------- SCAN SUMMARY ----------- Known viruses: 6539461 Engine version: 0.99.4 Scanned directories: 9 Scanned files: 13 Infected files: 0 Data scanned: 21.52 MB Data read: 169.16 MB (ratio 0.13:1) Time: 16.608 sec (0 m 16 s) ##### Thursday, June 07, 2018 2:12:13 PM (jereme.powers@POS-LOGFIL1) ------------------------------------------------------------------------------- ----------- SCAN SUMMARY ----------- Known viruses: 6539461 Engine version: 0.99.4 Scanned directories: 9 Scanned files: 13 Infected files: 0 Data scanned: 21.52 MB Data read: 169.16 MB (ratio 0.13:1) Time: 16.234 sec (0 m 16 s) ##################################################################################################################### I have tried many different regex combinations for the headerline but so far I have not been able to parse everything together into one line. ##################################################################################################################### Here is my nxlog config... define ROOT C:\Program Files (x86)\nxlog define LOGFILE %ROOT%\data\nxlog.log Moduledir %ROOT%\modules CacheDir %ROOT%\data Pidfile %ROOT%\data\nxlog.pid SpoolDir %ROOT%\data LogFile %ROOT%\data\nxlog.log ################################################################################## ################## # Extensions # ################## <Extension json> Module xm_json </Extension> <Extension _syslog> Module xm_syslog </Extension> <Extension charconv> Module xm_charconv </Extension> <Extension Eventlog_CSV> Module xm_csv Fields $EventTime, $EventType, $Severity, $Channel, $Hostname, $EventID, $SourceName, $AccountName, $AccountType, $Domain, $Message, $Task, $Category, $Keywords, $UserID, $SeverityValue, $ProviderGuid, $Version, $OpcodeValue, $Opcode, $ActivityID, $RelatedActivityID, $ProcessID, $ThreadID, $RecordNumber FieldTypes string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string Delimiter ; </Extension> <Extension multiline_CLAMWIN-ScanLog> Module xm_multiline Exec if $raw_event =~ /^\s*$/ drop(); else { $raw_event = replace($raw_event, "\r", " "); } HeaderLine /^\w{4}\s\w{7}\s\w{0,5}\s\w{0,4}\s\d{2}\s\d{2}:\d{2}:\d{2}\s\d{4}/ </Extension> <Extension multiline_CLAMWIN-UpdateLog> Module xm_multiline Exec if $raw_event =~ /^\s*$/ drop(); else { $raw_event = replace($raw_event, "\r", " "); } HeaderLine /^--------------------------------------/ </Extension> <Extension multiline_CLAMSentinel-DriveAddLog> Module xm_multiline Exec if $raw_event =~ /^\s*$/ drop(); else { $raw_event = replace($raw_event, "\r", " "); } HeaderLine /^[#]{5}\s\w{6,9},\s\w{3,9}\s\d{2},\s\d{4}\s\d+:\d+:\d+\s\w{2}\s[(].[)]$/ </Extension> ################################################################################## ##################### # WINDOWS Events # ##################### <Processor Filter_XML> Module pm_pattern PatternFile %ROOT%\conf\patterndb.xml </Processor> <Input IN_Eventlog> Module im_msvistalog Query <QueryList> <Query Id="0"> <Select Path="Security"></Select> <Select Path="Application"></Select> <Select Path="System"></Select> <Select Path="Setup">*</Select> </Query> </QueryList> ######################################### # Windows Events - Filter Application # ######################################### #Exec if ($Application =~ /appdata\roaming\dropbox\bin\dropbox.exe/) drop(); #Exec if ($Application =~ /nxlog\nxlog.exe/) drop(); #Exec if ($Application =~ /windows\system32\spoolsv.exe/) drop(); ######################################### # Filter by Source and Destination IP # ######################################### #Exec if ($SourceAddress =~ /8.8.8.8/) drop(); #Exec if ($DestAddress =~ /8.8.8.8/) drop(); ######################### # Filter by EventID # ######################### Exec if ($EventID == 4656 or $EventID == 4663) drop(); </Input> <Output OUT_Eventlog> Module om_file File "C:\ProgramData\.clamwin\log\WINEVENTLOGS.log" Exec if not defined $PatternID or not defined $Message { drop(); } Exec $Message = replace($Message, "\t", " "); $Message = replace($Message, "\n", " "); $Message = replace($Message, "\r", " "); Exec if not defined $AccountName { $AccountName = "-"; } Exec if not defined $AccountType { $AccountType = "-"; } Exec if not defined $Domain { $Domain = "-"; } Exec Eventlog_CSV->to_csv(); $raw_event = $Hostname + ' WIN-NXLOG ' + $raw_event + ' Task: ' + $Task + ' Category: '+ $Category + ' Keywords: ' + $Keywords + ' UserID: ' + $UserID + ' Severity: ' + $SeverityValue + ' ProviderGuid: ' + $ProviderGuid + ' Version: ' + $Version + ' OpcodeValue: ' + $OpcodeValue + ' Opcode: ' + $Opcode + ' ActivityID: ' + $ActivityID + ' RelatedActivityID: ' + $RelatedActivityID + ' ProcessID: ' + $ProcessID + ' ThreadID: ' + $ThreadID + ' RecordNumber: ' + $RecordNumber; </Output> <Route route_winevents_logs> Path IN_Eventlog => Filter_XML => OUT_Eventlog </Route> ################################################################################## ############### # CLAMWIN # ############### ############## # ScanLog # ############## <Input IN_CLAMWIN-ScanLog> Module im_file File "C:\ProgramData\.clamwin\log\ClamScanLog.txt" InputType multiline_CLAMWIN-ScanLog SavePos FALSE </Input> <Output OUT_CLAMWIN-ScanLog> Module om_file File "C:\ProgramData\.clamwin\log\CLAMWIN-ScanLog.log" Exec $Hostname = hostname_fqdn(); Exec $raw_event = $Hostname + " " + $raw_event + " " + 'CLAM-NXLOG'; </Output> <Route route_CLAMWIN-ScanLog> Path IN_CLAMWIN-ScanLog => OUT_CLAMWIN-ScanLog </Route> ################# # UpdateLog # ################# <Input IN_CLAMWIN-UpdateLog> Module im_file File "C:\ProgramData\.clamwin\log\ClamUpdateLog.txt" InputType multiline_CLAMWIN-UpdateLog SavePos FALSE </Input> <Output OUT_CLAMWIN-UpdateLog> Module om_file File "C:\ProgramData\.clamwin\log\CLAMWIN-UpdateLog.log" Exec $Hostname = hostname_fqdn(); Exec $raw_event = $Hostname + " " + $raw_event + " " + 'CLAM-NXLOG'; </Output> <Route route_CLAMWIN-UpdateLog> Path IN_CLAMWIN-UpdateLog => OUT_CLAMWIN-UpdateLog </Route> ################################################################################## ##################### # CLAMSentinel # ##################### ################### # DriveAddLog # ################### <Input IN_CLAMSentinel-DriveAddLog> Module im_file File "C:\ProgramData\.clamwin\log\ClamSentinel_DriveAddLog.txt" InputType multiline_CLAMSentinel-DriveAddLog SavePos FALSE </Input> <Output OUT_CLAMSentinel-DriveAddLog> Module om_file File "C:\ProgramData\.clamwin\log\CLAMSentinel-DriveAddLog.log" Exec $Hostname = hostname_fqdn(); Exec $raw_event = $Hostname + " " + $raw_event + " " + 'CLAM-NXLOG'; </Output> <Route route_CLAMSentinel-DriveAddLog> Path IN_CLAMSentinel-DriveAddLog => OUT_CLAMSentinel-DriveAddLog </Route> ##################################################################################################################### Everything is working but the DriveAddLog section. Please help!

[#]{5}\s\w{6,9},\s\w{3,9}\s\d{2},\s\d{4}\s\d+:\d+:\d+\s\w{2}\s[(].*[)]

Is the last regex configuration I have tried and not had any luck with.

Do the ##### pound symbols cause problems for NXLog? I have tried just using ##### and also ##### and had no luck.

I have also tried (.*) on the end without success.