xm_multiline help
I am having trouble getting multiline to work with ClamSentinel DriveAdd logs.
below is the raw log as ClamSentinel outputs the information.
#####################################################################################################################
##### Thursday, June 07, 2018 2:05:09 PM (jereme.powers@POS-LOGFIL1)
-------------------------------------------------------------------------------
----------- SCAN SUMMARY -----------
Known viruses: 6539461
Engine version: 0.99.4
Scanned directories: 9
Scanned files: 13
Infected files: 0
Data scanned: 21.52 MB
Data read: 169.16 MB (ratio 0.13:1)
Time: 17.069 sec (0 m 17 s)
##### Thursday, June 07, 2018 2:10:31 PM (jereme.powers@POS-LOGFIL1)
-------------------------------------------------------------------------------
----------- SCAN SUMMARY -----------
Known viruses: 6539461
Engine version: 0.99.4
Scanned directories: 9
Scanned files: 13
Infected files: 0
Data scanned: 21.52 MB
Data read: 169.16 MB (ratio 0.13:1)
Time: 16.608 sec (0 m 16 s)
##### Thursday, June 07, 2018 2:12:13 PM (jereme.powers@POS-LOGFIL1)
-------------------------------------------------------------------------------
----------- SCAN SUMMARY -----------
Known viruses: 6539461
Engine version: 0.99.4
Scanned directories: 9
Scanned files: 13
Infected files: 0
Data scanned: 21.52 MB
Data read: 169.16 MB (ratio 0.13:1)
Time: 16.234 sec (0 m 16 s)
#####################################################################################################################
I have tried many different regex combinations for the headerline but so far I have not been able to parse everything together into one line.
#####################################################################################################################
Here is my nxlog config...
define ROOT C:\Program Files (x86)\nxlog
define LOGFILE %ROOT%\data\nxlog.log
Moduledir %ROOT%\modules
CacheDir %ROOT%\data
Pidfile %ROOT%\data\nxlog.pid
SpoolDir %ROOT%\data
LogFile %ROOT%\data\nxlog.log
##################################################################################
##################
# Extensions #
##################
<Extension json>
Module xm_json
</Extension>
<Extension _syslog>
Module xm_syslog
</Extension>
<Extension charconv>
Module xm_charconv
</Extension>
<Extension Eventlog_CSV>
Module xm_csv
Fields $EventTime, $EventType, $Severity, $Channel, $Hostname, $EventID, $SourceName, $AccountName, $AccountType, $Domain, $Message, $Task, $Category, $Keywords, $UserID, $SeverityValue, $ProviderGuid, $Version, $OpcodeValue, $Opcode, $ActivityID, $RelatedActivityID, $ProcessID, $ThreadID, $RecordNumber
FieldTypes string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string
Delimiter ;
</Extension>
<Extension multiline_CLAMWIN-ScanLog>
Module xm_multiline
Exec if $raw_event =~ /^\s*$/ drop();
else
{
$raw_event = replace($raw_event, "\r", " ");
}
HeaderLine /^\w{4}\s\w{7}\s\w{0,5}\s\w{0,4}\s\d{2}\s\d{2}:\d{2}:\d{2}\s\d{4}/
</Extension>
<Extension multiline_CLAMWIN-UpdateLog>
Module xm_multiline
Exec if $raw_event =~ /^\s*$/ drop();
else
{
$raw_event = replace($raw_event, "\r", " ");
}
HeaderLine /^--------------------------------------/
</Extension>
<Extension multiline_CLAMSentinel-DriveAddLog>
Module xm_multiline
Exec if $raw_event =~ /^\s*$/ drop();
else
{
$raw_event = replace($raw_event, "\r", " ");
}
HeaderLine /^[#]{5}\s\w{6,9},\s\w{3,9}\s\d{2},\s\d{4}\s\d+:\d+:\d+\s\w{2}\s[(].[)]$/
</Extension>
##################################################################################
#####################
# WINDOWS Events #
#####################
<Processor Filter_XML>
Module pm_pattern
PatternFile %ROOT%\conf\patterndb.xml
</Processor>
<Input IN_Eventlog>
Module im_msvistalog
Query <QueryList>
<Query Id="0">
<Select Path="Security"></Select>
<Select Path="Application"></Select>
<Select Path="System"></Select>
<Select Path="Setup">*</Select>
</Query>
</QueryList>
#########################################
# Windows Events - Filter Application #
#########################################
#Exec if ($Application =~ /appdata\roaming\dropbox\bin\dropbox.exe/) drop();
#Exec if ($Application =~ /nxlog\nxlog.exe/) drop();
#Exec if ($Application =~ /windows\system32\spoolsv.exe/) drop();
#########################################
# Filter by Source and Destination IP #
#########################################
#Exec if ($SourceAddress =~ /8.8.8.8/) drop();
#Exec if ($DestAddress =~ /8.8.8.8/) drop();
#########################
# Filter by EventID #
#########################
Exec if ($EventID == 4656 or $EventID == 4663) drop();
</Input>
<Output OUT_Eventlog>
Module om_file
File "C:\ProgramData\.clamwin\log\WINEVENTLOGS.log"
Exec if not defined $PatternID or not defined $Message { drop(); }
Exec $Message = replace($Message, "\t", " "); $Message = replace($Message, "\n", " "); $Message = replace($Message, "\r", " ");
Exec if not defined $AccountName { $AccountName = "-"; }
Exec if not defined $AccountType { $AccountType = "-"; }
Exec if not defined $Domain { $Domain = "-"; }
Exec Eventlog_CSV->to_csv(); $raw_event = $Hostname + ' WIN-NXLOG ' + $raw_event + ' Task: ' + $Task + ' Category: '+ $Category + ' Keywords: ' + $Keywords + ' UserID: ' + $UserID + ' Severity: ' + $SeverityValue + ' ProviderGuid: ' + $ProviderGuid + ' Version: ' + $Version + ' OpcodeValue: ' + $OpcodeValue + ' Opcode: ' + $Opcode + ' ActivityID: ' + $ActivityID + ' RelatedActivityID: ' + $RelatedActivityID + ' ProcessID: ' + $ProcessID + ' ThreadID: ' + $ThreadID + ' RecordNumber: ' + $RecordNumber;
</Output>
<Route route_winevents_logs>
Path IN_Eventlog => Filter_XML => OUT_Eventlog
</Route>
##################################################################################
###############
# CLAMWIN #
###############
##############
# ScanLog #
##############
<Input IN_CLAMWIN-ScanLog>
Module im_file
File "C:\ProgramData\.clamwin\log\ClamScanLog.txt"
InputType multiline_CLAMWIN-ScanLog
SavePos FALSE
</Input>
<Output OUT_CLAMWIN-ScanLog>
Module om_file
File "C:\ProgramData\.clamwin\log\CLAMWIN-ScanLog.log"
Exec $Hostname = hostname_fqdn();
Exec $raw_event = $Hostname + " " + $raw_event + " " + 'CLAM-NXLOG';
</Output>
<Route route_CLAMWIN-ScanLog>
Path IN_CLAMWIN-ScanLog => OUT_CLAMWIN-ScanLog
</Route>
#################
# UpdateLog #
#################
<Input IN_CLAMWIN-UpdateLog>
Module im_file
File "C:\ProgramData\.clamwin\log\ClamUpdateLog.txt"
InputType multiline_CLAMWIN-UpdateLog
SavePos FALSE
</Input>
<Output OUT_CLAMWIN-UpdateLog>
Module om_file
File "C:\ProgramData\.clamwin\log\CLAMWIN-UpdateLog.log"
Exec $Hostname = hostname_fqdn();
Exec $raw_event = $Hostname + " " + $raw_event + " " + 'CLAM-NXLOG';
</Output>
<Route route_CLAMWIN-UpdateLog>
Path IN_CLAMWIN-UpdateLog => OUT_CLAMWIN-UpdateLog
</Route>
##################################################################################
#####################
# CLAMSentinel #
#####################
###################
# DriveAddLog #
###################
<Input IN_CLAMSentinel-DriveAddLog>
Module im_file
File "C:\ProgramData\.clamwin\log\ClamSentinel_DriveAddLog.txt"
InputType multiline_CLAMSentinel-DriveAddLog
SavePos FALSE
</Input>
<Output OUT_CLAMSentinel-DriveAddLog>
Module om_file
File "C:\ProgramData\.clamwin\log\CLAMSentinel-DriveAddLog.log"
Exec $Hostname = hostname_fqdn();
Exec $raw_event = $Hostname + " " + $raw_event + " " + 'CLAM-NXLOG';
</Output>
<Route route_CLAMSentinel-DriveAddLog>
Path IN_CLAMSentinel-DriveAddLog => OUT_CLAMSentinel-DriveAddLog
</Route>
#####################################################################################################################
Everything is working but the DriveAddLog section. Please help!
[#]{5}\s\w{6,9},\s\w{3,9}\s\d{2},\s\d{4}\s\d+:\d+:\d+\s\w{2}\s[(].*[)]
Is the last regex configuration I have tried and not had any luck with.
Do the ##### pound symbols cause problems for NXLog? I have tried just using ##### and also ##### and had no luck.
I have also tried (.*) on the end without success.