xm_multiline help

View thread

Jereme.Powers

I am having trouble getting multiline to work with ClamSentinel DriveAdd logs.

below is the raw log as ClamSentinel outputs the information.

#####################################################################################################################

##### Thursday, June 07, 2018 2:05:09 PM (jereme.powers@POS-LOGFIL1)

-------------------------------------------------------------------------------


----------- SCAN SUMMARY -----------
Known viruses: 6539461
Engine version: 0.99.4
Scanned directories: 9
Scanned files: 13
Infected files: 0
Data scanned: 21.52 MB
Data read: 169.16 MB (ratio 0.13:1)
Time: 17.069 sec (0 m 17 s)

##### Thursday, June 07, 2018 2:10:31 PM (jereme.powers@POS-LOGFIL1)

-------------------------------------------------------------------------------


----------- SCAN SUMMARY -----------
Known viruses: 6539461
Engine version: 0.99.4
Scanned directories: 9
Scanned files: 13
Infected files: 0
Data scanned: 21.52 MB
Data read: 169.16 MB (ratio 0.13:1)
Time: 16.608 sec (0 m 16 s)

##### Thursday, June 07, 2018 2:12:13 PM (jereme.powers@POS-LOGFIL1)

-------------------------------------------------------------------------------


----------- SCAN SUMMARY -----------
Known viruses: 6539461
Engine version: 0.99.4
Scanned directories: 9
Scanned files: 13
Infected files: 0
Data scanned: 21.52 MB
Data read: 169.16 MB (ratio 0.13:1)
Time: 16.234 sec (0 m 16 s)

#####################################################################################################################

I have tried many different regex combinations for the headerline but so far I have not been able to parse everything together into one line.

#####################################################################################################################

Here is my nxlog config...


define ROOT C:\Program Files (x86)\nxlog
define LOGFILE %ROOT%\data\nxlog.log

Moduledir %ROOT%\modules
CacheDir %ROOT%\data
Pidfile %ROOT%\data\nxlog.pid
SpoolDir %ROOT%\data
LogFile %ROOT%\data\nxlog.log

##################################################################################

##################
# Extensions #
##################

<Extension json>
Module xm_json
</Extension>

<Extension _syslog>
Module xm_syslog
</Extension>

<Extension charconv>
Module xm_charconv
</Extension>

<Extension Eventlog_CSV>
Module xm_csv
Fields $EventTime, $EventType, $Severity, $Channel, $Hostname, $EventID, $SourceName, $AccountName, $AccountType, $Domain, $Message, $Task, $Category, $Keywords, $UserID, $SeverityValue, $ProviderGuid, $Version, $OpcodeValue, $Opcode, $ActivityID, $RelatedActivityID, $ProcessID, $ThreadID, $RecordNumber
FieldTypes string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string, string
Delimiter ;
</Extension>

<Extension multiline_CLAMWIN-ScanLog>
Module xm_multiline
Exec if $raw_event =~ /^\s*$/ drop();

else

{

$raw_event = replace($raw_event, "\r", " ");

}
HeaderLine /^\w{4}\s\w{7}\s\w{0,5}\s\w{0,4}\s\d{2}\s\d{2}:\d{2}:\d{2}\s\d{4}/
</Extension>

<Extension multiline_CLAMWIN-UpdateLog>
Module xm_multiline
Exec if $raw_event =~ /^\s*$/ drop();

else

{

$raw_event = replace($raw_event, "\r", " ");

}
HeaderLine /^--------------------------------------/
</Extension>

<Extension multiline_CLAMSentinel-DriveAddLog>
Module xm_multiline
Exec if $raw_event =~ /^\s*$/ drop();

else

{

$raw_event = replace($raw_event, "\r", " ");

}
HeaderLine /^[#]{5}\s\w{6,9},\s\w{3,9}\s\d{2},\s\d{4}\s\d+:\d+:\d+\s\w{2}\s[(].[)]$/
</Extension>

##################################################################################

#####################
# WINDOWS Events #
#####################

<Processor Filter_XML>
Module pm_pattern
PatternFile %ROOT%\conf\patterndb.xml
</Processor>

<Input IN_Eventlog>
Module im_msvistalog
Query <QueryList>

<Query Id="0">

<Select Path="Security">
</Select>

<Select Path="Application"></Select>

<Select Path="System">
</Select>

<Select Path="Setup">*</Select>

</Query>

</QueryList>


#########################################

# Windows Events - Filter Application #
#########################################

#Exec if ($Application =~ /appdata\roaming\dropbox\bin\dropbox.exe/) drop();
#Exec if ($Application =~ /nxlog\nxlog.exe/) drop();
#Exec if ($Application =~ /windows\system32\spoolsv.exe/) drop();

#########################################
# Filter by Source and Destination IP #
#########################################

#Exec if ($SourceAddress =~ /8.8.8.8/) drop();
#Exec if ($DestAddress =~ /8.8.8.8/) drop();

#########################
# Filter by EventID #
#########################

Exec if ($EventID == 4656 or $EventID == 4663) drop();
</Input>

<Output OUT_Eventlog>
Module om_file
File "C:\ProgramData\.clamwin\log\WINEVENTLOGS.log"
Exec if not defined $PatternID or not defined $Message { drop(); }
Exec $Message = replace($Message, "\t", " "); $Message = replace($Message, "\n", " "); $Message = replace($Message, "\r", " ");
Exec if not defined $AccountName { $AccountName = "-"; }
Exec if not defined $AccountType { $AccountType = "-"; }
Exec if not defined $Domain { $Domain = "-"; }
Exec Eventlog_CSV->to_csv(); $raw_event = $Hostname + ' WIN-NXLOG ' + $raw_event + ' Task: ' + $Task + ' Category: '+ $Category + ' Keywords: ' + $Keywords + ' UserID: ' + $UserID + ' Severity: ' + $SeverityValue + ' ProviderGuid: ' + $ProviderGuid + ' Version: ' + $Version + ' OpcodeValue: ' + $OpcodeValue + ' Opcode: ' + $Opcode + ' ActivityID: ' + $ActivityID + ' RelatedActivityID: ' + $RelatedActivityID + ' ProcessID: ' + $ProcessID + ' ThreadID: ' + $ThreadID + ' RecordNumber: ' + $RecordNumber;
</Output>

<Route route_winevents_logs>
Path IN_Eventlog => Filter_XML => OUT_Eventlog
</Route>

##################################################################################

###############
# CLAMWIN #
###############

##############
# ScanLog #
##############

<Input IN_CLAMWIN-ScanLog>
Module im_file
File "C:\ProgramData\.clamwin\log\ClamScanLog.txt"
InputType multiline_CLAMWIN-ScanLog
SavePos FALSE
</Input>

<Output OUT_CLAMWIN-ScanLog>
Module om_file
File "C:\ProgramData\.clamwin\log\CLAMWIN-ScanLog.log"
Exec $Hostname = hostname_fqdn();
Exec $raw_event = $Hostname + " " + $raw_event + " " + 'CLAM-NXLOG';
</Output>

<Route route_CLAMWIN-ScanLog>
Path IN_CLAMWIN-ScanLog => OUT_CLAMWIN-ScanLog
</Route>

#################
# UpdateLog #
#################

<Input IN_CLAMWIN-UpdateLog>
Module im_file
File "C:\ProgramData\.clamwin\log\ClamUpdateLog.txt"
InputType multiline_CLAMWIN-UpdateLog
SavePos FALSE
</Input>

<Output OUT_CLAMWIN-UpdateLog>
Module om_file
File "C:\ProgramData\.clamwin\log\CLAMWIN-UpdateLog.log"
Exec $Hostname = hostname_fqdn();
Exec $raw_event = $Hostname + " " + $raw_event + " " + 'CLAM-NXLOG';
</Output>

<Route route_CLAMWIN-UpdateLog>
Path IN_CLAMWIN-UpdateLog => OUT_CLAMWIN-UpdateLog
</Route>

##################################################################################

#####################
# CLAMSentinel #
#####################

###################
# DriveAddLog #
###################

<Input IN_CLAMSentinel-DriveAddLog>
Module im_file
File "C:\ProgramData\.clamwin\log\ClamSentinel_DriveAddLog.txt"
InputType multiline_CLAMSentinel-DriveAddLog
SavePos FALSE
</Input>

<Output OUT_CLAMSentinel-DriveAddLog>
Module om_file
File "C:\ProgramData\.clamwin\log\CLAMSentinel-DriveAddLog.log"
Exec $Hostname = hostname_fqdn();
Exec $raw_event = $Hostname + " " + $raw_event + " " + 'CLAM-NXLOG';
</Output>

<Route route_CLAMSentinel-DriveAddLog>
Path IN_CLAMSentinel-DriveAddLog => OUT_CLAMSentinel-DriveAddLog
</Route>

#####################################################################################################################

Everything is working but the DriveAddLog section. Please help!