Apache log with custom log
#1
comoalt
Hello guyz, i am setting up nxlog service in our network and i am focusing to work with apache custom log and nxlog. Since Apache is at the end of a reverse proxies chain, the only way to keep forwarders ip adresses, is to use an Apache conditional variable (X-Forwarded-For) able to switch between combined and custom log (as explained in details here: http://www.techstacks.com/howto/log-client-ip-and-xforwardedfor-ip-in-apache.html). Apache log variable %h is replaced by %{X-Forwarded-For}i which is a string empty or containing one / more ip addresses. In this specific case when nxlog hit a custom log, parse_kvp error is reported. My Apache conf includes: LogFormat "%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-Agent}i"" combined LogFormat "%{X-Forwarded-For}i %l %u %t "%r" %>s %b "%{Referer}i" "%{User-Agent}i"" proxy SetEnvIf X-Forwarded-For "^......." forwarded CustomLog "logs/access.log" combined env=!forwarded CustomLog "logs/access.log" proxy env=forwarded On the net i was able to find working examples with Apache common log only. My config:
define ROOT C:\\Program Files (x86)\\nxlog
define ROOT_STRING C:\\Program Files (x86)\\nxlog
define CERTDIR %ROOT%\\cert
Moduledir %ROOT%\\modules
CacheDir %ROOT%\\data
Pidfile %ROOT%\\data\\nxlog.pid
SpoolDir %ROOT%\\data
LogFile %ROOT%\\data\\nxlog.log
<extension fileop="">
Module xm_fileop
</extension><extension json="">
Module xm_json
</extension><extension syslog="">
Module xm_syslog
</extension><extension exec="">
Module xm_exec
</extension>
## Create the parse rule for IIS logs. You can copy these from the header of the IIS log file.
<extension w3c="">
Module xm_csv
Fields $date, $time, $s-ip, $cs-method, $cs-uri-stem, $cs-uri-query, $s-port, $cs-username, $c-ip, $csUser-Agent, $csReferer, $sc-status, $sc-substatus, $sc-win32-status, $time-taken
FieldTypes string, string, string, string, string, string, string, string, string, string, string, string, string, string, string
Delimiter ' '
UndefValue -
</extension>
## In questa macchina con frontend Apache usiamo ext kvp invece di w3c
<extension kvp="">
Module xm_kvp
KVPDelimiter &
KVDelimiter =
</extension><extension kvp2="">
Module xm_kvp
KVPDelimiter ;
KVDelimiter =
#QuoteMethod None
</extension><input apache="" />
Module im_file
File "C:\Apache_install\httpd-2.4.25-win64-VC14\Apache24\logs\access.log"
Exec if $raw_event =~ /^(\S+) (\S+) (\S+) \[([^\]]+)\] \"(\S+) (.+) HTTP.\d\.\d\" (\d+) (\d+) \"([^\"]+)\" \"([^\"]+)\"/\
{ \
$Hostname = $1; \
if $3 != '-' $AccountName = $3; \
$EventTime = parsedate($4); \
$HTTPMethod = $5; \
$HTTPURL = $6; \
$HTTPResponseStatus = $7; \
$FileSize = $8; \
$HTTPReferer = $9; \
$HTTPUserAgent = $10; \
}
#Exec if $raw_event =~ /^(\S+) (\S+) (\S+) \[([^\]]+)\] \"(\S+) (.+) HTTP.\d\.\d\" (\d+) (\d+) \"([^\"]+)\" \"([^\"]+)\"/\
# { \
# $Hostname = $1; \
# if $3 != '-' $AccountName = $3; \
# $EventTime = parsedate($4); \
# $HTTPMethod = $5; \
# $HTTPURL = $6; \
# $HTTPResponseStatus = $7; \
# $FileSize = $8; \
# $HTTPReferer = $9; \
# $HTTPUserAgent = $10; \
# if $HTTPURL =~ /\?(.+)/ { $HTTPParams = $1; } \
# kvp->parse_kvp($HTTPParams); \
# delete($EventReceivedTime); \
# kvp2->to_kvp(); \
# }
<input internal="" />
Module im_internal
Exec $Message = to_json();
# Windows Event Log
<input eventlog="" />
Module im_msvistalog
# Query per ridurre Event Log . Usato il QueryXML non il comando Query
# I commenti nella quey vanno indicati in XML: <!-- stringa -->
<queryxml><querylist><query id="0"><!-- Select --><select path="Application">*[System[(Level=1 or Level=2 or Level=3)]]</select><select path="Security">*[System[(Level=1 or Level=2 or Level=3)]]</select><select path="System">*[System[(Level=1 or Level=2 or Level=3)]]</select><select path="ForwardedEvents">*</select><select path="Setup">*</select><select path="HardwareEvents">*</select><select path="Microsoft-Windows-PowerShell/Operational">*[System[(Level=1 or Level=2 or Level=3)]]</select><select path="Microsoft-Windows-TaskScheduler/Operational">*[System[(Level=1 or Level=2 or Level=3)]]</select><!-- Suppress --><suppress path="Security">*[System[(EventID=4689 or EventID=5158 or EventID=5440 or EventID=5444)]]</suppress><suppress path="Windows PowerShell">*[System[(EventID=501 or EventID=400 or EventID=600)]]</suppress></query></querylist></queryxml>
Exec $EventReceivedTime = integer($EventReceivedTime) / 1000000;
Exec to_json();
# 100Mb disk buffer
<processor buffer="">
Module pm_buffer
MaxSize 102400
Type disk
</processor>
# RFC5424 come indicato https://www.scip.ch/en/?labs.20141106
<processor rfc5424="">
Module pm_transformer
Exec $Hostname = hostname();
Outputformat syslog_rfc5424
</processor><output ssl_out="">
Module om_ssl
Host IP.IP.IP.IP
Port 443
CAFile %CERTDIR%/nxlog_rootCA.crt
CertFile %CERTDIR%/client.crt
CertKeyFile %CERTDIR%/client.key
KeyPass secret
AllowUntrusted TRUE
OutputType Binary
Exec to_syslog_ietf();
# Rimuovo CRLF LF TAB - lato server in om_file non funziona
Exec $raw_event =~ s/(\t|\r|\n)//g; $raw_event = replace($raw_event, '{', '[" "] {', 1);
#tag windows
Exec $raw_event =~ s/(\[.*])//g; $raw_event = replace($raw_event, '{', '[tag="windows"] {', 1);
#Use the following line for debugging (uncomment the fileop extension above as well)
#Exec file_write("C:\\Program Files (x86)\\nxlog\\data\\nxlog_output.log", $raw_event);
</output><route>
Path Apache, internal, eventlog => rfc5424 => buffer => ssl_out
</route>
```
Is there any solution for nxlog conf to work with this kind of apache custom log?
Thanks in advance
#1
comoalt
Hello guyz,
i am setting up nxlog service in our network and i am focusing to work with apache custom log and nxlog.
Since Apache is at the end of a reverse proxies chain, the only way to keep forwarders ip adresses, is to use an Apache conditional variable (X-Forwarded-For) able to switch between combined and custom log (as explained in details here: http://www.techstacks.com/howto/log-client-ip-and-xforwardedfor-ip-in-apache.html).
Apache log variable %h is replaced by %{X-Forwarded-For}i which is a string empty or containing one / more ip addresses.
In this specific case when nxlog hit a custom log, parse_kvp error is reported.
My Apache conf includes:
LogFormat "%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-Agent}i"" combined
LogFormat "%{X-Forwarded-For}i %l %u %t "%r" %>s %b "%{Referer}i" "%{User-Agent}i"" proxy
SetEnvIf X-Forwarded-For "^......." forwarded
CustomLog "logs/access.log" combined env=!forwarded
CustomLog "logs/access.log" proxy env=forwarded
On the net i was able to find working examples with Apache common log only. My config:
define ROOT C:\\Program Files (x86)\\nxlog
define ROOT_STRING C:\\Program Files (x86)\\nxlog
define CERTDIR %ROOT%\\cert
Moduledir %ROOT%\\modules
CacheDir %ROOT%\\data
Pidfile %ROOT%\\data\\nxlog.pid
SpoolDir %ROOT%\\data
LogFile %ROOT%\\data\\nxlog.log
<extension fileop="">
Module xm_fileop
</extension><extension json="">
Module xm_json
</extension><extension syslog="">
Module xm_syslog
</extension><extension exec="">
Module xm_exec
</extension>
## Create the parse rule for IIS logs. You can copy these from the header of the IIS log file.
<extension w3c="">
Module xm_csv
Fields $date, $time, $s-ip, $cs-method, $cs-uri-stem, $cs-uri-query, $s-port, $cs-username, $c-ip, $csUser-Agent, $csReferer, $sc-status, $sc-substatus, $sc-win32-status, $time-taken
FieldTypes string, string, string, string, string, string, string, string, string, string, string, string, string, string, string
Delimiter ' '
UndefValue -
</extension>
## In questa macchina con frontend Apache usiamo ext kvp invece di w3c
<extension kvp="">
Module xm_kvp
KVPDelimiter &
KVDelimiter =
</extension><extension kvp2="">
Module xm_kvp
KVPDelimiter ;
KVDelimiter =
#QuoteMethod None
</extension><input apache="" />
Module im_file
File "C:\Apache_install\httpd-2.4.25-win64-VC14\Apache24\logs\access.log"
Exec if $raw_event =~ /^(\S+) (\S+) (\S+) \[([^\]]+)\] \"(\S+) (.+) HTTP.\d\.\d\" (\d+) (\d+) \"([^\"]+)\" \"([^\"]+)\"/\
{ \
$Hostname = $1; \
if $3 != '-' $AccountName = $3; \
$EventTime = parsedate($4); \
$HTTPMethod = $5; \
$HTTPURL = $6; \
$HTTPResponseStatus = $7; \
$FileSize = $8; \
$HTTPReferer = $9; \
$HTTPUserAgent = $10; \
}
#Exec if $raw_event =~ /^(\S+) (\S+) (\S+) \[([^\]]+)\] \"(\S+) (.+) HTTP.\d\.\d\" (\d+) (\d+) \"([^\"]+)\" \"([^\"]+)\"/\
# { \
# $Hostname = $1; \
# if $3 != '-' $AccountName = $3; \
# $EventTime = parsedate($4); \
# $HTTPMethod = $5; \
# $HTTPURL = $6; \
# $HTTPResponseStatus = $7; \
# $FileSize = $8; \
# $HTTPReferer = $9; \
# $HTTPUserAgent = $10; \
# if $HTTPURL =~ /\?(.+)/ { $HTTPParams = $1; } \
# kvp->parse_kvp($HTTPParams); \
# delete($EventReceivedTime); \
# kvp2->to_kvp(); \
# }
<input internal="" />
Module im_internal
Exec $Message = to_json();
# Windows Event Log
<input eventlog="" />
Module im_msvistalog
# Query per ridurre Event Log . Usato il QueryXML non il comando Query
# I commenti nella quey vanno indicati in XML: <!-- stringa -->
<queryxml><querylist><query id="0"><!-- Select --><select path="Application">*[System[(Level=1 or Level=2 or Level=3)]]</select><select path="Security">*[System[(Level=1 or Level=2 or Level=3)]]</select><select path="System">*[System[(Level=1 or Level=2 or Level=3)]]</select><select path="ForwardedEvents">*</select><select path="Setup">*</select><select path="HardwareEvents">*</select><select path="Microsoft-Windows-PowerShell/Operational">*[System[(Level=1 or Level=2 or Level=3)]]</select><select path="Microsoft-Windows-TaskScheduler/Operational">*[System[(Level=1 or Level=2 or Level=3)]]</select><!-- Suppress --><suppress path="Security">*[System[(EventID=4689 or EventID=5158 or EventID=5440 or EventID=5444)]]</suppress><suppress path="Windows PowerShell">*[System[(EventID=501 or EventID=400 or EventID=600)]]</suppress></query></querylist></queryxml>
Exec $EventReceivedTime = integer($EventReceivedTime) / 1000000;
Exec to_json();
# 100Mb disk buffer
<processor buffer="">
Module pm_buffer
MaxSize 102400
Type disk
</processor>
# RFC5424 come indicato https://www.scip.ch/en/?labs.20141106
<processor rfc5424="">
Module pm_transformer
Exec $Hostname = hostname();
Outputformat syslog_rfc5424
</processor><output ssl_out="">
Module om_ssl
Host IP.IP.IP.IP
Port 443
CAFile %CERTDIR%/nxlog_rootCA.crt
CertFile %CERTDIR%/client.crt
CertKeyFile %CERTDIR%/client.key
KeyPass secret
AllowUntrusted TRUE
OutputType Binary
Exec to_syslog_ietf();
# Rimuovo CRLF LF TAB - lato server in om_file non funziona
Exec $raw_event =~ s/(\t|\r|\n)//g; $raw_event = replace($raw_event, '{', '[" "] {', 1);
#tag windows
Exec $raw_event =~ s/(\[.*])//g; $raw_event = replace($raw_event, '{', '[tag="windows"] {', 1);
#Use the following line for debugging (uncomment the fileop extension above as well)
#Exec file_write("C:\\Program Files (x86)\\nxlog\\data\\nxlog_output.log", $raw_event);
</output><route>
Path Apache, internal, eventlog => rfc5424 => buffer => ssl_out
</route>
```
Is there any solution for nxlog conf to work with this kind of apache custom log?
Thanks in advance
If the question is about the following then I suggest tweaking the regexp or validating $HTTPParams before passing it to parse_kvp().
kvp->parse_kvp($HTTPParams);
Otherwise you should rephrase your question and get to the point. You'd be very lucky if anyone would start setting up a similar environment and try to figure out what the exact problem is to solve this for you.
