Apache log with custom log


#1 comoalt

Hello guyz, i am setting up nxlog service in our network and i am focusing to work with apache custom log and nxlog. Since Apache is at the end of a reverse proxies chain, the only way to keep forwarders ip adresses, is to use an Apache conditional variable (X-Forwarded-For) able to switch between combined and custom log (as explained in details here: http://www.techstacks.com/howto/log-client-ip-and-xforwardedfor-ip-in-apache.html). Apache log variable %h is replaced by %{X-Forwarded-For}i which is a string empty or containing one / more ip addresses. In this specific case when nxlog hit a custom log, parse_kvp error is reported. My Apache conf includes: LogFormat "%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-Agent}i"" combined LogFormat "%{X-Forwarded-For}i %l %u %t "%r" %>s %b "%{Referer}i" "%{User-Agent}i"" proxy SetEnvIf X-Forwarded-For "^......." forwarded CustomLog "logs/access.log" combined env=!forwarded CustomLog "logs/access.log" proxy env=forwarded On the net i was able to find working examples with Apache common log only. My config:

define ROOT C:\\Program Files (x86)\\nxlog
define ROOT_STRING C:\\Program Files (x86)\\nxlog
define CERTDIR %ROOT%\\cert

Moduledir %ROOT%\\modules
CacheDir %ROOT%\\data
Pidfile %ROOT%\\data\\nxlog.pid
SpoolDir %ROOT%\\data
LogFile %ROOT%\\data\\nxlog.log

<extension fileop="">&#13;
 Module xm_fileop&#13;
</extension><extension json="">&#13;
 Module xm_json&#13;
</extension><extension syslog="">&#13;
 Module xm_syslog&#13;
</extension><extension exec="">&#13;
    Module  xm_exec&#13;
</extension>&#13;
&#13;
## Create the parse rule for IIS logs. You can copy these from the header of the IIS log file.&#13;
<extension w3c="">&#13;
	Module xm_csv&#13;
	Fields $date, $time, $s-ip, $cs-method, $cs-uri-stem, $cs-uri-query, $s-port, $cs-username, $c-ip, $csUser-Agent, $csReferer, $sc-status, $sc-substatus, $sc-win32-status, $time-taken&#13;
	FieldTypes string, string, string, string, string, string, string, string, string, string, string, string, string, string, string&#13;
	Delimiter ' '&#13;
	UndefValue -&#13;
</extension>&#13;
&#13;
## In questa macchina con frontend Apache usiamo ext kvp invece di w3c&#13;
<extension kvp="">&#13;
    Module          xm_kvp&#13;
    KVPDelimiter    &amp;&#13;
    KVDelimiter     =&#13;
</extension><extension kvp2="">&#13;
    Module          xm_kvp&#13;
    KVPDelimiter    ;&#13;
    KVDelimiter     =&#13;
    #QuoteMethod    None&#13;
</extension><input apache="" />&#13;
    Module          im_file&#13;
    File            "C:\Apache_install\httpd-2.4.25-win64-VC14\Apache24\logs\access.log"&#13;
	Exec    if $raw_event =~ /^(\S+) (\S+) (\S+) \[([^\]]+)\] \"(\S+) (.+) HTTP.\d\.\d\" (\d+) (\d+) \"([^\"]+)\" \"([^\"]+)\"/\&#13;
		{ \&#13;
		  $Hostname = $1; \&#13;
		  if $3 != '-' $AccountName = $3; \&#13;
		  $EventTime = parsedate($4); \&#13;
		  $HTTPMethod = $5; \&#13;
		  $HTTPURL = $6; \&#13;
		  $HTTPResponseStatus = $7; \&#13;
		  $FileSize = $8; \&#13;
		  $HTTPReferer = $9; \&#13;
		  $HTTPUserAgent = $10; \&#13;
		}&#13;
	&#13;
	#Exec if $raw_event =~ /^(\S+) (\S+) (\S+) \[([^\]]+)\] \"(\S+) (.+) HTTP.\d\.\d\" (\d+) (\d+) \"([^\"]+)\" \"([^\"]+)\"/\&#13;
	#	{ 													\&#13;
	#		$Hostname = $1; 								\&#13;
	#		if $3 != '-' $AccountName = $3; 				\&#13;
	#		$EventTime = parsedate($4); 					\&#13;
	#		$HTTPMethod = $5; 								\&#13;
	#		$HTTPURL = $6; 									\&#13;
	#		$HTTPResponseStatus = $7; 						\&#13;
	#		$FileSize = $8; 								\&#13;
	#		$HTTPReferer = $9; 								\&#13;
	#		$HTTPUserAgent = $10; 							\&#13;
	#		if $HTTPURL =~ /\?(.+)/ { $HTTPParams = $1; }	\&#13;
	#		kvp-&gt;parse_kvp($HTTPParams); 					\&#13;
	#		delete($EventReceivedTime); 					\&#13;
	#		kvp2-&gt;to_kvp(); 								\&#13;
	#	}&#13;
&#13;
&#13;
<input internal="" />&#13;
 Module im_internal&#13;
 Exec $Message = to_json(); &#13;
&#13;
 &#13;
# Windows Event Log&#13;
<input eventlog="" />&#13;
	Module im_msvistalog&#13;
	# Query per ridurre Event Log . Usato il QueryXML non il comando Query&#13;
	# I commenti nella quey vanno indicati in XML: <!-- stringa -->&#13;
	<queryxml><querylist><query id="0"><!-- Select --><select path="Application">*[System[(Level=1  or Level=2 or Level=3)]]</select><select path="Security">*[System[(Level=1  or Level=2 or Level=3)]]</select><select path="System">*[System[(Level=1  or Level=2 or Level=3)]]</select><select path="ForwardedEvents">*</select><select path="Setup">*</select><select path="HardwareEvents">*</select><select path="Microsoft-Windows-PowerShell/Operational">*[System[(Level=1  or Level=2 or Level=3)]]</select><select path="Microsoft-Windows-TaskScheduler/Operational">*[System[(Level=1  or Level=2 or Level=3)]]</select><!-- Suppress --><suppress path="Security">*[System[(EventID=4689 or EventID=5158 or EventID=5440 or EventID=5444)]]</suppress><suppress path="Windows PowerShell">*[System[(EventID=501 or EventID=400 or EventID=600)]]</suppress></query></querylist></queryxml>&#13;
	Exec $EventReceivedTime = integer($EventReceivedTime) / 1000000;&#13;
	Exec to_json();&#13;
&#13;
&#13;
# 100Mb disk buffer&#13;
<processor buffer="">&#13;
	Module pm_buffer&#13;
	MaxSize 102400&#13;
	Type disk&#13;
</processor>&#13;
&#13;
# RFC5424 come indicato https://www.scip.ch/en/?labs.20141106&#13;
<processor rfc5424="">&#13;
	Module pm_transformer&#13;
	Exec $Hostname = hostname();&#13;
	Outputformat syslog_rfc5424&#13;
</processor><output ssl_out="">&#13;
    Module	om_ssl&#13;
    Host	IP.IP.IP.IP&#13;
    Port	443&#13;
    CAFile		%CERTDIR%/nxlog_rootCA.crt&#13;
    CertFile	%CERTDIR%/client.crt&#13;
    CertKeyFile	%CERTDIR%/client.key&#13;
    KeyPass	secret&#13;
    AllowUntrusted TRUE&#13;
    OutputType	Binary&#13;
	&#13;
	Exec to_syslog_ietf();&#13;
	# Rimuovo CRLF LF TAB - lato server in om_file non funziona&#13;
	Exec $raw_event =~ s/(\t|\r|\n)//g; $raw_event = replace($raw_event, '{', '[" "] {', 1);&#13;
	 &#13;
	#tag windows&#13;
	Exec $raw_event =~ s/(\[.*])//g; $raw_event = replace($raw_event, '{', '[tag="windows"] {', 1);&#13;
&#13;
	#Use the following line for debugging (uncomment the fileop extension above as well)&#13;
	#Exec file_write("C:\\Program Files (x86)\\nxlog\\data\\nxlog_output.log", $raw_event);&#13;
</output><route>&#13;
 Path  Apache, internal, eventlog =&gt; rfc5424 =&gt; buffer =&gt; ssl_out&#13;
</route>&#13;
```&#13;
Is there any solution for nxlog conf to work with this kind of apache custom log?&#13;
&#13;
Thanks in advance
#2 b0ti Nxlog ✓
#1 comoalt
Hello guyz, i am setting up nxlog service in our network and i am focusing to work with apache custom log and nxlog. Since Apache is at the end of a reverse proxies chain, the only way to keep forwarders ip adresses, is to use an Apache conditional variable (X-Forwarded-For) able to switch between combined and custom log (as explained in details here: http://www.techstacks.com/howto/log-client-ip-and-xforwardedfor-ip-in-apache.html). Apache log variable %h is replaced by %{X-Forwarded-For}i which is a string empty or containing one / more ip addresses. In this specific case when nxlog hit a custom log, parse_kvp error is reported. My Apache conf includes: LogFormat "%h %l %u %t "%r" %>s %b "%{Referer}i" "%{User-Agent}i"" combined LogFormat "%{X-Forwarded-For}i %l %u %t "%r" %>s %b "%{Referer}i" "%{User-Agent}i"" proxy SetEnvIf X-Forwarded-For "^......." forwarded CustomLog "logs/access.log" combined env=!forwarded CustomLog "logs/access.log" proxy env=forwarded On the net i was able to find working examples with Apache common log only. My config: define ROOT C:\\Program Files (x86)\\nxlog&#13; define ROOT_STRING C:\\Program Files (x86)\\nxlog&#13; define CERTDIR %ROOT%\\cert&#13; &#13; Moduledir %ROOT%\\modules&#13; CacheDir %ROOT%\\data&#13; Pidfile %ROOT%\\data\\nxlog.pid&#13; SpoolDir %ROOT%\\data&#13; LogFile %ROOT%\\data\\nxlog.log&#13; &#13; <extension fileop="">&#13; Module xm_fileop&#13; </extension><extension json="">&#13; Module xm_json&#13; </extension><extension syslog="">&#13; Module xm_syslog&#13; </extension><extension exec="">&#13; Module xm_exec&#13; </extension>&#13; &#13; ## Create the parse rule for IIS logs. You can copy these from the header of the IIS log file.&#13; <extension w3c="">&#13; Module xm_csv&#13; Fields $date, $time, $s-ip, $cs-method, $cs-uri-stem, $cs-uri-query, $s-port, $cs-username, $c-ip, $csUser-Agent, $csReferer, $sc-status, $sc-substatus, $sc-win32-status, $time-taken&#13; FieldTypes string, string, string, string, string, string, string, string, string, string, string, string, string, string, string&#13; Delimiter ' '&#13; UndefValue -&#13; </extension>&#13; &#13; ## In questa macchina con frontend Apache usiamo ext kvp invece di w3c&#13; <extension kvp="">&#13; Module xm_kvp&#13; KVPDelimiter &amp;&#13; KVDelimiter =&#13; </extension><extension kvp2="">&#13; Module xm_kvp&#13; KVPDelimiter ;&#13; KVDelimiter =&#13; #QuoteMethod None&#13; </extension><input apache="" />&#13; Module im_file&#13; File "C:\Apache_install\httpd-2.4.25-win64-VC14\Apache24\logs\access.log"&#13; Exec if $raw_event =~ /^(\S+) (\S+) (\S+) \[([^\]]+)\] \"(\S+) (.+) HTTP.\d\.\d\" (\d+) (\d+) \"([^\"]+)\" \"([^\"]+)\"/\&#13; { \&#13; $Hostname = $1; \&#13; if $3 != '-' $AccountName = $3; \&#13; $EventTime = parsedate($4); \&#13; $HTTPMethod = $5; \&#13; $HTTPURL = $6; \&#13; $HTTPResponseStatus = $7; \&#13; $FileSize = $8; \&#13; $HTTPReferer = $9; \&#13; $HTTPUserAgent = $10; \&#13; }&#13; &#13; #Exec if $raw_event =~ /^(\S+) (\S+) (\S+) \[([^\]]+)\] \"(\S+) (.+) HTTP.\d\.\d\" (\d+) (\d+) \"([^\"]+)\" \"([^\"]+)\"/\&#13; # { \&#13; # $Hostname = $1; \&#13; # if $3 != '-' $AccountName = $3; \&#13; # $EventTime = parsedate($4); \&#13; # $HTTPMethod = $5; \&#13; # $HTTPURL = $6; \&#13; # $HTTPResponseStatus = $7; \&#13; # $FileSize = $8; \&#13; # $HTTPReferer = $9; \&#13; # $HTTPUserAgent = $10; \&#13; # if $HTTPURL =~ /\?(.+)/ { $HTTPParams = $1; } \&#13; # kvp-&gt;parse_kvp($HTTPParams); \&#13; # delete($EventReceivedTime); \&#13; # kvp2-&gt;to_kvp(); \&#13; # }&#13; &#13; &#13; <input internal="" />&#13; Module im_internal&#13; Exec $Message = to_json(); &#13; &#13; &#13; # Windows Event Log&#13; <input eventlog="" />&#13; Module im_msvistalog&#13; # Query per ridurre Event Log . Usato il QueryXML non il comando Query&#13; # I commenti nella quey vanno indicati in XML: <!-- stringa -->&#13; <queryxml><querylist><query id="0"><!-- Select --><select path="Application">*[System[(Level=1 or Level=2 or Level=3)]]</select><select path="Security">*[System[(Level=1 or Level=2 or Level=3)]]</select><select path="System">*[System[(Level=1 or Level=2 or Level=3)]]</select><select path="ForwardedEvents">*</select><select path="Setup">*</select><select path="HardwareEvents">*</select><select path="Microsoft-Windows-PowerShell/Operational">*[System[(Level=1 or Level=2 or Level=3)]]</select><select path="Microsoft-Windows-TaskScheduler/Operational">*[System[(Level=1 or Level=2 or Level=3)]]</select><!-- Suppress --><suppress path="Security">*[System[(EventID=4689 or EventID=5158 or EventID=5440 or EventID=5444)]]</suppress><suppress path="Windows PowerShell">*[System[(EventID=501 or EventID=400 or EventID=600)]]</suppress></query></querylist></queryxml>&#13; Exec $EventReceivedTime = integer($EventReceivedTime) / 1000000;&#13; Exec to_json();&#13; &#13; &#13; # 100Mb disk buffer&#13; <processor buffer="">&#13; Module pm_buffer&#13; MaxSize 102400&#13; Type disk&#13; </processor>&#13; &#13; # RFC5424 come indicato https://www.scip.ch/en/?labs.20141106&#13; <processor rfc5424="">&#13; Module pm_transformer&#13; Exec $Hostname = hostname();&#13; Outputformat syslog_rfc5424&#13; </processor><output ssl_out="">&#13; Module om_ssl&#13; Host IP.IP.IP.IP&#13; Port 443&#13; CAFile %CERTDIR%/nxlog_rootCA.crt&#13; CertFile %CERTDIR%/client.crt&#13; CertKeyFile %CERTDIR%/client.key&#13; KeyPass secret&#13; AllowUntrusted TRUE&#13; OutputType Binary&#13; &#13; Exec to_syslog_ietf();&#13; # Rimuovo CRLF LF TAB - lato server in om_file non funziona&#13; Exec $raw_event =~ s/(\t|\r|\n)//g; $raw_event = replace($raw_event, '{', '[" "] {', 1);&#13; &#13; #tag windows&#13; Exec $raw_event =~ s/(\[.*])//g; $raw_event = replace($raw_event, '{', '[tag="windows"] {', 1);&#13; &#13; #Use the following line for debugging (uncomment the fileop extension above as well)&#13; #Exec file_write("C:\\Program Files (x86)\\nxlog\\data\\nxlog_output.log", $raw_event);&#13; </output><route>&#13; Path Apache, internal, eventlog =&gt; rfc5424 =&gt; buffer =&gt; ssl_out&#13; </route>&#13; ```&#13; Is there any solution for nxlog conf to work with this kind of apache custom log?&#13; &#13; Thanks in advance

If the question is about the following then I suggest tweaking the regexp or validating $HTTPParams before passing it to parse_kvp().

kvp->parse_kvp($HTTPParams); 

Otherwise you should rephrase your question and get to the point. You'd be very lucky if anyone would start setting up a similar environment and try to figure out what the exact problem is to solve this for you.