1
response

Hi,
I'm using Graylog as logging platform and in the error logs I'm facing an issue related to the way Nxlog is forwarding messages to Graylog.

Here the stacktrace followed by the nxlog configuration:

2021-10-19T17:00:24.464+02:00 ERROR [DecodingProcessor] Error processing message RawMessage{id=49ba6ee1-30ed-11ec-a554-005056a17083, journalOffset=140463071838, codec=gelf, payloadSize=249, timestamp=2021-10-19T15:00:24.398Z, remoteAddress=/10.181.8.23:39018}
java.lang.IllegalArgumentException: GELF message <49ba6ee1-30ed-11ec-a554-005056a17083> (received from <10.181.8.23:39018>) has empty mandatory "short_message" field.
at org.graylog2.inputs.codecs.GelfCodec.validateGELFMessage(GelfCodec.java:258) ~[graylog.jar:?]
at org.graylog2.inputs.codecs.GelfCodec.decode(GelfCodec.java:140) ~[graylog.jar:?]
at org.graylog2.shared.buffers.processors.DecodingProcessor.processMessage(DecodingProcessor.java:149) ~[graylog.jar:?]
at org.graylog2.shared.buffers.processors.DecodingProcessor.onEvent(DecodingProcessor.java:90) [graylog.jar:?]
at org.graylog2.shared.buffers.processors.ProcessBufferProcessor.onEvent(ProcessBufferProcessor.java:90) [graylog.jar:?]
at org.graylog2.shared.buffers.processors.ProcessBufferProcessor.onEvent(ProcessBufferProcessor.java:47) [graylog.jar:?]
at com.lmax.disruptor.WorkProcessor.run(WorkProcessor.java:143) [graylog.jar:?]
at com.codahale.metrics.InstrumentedThreadFactory$InstrumentedRunnable.run(InstrumentedThreadFactory.java:66) [graylog.jar:?]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_292]
2021-10-19T17:00:24.464+02:00 ERROR [DecodingProcessor] Unable to decode raw message RawMessage{id=49ba47dc-30ed-11ec-a554-005056a17083, journalOffset=140463071836, codec=gelf, payloadSize=249, timestamp=2021-10-19T15:00:24.397Z, remoteAddress=/10.181.8.23:39018} on input <581344a7a0474e1cf763f7f4>.
2021-10-19T17:00:24.469+02:00 ERROR [DecodingProcessor] Error processing message RawMessage{id=49bb3232-30ed-11ec-a554-005056a17083, journalOffset=140463071886, codec=gelf, payloadSize=249, timestamp=2021-10-19T15:00:24.403Z, remoteAddress=/10.181.8.23:39018}
java.lang.IllegalArgumentException: GELF message <49bb3232-30ed-11ec-a554-005056a17083> (received from <10.181.8.23:39018>) has empty mandatory "short_message" field.
at org.graylog2.inputs.codecs.GelfCodec.validateGELFMessage(GelfCodec.java:258) ~[graylog.jar:?]
at org.graylog2.inputs.codecs.GelfCodec.decode(GelfCodec.java:140) ~[graylog.jar:?]
at org.graylog2.shared.buffers.processors.DecodingProcessor.processMessage(DecodingProcessor.java:149) ~[graylog.jar:?]
at org.graylog2.shared.buffers.processors.DecodingProcessor.onEvent(DecodingProcessor.java:90) [graylog.jar:?]
at org.graylog2.shared.buffers.processors.ProcessBufferProcessor.onEvent(ProcessBufferProcessor.java:90) [graylog.jar:?]
at org.graylog2.shared.buffers.processors.ProcessBufferProcessor.onEvent(ProcessBufferProcessor.java:47) [graylog.jar:?]
at com.lmax.disruptor.WorkProcessor.run(WorkProcessor.java:143) [graylog.jar:?]
at com.codahale.metrics.InstrumentedThreadFactory$InstrumentedRunnable.run(InstrumentedThreadFactory.java:66) [graylog.jar:?]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_292]
@
----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------

define ROOT /usr/bin
#define REGEX /^(?x)^\[(?<EventTime>\d{4}\-\d{2}\-\d{2}\ \d{2}\:\d{2}\:\d{2})\,\d{3}\]/
define REGEX /(?x)^\[(?<EventTime2>\d{4}\-\d{2}\-\d{2}\ \d{2}\:\d{2}\:\d{2})\,\d{3}\]\ (?<Severity>\S+)\ (?<Class>\S+)\ \-\ (?<Message>[\s\S]+)/
define REGEX_CATALINA /(?x)^(?<EventTime>\d{2}\-\w{3}\-\d{4}\ \d{2}\:\d{2}\:\d{2})\.\d{3}\ (?<Message>[\s\S]+) | (?x)^(?<EventTime2>\w{3}\ \w{3}\ \d{2}\ \d{2}\:\d{2}\:\d{2})\ (?<Message2>[\s\S]+)/

<Extension gelfExt>
Module xm_gelf
# Avoid truncation of the short_message field to 64 characters.
ShortMessageLength 65536
</Extension>

<Extension syslogExt>
Module xm_syslog
</Extension>

<Extension multiline>
Module xm_multiline
HeaderLine %REGEX%
</Extension>

User nxlog
Group sfd

#Moduledir /usr/lib/nxlog/modules
Moduledir /usr/libexec/nxlog/modules
CacheDir /var/spool/nxlog
PidFile /var/run/nxlog/nxlog.pid
LogFile /var/log/nxlog/nxlog.log
LogLevel INFO

<Input file_catalina1>
Module im_file
File '/opt/tomcat/logs/catalina.out'
PollInterval 1
SavePos True
ReadFromLast True
Recursive False
RenameCheck False
Exec $FileName = file_name(); # Send file name with each message
</Input>
<Input file_catalina2>
Module im_file
File '/opt/tomcat8081/logs/catalina.out'
PollInterval 1
SavePos True
ReadFromLast True
Recursive False
RenameCheck False
Exec $FileName = file_name(); # Send file name with each message
</Input>
<Input file_catalina3>
Module im_file
File '/opt/tomcat8082/logs/catalina.out'
PollInterval 1
SavePos True
ReadFromLast True
Recursive False
RenameCheck False
Exec $FileName = file_name(); # Send file name with each message
</Input>
#<Input file_catalina>
# Module im_file
# File "/opt/tomcat/logs/catalina.out"
# InputType multiline
# <Exec>
# if $raw_event =~ %REGEX_CATALINA% $EventTime = parsedate($1);
# </Exec>
#</Input>

<Input log4j1>
Module im_file
File "/opt/tomcat/logs/*error*.log"
InputType multiline
<Exec>
if $raw_event =~ %REGEX% $EventTime = parsedate($1);
</Exec>
</Input>
<Input log4j2>
Module im_file
File "/opt/tomcat8081/logs/*error*.log"
InputType multiline
<Exec>
if $raw_event =~ %REGEX% $EventTime = parsedate($1);
</Exec>
</Input>
<Input log4j3>
Module im_file
File "/opt/tomcat8082/logs/*error*.log"
InputType multiline
<Exec>
if $raw_event =~ %REGEX% $EventTime = parsedate($1);
</Exec>
</Input>

#<Input file_logs>
# Module im_file
# File '/opt/tomcat8081/logs/*error*.log'
# PollInterval 1
# SavePos True
# ReadFromLast True
# Recursive False
# RenameCheck False
# Exec $FileName = file_name(); # Send file name with each message
#</Input>

#<Input syslog-udp>
# Module im_udp
# Host 127.0.0.1
# Port 514
# Exec parse_syslog_bsd();
#</Input>

<Output gelf>
Module om_udp
Host graylog.intern.ti8m.ch
Port 1515
OutputType GELF
<Exec>
# These fields are needed for Graylog
$gl2_source_collector = 'fb04f873-aed9-441a-b3a9-7545d4c728f6';
$collector_node_id = 'app-01.prd.sfd.intern.ti8m.ch';
$Hostname = 'app-01.prd.sfd.intern.ti8m.ch';
</Exec>
</Output>

<Route route-1>
Path file_catalina1 => gelf
</Route>
<Route route-2>
Path log4j1 => gelf
</Route>

<Route route-3>
Path file_catalina2 => gelf
</Route>
<Route route-4>
Path log4j2 => gelf
</Route>

<Route route-5>
Path file_catalina3 => gelf
</Route>
<Route route-6>
Path log4j3 => gelf

</Route>

#<Route route-7>
# Path syslog-udp => gelf
#</Route>

Hope you can help solving the issue.

Thank you for your time.

Best regards,

Stefano Paci

AskedOctober 19, 2021 - 5:26pm

Answer (1)