responses
Hi all,
I'm trying to integrate the Exchange Logs to NXLog CE using the example config from https://nxlog.co/documentation/nxlog-user-guide/exchange.html which does'nt work. After reading through the forums I have come up with the following nxlog.conf which still doesn't work producing the errors at the end of this post. Is there any comprehensive guide to setup it up?
---------------- nxlog.conf ----------------------
Panic Soft
#NoFreeOnExit TRUE
define ROOT C:\Program Files (x86)\nxlog
define CERTDIR %ROOT%\cert
define CONFDIR %ROOT%\conf
define LOGDIR %ROOT%\data
define LOGFILE %LOGDIR%\nxlog.log
define BASEDIR E:\Exchange Server
LogFile %LOGFILE%
Moduledir %ROOT%\modules
CacheDir %ROOT%\data
Pidfile %ROOT%\data\nxlog.pid
SpoolDir %ROOT%\data
<Extension _syslog>
Module xm_syslog
</Extension>
<Extension _charconv>
Module xm_charconv
AutodetectCharsets iso8859-2, utf-8, utf-16, utf-32
</Extension>
<Extension _exec>
Module xm_exec
</Extension>
<Extension _fileop>
Module xm_fileop
# Check the size of our log file hourly, rotate if larger than 5MB
<Schedule>
Every 1 hour
Exec if (file_exists('%LOGFILE%') and \
(file_size('%LOGFILE%') >= 5M)) \
file_cycle('%LOGFILE%', 8);
</Schedule>
# Rotate our log file every week on Sunday at midnight
<Schedule>
When @weekly
Exec if file_exists('%LOGFILE%') file_cycle('%LOGFILE%', 8);
</Schedule>
</Extension>
<Extension csv_parser>
Module xm_csv
Fields date-time, client-ip, client-hostname, server-ip, server-hostname, \
source-context, connector-id, source, event-id, \
internal-message-id, message-id, network-message-id, \
recipient-address, recipient-status, total-bytes, recipient-count, \
related-recipient-address, reference, message-subject, \
sender-address, return-path, message-info, directionality, \
tenant-id, original-client-ip, original-server-ip, custom-data, \
transport-traffic-type, log-id, schema-version, \
session, source, Destination, direction, description, \
session-id, sequence-number, \
local-endpoint, remote-endpoint, event, data, context
Exec rename_field('sender-address', 'sender_address'); $sender_address = lc($sender_address);
Exec rename_field('date-time', 'date_time'); $date_time = lc($date_time);
</Extension>
<Input messagetracking>
Module im_file
File '%BASEDIR%\TransportRoles\Logs\MessageTracking\MSGTRK*.LOG'
<Exec>
if $raw_event =~ /^(\xEF\xBB\xBF)?(date-time,|#)/ drop();
else
{
csv_parser->parse_csv();
$EventTime = parsedate($date_time);
}
</Exec>
</Input>
<Input connectivity>
Module im_file
File '%BASEDIR%\TransportRoles\Logs\Hub\Connectivity\CONNECTLOG*.LOG'
<Exec>
if $raw_event =~ /^(\xEF\xBB\xBF)?(date-time,|#)/ drop();
else
{
csv_parser->parse_csv();
$EventTime = parsedate($date_time);
}
</Exec>
</Input>
<Input smtp_receive>
Module im_file
File '%BASEDIR%\TransportRoles\Logs\Hub\ProtocolLog\SmtpReceive\RECV*.LOG'
<Exec>
if $raw_event =~ /^(\xEF\xBB\xBF)?(date-time,|#)/ drop();
else
{
csv_parser->parse_csv();
$EventTime = parsedate($date_time);
}
</Exec>
</Input>
<Input smtp_send>
Module im_file
File '%BASEDIR%\TransportRoles\Logs\Hub\ProtocolLog\SmtpSend\SEND*.LOG'
<Exec>
if $raw_event =~ /^(\xEF\xBB\xBF)?(date-time,|#)/ drop();
else
{
csv_parser->parse_csv();
$EventTime = parsedate($date_time);
}
</Exec>
</Input>
<Extension _json>
Module xm_json
</Extension>
<Output out>
Module om_http
URL http://kibanaip:9200
ContentType application/json
<Exec>
set_http_request_path(strftime($EventTime, "/nxlog-%Y%m%d/" +
$SourceModuleName));
rename_field("timestamp", "@timestamp");
to_json();
</Exec>
</Output>
---------------- nxlog.conf ----------------------
Errors:
2021-03-05 11:46:03 WARNING no routes defined!
2021-03-05 11:46:03 WARNING not starting unused module messagetracking
2021-03-05 11:46:03 WARNING not starting unused module connectivity
2021-03-05 11:46:03 WARNING not starting unused module smtp_receive
2021-03-05 11:46:03 WARNING not starting unused module smtp_send
2021-03-05 11:46:03 WARNING not starting unused module out
2021-03-05 11:46:03 INFO nxlog-ce-2.10.2150 started
Thank You in advance!
Gregory
Comments (1)
Now I have changed my config according to various community posts to the below conf file:
------------------------------------------------------conf file---------------------------------------------------------------------
Panic Soft
#NoFreeOnExit TRUE
define ROOT C:\Program Files (x86)\nxlog
define CERTDIR %ROOT%\cert
define CONFDIR %ROOT%\conf
define LOGDIR %ROOT%\data
define LOGFILE %LOGDIR%\nxlog.log
define BASEDIR E:\Exchange Server
LogFile %LOGFILE%
#LogLevel DEBUG
Moduledir %ROOT%\modules
CacheDir %ROOT%\data
Pidfile %ROOT%\data\nxlog.pid
SpoolDir %ROOT%\data
<Extension _syslog>
Module xm_syslog
</Extension>
<Extension _charconv>
Module xm_charconv
AutodetectCharsets iso8859-2, utf-8, utf-16, utf-32
</Extension>
<Extension _exec>
Module xm_exec
</Extension>
<Extension _fileop>
Module xm_fileop
# Check the size of our log file hourly, rotate if larger than 5MB
<Schedule>
Every 1 hour
Exec if (file_exists('%LOGFILE%') and \
(file_size('%LOGFILE%') >= 5M)) \
file_cycle('%LOGFILE%', 8);
</Schedule>
# Rotate our log file every week on Sunday at midnight
<Schedule>
When @weekly
Exec if file_exists('%LOGFILE%') file_cycle('%LOGFILE%', 8);
</Schedule>
</Extension>
<Extension csv_parser_mt>
Module xm_csv
Fields $date-time,$client-ip,$client-hostname,$server-ip,$server-hostname,$source-context,$connector-id,$source,$event-id,$internal-message-id,$message-id,$network-message-id,$recipient-address,$recipient-status,$total-bytes,$recipient-count,$related-recipient-address,$reference,$message-subject,$sender-address,$return-path,$message-info,$directionality,$tenant-id,$original-client-ip,$original-server-ip,$custom-data
FieldTypes datetime,ip4addr,String,ip4addr,String,String,String,String,String,String,String,String,String,String,Integer,String,String,String,String,String,String,String,String,String,ip4addr,ip4addr,String
Exec rename_field('sender-address', 'sender_address'); $sender_address = lc($sender_address);
Exec rename_field('date-time', 'date_time'); $date_time = lc($date_time);
</Extension>
<Extension csv_parser_con>
Module xm_csv
Fields date-time, session, source, Destination, direction, description
Exec rename_field('date-time', 'date_time'); $date_time = lc($date_time);
</Extension>
<Extension csv_parser_smtp>
Module xm_csv
Fields date-time, connector-id, session-id, sequence-number, \
local-endpoint, remote-endpoint, event, data, context
Exec rename_field('date-time', 'date_time'); $date_time = lc($date_time);
</Extension>
<Input messagetracking>
Module im_file
File '%BASEDIR%\TransportRoles\Logs\MessageTracking\MSGTRK*.LOG'
SavePos TRUE
Exec if $raw_event =~ /HealthMailbox/ drop();
Exec if $raw_event =~ /^#/ drop();
</Input>
<Input connectivity>
Module im_file
File '%BASEDIR%\TransportRoles\Logs\Hub\Connectivity\CONNECTLOG*.LOG'
SavePos TRUE
Exec if $raw_event =~ /HealthMailbox/ drop();
Exec if $raw_event =~ /^#/ drop();
</Input>
<Input smtp_receive>
Module im_file
File '%BASEDIR%\TransportRoles\Logs\Hub\ProtocolLog\SmtpReceive\RECV*.LOG'
SavePos TRUE
Exec if $raw_event =~ /HealthMailbox/ drop();
Exec if $raw_event =~ /^#/ drop();
</Input>
<Input smtp_send>
Module im_file
File '%BASEDIR%\TransportRoles\Logs\Hub\ProtocolLog\SmtpSend\SEND*.LOG'
SavePos TRUE
Exec if $raw_event =~ /HealthMailbox/ drop();
Exec if $raw_event =~ /^#/ drop();
</Input>
<Extension _json>
Module xm_json
</Extension>
<Output out>
Module om_http
URL http://kibana:9200
ContentType application/json
<Exec>
set_http_request_path(strftime($date_time, "/nxlog-%Y%m%d/" + $SourceModuleName));
rename_field("timestamp", "@timestamp");
to_json();
</Exec>
</Output>
<Route 1>
Path messagetracking, connectivity, smtp_receive, smtp_send => out
</Route>
------------------------------------------------------conf file---------------------------------------------------------------------
Still no data come out to Kibana and I get the following errors in the logs:
------------------------------------------------------Logs---------------------------------------------------------------------
2021-03-08 12:41:28 INFO reconnecting in 0 seconds
2021-03-08 12:41:28 ERROR http response timeout from server
2021-03-08 12:41:28 ERROR procedure 'set_http_request_path' failed at line 110, character 90 in C:\Program Files (x86)\nxlog\conf\nxlog.conf. statement execution has been aborted; function 'strftime' failed at line 110, character 88 in C:\Program Files (x86)\nxlog\conf\nxlog.conf. expression evaluation has been aborted; got 'unknown' for first argument of function 'strftime(datetime, string)'
2021-03-08 12:41:28 INFO connecting to kibanaip:9200
2021-03-08 12:41:28 ERROR HTTP response status is not OK: 405 Method Not Allowed
------------------------------------------------------Logs---------------------------------------------------------------------
I believe there is a problem recognising the date-time data type and it produces a json not what Kibana wants....
Since I'm a complete newbie, does anybody has any insight??