Buffering not working


#1 billychua

Hi,

I had this situation whereby when I unplugged the cable of the outgoing interface of the nxlog server, the logs went into the cache folder. When I connected back to the interface, the new logs continues to flow to the external logger,  but the cached logs remain inside the cache folder and do not forward out to the external logger.

below is my configuration.  Looking forward to any help, please. thank you

User rootGroup rootPanic Soft##modify change# default values:# PidFile   /opt/nxlog/var/run/nxlog/nxlog.pid# CacheDir  /opt/nxlog/var/spool/nxlog# ModuleDir /opt/nxlog/lib/nxlog/modules# SpoolDir /opt/nxlog/var/spool/nxlogCacheDir /data/nxlogcache

define CERTDIR /opt/nxlog/var/lib/nxlog/certdefine CONFDIR /opt/nxlog/var/lib/nxlog

# Note that these two lines define constants only; the log file location# is ultimately set by the `LogFile` directive (see below). The# `MYLOGFILE` define is also used to rotate the log file automatically# (see the `_fileop` block).define LOGDIR /opt/nxlog/var/log/nxlogdefine MYLOGFILE %LOGDIR%/nxlog.log

# By default, `LogFile %MYLOGFILE%` is set in log4ensics.conf. This# allows the log file location to be modified via NXLog Manager. If you# are not using NXLog Manager, you can instead set `LogFile` below and# disable the `include` line.LogFile %MYLOGFILE%#include %CONFDIR%/log4ensics.conf

<Extension _syslog>   Module  xm_syslog</Extension>

<Extension exec>        Module  xm_exec</Extension>

<Extension _leef>Module xm_leef</Extension>

<Output Egress_To_xxx>       Module om_tcp       Port    1524       Host    xxx.xxx.xxx.xxx</Output>

# Set port and Ip Address to listen to traffic<Input External_Ingress>       Module im_tcp       Port    1524       Host    0.0.0.0</Input><Output External_Log_Locally>       Module om_file       File    '/data/store/External/External_logs.txt'

#Set log rotation to run daily and keep 90 days worth of logs               <Schedule>               when    @daily               <Exec>                       # Create year/month directories if necessary                                               dir_make('/data/store/External/' + strftime(now() - 86400, '%Y-%m'));

                                               # Rotate current file into the correct directory                                               rotate_to('/data/store/External/' + strftime(now() - 86400, '%Y-%m/External_logs_%Y-%m-%d.txt'));

                                               #Remove files older than 90 days                                               exec_async("/usr/bin/find", "/data/store/External", "-mtime", "+90", "-type", "f", "-delete");                                                              </Exec>       </Schedule>

</Output>

<Processor External_To_xxx_Buffer>       Module  pm_buffer       Type    Disk       #130MiB buffer       MaxSize 130000000       WarnLimit       100000000</Processor>

<Route External_To_xxx_Path>       Path    External_Ingress => External_To_xxx_Buffer => Egress_To_xxx</Route>

<Route External_To_Local>       Path    External_Ingress => External_Log_Locally</Route>

<Input Internal_Ingress>       Module im_tcp       Port    2524       Host    0.0.0.0

       <Exec>           # edit syslog header for xxxx           if $raw_event =~ /\d\d:\d\d:\d\d\s+(xxxx)\s+/           {               $Hostname = $1;               to_syslog_bsd();           }

       </Exec>

</Input>

<Output Internal_Log_Locally>        Module om_file        File   '/data/store/Internal/Internal_logs.txt'

                #Set log rotation to run daily and keep 90 days worth of logs       <Schedule>               when    @daily               <Exec>                       # Create year/month directories if necessary                       dir_make('/data/store/Internal/' + strftime(now() - 86400, '%Y-%m'));

                       # Rotate current file into the correct directory                       rotate_to('/data/store/Internal/' + strftime(now() - 86400, '%Y-%m/Internal_logs_%Y-%m-%d.txt'));

                       #Remove files older than 90 days                       exec_async("/usr/bin/find", "/data/store/Internal", "-mtime", "+90", "-type", "f", "-delete");

               </Exec>

       </Schedule></Output>

<Processor Internal_To_xxx_Buffer>        Module pm_buffer        Type   Disk        #130MiB buffer        MaxSize        130000000        WarnLimit      100000000</Processor>

<Route Internal_To_xxx_Path>       Path    Internal_Ingress => Internal_To_xxx_Buffer => Egress_To_xxx</Route>

<Route Internal_To_Local>       Path    Internal_Ingress => Internal_Log_Locally</Route>

<Input xxx_UDP_Ingress>       Module im_udp       Port   514       Host   0.0.0.0

<Exec> # edit syslog header for xxx else if $MessageSourceAddress == 'xx.xxx.x.x' {  $Hostname = 'xxx';  to_syslog_bsd(); }

 # edit syslog header for xxx else if $MessageSourceAddress == 'xx.xxx.xx.xx' {  $Hostname = 'xxx';  to_syslog_bsd(); }

              # edit syslog header for xxx               else if $MessageSourceAddress == 'xx.xx.xx'               {                       $Hostname = 'xxx';                       to_syslog_bsd();               }

</Exec>

</Input>

<Output xxx_UDP_Log_Locally>        Module om_file        File   '/data/store/UDP/xxx_UDP_logs.txt'

                #Set log rotation to run daily and keep 90 days worth of logs               <Schedule>               when    @daily               <Exec>                       # Create year/month directories if necessary                                               dir_make('/data/store/UDP/' + strftime(now() - 86400, '%Y-%m'));

                                               # Rotate current file into the correct directory                                               rotate_to('/data/store/UDP/' + strftime(now() - 86400, '%Y-%m/xxxx_UDP_logs_%Y-%m-%d.txt'));

                                               #Remove files older than 90 days                                               exec_async("/usr/bin/find", "/data/store/UDP", "-mtime", "+90", "-type", "f", "-delete");

               </Exec>       </Schedule></Output>

<Processor xxx_UDP_To_xxx_Buffer>        Module pm_buffer        Type   Disk        #130MiB buffer        MaxSize        130000000        WarnLimit      100000000</Processor>

<Route xxx_UDP_To_xxx_Path>       Path    xxx_UDP_Ingress => xxx_UDP_To_xxx_Buffer => Egress_To_xxx</Route>

<Route xxx_UDP_To_Local>       Path    xxx_UDP_Ingress => xxx_UDP_Log_Locally</Route>

<Input audit_log>       Module im_file       File    '/var/log/secure'       <Exec>               to_syslog_bsd();       </Exec></Input>

<Processor xxx_Audit_To_xxx_Buffer>        Module pm_buffer        Type   Disk        #130MiB buffer        MaxSize        130000000        WarnLimit      100000000</Processor>

<Route xx_Audit_To_xx_Path>       Path    audit_log => xxx_Audit_To_xxx_Buffer => Egress_To_xxx</Route>

<Input audit_log_2>       Module im_file       File    '/var/log/audit/audit.log'       <Exec>               to_syslog_bsd();       </Exec></Input>

<Processor xxx_Audit_2_To_xxx_Buffer>        Module pm_buffer        Type   Disk        #130MiB buffer        MaxSize        130000000        WarnLimit      100000000</Processor>

<Route xxx_Audit2_To_xxxx_Path>       Path    audit_log_2 => xxxx_Audit_2_To_xxxx_Buffer => Egress_To_xxxx</Route>

# This block rotates `%MYLOGFILE%` on a schedule. Note that if `LogFile`# is changed in log4ensics.conf via NXLog Manager, rotation of the new# file should also be configured there.<Extension _fileop>   Module  xm_fileop

   # Check the size of our log file hourly, rotate if larger than 5MB   <Schedule>       Every   1 hour       <Exec>           if ( file_exists('%MYLOGFILE%') and                (file_size('%MYLOGFILE%') >= 5M) )           {                file_cycle('%MYLOGFILE%', 8);           }       </Exec>   </Schedule>

   # Rotate our log file every week on Sunday at midnight   <Schedule>       When    @weekly       Exec    if file_exists('%MYLOGFILE%') file_cycle('%MYLOGFILE%', 8);   </Schedule></Extension>

#2 gahorvath Nxlog ✓

Hello Billy,

This looks like you are running NXLog EE, is that right?

What version?

Would you be so kind to repost your config in a code block? Use 3 backticks (```) to open and close the block.