nxlog wineventlog + kinesis tap service: contention and possible file locks


#1 bmac

We are utilizing nxlog win with the im_msvistalog module. Fairly simplified json output file (om_file) with an hourly rotation. AWS Kinesis Tap is configured to read from this file as a source. We continue to experience strange situations where nxlog will appear to be running healthy as a service, but will be writing 0 lines to the output file as if the mswineventlog back end is just not functioning. When this occurs the nxlog configuration schedule will executed as expected, and rotate 0KB files into the destination folder. This is running on two domain controllers with wineventlog seeing anywhere from 20-30 individual logs per second.

Is anyone utilizing this type of configuration and experiencing any similar issues? Strangely enough a search in the forum for "kinesis" was returned with 0 results.

Panic Soft
#NoFreeOnExit TRUE

define ROOT     C:\Program Files (x86)\nxlog
define CERTDIR  %ROOT%\cert
define CONFDIR  %ROOT%\conf
define LOGDIR   %ROOT%\data
define LOGFILE  %LOGDIR%\nxlog.log
LogFile %LOGFILE%


Moduledir %ROOT%\modules
CacheDir  %ROOT%\data
Pidfile   %ROOT%\data\nxlog.pid
SpoolDir  %ROOT%\data

<Extension _json>
    Module	xm_json
</Extension>

<Extension _exec>
    Module      xm_exec
</Extension>

<Extension _fileop>
    Module      xm_fileop

    # Check the size of our log file hourly, rotate if larger than 5MB
    <Schedule>
        Every   1 hour
        Exec    if (file_exists('%LOGFILE%') and \
                   (file_size('%LOGFILE%') >= 5M)) \
                    file_cycle('%LOGFILE%', 8);
    </Schedule>

    # Rotate our log file every week on Sunday at midnight
    <Schedule>
        When    @weekly
        Exec    if file_exists('%LOGFILE%') file_cycle('%LOGFILE%', 8);
    </Schedule>
</Extension>

#Get logs from Windows EventLog API
<Input eventlog> 
	Module 		im_msvistalog
</Input>


#Send logs to customized file
<Output out>
	#define OUTDIR 	C:\Program Files (x86)\logs\data_nxlogs
	define OUTDIR D:\nxlog-logs
	Exec create_var('offset');
	Module 		om_file
	File		'%OUTDIR%\output.json'

	<Schedule>
		When @hourly
		<Exec>
			if not dir_exists('%OUTDIR%\Older_Logs')
				dir_make('%OUTDIR%\Older_Logs');
			rotate_to('%OUTDIR%\Older_Logs\' +strftime(now(), '%m_%d_%Y-%H_%M.log'));
		</Exec>
	</Schedule>

	<Exec>
		$EventTime = strftime($EventTime, '%Y-%m-%d %H:%M:%S %z');
	 	to_json();
	</Exec>

#Check back the previous logs and delete all logs older than 24hrs
	<Schedule>
		When @hourly
		<Exec>
			#log_info(now());
			#log_info(now() - 86400);
			set_var('offset', now() - 86400);
			if file_exists('D:\nxlog-logs\Older_Logs\' +strftime(get_var('offset'), '%m_%d_%Y-%H_%M.log'))
				file_remove('D:\nxlog-logs\Older_Logs\' +strftime(get_var('offset'), '%m_%d_%Y-%H_%M.log'));
		</Exec>
	</Schedule>
	
</Output>

<Route eventlog_to_out>
   Path eventlog => out
</Route>
#2 manuel.munozDeactivated Nxlog ✓
#1 bmac
We are utilizing nxlog win with the im_msvistalog module. Fairly simplified json output file (om_file) with an hourly rotation. AWS Kinesis Tap is configured to read from this file as a source. We continue to experience strange situations where nxlog will appear to be running healthy as a service, but will be writing 0 lines to the output file as if the mswineventlog back end is just not functioning. When this occurs the nxlog configuration schedule will executed as expected, and rotate 0KB files into the destination folder. This is running on two domain controllers with wineventlog seeing anywhere from 20-30 individual logs per second. Is anyone utilizing this type of configuration and experiencing any similar issues? Strangely enough a search in the forum for "kinesis" was returned with 0 results. Panic Soft #NoFreeOnExit TRUE define ROOT C:\Program Files (x86)\nxlog define CERTDIR %ROOT%\cert define CONFDIR %ROOT%\conf define LOGDIR %ROOT%\data define LOGFILE %LOGDIR%\nxlog.log LogFile %LOGFILE% Moduledir %ROOT%\modules CacheDir %ROOT%\data Pidfile %ROOT%\data\nxlog.pid SpoolDir %ROOT%\data <Extension _json> Module xm_json </Extension> <Extension _exec> Module xm_exec </Extension> <Extension _fileop> Module xm_fileop # Check the size of our log file hourly, rotate if larger than 5MB <Schedule> Every 1 hour Exec if (file_exists('%LOGFILE%') and \ (file_size('%LOGFILE%') >= 5M)) \ file_cycle('%LOGFILE%', 8); </Schedule> # Rotate our log file every week on Sunday at midnight <Schedule> When @weekly Exec if file_exists('%LOGFILE%') file_cycle('%LOGFILE%', 8); </Schedule> </Extension> #Get logs from Windows EventLog API <Input eventlog> Module im_msvistalog </Input> #Send logs to customized file <Output out> #define OUTDIR C:\Program Files (x86)\logs\data_nxlogs define OUTDIR D:\nxlog-logs Exec create_var('offset'); Module om_file File '%OUTDIR%\output.json' <Schedule> When @hourly <Exec> if not dir_exists('%OUTDIR%\Older_Logs') dir_make('%OUTDIR%\Older_Logs'); rotate_to('%OUTDIR%\Older_Logs\' +strftime(now(), '%m_%d_%Y-%H_%M.log')); </Exec> </Schedule> <Exec> $EventTime = strftime($EventTime, '%Y-%m-%d %H:%M:%S %z'); to_json(); </Exec> #Check back the previous logs and delete all logs older than 24hrs <Schedule> When @hourly <Exec> #log_info(now()); #log_info(now() - 86400); set_var('offset', now() - 86400); if file_exists('D:\nxlog-logs\Older_Logs\' +strftime(get_var('offset'), '%m_%d_%Y-%H_%M.log')) file_remove('D:\nxlog-logs\Older_Logs\' +strftime(get_var('offset'), '%m_%d_%Y-%H_%M.log')); </Exec> </Schedule> </Output> <Route eventlog_to_out> Path eventlog => out </Route>

Hello Barry,

I have used same config file as you and cannot see any problem. Files are getting populated and rotated every hour.