3
responses

Hi,

We are using nxlog ce-2.11.2190.msi to forward windows server event logs to our SIEM. And now we are seeing an issue that the $message of events with ID 4624/4625/4xxx are missing, while those of event with ID 7xxx are shown completely. This issue will be temporarily fixed if we restarted the nxlog service, but the same problem comes back after nxlog service running for a while.

This is what we see on SIEM. log prtsc

And the complete events should be like: [Our server’s system language is Traditional Chinese, so the $message is displayed as bytes]

Apr 12 18:19:36 win2k12.ahsuhome.local Microsoft-Windows-Security-Auditing[500]: Microsoft-Windows-Security-Auditing: 4634: \0xe5\0xb8\0xb3\0xe6\0x88\0xb6\0xe5\0xb7\0xb2\0xe7\0x99\0xbb\0xe5\0x87\0xba\0xe3\0x80\0x82 \0xe4\0xb8\0xbb\0xe6\0x97\0xa8: \0x09\0xe5\0xae\0x89\0xe5\0x85\0xa8\0xe6\0x80\0xa7\0xe8\0xad\0x98\0xe5\0x88\0xa5\0xe7\0xa2\0xbc:\0x09\0x09S-1-5-18 \0x09\0xe5\0xb8\0xb3\0xe6\0x88\0xb6\0xe5\0x90\0x8d\0xe7\0xa8\0xb1:\0x09\0x09WIN2K12$ \0x09\0xe5\0xb8\0xb3\0xe6\0x88\0xb6\0xe7\0xb6\0xb2\0xe5\0x9f\0x9f:\0x09\0x09AHSUHOME \0x09\0xe7\0x99\0xbb\0xe5\0x85\0xa5\0xe8\0xad\0x98\0xe5\0x88\0xa5\0xe7\0xa2\0xbc:\0x09\0x090x367342 \0xe7\0x99\0xbb\0xe5\0x85\0xa5\0xe9\0xa1\0x9e\0xe5\0x9e\0x8b:\0x09\0x09\0x093 \0xe7\0x95\0xb6\0xe7\0x99\0xbb\0xe5\0x85\0xa5\0xe5\0xb7\0xa5\0xe4\0xbd\0x9c\0xe9\0x9a\0x8e\0xe6\0xae\0xb5\0xe6\0x90\0x8d\0xe6\0xaf\0x80\0xe6\0x99\0x82\0xef\0xbc\0x8c\0xe5\0xb0\0xb1\0xe6\0x9c\0x83\0xe7\0x94\0xa2\0xe7\0x94\0x9f\0xe9\0x80\0x99\0xe5\0x80\0x8b\0xe4\0xba\0x8b\0xe4\0xbb\0xb6\0xe3\0x80\0x82\0xe9\0x80\0x99\0xe5\0x80\0x8b\0xe4\0xba\0x8b\0xe4\0xbb\0xb6\0xe5\0x8f\0xaf\0xe8\0x83\0xbd\0xe8\0x88\0x87\0xe4\0xbd\0xbf\0xe7\0x94\0xa8\0xe7\0x99\0xbb\0xe5\0x85\0xa5\0xe8\0xad\0x98\0xe5\0x88\0xa5\0xe7\0xa2\0xbc\0xe6\0x95\0xb8\0xe5\0x80\0xbc\0xe7\0x9a\0x84\0xe7\0x99\0xbb\0xe5\0x85\0xa5\0xe4\0xba\0x8b\0xe4\0xbb\0xb6\0xe6\0xad\0xa3\0xe9\0x9d\0xa2\0xe7\0x9b\0xb8\0xe9\0x97\0x9c\0xe3\0x80\0x82\0xe7\0x99\0xbb\0xe5\0x85\0xa5\0xe8\0xad\0x98\0xe5\0x88\0xa5\0xe7\0xa2\0xbc\0xe5\0x83\0x85\0xe6\0x9c\0x89\0xe5\0x9c\0xa8\0xe9\0x87\0x8d\0xe6\0x96\0xb0\0xe5\0x95\0x9f\0xe5\0x8b\0x95\0xe7\0x9b\0xb8\0xe5\0x90\0x8c\0xe9\0x9b\0xbb\0xe8\0x85\0xa6\0xe4\0xb9\0x8b\0xe9\0x96\0x93\0xe6\0x89\0x8d\0xe6\0x9c\0x83\0xe6\0x98\0xaf\0xe5\0x94\0xaf\0xe4\0xb8\0x80\0xe7\0x9a\0x84\0xe3\0x80\0x82

Our config is as below:

define SIEM_IP   10.0.0.253
define ROOT     C:\Program Files (x86)\nxlog
define CERTDIR  %ROOT%\cert
define CONFDIR  %ROOT%\conf
define LOGDIR   %ROOT%\data
define LOGFILE  %LOGDIR%\nxlog.log
LogFile %LOGFILE%

Moduledir %ROOT%\modules
CacheDir   %ROOT%\data
Pidfile        %ROOT%\data\nxlog.pid
SpoolDir    %ROOT%\data

<Extension syslog>
  Module    xm_syslog
</Extension>

<Input in_eventlog>
  Module            im_msvistalog
  ReadFromLast TRUE
  SavePos           TRUE
  Query        <QueryList> \
    <Query Id="0"> \
      <Select Path="Security">*[System[(EventID=1100 or EventID=1102)]]</Select> \
      <Select Path="Security">*[System[(EventID=4768 or EventID=4769 or EventID=4771)]]</Select> \
      <Select Path="Security">*[System[(EventID=4616 or EventID=4657)]]</Select> \
      <Select Path="Security">*[System[(EventID=4624 or EventID=4625 or EventID=4634 or EventID=4647 or EventID=4648)]]</Select> \
      <Select Path="Security">*[System[(EventID=5140 or EventID=5142 or EventID=5143 or EventID=5144 or EventID=5145 or EventID=5168)]]</Select> \
      <Select Path="Security">*[System[(EventID=4656 or EventID=4658 or EventID=4660 or EventID=4663 or EventID=4664 or EventID=4985 or EventID=5051 or EventID=4670)]]</Select> \
      <Select Path="Security">*[System[(EventID=4719 or EventID=4739)]]</Select> \
      <Select Path="Security">*[System[(EventID=4720 or EventID=4722 or EventID=4723 or EventID=4724 or EventID=4725 or EventID=4726 or EventID=4738 or EventID=4740 or EventID=4767)]]</Select> \
      <Select Path="Security">*[System[(EventID=4727 or EventID=4728 or EventID=4729 or EventID=4730 or EventID=4731 or EventID=4732 or EventID=4733 or EventID=4734 or EventID=4735 or EventID=4737 or EventID=4764)]]</Select> \
      <Select Path="Security">*[System[(EventID=4741 or EventID=4742 or EventID=4743)]]</Select> \
      <Select Path="Security">*[System[(EventID=4744 or EventID=4745 or EventID=4748)]]</Select> \
      <Select Path="Security">*[System[(EventID=4749 or EventID=4750 or EventID=4753)]]</Select> \
      <Select Path="Security">*[System[(EventID=4754 or EventID=4755 or EventID=4756 or EventID=4758 or EventID=4759 or EventID=4760 or EventID=4763)]]</Select> \
      <Select Path="Security">*[System[(EventID=4778)]]</Select> \
      <Select Path="Security">*[System[(EventID=4783)]]</Select> \
      <Select Path="Security">*[System[(EventID=4800 or EventID=4801)]]</Select> \
      <Select Path="System">*[System[(EventID=7036)]]</Select> \
    </Query> \
  </QueryList> 
</Input>

<Output out_eventlog>
  Module om_udp
  Host   %SIEM_IP%
  Port   514
  Exec $SyslogFacilityValue = 17;
  Exec $Message = string($SourceName) + ": " + string($EventID) + ": " + $Message;
  Exec if ($EventType == 'ERROR' or $EventType == 'AUDIT_FAILURE') { $SyslogSeverityValue = 3; } \
       else if ($EventType == 'WARNING')  { $SyslogSeverityValue = 4; } \
       else if ($EventType == 'INFO' or $EventType == 'AUDIT_SUCCESS')  { $SyslogSeverityValue = 5; } 
  Exec to_syslog_bsd();
</Output>

<Route eventlog>
  Path  in_eventlog => out_eventlog
</Route>

Any ideas about how can this happen will be appreciated.

AskedApril 14, 2022 - 5:38am

Answer (1)

Although your XPath query seems a bit odd because of multiple rows with:
<Select Path="Security">*[System[(.....
it shouldn't be a problem....but you could re-consider the way you are filtering the events.

The fact that it starts working well after a service restart means it's probably not related to your configuration. A record or a log about the event that actually triggers the issue has to be written somewhere... So, the best thing you could do is to share your NXLog's internal logs. They are located in your:
C:\Program Files (x86)\nxlog\data\nxlog.log file.

Also, please tell us something more about your use case. On how many servers does the nxlog service run? Is the nxlog configuration the same for all of them?

Best regards,

Nenad

Comments (2)

  • aorta's picture

    Hi,

    Thank you for the reply.

    I have uploaded two log files to google drive. They were captured from two different servers which occur the same event message dissapearing issue. The number of hosts are around 10 and 80 and they are all running the same configuration.

  • NenadM's picture
    (NXLog)

    Hello

    Thanks for sharing the logs. Unfortunately, there's nothing in them indicating that NXLog's modules in use have a problem. I've found only information about nxlog service being stopped and started.
    This could be an NXLog related issue or something with the OS...but without detailed logs I can just make assumptions and ask you to do more tests.

    So, there are two things you can do now and I'd strongly recommend setting up a TEST SERVER for both of them:
    1. In case that the problem is with the XPath language used for Event filtering, you can optimize the code in the following way:
    Add an extra define directive like in the following example:
    https://docs.nxlog.co/userguide/integrate/windows-eventlog.html#eventids-examples

    Such a configuration reduces the number of the SELECT queries being executed by NXLOG and provides a much wider scope of log collection. It also makes it easier to amend the list of the Events being collected. In this particular case, the config should look like the following one.
    !!NOTE: Make sure to replace the EventID list with your actual IDs!!

    # define Security Events
    define SecurityEvents 4624, 4634, 4648, 4656, 4658, 4660, 4663, 4672, \
    4673, 4688, 4689, 4698, 4720, 4768, 4769, 4946, \
    5140, 5142, 5144, 5145, 5154, 5156, 5447, 8222
    # define Other Events
    define OtherEvents 7036

    define SIEM_IP 10.0.0.253
    define ROOT C:\Program Files (x86)\nxlog
    define CERTDIR %ROOT%\cert
    define CONFDIR %ROOT%\conf
    define LOGDIR %ROOT%\data
    define LOGFILE %LOGDIR%\nxlog.log
    LogFile %LOGFILE%

    Moduledir %ROOT%\modules
    CacheDir %ROOT%\data
    Pidfile %ROOT%\data\nxlog.pid
    SpoolDir %ROOT%\data

    <Extension syslog>
    Module xm_syslog
    </Extension>

    <Input in_eventlog>
    Module im_msvistalog
    ReadFromLast TRUE
    SavePos TRUE
    Query <QueryList> \
    <Query Id="0"> \
    <Select Path="Security">*</Select>\
    <Select Path="System">*</Select> \
    </Query> \
    </QueryList>
    Exec if ($EventID NOT IN (%SecurityEvents%)) and \
    ($EventID NOT IN (%OtherEvents%)) drop();
    </Input>

    <Output out_eventlog>
    Module om_udp
    Host %SIEM_IP%
    Port 514
    Exec $SyslogFacilityValue = 17;
    Exec $Message = string($SourceName) + ": " + string($EventID) + ": " + $Message;
    Exec if ($EventType == 'ERROR' or $EventType == 'AUDIT_FAILURE') { $SyslogSeverityValue = 3; } \
    else if ($EventType == 'WARNING') { $SyslogSeverityValue = 4; } \
    else if ($EventType == 'INFO' or $EventType == 'AUDIT_SUCCESS') { $SyslogSeverityValue = 5; }
    Exec to_syslog_bsd();
    </Output>

    <Route eventlog>
    Path in_eventlog => out_eventlog
    </Route>

    2. In order to get some more details in the logs - you can increase the LogLevel from INFO to DEBUG. Just add the last line to your config file:

    define SIEM_IP 10.0.0.253
    define ROOT C:\Program Files (x86)\nxlog
    define CERTDIR %ROOT%\cert
    define CONFDIR %ROOT%\conf
    define LOGDIR %ROOT%\data
    define LOGFILE %LOGDIR%\nxlog.log
    LogFile %LOGFILE%
    LogLevel DEBUG