currently running 2.10.2150 on windows with a config that reads a debug DNS log (on c:), parses the logs, drops 99% of the logs, and writes the remainder out with file_write()

we are seeing this memory pool allocation error and looking for info about if it's a known issue, something that we should be changing the config to deal with, or what's happening.

our current config is something very close to (filenames may vary):

Panic Soft #NoFreeOnExit TRUE

define ROOT C:\Program Files (x86)\nxlog define CERTDIR %ROOT%\cert #define CONFDIR %ROOT%\conf define CONFDIR "C:\Program Files (x86)\nxlog\conf define LOGDIR %ROOT%\data #define LOGFILE %LOGDIR%\nxlog.log define LOGFILE %LOGDIR%\nxlog.log LogFile %LOGFILE% #LogLevel DEBUG define OUTFILE H:\dnsadvlogs\dns-filtered.log

Moduledir %ROOT%\modules CacheDir %ROOT%\data Pidfile %ROOT%\data\nxlog.pid SpoolDir %ROOT%\data

<Extension _syslog> Module xm_syslog </Extension>

<Extension _json> Module xm_json </Extension>

<Extension _charconv> Module xm_charconv AutodetectCharsets iso8859-2, utf-8, utf-16, utf-32 </Extension>

<Extension _exec> Module xm_exec </Extension>

<Extension _fileop> Module xm_fileop

# Check the log file size every hour and rotate if larger than 5 MB
<Schedule>
    Every 1 hour
    <Exec>
        if (file_exists('%LOGFILE%') and file_size('%LOGFILE%') >= 5M)
            file_cycle('%LOGFILE%', 8);
    </Exec>
</Schedule>

# Rotate log file every week on Sunday at midnight
<Schedule>
    When    @weekly
    Exec    if file_exists('%LOGFILE%') file_cycle('%LOGFILE%', 8);
</Schedule>

</Extension>

<Extension _fileop2> Module xm_fileop

# Rotate log file every week on Sunday at midnight
<Schedule>
    Every 5 min
    Exec    if file_exists('%OUTFILE') file_cycle('%OUTFILE%', 8);
</Schedule>

</Extension>

<Extension win_dns_parser> Module xm_multiline # look for a date at the start of the line HeaderLine /^\d+/\d+/\d+/ #filter blank and header lines from the input Exec if $raw_event =~ /^\s*$/ drop(); Exec if $raw_event =~ /^DNS Server log file creation/ drop(); Exec if $raw_event =~ /^Log file wrap at / drop(); Exec if $raw_event =~ /^Message logging key/ drop(); Exec if $raw_event =~ /^\sField # Information Values/ drop(); Exec if $raw_event =~ /^\s------- ----------- ------/ drop(); Exec if $raw_event =~ /^\s1 Date/ drop(); Exec if $raw_event =~ /^\s2 Time/ drop(); Exec if $raw_event =~ /^\s3 Thread ID$/ drop(); Exec if $raw_event =~ /^\s4 Context$/ drop(); Exec if $raw_event =~ /^\s5 Internal packet identifier$/ drop(); Exec if $raw_event =~ /^\s6 UDP/TCP indicator$/ drop(); Exec if $raw_event =~ /^\s7 Send/Receive indicator$/ drop(); Exec if $raw_event =~ /^\s8 Remote IP$/ drop(); Exec if $raw_event =~ /^\s9 Xid (hex)$/ drop(); Exec if $raw_event =~ /^\s10 Query/Response R = Response$/ drop(); Exec if $raw_event =~ /^\sblank = Query$/ drop(); Exec if $raw_event =~ /^\s11 Opcode Q = Standard Query$/ drop(); Exec if $raw_event =~ /^\sN = Notify$/ drop(); Exec if $raw_event =~ /^\sU = Update$/ drop(); Exec if $raw_event =~ /^\s*? = Unknown$/ drop(); Exec if $raw_event =~ /^\s12 [ Flags (hex)$/ drop(); Exec if $raw_event =~ /^\s13 Flags (char codes) A = Authoritative Answer$/ drop(); Exec if $raw_event =~ /^\sT = Truncated Response$/ drop(); Exec if $raw_event =~ /^\sD = Recursion Desired$/ drop(); Exec if $raw_event =~ /^\sR = Recursion Available$/ drop(); Exec if $raw_event =~ /^\s14 ResponseCode ]$/ drop(); Exec if $raw_event =~ /^\s15 Question Type$/ drop(); Exec if $raw_event =~ /^\s16 Question Name/ drop(); </Extension>

<Input win_dns> Module im_file #File "H:\dnsadvlogs\dns.log" File "H:\dnsadvlogs\dns.log" # for testing we want to re-read from the start of the file each time SavePos False ReadFromLast True InputType win_dns_parser <Exec> $Message = $raw_event; $Message = replace($Message, "\r", ""); </Exec> </Input>

<Output win_dns_trimmed> Module om_file

File "H:\dnsadvlogs\dns-trimmed.log"

File      "H:\dnsadvlogs\dns-trimmed.log"
<Exec>
    # manipulate the log entry here, not in input so that we can do other things with the raw input as well
    $Message =~ /^(?<timestamp>\d+\/\d+\/\d+ \d+\:\d+\:\d+\s+\S+)\s+(?<pid>\d+)\s+(?<win_dns_type>[^ ]*)\s+(?<win_dns_packetID>[^ ]*)\s+(?<win_dns_protocol>[^ ]*)\s+(?<win_dns_direction>[^ ]*)\s+(?<win_dns_IP>[^ ]*)\s+(?<win_dns_hexID>[^ ]*) (?<win_dns_qr>.) (?P<win_dns_opcode>.) \[(?<win_dns_flags_hex>\S+) (?<win_dns_flags>.*) (?<win_dns_resultcode>\S+)\]\s+(?<win_dns_recordType>\S+)\s+(?<win_dns_query>\S*).*ANSWER SECTION:.(?<answer>.+)\s+AUTHORITY SECTION:/s; 
    $timestamp=$1;
    $pid=$2;
    $win_dns_type=$3;
    $win_dns_packetID=$4;
    $win_dns_protocol=$5;
    $win_dns_direction=$6;
    $win_dns_IP=$7;
    $win_dns_hexID=$8;
    $win_dns_qr=$9;
    $win_dns_opcode=$10;
    $win_dns_flags_hex=$11;
    $win_dns_flags=$12;
    $win_dns_resultcode=$13;
    $win_dns_recordType=$14;
    $win_dns_query=$15;
    $answer=$16;
    # drop messages if they are not replies (since the replies contain the query info)
    if $win_dns_qr == " " drop();
    # drop logs that have no answer info
    #if $answer =~ /^\s+empty\s+$/ drop();
    # drop logs that don't parse (if we don't have a requestion IP address, the log is worthless to UBA)
    if not defined $win_dns_IP drop();
    # drop logs from dnsmasq caching servers.
    #if $win_dns_IP IN ("10.16.169.32","10.49.58.4","10.49.58.3","10.16.6.22","199.47.139.239","199.47.139.238","199.47.139.182") drop();
    # for the first pass, just filter the logs, don't change the format
    # this greatly simplifies the Splunk changes needed as the log parsing doesn't need to change
    $orig = replace($raw_event, "\r", "\r\n") + "\r\n";
    file_write("%OUTFILE%",$orig); drop();
    delete($orig); delete($Message);
    delete($EventReceivedTime); delete($SourceModuleName); delete($SourceModuleType); delete($pid);
    delete($win_dns_type); delete($win_dns_packetID); delete($win_dns_protocol); delete($win_dns_direction);
    delete($win_dns_hexID); delete($win_dns_qr); delete($win_dns_opcode); delete($win_dns_flags_hex);
    delete($win_dns_flags);
    $answer = replace($answer,"(3)",".");
    $answer = replace($answer,"(6)",".");
    $answer = replace($answer,"(9)","");
    $answer = replace($answer,"(0)","");
    $answer =~ s/\[\S\S\S\S\]//g;
    $win_dns_query = replace($win_dns_query,"(3)",".");
    $win_dns_query = replace($win_dns_query,"(6)",".");
    $win_dns_query = replace($win_dns_query,"(9)","");
    $win_dns_query =~ s/\(0\)UDP//;
    $win_dns_query =~ s/\(0\)TCP//;
    $win_dns_query =~ s/\[\S\S\S\S\]//g;
    rename_field("win_dns_query","q");
    rename_field("win_dns_resultcode","rc");
    rename_field("win_dns_IP","ip");
    rename_field("win_dns_recordType","type");
    if $rc == "NXDOMAIN" delete($answer);
    to_json();
</Exec>

</Output> <Route win_dns_route> Path win_dns => win_dns_trimmed </Route>