responses
Hi, my logs:
<188>date=2017-09-26 time=10:53:04 devname=FT01 devid=************* logid=0000000011 type=traffic subtype=forward level=warning vd=root srcip=10.25d.1dd.ddd srcport=59ddd srcintf="portB" dstip=2dd.1dd.dd5.ddd dstport=dd3 dstintf="port9" poluuid=16594dae-dddd-51e7-dddd-da81ec23cc23 sessionid=26ddd5925 proto=6 action=ip-conn policyid=103 policytype=policy appcat="unscanned" crscore=5 craction=262144 crlevel=low devtype="Router/NAT Device" mastersrcmac=00:dd:14:dd:c8:dd srcmac=00:dd:14:dd:c8:ddt="default" appa
<189>date=2017-09-26 time=10:53:04 devname=FT01 devid=************* logid=0000000013 type=traffic subtype=forward level=notice vd=root srcip=10.25d.1dd.ddd srcport=59674 srcintf="portB" dstip=2dd.1dd.dd5.ddd dstport=443 dstintf="port9" poluuid=16594dae-dddd-51e7-dddd-da81ec23cc23 sessionid=260ddd925 proto=6 action=close policyid=103 policytype=policy dstcountry="United States" srccountry="Reserved" trandisp=snat transip=10.2dd.1dd.2dd transport=59674 service="HTTPS" appid=40568 app="HTTPS.BROWSER" appcat="Web.Client" apprisk=medium applist="default" appact=detected duration=140 sentbyte=1244 rcvdbyte=770 sentpkt=9 rcvdpkt=6 devtype="Router/NAT Device" mastersrcmac=00:2d:dd:6b:dd:60 srcmac=00:dd:14:dd:c8:ddc=88:ad:d2:88:eb
<189>date=2017-09-26 time=10:53:04 devname=FT01 devid=************* logid=0000000013 type=traffic subtype=forward level=notice vd=root srcip=10.2dd.dd.dd srcport=42768 srcintf="portB" dstip=2dd.1dd.dd.dd dstport=53 dstintf="port9" poluuid=0dddda60-dddd-51e7-dddd-56c9d0ddde2f sessionid=260ddd113 proto=17 action=accept policyid=84 policytype=policy dstcountry="Hong Kong" srccountry="Reserved" trandisp=snat transip=10.2dd.dd.2d transport=42768 service="DNS" appid=16195 app="DNS" appcat="Network.Service" apprisk=elevated applist="default" appact=detected duration=180 sentbyte=77 rcvdbyte=93 sentpkt=1 rcvdpkt=1 devtype="Router/NAT Device" mastersrcmac=00:dd:14:dd:c8:dd srcmac=00:dd:14:dd:c8:dddd:6060 srcmac=0
I want to use regular expressions:
field >> regex
action = ^.+\saction=(\S+)\s
app = ^.+\sapp=\"(.+?)\"
appcat = ^.+\sappcat=\"(.+?)\"
applist = ^.+\sapplist=\"(.+?)\"
attack = ^.+\sattack=\"(.+?)\"
devid = ^.+\sdevid=(\S+)\s
dir = ^.+\sdir=(\S+)\s
dstcountry = ^.+\sdstcountry=\"(.+?)\"
dstintf = ^.+\sdstintf=\"(.+?)\"
dstip = ^.+\sdstip=(\S+)\s
dstport = ^.+\sdstport=(\S+)\s
... 175 more
What configuration to use?
<Input i.forti.log>
Module im_file
File "/var/log/forti.log"
InputType LineBased
</Input>
<Output o.forti.log>
Module om_tcp
Host 192.168.00.00
Port XXXXX
CAFile /data/conf/ca.crt
AllowUntrusted TRUE
OutputType LineBased
</Output>
<Route r.forti.log>
Path i.forti.log => o.forti.log
</Route>
Thank you very much!!
Comments (2)
previously use:
<Extension kvp>
Module xm_kvp
KVPDelimiter \s
KVDelimiter =
EscapeChar \\
</Extension>
<Extension json>
Module xm_json
</Extension>
<Input i.ftp.log>
Module im_file
File "/var/log/forti.log"
SavePos FALSE
ReadFromLast FALSE
Exec if $raw_event =~ /^#/ drop(); \ ##<< replacement?
else \
{ \
kvp->parse_kvp(); \
delete($EventReceivedTime); \
delete($SourceModuleName); \
delete($SourceModuleType); \
if ( integer($Weight) > integer($Height) - 100 ) $Overweight = TRUE; \
to_json();\
}
</Input>
<Output o.ftp.log>
Module om_ssl
Host 192.168.0.00
Port XXXXX
CAFile /data/conf/ca.crt
AllowUntrusted TRUE
OutputType LineBased
</Output>
<Route r.ftp.log>
Path i.ftp.log => o.ftp.log
</Route>
BUT the result is:
{"<189>date":"2017-09-21","time":"20:54:43","devname":"FT01","devid":"****************","logid":"0000000013","type":"traffic","subtype":"forward","level":"notice","vd":"root","srcip":"10.dd6.d6.dd4","srcport":"137","srcintf":"portB","dstip":"1dd.255.255.255","dstport":"137","dstintf":"port9","sessionid":"24d239345","proto":"17","action":"deny","policyid":"0","policytype":"policy","dstcountry":"Reserved","srccountry":"Reserved","trandisp":"noop","service":"SMB_UDP_137","duration":"0","sentbyte":"0","rcvdbyte":"0","sentpkt":"0","appcat":"unscanned","crscore":"30","craction":"131072","crlevel":"high","devtype":"Router/NAT Device","mastersrcmac":"00:dd:14:db:dd:dd","srcmac":"00:dd:dd:6b:cd:60"}
i need fields like a:
<Extension csv.log>
Module xm_csv
Fields $timestamp,$uid,$srcip,$srcport,$dstip,$dstport,$service
FieldsType string,string,string,string,string,string,string
Delimiter \t
</Extension>
<Input i.log>
Module im_file
File "/bla/bla.log"
ReadFromLast TRUE
Exec csv.log->parse_csv();
</Input>
<Output o.log>
Module om_ssl
Host 1dd.ddd.dd.dd
Port XXXX
OutputType GELF_TCP
CAFile /data/conf/ca.crt
AllowUntrusted TRUE
</Output>
<Route r.log>
Path i.log => o.log
</Route>
The number of fields in each log varies and are different in each log.
thank you.
You will need to rewrite $raw_event using to_json() or similar depending on what you need in the output.