butler | Bonjour,
Je met actuellement en place l'ensemble rsyslogd / elasticsearch / kibana 4
Pour le parsing des données entre rsyslog et Elastic, j'utilise le module rsyslog omelasticsearch et non logstash comme c'est souvent l'usage (meilleur perf)
Je rencontre des problèmes lors du pasing des données depuis rsyslog, les valeurs issues depuis de mes ruleset ne sont pas intégrées correctement au template, et je ne trouve pas d'ou vient le problème
Voici ma conf :
Conf rsyslog.conf
Code :
- module(load="mmnormalize" )
- module(load="omelasticsearch" )
- module(load="imudp" ) # needs to be done just once
- module(load="imtcp" MaxSessions="500" )
- template(name="RemoteHost" type="string" string="/var/log/syslog" )
- # REMOTE RULES
- ruleset(name="remote" ) {
- $IncludeConfig /etc/rsyslog.d/*.remote
- action(type="omfile" DynaFile="RemoteHost" )
- }
- input(type="imudp" port="514" ruleset="remote" )
- input(type="imtcp" port="514" ruleset="remote" )
|
/etc/rsyslog.d/10-webservers.remote
Code :
- if $programname == 'apache-access' then {
- action(type="mmnormalize" userawmsg="off" rulebase="/etc/rsyslog.d/apacheaccess.rule" )
- action(type="omfile" DynaFile="WebFiles" template="logstash-accesslog" DirCreateMode="0755" FileCreateMode="0644" )
- action(type="omelasticsearch"
- server="localhost"
- serverport="9200"
- template="logstash-accesslog"
- searchIndex="logstash-index"
- dynSearchIndex="on"
- searchType="logstash-index"
- bulkmode="on"
- queue.type="linkedlist"
- queue.size="5000"
- queue.dequeuebatchsize="300"
- action.resumeretrycount="-1"
- errorFile="/var/log/rsyslog.es-error.log" )
- stop
- }
|
/etc/rsyslog.d/apacheaccess.rule
Code :
- version=2
- prefix=%rcvdat:date-rfc3164% %rcvdfrom:word%
- rule=: %tag:word% %ip:word% - %identd:word% [%timestamp:char-to:]%] "%method:word% %url:word% %pver:char-to:"%" %status:word% %bytesend:word% %referer:word% %useragent:quoted-string%
|
/etc/rsyslog.d/logstash.template
Code :
- template(name="WebFiles" type="string" string="/var/log/rsyslog/%fromhost-ip%/httpd/%$YEAR%/%$MONTH%/access_%$!vhost%-%$DAY%.log" )
- template(name="logstash-accesslog" type="list" option.json="on" ) {
- constant(value="{" )
- constant(value="\"@timestamp\":\"" ) property(name="timereported" dateFormat="rfc3339" )
- constant(value="\",\"message\":\"" ) property(name="msg" position.from="2" spifno1stsp="off" )
- constant(value="\",\"host\":\"" ) property(name="fromhost-ip" )
- constant(value="\",\"@source_host\":\"" ) property(name="hostname" )
- constant(value="\",\"tag\":\"" ) property(name="syslogtag" )
- constant(value="\",\"vhost\":\"" ) property(name="$!vhost" )
- constant(value="\",\"bytes\":\"" ) property(name="$!bytesend" )
- constant(value="\",\"clientip\":\"" ) property(name="$!ip" )
- constant(value="\",\"method\":\"" ) property(name="$!method" )
- constant(value="\",\"request\":\"" ) property(name="$!url" )
- constant(value="\",\"pversion\":\"" ) property(name="$!pver" )
- constant(value="\",\"referrer\":\"" ) property(name="$!referer" )
- constant(value="\",\"useragent\":\"" ) property(name="$!useragent" )
- constant(value="\",\"status\":\"" ) property(name="$!status" )
- constant(value="\"}" )
- }
|
Exemple de Log en entrée :
Code :
- Nov 25 15:09:18 rp4 apache-access 192.168.102.233 - - [25/Nov/2015:15:09:18 +0100] "GET /mac/customer/ticket/tickets-data-loaded.do?clientTime=1448460559152&lastCallTime=1448460549090&_=1448460559153 HTTP/1.1" 200 2184 "http://xxxxxxxx/mac/customer/orderbill/order/line/create/flow.do?execution=e1s1" "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:42.0) Gecko/20100101 Firefox/42.0"
|
Le parsing est OK :
Code :
- head -n1 /var/log/rsyslog/apache-exemple | /usr/lib/lognorm/lognormalizer -r /etc/rsyslog.d/apacheaccess.rule
- [cee@115 useragent="Mozilla/5.0 (Windows NT 6.1; WOW64; rv:42.0) Gecko/20100101 Firefox/42.0" referer="\"http://xxxxxxxxx/mac/customer/orderbill/order/line/create/flow.do?execution=e1s1\"" bytesend="2184" status="200" pver="HTTP/1.1" url="/mac/customer/ticket/tickets-data-loaded.do?clientTime=1448460559152&lastCallTime=1448460549090&_=1448460559153" method="GET" timestamp="25/Nov/2015:15:09:18 +0100" identd="-" ip="192.168.102.233" tag="apache-access" rcvdfrom="rp4" rcvdat="Nov 25 15:09:18"]
|
Et enfin, exemple de log au format JSON en sortie de Rsyslog pour elastic donc
Code :
- {"@timestamp":"2015-11-27T15:49:59+01:00","message":"192.168.101.62 - - [27/Nov/2015:15:49:58 +0100] \"GET /mac/customer/ticket/tickets-data-loaded.do?clientTime=1448635799539&lastCallTime=1448635789486&_=1448635799540 HTTP/1.1\" 200 2184 \"http://xxxxxxxxx/mac/customer/home.do\" \"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:39.0) Gecko/20100101 Firefox/39.0\" ","host":"193.56.56.56","@source_host":"rp4","tag":"apache-access","vhost":"","bytes":"","clientip":"","method":"","request":"","pversion":"","referrer":"","useragent":"","status":""}
|
Avec un parseur :
Code :
- {
- "@timestamp":"2015-11-27T15:49:59+01:00",
- "message":"192.168.101.62 - - [27/Nov/2015:15:49:58 +0100] \"GET /mac/customer/ticket/tickets-data-loaded.do?clientTime=1448635799539&lastCallTime=1448635789486&_=1448635799540 HTTP/1.1\" 200 2184 \"http://xxxxxxxx/mac/customer/home.do\" \"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:39.0) Gecko/20100101 Firefox/39.0\" ",
- "host":"193.56.56.56",
- "@source_host":"rp4",
- "tag":"apache-access",
- "vhost":"",
- "bytes":"",
- "clientip":"",
- "method":"",
- "request":"",
- "pversion":"",
- "referrer":"",
- "useragent":"",
- "status":""
- }
|
Donc en ce que j'en comprends, les valeurs que je récupère sont uniquement celle disponible par défaut avec Rsyslog
http://www.rsyslog.com/doc/master/ [...] rties.html
Les valeurs issues de ma rule ne semblent donc pas remontées dans le template
En espérant que quelqu'un puisse m'aider |
![[:kiki]](https://forum-images.hardware.fr/images/perso/kiki.gif "[:kiki]")