Query log analytics - using logstash, elasticsearch and kibana 28.11.2013

download Query log analytics - using logstash, elasticsearch and kibana 28.11.2013

of 27

  • date post

    11-Jun-2015
  • Category

    Technology

  • view

    1.957
  • download

    0

Embed Size (px)

description

As a search consultant I need to understand how a search application is used with the end goal of providing a better search experience for the end user. That story can come from many places and part of that story can be found in the query logs. Blog post about the same topic: http://nhhagen.wordpress.com/2013/11/28/query-log-analysis-using-logstash-elasticsearch-and-kibana/

Transcript of Query log analytics - using logstash, elasticsearch and kibana 28.11.2013

  • 1. Query log analytics 28.11.2013 Niels Henrik HagenOSLO STOCKHOLMLONDONBOSTON

2. Who am I? Niels Henrik Hagen @nhhagen Search enthusiast 3. Analyzing query logs brings insight into how a search application is used, 4. use that insight to improve the search application 5. What is a query log? 6. {"message":{"time":3016,"hitCounts":{"directmail":27,"refiners":0,"products":2529},"searchQuery":{"attributes":{"querystring":"","sortby":"kawdefault","sortorder":"","offset":0},"metaAttributes":{"queryModel":{}}}},"@timestamp":"2013-10-22T11:07:02.011+00:00","@version":1,"source_host":"search3.kaw. {"message":{"time":492,"hitCounts":{"directmail":0,"refiners":0,"products":5},"searchQuery":{"attributes":{"querystring":"fisk","sortby":"kawdefault","sortorder":"","offset":0},"metaAttributes":{"queryModel":{}}}},"@timestamp":"2013-10-22T11:07:19.714+00:00","@version":1,"source_host":"search3.kaw.no {"message":{"time":459,"hitCounts":{"directmail":27,"refiners":0,"products":2529},"searchQuery":{"attributes":{"querystring":"","sortby":"kawdefault","hits":30,"sortorder":"","offset":0},"metaAttributes":{"queryModel":{}}}},"@timestamp":"2013-10-22T11:11:37.852+00:00","@version":1,"source_host":"sea {"message":{"time":157,"hitCounts":{"directmail":1},"searchQuery":{"attributes":{"hits":1,"querystring":"","sortby":"","navigators":[],"sortorder":"","productid":"112530","offset":0,"filter s":[]},"metaAttributes":{"queryModel":{}}}},"@timestamp":"2013-10-22T11:11:43.858+00:00","@version":1,"source_ho {"message":{"time":155,"hitCounts":{"products":2524},"searchQuery":{"attributes":{"querystring":"skjenk hyller og kommoder ","sortby":"kawdefault","hits":9,"sortorder":"","similarto":"112530","offset":0},"metaAttributes":{"queryModel":{}}}},"@timestamp":"2013-10-22T11:11:44.810+00:00","@version":1,"s {"message":{"time":252,"hitCounts":{"directmail":0,"refiners":0,"products":5},"searchQuery":{"attributes":{"querystring":"fisk","sortby":"kawdefault","hits":30,"sortorder":"","offset":0},"metaAttributes":{"queryModel":{}}}},"@timestamp":"2013-10-22T11:12:03.340+00:00","@version":1,"source_host":"sear {"message":{"time":460,"hitCounts":{"directmail":27,"refiners":0,"products":2529},"searchQuery":{"attributes":{"querystring":"","sortby":"kawdefault","hits":30,"sortorder":"","offset":0},"metaAttributes":{"queryModel":{}}}},"@timestamp":"2013-10-22T11:12:47.508+00:00","@version":1,"source_host":"sea {"message":{"time":303,"hitCounts":{"directmail":5,"refiners":0,"products":122},"searchQuery":{"attributes":{"querystring":"leker","sortby":"kawdefault","hits":30,"sortorder":"","offset":0},"metaAttributes":{"queryModel":{}}}},"@timestamp":"2013-10-22T11:12:49.918+00:00","@version":1,"source_host":"s {"message":{"time":386,"hitCounts":{"directmail":1,"refiners":0,"products":91},"searchQuery":{"attributes":{"querystring":"","sortby":"kawdefault","hits":30,"sortorder":"","navigators":[{"key":"shop","displayName":"Butikk","interval":false,"upperValue":null,"lowerValue":null,"modifiers":[{"value":"E {"message":{"time":22,"hitCounts":{"directmail":1},"searchQuery":{"attributes":{"hits":1,"querystring":"","sortby":"","navigators":[],"sortorder":"","productid":"112173","offset":0,"filter s":[]},"metaAttributes":{"queryModel":{}}}},"@timestamp":"2013-10-22T11:13:19.196+00:00","@version":1,"source_hos {"message":{"time":152,"hitCounts":{"products":58},"searchQuery":{"attributes":{"querystring":"peisovn jtul ","sortby":"kawdefault","hits":9,"sortorder":"","similarto":"112173","offset":0},"metaAttributes":{"queryModel":{}}}},"@timestamp":"2013-10-22T11:13:21.289+00:00","@version":1,"source_host":"s {"message":{"time":46,"hitCounts":{"directmail":0},"searchQuery":{"attributes":{"hits":1,"querystring":"","sortby":"","navigators":[],"sortorder":"","offset":0,"dmid":"1201","filters":[]}, "metaAttributes":{"queryModel":{}}}},"@timestamp":"2013-10-22T11:13:54.993+00:00","@version":1,"source_host":"sea {"message":{"time":292,"hitCounts":{"directmail":27,"refiners":0,"products":2529},"searchQuery":{"attributes":{"querystring":"","sortby":"kawdefault","hits":30,"sortorder":"","offset":0},"metaAttributes":{"queryModel":{}}}},"@timestamp":"2013-10-22T11:14:08.393+00:00","@version":1,"source_host":"sea {"message":{"time":335,"hitCounts":{"directmail":0,"refiners":0,"products":8},"searchQuery":{"attributes":{"querystring":"dyner","sortby":"kawdefault","hits":30,"sortorder":"","offset":0},"metaAttributes":{"queryModel":{}}}},"@timestamp":"2013-10-22T11:14:20.384+00:00","@version":1,"source_host":"sea {"message":{"time":326,"hitCounts":{"directmail":27,"refiners":0,"products":2529},"searchQuery":{"attributes":{"querystring":"","sortby":"kawdefault","hits":30,"sortorder":"","offset":0},"metaAttributes":{"queryModel":{}}}},"@timestamp":"2013-10-22T11:14:36.072+00:00","@version":1,"source_host":"sea {"message":{"time":39,"hitCounts":{"directmail":0},"searchQuery":{"attributes":{"hits":1,"querystring":"","sortby":"","navigators":[],"sortorder":"","offset":0,"dmid":"1201","filters":[]}, "metaAttributes":{"queryModel":{}}}},"@timestamp":"2013-10-22T11:14:42.194+00:00","@version":1,"source_host":"sea {"message":{"time":289,"hitCounts":{"directmail":2,"refiners":0,"products":222},"searchQuery":{"attributes":{"querystring":"","sortby":"kawdefault","hits":30,"sortorder":"","navigators":[{"key":"shop","displayName":"Butikk","interval":false,"upperValue":null,"lowerValue":null,"modifiers":[{"value":" {"message":{"time":333,"hitCounts":{"directmail":27,"refiners":0,"products":2529},"searchQuery":{"attributes":{"querystring":"","sortby":"kawdefault","hits":30,"sortorder":"","offset":0},"metaAttributes":{"queryModel":{}}}},"@timestamp":"2013-10-22T11:15:22.236+00:00","@version":1,"source_host":"sea {"message":{"time":316,"hitCounts":{"directmail":2,"refiners":0,"products":47},"searchQuery":{"attributes":{"querystring":"","sortby":"kawdefault","hits":30,"sortorder":"","navigators":[{"key":"dmcategory","displayName":"Kategorier","interval":false,"upperValue":null,"lowerValue":null,"modifiers":[{ {"message":{"time":24,"hitCounts":{"directmail":1},"searchQuery":{"attributes":{"hits":1,"querystring":"","sortby":"","navigators":[],"sortorder":"","offset":0,"dmid":"1447","filters":[]}, "metaAttributes":{"queryModel":{}}}},"@timestamp":"2013-10-22T11:15:55.959+00:00","@version":1,"source_host":"sea {"message":{"time":294,"hitCounts":{"directmail":0,"refiners":0,"products":12},"searchQuery":{"attributes":{"querystring":"dusjkabinett","sortby":"kawdefault","hits":30,"sortorder":"","offset":0},"metaAttributes":{"queryModel":{}}}},"@timestamp":"2013-10-22T11:16:13.633+00:00","@version":1,"source_ho {"message":{"time":22,"hitCounts":{"directmail":0},"searchQuery":{"attributes":{"hits":1,"querystring":"","sortby":"","navigators":[],"sortorder":"","offset":0,"dmid":"1023","filters":[]}, "metaAttributes":{"queryModel":{}}}},"@timestamp":"2013-10-22T11:16:15.842+00:00","@version":1,"source_host":"sea {"message":{"time":260,"hitCounts":{"directmail":0,"refiners":0,"products":0,"expiredproducts":0},"searchQuery":{"attributes":{"querystring":"cd591700627no","sortby":"kawdefault","hits":30,"sortorder":"","offset":0}," metaAttributes":{"queryModel":{}}}},"@timestamp":"2013-10-22T11:16:30.752+00:00","@v {"message":{"time":20,"hitCounts":{"directmail":1},"searchQuery":{"attributes":{"hits":1,"querystring":"","sortby":"","navigators":[],"sortorder":"","offset":0,"dmid":"1447","filters":[]}, "metaAttributes":{"queryModel":{}}}},"@timestamp":"2013-10-22T11:16:31.534+00:00","@version":1,"source_host":"sea {"message":{"time":26,"hitCounts":{"directmail":1},"searchQuery":{"attributes":{"hits":1,"querystring":"","sortby":"","navigators":[],"sortorder":"","productid":"112597","offset":0,"filter s":[]},"metaAttributes":{"queryModel":{}}}},"@timestamp":"2013-10-22T11:16:34.619+00:00","@version":1,"source_hos {"message":{"time":140,"hitCounts":{"products":132},"searchQuery":{"attributes":{"querystring":"dusjkabinett baderomsinnredning ","sortby":"kawdefault","hits":9,"sortorder":"","similarto":"112597","offset":0},"metaAttributes":{"queryModel":{}}}},"@timestamp":"2013-10-22T11:16:37.566+00:00","@version" {"message":{"time":298,"hitCounts":{"directmail":0,"refiners":0,"products":30},"searchQuery":{"attributes":{"querystring":"kjkkenutstyr","sortby":"kawdefault","hits":30,"sortorder":"","offset":0},"metaAttributes":{"queryModel":{}}}},"@timestamp":"2013-10-22T11:16:42.008+00:00","@version":1,"source_h {"message":{"time":20,"hitCounts":{"directmail":0},"searchQuery":{"attributes":{"hits":1,"querystring":"","sortby":"","navigators":[],"sortorder":"","offset":0,"dmid":"1062","filters":[]}, "metaAttributes":{"queryModel":{}}}},"@timestamp":"2013-10-22T11:16:46.663+00:00","@version":1,"source_host":"sea {"message":{"time":22,"hitCounts":{"directmail":1},"searchQuery":{"attributes":{"hits":1,"querystring":"","sortby":"","navigators":[],"sortorder":"","offset":0,"dmid":"1447","filters":[]}, "metaAttributes":{"queryModel":{}}}},"@timestamp":"2013-10-22T11:16:52.785+00:00","@version":1,"source_host":"sea {"message":{"time":20,"hitCounts":{"directmail":1},"searchQuery":{"attributes":{"hits":1,"querystring":"","sortby":"","navigators":[],"sortorder":"","productid":"111811","offset":0,"filter s":[]},"metaAttributes":{"queryModel":{}}}},"@timestamp":"2013-10-22T11:17:02.328+00:00","@version":1,"source_hos {"message":{"time":146,"hitCounts":{"products":135},"searchQuery":{"attributes":{"querystring":"dusjkabinett element baderomsinnredning ","sortby":"kawdefault","hits":9,"sortorder":"","similarto":"111811","offset":0},"metaAttributes":{"queryModel":{}}}},"@timestamp":"2013-10-22T11:17:04.688+00:00","@ {"message":{"time":22,"hitCounts":{"directmail":1},"searchQuery":{"attributes":{"hits":1,"querystring":"","sortby":"","navigators":[],"sortorder":"","offset":0,"dmid":"1447","filters":[]}, "metaAttributes":{"queryModel":{}}}},"@timestamp":"2013-10-22T11:17:04.729+00:00","@version":1,"source_host":"sea