月份：2016年1月

OpenResty(Nginx Lua)统计网站访问信息

背景

之前的一篇文章openresty(nginx lua)统计域名状态码、平均响应时间和流量实现了对域名状态码，平均响应时间和流量的统计。但之前的统计方法没有实现当某一域名404或500等状态码超过一定数量后发送具体的url来快速定位位置。这个功能我们其实是通过统计网站日志来实现了。为了摆脱对网站日志的依赖以及提高统计性能，我们尝试把此功能也用nginx lua来实现。具体的使用方法与之前的文章一样，这里只是更新了两个lua脚本。

使用方法

1、获取域名devops.webres.wang 404状态码数量

  1. curl -s "localhost/domain_status?count=status&host=devops.webres.wang&status=404"

输出:
10 688
第一列为状态码数量，第二列为域名请求总数
2、获取当域名devops.webres.wang 404状态码超过50个时,输出前10个url

  1. curl -s "localhost/domain_status?count=statusUrl&host=devops.webres.wang&status=404&exceed=50&output=10"

输出:
/hello-world 90
/centos 10
第一列为url，第二列为url请求次数。
3、获取域名devops.webres.wang upstream一分钟内平均耗时

  1. curl -s "localhost/domain_status?count=upT&host=devops.webres.wang"

输出:
0.02 452
第一列为upstream平均耗时，第二列为域名总请求次数。
4、获取当域名devops.webres.wang upstream平均耗时超过0.5秒时,输出其url

  1. curl -s "localhost/domain_status?count=upTUrl&host=devops.webres.wang&exceed=0.5"

输出:
/hello.php 0.82 52
第一列为url，第二列为此url平均耗时,第三列为此url请求次数。监控此接口数据可以快速定位出具体哪些url慢了。
5、获取域名devops.webres.wang request time平均耗时

  1. curl -s "localhost/domain_status?count=reqT&host=devops.webres.wang"

输出:
1.82 52
第一列为平均耗时，第二列为域名请求数。request time是指完成整个请求所需要的时间（包括把数据传输到用户浏览器的时间）。对于php请求，upstream time指的是nginx把php请求传给fastcgi到完成数据接收所需时间。所以request time永远大于upstream time。
6、获取域名devops.webres.wang占用的带宽(单位:字节/秒)

  1. curl -s "localhost/domain_status?count=flow&host=devops.webres.wang"

输出:
1024 52
第一列为此域名一分钟内平均传输速率，单位为字节/秒，第二列为域名请求总数。

相关脚本

log_acesss.lua

  1. local access = ngx.shared.access
  2. local host = ngx.var.host or "unknow"
  3. local status = ngx.var.status
  4. local body_bytes_sent = ngx.var.body_bytes_sent
  5. local request_time = ngx.var.request_time
  6. local upstream_response_time = ngx.var.upstream_response_time or 0
  7. local request_uri = ngx.var.request_uri or "/unknow"
  8. local timestamp = ngx.time()
  9. local expire_time = 70
  10.  
  11. local status_key = table.concat({host,"-",status,"-",timestamp})
  12. local flow_key = table.concat({host,"-flow-",timestamp})
  13. local req_time_key = table.concat({host,"-reqt-",timestamp})
  14. local up_time_key = table.concat({host,"-upt-",timestamp})
  15. local total_key = table.concat({host,"-total-",timestamp})
  16.  
  17. — 域名总请求数
  18. local n,e = access:incr(total_key,1)
  19. if not n then
  20. access:set(total_key, 1, expire_time)
  21. end
  22.  
  23. — 域名状态码请求数
  24. local n,e = access:incr(status_key,1)
  25. if not n then
  26. access:set(status_key, 1, expire_time)
  27. end
  28.  
  29. — 域名流量
  30. local n,e = access:incr(flow_key,body_bytes_sent)
  31. if not n then
  32. access:set(flow_key, body_bytes_sent, expire_time)
  33. end
  34.  
  35. — 域名请求耗时
  36. local n,e = access:incr(req_time_key,request_time)
  37. if not n then
  38. access:set(req_time_key, request_time, expire_time)
  39. end
  40.  
  41. — 域名upstream耗时
  42. local n,e = access:incr(up_time_key,upstream_response_time)
  43. if not n then
  44. access:set(up_time_key, upstream_response_time, expire_time)
  45. end
  46.  
  47. — 获取不带参数的uri
  48. local m, err = ngx.re.match(request_uri, "(.*?)\?")
  49. local request_without_args = m and m[1] or request_uri
  50.  
  51. — 存储状态码大于400的url
  52. if tonumber(status) >= 400 then
  53. — 拼接url,状态码,字节数等字段
  54. local request_log_t = {}
  55. table.insert(request_log_t,host)
  56. table.insert(request_log_t,request_without_args)
  57. table.insert(request_log_t,status)
  58. local request_log = table.concat(request_log_t," ")
  59.  
  60. — 把拼接的字段储存在字典中
  61. local log_key = table.concat({"status-",timestamp})
  62. local request_log_dict = access:get(log_key) or ""
  63. if request_log_dict == "" then
  64. request_log_dict = request_log
  65. else
  66. request_log_dict = table.concat({request_log_dict,"n",request_log})
  67. end
  68. access:set(log_key, request_log_dict, expire_time)
  69. end
  70.  
  71. — 存储upstream time大于0.5的url
  72. if tonumber(upstream_response_time) > 0.5 then
  73. — 拼接url,状态码,字节数等字段
  74. local request_log_t = {}
  75. table.insert(request_log_t,host)
  76. table.insert(request_log_t,request_without_args)
  77. table.insert(request_log_t,upstream_response_time)
  78. local request_log = table.concat(request_log_t," ")
  79.  
  80. — 把拼接的字段储存在字典中
  81. local log_key = table.concat({"upt-",timestamp})
  82. local request_log_dict = access:get(log_key) or ""
  83. if request_log_dict == "" then
  84. request_log_dict = request_log
  85. else
  86. request_log_dict = table.concat({request_log_dict,"n",request_log})
  87. end
  88. access:set(log_key, request_log_dict, expire_time)
  89. end

domain_status.lua

  1. — 各参数用法:
  2. — count=status,host=xxx.com,status=404 统计xxx.com域名一分钟内404状态码个数.
  3. — count=statusUrl,host=xxx.com,status=404,exceed=50,output=30 当xxx.com域名404状态码一分钟内超过50个时,输出前30个url，否则返回空.
  4.  
  5. — count=upT,host=xxx.com 统计xxx.com域名一分钟内平均upsteam耗时
  6. — count=upTUrl,host=xxx.com,exceed=0.5 输出upstreamTime超过0.5秒的url,没有就返回空
  7.  
  8. — count=reqT,host=xxx.com 统计xxx.com域名一分钟内平均请求耗时
  9. — count=flow,host=xxx.com 统计xxx.com域名一分钟内流量(单位字节/秒)
  10.  
  11. — 函数: 获取迭代器值
  12. local get_field = function(iterator)
  13.     local m,err = iterator
  14.     if err then
  15.         ngx.log(ngx.ERR, "get_field iterator error: ", err)
  16.         ngx.exit(ngx.HTTP_OK)
  17.     end
  18.     return m[0]
  19. end
  20.  
  21. — 函数: 按值排序table
  22. local getKeysSortedByValue = function (tbl, sortFunction)
  23.   local keys = {}
  24.   for key in pairs(tbl) do
  25.     table.insert(keys, key)
  26.   end
  27.  
  28.   table.sort(keys, function(a, b)
  29.     return sortFunction(tbl[a], tbl[b])
  30.   end)
  31.  
  32.   return keys
  33. end
  34.  
  35. — 函数: 判断table是否存在某元素
  36. local tbl_contain = function(table,element)
  37.     for k in pairs(table) do
  38.         if k == element then
  39.             return true
  40.         end
  41.     end
  42.     return false
  43. end
  44.  
  45. local access = ngx.shared.access
  46. local now = ngx.time()
  47. local one_minute_ago = now – 60
  48.  
  49. — 获取参数
  50. local args = ngx.req.get_uri_args()
  51. local count_arg = args["count"]
  52. local host_arg = args["host"]
  53. local status_arg = args["status"]
  54. local exceed_arg = args["exceed"]
  55. local output_arg = args["output"]
  56. local count_t = {["status"]=0,["statusUrl"]=0,["upT"]=0,["upTUrl"]=0,["reqT"]=0,["flow"]=0}
  57.  
  58. — 检查参数是否满足
  59. if not tbl_contain(count_t,count_arg) then
  60.     ngx.print("count arg invalid.")
  61.     ngx.exit(ngx.HTTP_OK)
  62. end
  63.  
  64. if not host_arg then
  65.     ngx.print("host arg not found.")
  66.     ngx.exit(ngx.HTTP_OK)
  67. end
  68.  
  69. if count_arg == "status" and not status_arg then
  70.     ngx.print("status arg not found.")
  71.     ngx.exit(ngx.HTTP_OK)
  72. end
  73.  
  74. if count_arg == "statusUrl" and not (status_arg and exceed_arg and output_arg)  then
  75.     ngx.print("status or exceed or output arg not found.")
  76.     ngx.exit(ngx.HTTP_OK)
  77. end
  78.  
  79. if count_arg == "upTUrl" and not exceed_arg then
  80.     ngx.print("exceed arg not found.")
  81.     ngx.exit(ngx.HTTP_OK)
  82. end
  83.  
  84. — 检查参数是否合法
  85. if status_arg and ngx.re.find(status_arg, "^[0-9]{3}$") == nil then
  86.     ngx.print("status arg must be a valid httpd code.")
  87.     ngx.exit(ngx.HTTP_OK)
  88. end
  89.  
  90. if exceed_arg and ngx.re.find(exceed_arg, "^[0-9.]+$") == nil then
  91.     ngx.print("exceed arg must be a number.")
  92.     ngx.exit(ngx.HTTP_OK)
  93. end
  94.  
  95. if output_arg and ngx.re.find(output_arg, "^[0-9]+$") == nil then
  96.     ngx.print("output arg must be a number.")
  97.     ngx.exit(ngx.HTTP_OK)
  98. end
  99.  
  100. — 开始统计
  101. local url
  102. local status_code
  103. local upstream_time
  104. local status_total = 0
  105. local host
  106. local req_total = 0
  107. local flow_total = 0
  108. local reqtime_total = 0
  109. local upstream_total = 0
  110. local status_url_t = {}
  111. local upstream_url_t = {}
  112. local upstream_url_count_t = {}
  113.  
  114. local status_log
  115. local upt_log
  116.  
  117. for second_num=one_minute_ago,now do
  118.     local flow_key = table.concat({host_arg,"-flow-",second_num})
  119.     local req_time_key = table.concat({host_arg,"-reqt-",second_num})
  120.     local up_time_key = table.concat({host_arg,"-upt-",second_num})
  121.     local total_req_key = table.concat({host_arg,"-total-",second_num})
  122.     local log_key
  123.     local log_line
  124.  
  125.     — 合并状态码大于等于400的请求日志到变量status_log
  126.     log_key = table.concat({"status-",second_num})
  127.     log_line = access:get(log_key) or ""
  128.     if not (log_line == "") then
  129.         status_log = table.concat({log_line,"n",status_log})
  130.     end
  131.  
  132.     — 合并upstream time大于0.5秒的请求日志到变量upt_log
  133.     log_key = table.concat({"upt-",second_num})
  134.     log_line = access:get(log_key) or ""
  135.     if not (log_line == "") then
  136.         upt_log = table.concat({log_line,"n",upt_log})
  137.     end
  138.  
  139.     — 域名总请求数
  140.     local req_sum = access:get(total_req_key) or 0
  141.     req_total = req_total + req_sum
  142.  
  143.     if count_arg == "status" or count_arg == "statusUrl" then
  144.         local status_key = table.concat({host_arg,"-",status_arg,"-",second_num})
  145.         local status_sum = access:get(status_key) or 0
  146.         status_total = status_total + status_sum
  147.     end
  148.  
  149.     if count_arg == "flow" then
  150.         local flow_sum = access:get(flow_key) or 0
  151.         flow_total = flow_total + flow_sum
  152.     end
  153.  
  154.     if count_arg == "reqT" then
  155.         local req_time_sum = access:get(req_time_key) or 0
  156.         reqtime_total = reqtime_total + req_time_sum
  157.     end
  158.  
  159.     if count_arg == "upT" then
  160.         local up_time_sum = access:get(up_time_key) or 0
  161.         upstream_total = upstream_total + up_time_sum
  162.     end
  163. end
  164.  
  165. — 统计状态码url
  166. if count_arg == "statusUrl" and status_log and not (status_log == "") then
  167.     local iterator, err = ngx.re.gmatch(status_log,".+n")
  168.     if not iterator then
  169.         ngx.log(ngx.ERR, "status_log iterator error: ", err)
  170.         return
  171.     end
  172.     for line in iterator do
  173.         if not line[0] then
  174.             ngx.log(ngx.ERR, "line[0] is nil")
  175.             return
  176.         end
  177.         local iterator, err = ngx.re.gmatch(line[0],"[^ n]+")
  178.         if not iterator then
  179.             ngx.log(ngx.ERR, "line[0] iterator error: ", err)
  180.             return
  181.         end
  182.  
  183.         host = get_field(iterator())
  184.         url = get_field(iterator())
  185.         status_code = get_field(iterator())
  186.  
  187.         if status_code == status_arg then
  188.             if status_url_t[url] then
  189.                 status_url_t[url] = status_url_t[url] + 1
  190.             else
  191.                 status_url_t[url] = 1
  192.             end
  193.         end
  194.  
  195.     end   
  196. end
  197.  
  198. — 统计upstream time大于0.5秒url
  199. if count_arg == "upTUrl" and upt_log and not (upt_log == "") then
  200.     local iterator, err = ngx.re.gmatch(upt_log,".+n")
  201.     if not iterator then
  202.         ngx.log(ngx.ERR, "upt_log iterator error: ", err)
  203.         return
  204.     end
  205.     for line in iterator do
  206.         if not line[0] then
  207.             ngx.log(ngx.ERR, "line[0] is nil")
  208.             return
  209.         end
  210.         local iterator, err = ngx.re.gmatch(line[0],"[^ n]+")
  211.         if not iterator then
  212.             ngx.log(ngx.ERR, "line[0] iterator error: ", err)
  213.             return
  214.         end
  215.  
  216.         host = get_field(iterator())
  217.         url = get_field(iterator())
  218.         upstream_time = get_field(iterator())
  219.         upstream_time = tonumber(upstream_time) or 0
  220.  
  221.         — 统计各url upstream平均耗时
  222.         if host == host_arg then
  223.             if upstream_url_t[url] then
  224.                 upstream_url_t[url] = upstream_url_t[url] + upstream_time
  225.             else
  226.                 upstream_url_t[url] = upstream_time
  227.             end
  228.  
  229.             if upstream_url_count_t[url] then
  230.                 upstream_url_count_t[url] = upstream_url_count_t[url] + 1
  231.             else
  232.                 upstream_url_count_t[url] = 1
  233.             end
  234.         end   
  235.     end   
  236. end
  237.  
  238. — 输出结果
  239. if count_arg == "status" then
  240.     ngx.print(status_total," ",req_total)
  241.  
  242. elseif count_arg == "flow" then
  243.     ngx.print(flow_total," ",req_total)
  244.  
  245. elseif count_arg == "reqT" then
  246.     local reqt_avg = 0
  247.     if req_total == 0 then
  248.         reqt_avg = 0
  249.     else
  250.         reqt_avg = reqtime_total/req_total
  251.     end
  252.     ngx.print(reqt_avg," ",req_total)
  253.  
  254. elseif count_arg == "upT" then
  255.     local upt_avg = 0
  256.     if req_total == 0 then
  257.             upt_avg = 0
  258.     else
  259.             upt_avg = upstream_total/req_total
  260.     end
  261.     ngx.print(upt_avg," ",req_total)
  262.  
  263. elseif count_arg == "statusUrl" then
  264.     if status_total > tonumber(exceed_arg) then
  265.         — 排序table
  266.         status_url_t_key = getKeysSortedByValue(status_url_t, function(a, b) return a > b end)
  267.         local output_body = ""
  268.         for i, uri in ipairs(status_url_t_key) do
  269.             if output_body == "" then
  270.                 output_body = table.concat({uri," ",status_url_t[uri]})
  271.             else   
  272.                 output_body = table.concat({output_body,"n",uri," ",status_url_t[uri]})
  273.             end
  274.             if i >= tonumber(output_arg) then
  275.                 ngx.print(output_body)
  276.                 ngx.exit(ngx.HTTP_OK)
  277.             end               
  278.         end
  279.  
  280.         ngx.print(output_body)
  281.         ngx.exit(ngx.HTTP_OK)
  282.     end
  283.  
  284. elseif count_arg == "upTUrl" then
  285.     local max_output = 30
  286.     local total_time = 0
  287.     local total_count = 0
  288.     local output_body = ""
  289.     local i = 0
  290.     for url in pairs(upstream_url_t) do
  291.         i = i + 1
  292.         total_time = upstream_url_t[url]
  293.         total_count = upstream_url_count_t[url]
  294.         avg_time = upstream_url_t[url] / upstream_url_count_t[url]
  295.         if avg_time > tonumber(exceed_arg) then
  296.             output_body = table.concat({url," ",avg_time," ",total_count,"n",output_body})
  297.         end
  298.  
  299.         if i >= max_output then
  300.             ngx.print(output_body)
  301.             ngx.exit(ngx.HTTP_OK)
  302.         end           
  303.     end
  304.     ngx.print(output_body)
  305.     ngx.exit(ngx.HTTP_OK)
  306.  
  307. end