/usr/lib/imh-ultrastack-common/venv/lib/python3.13/site-packages/ngxutil/

/usr/lib/imh-ultrastack-common/venv/lib/python3.13/site-packages/ngxutil

Name	Size	Mode	Actions
__pycache__/	-	0755	rm
api.py	4723	0644	edit dl rm
cache.py	6938	0644	edit dl rm
cli.py	10702	0644	edit dl rm
info.py	14916	0644	edit dl rm
logparse.py	10537	0644	edit dl rm
profile.py	1758	0644	edit dl rm
report.py	14618	0644	edit dl rm
sendmail.py	1374	0644	edit dl rm
util.py	2888	0644	edit dl rm
vts.py	1710	0644	edit dl rm
__init__.py	405	0644	edit dl rm

Edit: /usr/lib/imh-ultrastack-common/venv/lib/python3.13/site-packages/ngxutil/report.py (14618B)

# vim: set ts=4 sw=4 expandtab syntax=python:
"""

ngxutil.report
ANSI & HTML report generator

@author J. Hipps <jacobh@inmotionhosting.com>

"""

import os
import sys
import logging
import re
from glob import glob

import arrow
from prettytable import PrettyTable

from ngxutil import default_cache_base
from ngxutil import logparse
from ngxutil.vts import parse_vts
from ngxutil.cache import read_cache_file
from ngxutil.util import *

# HTML table attribs
tab_attribs = {'style': "border: 3px solid black; border-collapse: collapse; width: 650px; "
                        "font-size: 16px; font-family: Helvetica,sans-serif; cell-padding: 10px"}

logger = logging.getLogger('ngxutil')

def repgen_header(vdat):
    """
    Generate server status header
    Returns: (plain_text, html)
    """
    # calc uptime
    try:
        uptime = format_uptime((vdat.get('nowMsec') - vdat.get('loadMsec')) / 1000.0)
    except:
        uptime = "???"

pt = PrettyTable()
    pt.field_names = ('Server', 'Version', 'Uptime')
    try:
        pt.add_row((vdat.get('hostName'), vdat.get('nginxVersion'), uptime))
    except:
        pass
    return (pt.get_string(), pt.get_html_string() + "<br/>")

def repgen_server(span=None, tail=None, infile='/var/log/nginx/access.log'):
    """
    Generate a server-wide report
    """
    vdata = parse_vts()
    ro, rh = repgen_header(vdata)
    rdom = repgen_domain('*', vdata, noheader=True, span=span, tail=tail, infile=infile)
    if rdom is None:
        return None
    ro += "\n\n" + rdom[0] + "\n\n"
    rh += "\n" + rdom[1] + "\n"
    return (ro, rh)

def repgen_domain(domain, vdata=None, noheader=False, span=None, tail=None, infile='/var/log/nginx/access.log'):
    """
    Generate a report for @domain
    """
    if vdata is None:
        vdata = parse_vts()

ro = ""; rh = ""

duser = get_domain_owner(domain)
    if not duser:
        return None

try:
        tvd = vdata['serverZones'][duser['vhost']]
    except:
        logger.error("Failed to retrieve status data for domain %s. Data not yet available.", domain)
        return None

# determine loglines to read
    if tail is not None:
        llines = tail
    else:
        if span:
            llines = None
        else:
            llines = vdata['connections']['requests']

# Get access.log data, then filter it based on the vhost field
    try:
        logger.info("Parsing log '%s' (%s)...", infile, format_size(os.stat(infile).st_size))
    except Exception as e:
        logger.error("Unable to stat input file: %s", str(e))
        sys.exit(100)
    aclog = logparse.parse_log(infile, span=span, lastlines=llines)
    logdata = logparse.filter_log(aclog, 'vhost', duser['vhost'].lower())

xpc = logparse.get_cache_hitrate(logdata)

# Domain header table
    if span:
        rqtot = len(logdata)
        rtimes = logparse.get_reqtime(logdata)
        rx_fmt = "---"
        tx_fmt = format_size(logparse.get_bytes(logdata))
        avgtime_fmt = "Min/Avg/Max"
        avgtime_str = "{min:01.03f} / {avg:01.03f} / {max:01.03f}".format(**rtimes)
    else:
        rqtot = tvd['requestCounter']
        rx_fmt = format_size(tvd['inBytes'])
        tx_fmt = format_size(tvd['outBytes'])
        avgtime_fmt = "Avg"
        avgtime_str = "{:01.03f}".format(float(sum(tvd['requestMsecs']['msecs'])) / \
                                         float(len(tvd['requestMsecs']['msecs'])) / 1000.0)
    pt = PrettyTable()
    pt.field_names = ('RX bytes', 'TX bytes', 'Requests', 'Req Time (%s)' % (avgtime_fmt))
    pt.add_row((rx_fmt, tx_fmt, rqtot, avgtime_str))

if span:
        rduration = format_uptime(int(span) * 3600)
    else:
        rduration = format_uptime((vdata.get('nowMsec') - vdata.get('loadMsec')) / 1000.0)

if not noheader:
        ro = strcolor('white', "<< {} / {} >>\n".format(domain, duser['vhost']))
        rh = "<h1 style=\"font: 48px Helvetica,sans-serif\">" + domain + " / " + duser['vhost'] + "</h1>\n" + \
             gen_email_header(rduration)
    ro += pt.get_string() + "\n\n"
    rh += strip_colors(pt.get_html_string(attributes=tab_attribs)) + "<br/>\n"

# Add in cache stats
    cinfo = repgen_cache_info(duser['owner'], duser['vhost'], domain)
    ro += cinfo[0]; rh += cinfo[1]

# Build response code table
    pt = PrettyTable()
    pt.field_names = ('1xx', '2xx', '3xx', '4xx', '5xx', 'TOTAL')
    if span:
        rpc = logparse.get_status_hitrate(logdata)
        pt.add_row((strcolor('magenta', mkpct(rpc.get('1xx', 0), rqtot)),
                    strcolor('green', mkpct(rpc.get('2xx', 0), rqtot)),
                    strcolor('blue', mkpct(rpc.get('3xx', 0), rqtot)),
                    strcolor('yellow', mkpct(rpc.get('4xx', 0), rqtot)),
                    strcolor('red', mkpct(rpc.get('5xx', 0), rqtot)),
                    strcolor('white', mkpct(rqtot, rqtot))))
    else:
        pt.add_row((strcolor('magenta', mkpct(tvd['responses']['1xx'], rqtot)),
                    strcolor('green', mkpct(tvd['responses']['2xx'], rqtot)),
                    strcolor('blue', mkpct(tvd['responses']['3xx'], rqtot)),
                    strcolor('yellow', mkpct(tvd['responses']['4xx'], rqtot)),
                    strcolor('red', mkpct(tvd['responses']['5xx'], rqtot)),
                    strcolor('white', mkpct(rqtot, rqtot))))
    ro += strcolor('white', "HTTP RESPONSE CODES\n") + pt.get_string() + "\n\n"
    rh += "<h2 style=\"font: 24px Helvetica,sans-serif\">HTTP Response Codes</h2>\n" + \
          strip_colors(pt.get_html_string(attributes=tab_attribs)) + "<br/>\n"

# Build UCS tables
    hit_dtot = rqtot - xpc.get('STATIC/TYPE', 0) - xpc.get('STATIC/PATH', 0) - xpc.get('DISABLED', 0) - xpc.get('BYPASS', 0)
    hit_stot = rqtot - xpc.get('DISABLED', 0) - xpc.get('BYPASS', 0) - xpc.get('HIT', 0)
    hit_btot = rqtot - xpc.get('STATIC/TYPE', 0) - xpc.get('STATIC/PATH', 0)

ptx = PrettyTable()
    ptx.field_names = ('HIT (Dynamic)', 'HIT (Static)', 'DISABLED+BYPASS')
    ptx.add_row((strcolor('green', mkpct(xpc.get('HIT', 0), hit_dtot)),
                strcolor('yellow', mkpct(xpc.get('STATIC/TYPE', 0) + xpc.get('STATIC/PATH', 0), hit_stot)),
                strcolor('red', mkpct(xpc.get('BYPASS', 0) + xpc.get('DISABLED', 0), hit_btot))))

pt = PrettyTable()
    pt.field_names = ('HIT', 'MISS', 'BYPASS', 'EXPIRED', 'UPDATING')
    pt.add_row((strcolor('green', mkpct(xpc.get('HIT', 0), rqtot)),
                strcolor('red', mkpct(xpc.get('MISS', 0), rqtot)),
                strcolor('magenta', mkpct(xpc.get('BYPASS', 0), rqtot)),
                strcolor('yellow', mkpct(xpc.get('EXPIRED', 0), rqtot)),
                strcolor('yellow', mkpct(xpc.get('UPDATING', 0), rqtot))))

pt2 = PrettyTable()
    pt2.field_names = ('STATIC/TYPE', 'STATIC/PATH', 'DISABLED', 'TOTAL')
    pt2.add_row((strcolor('green', mkpct(xpc.get('STATIC/TYPE', 0), rqtot)),
                 strcolor('green', mkpct(xpc.get('STATIC/PATH', 0), rqtot)),
                 strcolor('magenta', mkpct(xpc.get('DISABLED', 0), rqtot)),
                 strcolor('white', mkpct(rqtot, rqtot))))

ro += strcolor('white', "CACHE STATUS\n") + ptx.get_string() + "\n" + \
          pt.get_string() + "\n" + pt2.get_string() + "\n\n"
    rh += "<h2 style=\"font: 24px Helvetica,sans-serif\">Cache Status</h2>\n" + \
          strip_colors(ptx.get_html_string(attributes=tab_attribs)) + "<br/>\n" + \
          strip_colors(pt.get_html_string(attributes=tab_attribs)) + "<br/>\n" + \
          strip_colors(pt2.get_html_string(attributes=tab_attribs)) + "<br/>\n"

# Show NLP datas
    fullurl = (domain == '*')
    cinfo = repgen_nlp(logdata, user=duser['owner'], fullurl=fullurl)
    ro += cinfo[0]; rh += cinfo[1]

rh += gen_email_footer()
    return (ro, rh)

def repgen_cache_info(zone, vhost=None, domain=None, cbase=default_cache_base):
    """
    Return stats about currently cached pages
    """
    gpath = os.path.join(cbase, zone, '*', '*')
    tally = {'files': 0, 'size': 0}
    for tfile in glob(gpath):
        cdata = read_cache_file(tfile)
        if not cdata:
            continue
        if domain:
            if not re.match(r'^https?(GET|HEAD|OPTIONS)(' + domain.lower() + r'|' + vhost.lower() + r')/?.*$', cdata['key'], re.I):
                continue
        tally['files'] += 1
        tally['size'] += cdata['stat'].st_size

# Build cache stats info
    pt = PrettyTable()
    pt.field_names = ('Total Files', 'Total Size')
    pt.add_row((strcolor('white', tally['files']),
               strcolor('white', format_size(tally['size']))))
    ro = strcolor('white', "CACHE ZONE STATS\n") + pt.get_string() + "\n\n"
    rh = "<h2 style=\"font: 24px Helvetica,sans-serif\">Cache Zone Status</h2>\n" + \
         strip_colors(pt.get_html_string(attributes=tab_attribs)) + "<br/>\n"
    return (ro, rh)

def repgen_nlp(logdata=None, user=None, lastlines=10000, topcount=10, rresolve=True, maxwidth=65, fullurl=False):
    """
    Generate an ANSI and HTML report of top hits
    If @logdata is not supplied, then a fresh read of the log is done,
    pulling the last @lastlines lines from the file
    """
    if not logdata:
        logdata = logparse.parse_log('/var/log/nginx/access.log', lastlines=lastlines)

ro = ""; rh = ""

# Top Hosts
    utop = list(logparse.get_top_hosts(logdata, topcount=topcount).values())
    pt = PrettyTable()
    pt.add_column('HITS', [x['hits'] for x in utop])
    pt.add_column('HOST', [x['host'] for x in utop])
    pt.align['HITS'] = 'r'
    pt.align['HOST'] = 'l'

ro += strcolor('white', "TOP HOSTS\n") + pt.get_string() + "\n\n"
    rh += "<h2 style=\"font: 24px Helvetica,sans-serif\">Top Hosts</h2>\n" + \
          strip_colors(pt.get_html_string(attributes=tab_attribs)) + "<br/>\n"

# Top URIs
    utop = list(logparse.get_top_uris(logdata, czone=user, topcount=topcount).values())
    pt = PrettyTable()
    pt.add_column('HITS', [x['hits'] for x in utop])
    pt.add_column('METHOD', [x['method'] for x in utop])
    pt.add_column('STATUS', [x['status'] for x in utop])
    pt.add_column('CACHE', [x['cache'] for x in utop])
    if fullurl:
        pt.add_column('HOST/URI', [(x['host'] + x['uri'])[:maxwidth] for x in utop])
        pt.align['HOST/URI'] = 'l'
    else:
        pt.add_column('URI', [x['uri'][:maxwidth] for x in utop])
        pt.align['URI'] = 'l'
    pt.align['HITS'] = 'r'

ro += strcolor('white', "TOP URIs\n") + pt.get_string() + "\n\n"
    rh += "<h2 style=\"font: 24px Helvetica,sans-serif\">Top URIs</h2>\n" + \
          strip_colors(pt.get_html_string(attributes=tab_attribs)) + "<br/>\n"

# Top User-Agents
    utop = logparse.get_top_agents(logdata, topcount=topcount)
    pt = PrettyTable()
    pt.add_column('COUNT', list(utop.values()))
    pt.add_column('USER-AGENT', [x[:maxwidth] for x in list(utop.keys())])
    pt.align['COUNT'] = 'r'
    pt.align['USER-AGENT'] = 'l'

ro += strcolor('white', "TOP USER-AGENTS\n") + pt.get_string() + "\n\n"
    rh += "<h2 style=\"font: 24px Helvetica,sans-serif\">Top User-Agents</h2>\n" + \
          strip_colors(pt.get_html_string(attributes=tab_attribs)) + "<br/>\n"

# Top Locations
    if logdata and logdata[0].get('loc') is not None:
        utop = logparse.get_top_locations(logdata, topcount=topcount)
        pt = PrettyTable()
        pt.add_column('COUNT', list(utop.values()))
        pt.add_column('LOCATION', list(utop.keys()))
        pt.align['COUNT'] = 'r'
        pt.align['LOCATION'] = 'l'

ro += strcolor('white', "TOP LOCATIONS\n") + pt.get_string() + "\n\n"
        rh += "<h2 style=\"font: 24px Helvetica,sans-serif\">Top Locations</h2>\n" + \
              strip_colors(pt.get_html_string(attributes=tab_attribs)) + "<br/>\n"

# Top Client IPs
    utop = logparse.get_top_ips(logdata, rresolve=rresolve, topcount=topcount)
    pt = PrettyTable()
    pt.add_column('HITS', [x['hits'] for x in list(utop.values())])
    pt.add_column('IP', list(utop.keys()))
    pt.add_column('HOST/PTR', [x['reverse'] for x in list(utop.values())])
    pt.align['HITS'] = 'r'
    pt.align['HOST/PTR'] = 'l'

ro += strcolor('white', "TOP CLIENT IPs\n") + pt.get_string()
    rh += "<h2 style=\"font: 24px Helvetica,sans-serif\">Top Client IPs</h2>\n" + \
          strip_colors(pt.get_html_string(attributes=tab_attribs)) + "<br/>\n"

return (ro, rh)

def gen_email_header(tperiod):
    """
    Return email stats header
    """
    outstr = """
    <table style="border: 1px solid #444; border-collapse: collapse; width: 650px; font-size: 12px; font-family: Helvetica,sans-serif">
    <tr>
    <td>Generated on <b>%s</b></td>
    <td>Period: <b>%s</b></td>
    </tr>
    </table>
    """ % (arrow.now().format("YYYY-MM-DD HH:mm:ss ZZ"), tperiod)
    return outstr

def gen_email_footer():
    """
    Return email stats footer
    """
    outstr = """
    <h3 style="font: 18px Helvetica,sans-serif">Glossary</h3>
    <table style="border: 1px solid #aaa; border-collapse: collapse; width: 650px; font-size: 12px; font-family: Helvetica,sans-serif">
    <tr><td>
    <li><b>HIT</b> - A valid (not expired) page was served from the cache</li>
    <li><b>MISS</b> - No valid cached page was available. A request was made to the application, and possibly cached for later use</li>
    <li><b>BYPASS</b> - The cache was bypassed due to meeting certain criteria (such as being logged in, using admin or user portal, using a checkout page in an ecommerce store, etc.)</li>
    <li><b>EXPIRED</b> - An expired page was served from the cache. This can happen if the origin server/application threw an error. Instead of showing the user an error page, an expired page from the cache is shown.</li>
    <li><b>UPDATING</b> - An expired page was served from the cache. Another request is in progress (by another client), which will update the cached page. This status typically only happens during very high traffic periods</li>
    <li><b>STATIC/TYPE</b> - Static content that was served directly by NGINX. The file extension (type) was used to serve the file directly from NGINX, rather than requesting the resource from Apache</li>
    <li><b>STATIC/PATH</b> - Static content that was served directly by NGINX. A specifically-defined path was used to make this determination</li>
    <li><b>DISABLED</b> - Caching has been explicitly disabled by the user</li>
    <li><b>TOTAL</b> - Total number of all requests. All constituent statuses may not add up to 100%, since pages without a specific cache status are not reported</li>
    </td></tr>
    </table>
    """
    return outstr

# vim: set ts=4 sw=4 expandtab syntax=python: """ ngxutil.report ANSI & HTML report generator @author J. Hipps """ import os import sys import logging import re from glob import glob import arrow from prettytable import PrettyTable from ngxutil import default_cache_base from ngxutil import logparse from ngxutil.vts import parse_vts from ngxutil.cache import read_cache_file from ngxutil.util import * # HTML table attribs tab_attribs = {'style': "border: 3px solid black; border-collapse: collapse; width: 650px; " "font-size: 16px; font-family: Helvetica,sans-serif; cell-padding: 10px"} logger = logging.getLogger('ngxutil') def repgen_header(vdat): """ Generate server status header Returns: (plain_text, html) """ # calc uptime try: uptime = format_uptime((vdat.get('nowMsec') - vdat.get('loadMsec')) / 1000.0) except: uptime = "???" pt = PrettyTable() pt.field_names = ('Server', 'Version', 'Uptime') try: pt.add_row((vdat.get('hostName'), vdat.get('nginxVersion'), uptime)) except: pass return (pt.get_string(), pt.get_html_string() + "
") def repgen_server(span=None, tail=None, infile='/var/log/nginx/access.log'): """ Generate a server-wide report """ vdata = parse_vts() ro, rh = repgen_header(vdata) rdom = repgen_domain('*', vdata, noheader=True, span=span, tail=tail, infile=infile) if rdom is None: return None ro += "\n\n" + rdom[0] + "\n\n" rh += "\n" + rdom[1] + "\n" return (ro, rh) def repgen_domain(domain, vdata=None, noheader=False, span=None, tail=None, infile='/var/log/nginx/access.log'): """ Generate a report for @domain """ if vdata is None: vdata = parse_vts() ro = ""; rh = "" duser = get_domain_owner(domain) if not duser: return None try: tvd = vdata['serverZones'][duser['vhost']] except: logger.error("Failed to retrieve status data for domain %s. Data not yet available.", domain) return None # determine loglines to read if tail is not None: llines = tail else: if span: llines = None else: llines = vdata['connections']['requests'] # Get access.log data, then filter it based on the vhost field try: logger.info("Parsing log '%s' (%s)...", infile, format_size(os.stat(infile).st_size)) except Exception as e: logger.error("Unable to stat input file: %s", str(e)) sys.exit(100) aclog = logparse.parse_log(infile, span=span, lastlines=llines) logdata = logparse.filter_log(aclog, 'vhost', duser['vhost'].lower()) xpc = logparse.get_cache_hitrate(logdata) # Domain header table if span: rqtot = len(logdata) rtimes = logparse.get_reqtime(logdata) rx_fmt = "---" tx_fmt = format_size(logparse.get_bytes(logdata)) avgtime_fmt = "Min/Avg/Max" avgtime_str = "{min:01.03f} / {avg:01.03f} / {max:01.03f}".format(**rtimes) else: rqtot = tvd['requestCounter'] rx_fmt = format_size(tvd['inBytes']) tx_fmt = format_size(tvd['outBytes']) avgtime_fmt = "Avg" avgtime_str = "{:01.03f}".format(float(sum(tvd['requestMsecs']['msecs'])) / \ float(len(tvd['requestMsecs']['msecs'])) / 1000.0) pt = PrettyTable() pt.field_names = ('RX bytes', 'TX bytes', 'Requests', 'Req Time (%s)' % (avgtime_fmt)) pt.add_row((rx_fmt, tx_fmt, rqtot, avgtime_str)) if span: rduration = format_uptime(int(span) * 3600) else: rduration = format_uptime((vdata.get('nowMsec') - vdata.get('loadMsec')) / 1000.0) if not noheader: ro = strcolor('white', "<< {} / {} >>\n".format(domain, duser['vhost'])) rh = "

" + domain + " / " + duser['vhost'] + "

\n" + \ gen_email_header(rduration) ro += pt.get_string() + "\n\n" rh += strip_colors(pt.get_html_string(attributes=tab_attribs)) + "
\n" # Add in cache stats cinfo = repgen_cache_info(duser['owner'], duser['vhost'], domain) ro += cinfo[0]; rh += cinfo[1] # Build response code table pt = PrettyTable() pt.field_names = ('1xx', '2xx', '3xx', '4xx', '5xx', 'TOTAL') if span: rpc = logparse.get_status_hitrate(logdata) pt.add_row((strcolor('magenta', mkpct(rpc.get('1xx', 0), rqtot)), strcolor('green', mkpct(rpc.get('2xx', 0), rqtot)), strcolor('blue', mkpct(rpc.get('3xx', 0), rqtot)), strcolor('yellow', mkpct(rpc.get('4xx', 0), rqtot)), strcolor('red', mkpct(rpc.get('5xx', 0), rqtot)), strcolor('white', mkpct(rqtot, rqtot)))) else: pt.add_row((strcolor('magenta', mkpct(tvd['responses']['1xx'], rqtot)), strcolor('green', mkpct(tvd['responses']['2xx'], rqtot)), strcolor('blue', mkpct(tvd['responses']['3xx'], rqtot)), strcolor('yellow', mkpct(tvd['responses']['4xx'], rqtot)), strcolor('red', mkpct(tvd['responses']['5xx'], rqtot)), strcolor('white', mkpct(rqtot, rqtot)))) ro += strcolor('white', "HTTP RESPONSE CODES\n") + pt.get_string() + "\n\n" rh += "

HTTP Response Codes

\n" + \ strip_colors(pt.get_html_string(attributes=tab_attribs)) + "
\n" # Build UCS tables hit_dtot = rqtot - xpc.get('STATIC/TYPE', 0) - xpc.get('STATIC/PATH', 0) - xpc.get('DISABLED', 0) - xpc.get('BYPASS', 0) hit_stot = rqtot - xpc.get('DISABLED', 0) - xpc.get('BYPASS', 0) - xpc.get('HIT', 0) hit_btot = rqtot - xpc.get('STATIC/TYPE', 0) - xpc.get('STATIC/PATH', 0) ptx = PrettyTable() ptx.field_names = ('HIT (Dynamic)', 'HIT (Static)', 'DISABLED+BYPASS') ptx.add_row((strcolor('green', mkpct(xpc.get('HIT', 0), hit_dtot)), strcolor('yellow', mkpct(xpc.get('STATIC/TYPE', 0) + xpc.get('STATIC/PATH', 0), hit_stot)), strcolor('red', mkpct(xpc.get('BYPASS', 0) + xpc.get('DISABLED', 0), hit_btot)))) pt = PrettyTable() pt.field_names = ('HIT', 'MISS', 'BYPASS', 'EXPIRED', 'UPDATING') pt.add_row((strcolor('green', mkpct(xpc.get('HIT', 0), rqtot)), strcolor('red', mkpct(xpc.get('MISS', 0), rqtot)), strcolor('magenta', mkpct(xpc.get('BYPASS', 0), rqtot)), strcolor('yellow', mkpct(xpc.get('EXPIRED', 0), rqtot)), strcolor('yellow', mkpct(xpc.get('UPDATING', 0), rqtot)))) pt2 = PrettyTable() pt2.field_names = ('STATIC/TYPE', 'STATIC/PATH', 'DISABLED', 'TOTAL') pt2.add_row((strcolor('green', mkpct(xpc.get('STATIC/TYPE', 0), rqtot)), strcolor('green', mkpct(xpc.get('STATIC/PATH', 0), rqtot)), strcolor('magenta', mkpct(xpc.get('DISABLED', 0), rqtot)), strcolor('white', mkpct(rqtot, rqtot)))) ro += strcolor('white', "CACHE STATUS\n") + ptx.get_string() + "\n" + \ pt.get_string() + "\n" + pt2.get_string() + "\n\n" rh += "

Cache Status

\n" + \ strip_colors(ptx.get_html_string(attributes=tab_attribs)) + "
\n" + \ strip_colors(pt.get_html_string(attributes=tab_attribs)) + "
\n" + \ strip_colors(pt2.get_html_string(attributes=tab_attribs)) + "
\n" # Show NLP datas fullurl = (domain == '*') cinfo = repgen_nlp(logdata, user=duser['owner'], fullurl=fullurl) ro += cinfo[0]; rh += cinfo[1] rh += gen_email_footer() return (ro, rh) def repgen_cache_info(zone, vhost=None, domain=None, cbase=default_cache_base): """ Return stats about currently cached pages """ gpath = os.path.join(cbase, zone, '*', '*') tally = {'files': 0, 'size': 0} for tfile in glob(gpath): cdata = read_cache_file(tfile) if not cdata: continue if domain: if not re.match(r'^https?(GET|HEAD|OPTIONS)(' + domain.lower() + r'|' + vhost.lower() + r')/?.*$', cdata['key'], re.I): continue tally['files'] += 1 tally['size'] += cdata['stat'].st_size # Build cache stats info pt = PrettyTable() pt.field_names = ('Total Files', 'Total Size') pt.add_row((strcolor('white', tally['files']), strcolor('white', format_size(tally['size'])))) ro = strcolor('white', "CACHE ZONE STATS\n") + pt.get_string() + "\n\n" rh = "

Cache Zone Status

\n" + \ strip_colors(pt.get_html_string(attributes=tab_attribs)) + "
\n" return (ro, rh) def repgen_nlp(logdata=None, user=None, lastlines=10000, topcount=10, rresolve=True, maxwidth=65, fullurl=False): """ Generate an ANSI and HTML report of top hits If @logdata is not supplied, then a fresh read of the log is done, pulling the last @lastlines lines from the file """ if not logdata: logdata = logparse.parse_log('/var/log/nginx/access.log', lastlines=lastlines) ro = ""; rh = "" # Top Hosts utop = list(logparse.get_top_hosts(logdata, topcount=topcount).values()) pt = PrettyTable() pt.add_column('HITS', [x['hits'] for x in utop]) pt.add_column('HOST', [x['host'] for x in utop]) pt.align['HITS'] = 'r' pt.align['HOST'] = 'l' ro += strcolor('white', "TOP HOSTS\n") + pt.get_string() + "\n\n" rh += "

Top Hosts

\n" + \ strip_colors(pt.get_html_string(attributes=tab_attribs)) + "
\n" # Top URIs utop = list(logparse.get_top_uris(logdata, czone=user, topcount=topcount).values()) pt = PrettyTable() pt.add_column('HITS', [x['hits'] for x in utop]) pt.add_column('METHOD', [x['method'] for x in utop]) pt.add_column('STATUS', [x['status'] for x in utop]) pt.add_column('CACHE', [x['cache'] for x in utop]) if fullurl: pt.add_column('HOST/URI', [(x['host'] + x['uri'])[:maxwidth] for x in utop]) pt.align['HOST/URI'] = 'l' else: pt.add_column('URI', [x['uri'][:maxwidth] for x in utop]) pt.align['URI'] = 'l' pt.align['HITS'] = 'r' ro += strcolor('white', "TOP URIs\n") + pt.get_string() + "\n\n" rh += "

Top URIs

\n" + \ strip_colors(pt.get_html_string(attributes=tab_attribs)) + "
\n" # Top User-Agents utop = logparse.get_top_agents(logdata, topcount=topcount) pt = PrettyTable() pt.add_column('COUNT', list(utop.values())) pt.add_column('USER-AGENT', [x[:maxwidth] for x in list(utop.keys())]) pt.align['COUNT'] = 'r' pt.align['USER-AGENT'] = 'l' ro += strcolor('white', "TOP USER-AGENTS\n") + pt.get_string() + "\n\n" rh += "

Top User-Agents

\n" + \ strip_colors(pt.get_html_string(attributes=tab_attribs)) + "
\n" # Top Locations if logdata and logdata[0].get('loc') is not None: utop = logparse.get_top_locations(logdata, topcount=topcount) pt = PrettyTable() pt.add_column('COUNT', list(utop.values())) pt.add_column('LOCATION', list(utop.keys())) pt.align['COUNT'] = 'r' pt.align['LOCATION'] = 'l' ro += strcolor('white', "TOP LOCATIONS\n") + pt.get_string() + "\n\n" rh += "

Top Locations

\n" + \ strip_colors(pt.get_html_string(attributes=tab_attribs)) + "
\n" # Top Client IPs utop = logparse.get_top_ips(logdata, rresolve=rresolve, topcount=topcount) pt = PrettyTable() pt.add_column('HITS', [x['hits'] for x in list(utop.values())]) pt.add_column('IP', list(utop.keys())) pt.add_column('HOST/PTR', [x['reverse'] for x in list(utop.values())]) pt.align['HITS'] = 'r' pt.align['HOST/PTR'] = 'l' ro += strcolor('white', "TOP CLIENT IPs\n") + pt.get_string() rh += "

Top Client IPs

\n" + \ strip_colors(pt.get_html_string(attributes=tab_attribs)) + "
\n" return (ro, rh) def gen_email_header(tperiod): """ Return email stats header """ outstr = """

Generated on %s

Period: %s

""" % (arrow.now().format("YYYY-MM-DD HH:mm:ss ZZ"), tperiod) return outstr def gen_email_footer(): """ Return email stats footer """ outstr = """

Glossary

HIT - A valid (not expired) page was served from the cache

MISS - No valid cached page was available. A request was made to the application, and possibly cached for later use

BYPASS - The cache was bypassed due to meeting certain criteria (such as being logged in, using admin or user portal, using a checkout page in an ecommerce store, etc.)

EXPIRED - An expired page was served from the cache. This can happen if the origin server/application threw an error. Instead of showing the user an error page, an expired page from the cache is shown.

UPDATING - An expired page was served from the cache. Another request is in progress (by another client), which will update the cached page. This status typically only happens during very high traffic periods

STATIC/TYPE - Static content that was served directly by NGINX. The file extension (type) was used to serve the file directly from NGINX, rather than requesting the resource from Apache

STATIC/PATH - Static content that was served directly by NGINX. A specifically-defined path was used to make this determination

DISABLED - Caching has been explicitly disabled by the user

TOTAL - Total number of all requests. All constituent statuses may not add up to 100%, since pages without a specific cache status are not reported

""" return outstr