/usr/lib/imh-ultrastack-common/venv/lib/python3.13/site-packages/ngxutil
Edit: /usr/lib/imh-ultrastack-common/venv/lib/python3.13/site-packages/ngxutil/info.py (14916B)
# vim: set ts=4 sw=4 expandtab syntax=python:
"""
ngxutil.info
Implements the --info mode
Makes a request to the provided URL and provides
helpful information about cache info, cookies, errors, etc.
@author J. Hipps
"""
import os
import re
import logging
import subprocess
from urllib.parse import urlparse
import requests
import arrow
import yaml
from yaml import CDumper, CLoader
from ngxutil.vts import parse_vts
from ngxutil.cache import find_cache_item_url
from ngxutil.util import *
logger = logging.getLogger('ngxutil')
# Disable insecure warnings in newer versions of Requests module
if 'packages' in requests.__dict__:
# pylint: disable=no-member,no-name-in-module,import-error,wrong-import-position
from requests.packages.urllib3.exceptions import InsecureRequestWarning, InsecurePlatformWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
requests.packages.urllib3.disable_warnings(InsecurePlatformWarning)
def check_url(url):
"""
Fetch @url and analyze the headers and contents
"""
start_time = arrow.now().shift(seconds=-1)
try:
r = requests.get(url, verify=False)
except Exception as e:
logger.error("Failed to fetch URL [%s]: %s", url, str(e))
logger.error("Please NGINX is running, and that it is listening on ports 80 and 443")
return None
# Parse domain from URL and determine owner
# We need this to determine the cache zone and other info
domain = urlparse(url).netloc
uri = urlparse(url).path
duser = get_domain_owner(domain)
if not duser:
return None
# Load domain's ngxconf config
uconf = get_ngxconf_config(duser['owner'], duser['vhost'])
udata = get_userdata(duser['owner'], duser['vhost'])
# Status line
try:
st_color = {'1': "white", '2': "green", '3': "blue", '4': "yellow", '5': "red" }[str(r.status_code)[0]]
except:
st_color = "red"
print(strcolor('white', "GET %s" % (url)))
print("Elapsed: %2.03fs" % r.elapsed.total_seconds())
print("---")
print(strcolor(st_color, "%s %s" % (r.status_code, r.reason)))
# XPC header
xpc = r.headers.get('X-Proxy-Cache', "N/A")
try:
xpc_color = {'HIT': "green", 'MISS': "red", 'EXPIRED': "blue",
'BYPASS': "yellow", 'DISABLED': "red",
'STATIC/TYPE': "green", 'STATIC/PATH': "green" }[str(xpc)]
except:
xpc_color = "red"
print("Cache status: %s" % (strcolor(xpc_color, xpc)))
# Cache zone check
ccdata = find_cache_item_url(duser['owner'], url)
if ccdata:
if ccdata['expired']:
ccvalid = strcolor('yellow', "EXPIRED")
else:
ccvalid = strcolor('green', "VALID")
print("* %s -- expires %s (%s) [key: %s] " % (strcolor('green', "IN CACHE"),
ccdata['expiry_txt'], ccvalid, ccdata['key']))
print("Cache object: %s (%s)" % (ccdata['path'], format_size(ccdata['stat'].st_size)))
else:
print("* " + strcolor('red', "NOT IN CACHE"))
# Check for matching config patterns
rgx_matches = check_uri_matches(uri, uconf)
if rgx_matches:
print("* " + strcolor('green', "URI REGEX MATCH FROM CONFIG"))
map(lambda x: print("\t" + strcolor('cyan', x)), rgx_matches)
else:
if re.match(r'.+\.(ico|jpe?g|gif|png|bmp|svg|tiff|'
r'exe|dmg|zip|rar|7z|docx?|xlsx?|js|css|'
r'less|sass|scss|ttf|woff2?|mp3|mp4|mkv|'
r'avi|mov|mpe?g|aac|wav|flac)$', uri, re.I):
if uconf.get('accel_static_content'):
print("* " + strcolor('yellow', "Static content regex match (ACCELERATED)") + " -- served by NGINX")
if r.status_code == 404:
print(" TIP: Expecting this file to be served by a script or Apache htaccess rewrite?")
print(" If so, add a new force_passthru rule to the domain's ngxconf config")
else:
print("* " + strcolor('red', "Static content regex match (BYPASSED)") + " -- served by Apache")
print("---")
# Redirect
redir_loc = r.headers.get('Location')
if redir_loc:
print("* REDIRECT TO --> %s" % (strcolor('cyan', redir_loc)))
# Content-Encoding/gzip headers
gzenc = r.headers.get('Content-Encoding')
if gzenc and gzenc != 'identity':
print("* Content compressed/encoding via %s" % (strcolor('green', gzenc)))
else:
print("* " + strcolor('yellow', "No compression or content encoding"))
if "Accept-Encoding" in r.headers.get('Vary', ''):
print("* " + strcolor('green', "Vary on Accept-Encoding enabled"))
else:
print("* " + strcolor('red', "Vary on Accept-Encoding is NOT enabled"))
# Client cache-control headers
cch_list = ('Cache-Control', 'Expires', 'Pragma')
if set(cch_list).intersection(r.headers.keys()):
print("* Client cache-control headers present:")
for thead, tval in r.headers.items():
if thead in cch_list:
vcolor = ''
if thead.lower() in ('cache-control', 'pragma'):
if re.search(r'(no-cache|no-store|max\-age=0)', tval, re.I):
vcolor = 'red'
else:
vcolor = 'green'
elif thead.lower() == 'expires':
if tval == '-1' or '1970' in tval:
vcolor = 'red'
else:
vcolor = 'green'
print("\t%s: %s" % (thead, strcolor(vcolor, tval)))
if uconf.get('cache_honor_cc'):
print("* " + strcolor('yellow', "Cache-Control headers will be used to determine NGINX cache time") + \
" (cache_honor_cc == true)")
if uconf.get('cache_honor_expires'):
print("* " + strcolor('yellow', "Expires headers will be used to determine NGINX cache time") + \
" (cache_honor_expires == true)")
else:
print("* " + strcolor('yellow', "Client cache-control headers NOT present"))
# H2/Upgrade headers
h2upgrade = r.headers.get('Upgrade')
if h2upgrade:
print("* " + strcolor('yellow', "Upgrade header present: ") + strcolor('cyan', h2upgrade))
if 'h2' in h2upgrade and check_rpm_installed('ea-apache24-mod_http2'):
print("* " + strcolor('red', "HTTP/2 Upgrade header present; Apache mod_http2 installed"))
print(" TIP: Using Apache mod_http2 in combination with NGINX is not recommended,")
print(" as it can cause incorrect behavior in certain versions of Apple Safari.")
print(" Run `yum -y remove ea-apache24-mod_http2` to resolve")
print("---")
# Cookies
if len(r.cookies):
print("* " + strcolor('red', "One or more cookies are being set by the server"))
if uconf.get('cache_honor_cookies', True):
print(" TIP: Cookies will prevent the page from being cached, unless cache_honor_cookies")
print(" is set to false in the ngxconf config for this domain (default: true).")
print(" If session management is necessary, offloading these requests to an AJAX")
print(" endpoint via client-side Javascript will provide the best performance.")
else:
print("* " + strcolor('yellow', "Cookies are ignored for cache determination (cache_honor_cookies == false)"))
for ckname, ckvalue in dict(r.cookies).items():
print("\t%s = '%s'" % (ckname, ckvalue))
else:
print("* " + strcolor('green', "No cookies are being set by the server"))
print("---")
# Show additional information if a 4xx or 5xx error is encountered
if str(r.status_code)[0] in ('4', '5'):
eaphp = udata.get('phpversion', 'ea-php70')
# Show process status for certain 5xx error codes
if str(r.status_code) in ('502', '503', '504'):
show_running('Apache', '/var/run/apache2/httpd.pid')
show_running('NGINX', '/var/run/nginx.pid')
eapid = '/opt/cpanel/%s/root/usr/var/run/php-fpm/php-fpm.pid' % (eaphp)
show_running('PHP-FPM (%s)' % (eaphp), eapid)
print("---")
# Show log excerpts from PHP-FPM, Apache, and NGINX error logs
print("\n*** ERROR LOG EXCERPTS ***\n")
for tline in get_phpfpm_log(eaphp, start_time):
print("\t[fpm/%s] %s" % (eaphp, tline['msg']))
for tline in get_apache_log(start_time):
print("\t[apache] %s" % (tline['msg']))
for tline in get_nginx_log(start_time):
print("\t[nginx] %s" % (tline['msg']))
print("\n---")
def check_uri_matches(uri, uconf, ukey=None):
"""
Checks @ukey (if None, all keys) for matches of @uri against @uconf
"""
matchlist = []
for tkey, tlist in uconf.items():
if ukey is not None and ukey != tkey:
continue
if isinstance(tlist, list):
for trgx in tlist:
if re.match(r'^' + trgx, uri, re.I):
matchlist.append("%s -> %s" % (tkey, trgx))
return matchlist
def get_ngxconf_config(user, domain):
"""
Parse ngxconf config for the specified @user and @domain
"""
udir = '/home/%s/.imh/nginx' % (user)
tfile = '%s/%s.yml' % (udir, domain.lower().replace('.', '_'))
try:
with open(tfile, 'r') as f:
uconf = yaml.load(f, Loader=CLoader)
logger.debug("Parsed ngxconf configuration for domain from %s", tfile)
except Exception as e:
logger.error("Failed to parse ngxconf config for %s/%s: %s", user, domain, str(e))
return None
return uconf
def get_userdata(user, domain):
"""
Parse cPanel userdata for specified domain
"""
upath = '/var/cpanel/userdata/%s/%s' % (user, domain)
try:
with open(upath, 'r') as f:
udata = yaml.load(f, Loader=CLoader)
logger.debug("Parsed cPanel userdata for %s:%s", user, domain)
except Exception as e:
logger.error("Failed to parse cPanel userdata from %s: %s", upath, str(e))
return None
return udata
def get_phpfpm_log(phpver, start, lastlines=50):
"""
Find matching lines from PHP-FPM error log
"""
lpath = '/opt/cpanel/%s/root/usr/var/log/php-fpm/error.log' % (phpver)
try:
with open(lpath, 'r') as f:
rawlog = f.readlines()[-lastlines:]
logger.debug("Read %d lines from %s (requested %s)", len(rawlog), lpath, lastlines)
except Exception as e:
logger.error("Failed to read [%s]: %s", lpath, str(e))
return None
# parse timestamp from lines
logout = []
for tline in rawlog:
lmatch = re.match(r'^\[(?P[^\]]+)\] (?P[A-Z]+): (?P.+)$', tline)
if lmatch:
tinfo = lmatch.groupdict()
tinfo['timestamp'] = arrow.get(tinfo['timestr'], 'DD-MMM-YYYY HH:mm:ss', tzinfo='local')
if tinfo['timestamp'] > start:
logout.append(tinfo)
else:
logger.debug("Failed to parse line from PHP-FPM error log: '%s'", tline)
return logout
def get_apache_log(start, lastlines=50):
"""
Find matching lines from Apache error log
"""
lpath = '/usr/local/apache/logs/error_log'
try:
with open(lpath, 'r') as f:
rawlog = f.readlines()[-lastlines:]
logger.debug("Read %d lines from %s (requested %s)", len(rawlog), lpath, lastlines)
except Exception as e:
logger.error("Failed to read [%s]: %s", lpath, str(e))
return None
# parse timestamp from lines
logout = []
for tline in rawlog:
lmatch = re.match(r'^\[(?P[^\]]+)\] \[(?P[^:]*):(?P[^\]]+)\] '
r'\[pid (?P[0-9]+):tid (?P[0-9]+)\] (?P.+)$', tline)
if lmatch:
tinfo = lmatch.groupdict()
tinfo['timestamp'] = arrow.get(tinfo['timestr'], 'ddd MMM DD HH:mm:ss.SSSSSS YYYY', tzinfo='local')
if tinfo['timestamp'] > start:
logout.append(tinfo)
else:
logger.debug("Failed to parse line from Apache error log: '%s'", tline)
return logout
def get_nginx_log(start, lastlines=50):
"""
Find matching lines from Nginx error log
"""
lpath = '/var/log/nginx/error.log'
try:
with open(lpath, 'r') as f:
rawlog = f.readlines()[-lastlines:]
logger.debug("Read %d lines from %s (requested %s)", len(rawlog), lpath, lastlines)
except Exception as e:
logger.error("Failed to read [%s]: %s", lpath, str(e))
return None
# parse timestamp from lines
logout = []
for tline in rawlog:
lmatch = re.match(r'^(?P.+?) \[(?P[^\]]+)\] (?P[0-9]+)#(?P[0-9]+): '
r'\*(?P[0-9]+) (?P.+)$', tline)
if lmatch:
tinfo = lmatch.groupdict()
tinfo['timestamp'] = arrow.get(tinfo['timestr'], 'YYYY/MM/DD HH:mm:ss', tzinfo='local')
if tinfo['timestamp'] >= start:
logout.append(tinfo)
else:
logger.debug("Failed to parse line from Nginx error log: '%s'", tline)
return logout
def show_running(name, pidfile):
"""
Print info about running process @name, with PID
from @pidfile
"""
xpid = check_running(pidfile)
if xpid:
print("* %s -> %s (%s)" % (name, strcolor('green', "RUNNING"), strcolor('white', xpid)))
else:
print("* %s -> %s" % (name, strcolor('red', "NOT RUNNING")))
def check_running(pidfile):
"""
Check if process is running by sending SIG0 to PID
specified in @pidfile
"""
try:
with open(pidfile, 'r') as f:
praw = f.read()
pid = int(praw.strip())
except Exception as e:
logger.debug("Failed to read pidfile at %s: %s", pidfile, str(e))
return None
try:
os.kill(pid, 0)
except OSError as e:
if e.errno == 3:
logger.debug("Process %d is not running (pidfile: %s)", pid, pidfile)
return None
except Exception as e:
logger.warning("Encountered an error when checking PID %d (pidfile: %s): %s",
pid, pidfile, str(e))
return None
return pid
def check_rpm_installed(pkgname):
"""
Checks to see if an RPM is installed
"""
try:
outstr = subprocess.check_output(['/bin/rpm', '-qa', pkgname])
if outstr.strip().lower() == pkgname.lower():
return True
else:
return False
except Exception as e:
logger.debug("Failed to run RPM command: %s", str(e))
return False