CasperSecurity
#!/usr/bin/python2
"""
A better way to watch /proc/interrupts, especially on large NUMA machines with
so many CPUs that /proc/interrupts is wider than the screen. Press '0'-'9'
for node views, 't' for node totals
"""
import sys
import tty
import termios
from time import sleep
import subprocess
from optparse import OptionParser
import thread
import threading
KEYEVENT = threading.Event()
def gen_numa():
"""Generate NUMA info"""
cpunodes = {}
numacores = {}
out = subprocess.Popen('numactl --hardware | grep cpus', shell=True,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
errtxt = out.stderr.readline()
if errtxt:
print errtxt + '\r\n'
print "Is numactl installed?\r"
exit(1)
for line in out.stdout.readlines():
arr = line.split()
if arr[0] == "node" and arr[2] == "cpus:" and len(arr) > 3:
node = arr[1]
numacores[node] = arr[3:]
for core in arr[3:]:
cpunodes[core] = node
return numacores, cpunodes
# input character, passed between threads
INCHAR = ''
def wait_for_input():
"""Get a single character of input, validate"""
global INCHAR
acceptable_keys = ['0', '1', '2', '3', '4', '5', '6', '7',
'8', '9', '0', 't']
while True:
key = sys.stdin.read(1)
# simple just to exit on any invalid input
if not key in acceptable_keys:
thread.interrupt_main()
# set the new input and notify the main thread
INCHAR = key
KEYEVENT.set()
def filter_found(name, filter_list):
"""Check if IRQ name matches anything in the filter list"""
for filt in filter_list:
if filt in name:
return True
return False
def display_itop(seconds, rowcnt, iterations, sort, totals, dispnode, zero,
filters):
"""Main I/O loop"""
irqs = {}
cpunodes = {}
numacores = {}
loops = 0
width = len('NODEXX')
print ("interactive commands -- "
"t: view totals, 0-9: view node, any other key: quit")
while True:
# Grab the new display type at a time when nothing is in flux
if KEYEVENT.isSet():
KEYEVENT.clear()
dispnode = INCHAR if INCHAR in numacores.keys() else '-1'
out = open('/proc/interrupts', 'r')
header = out.readline()
cpus = []
for name in header.split():
num = name[3:]
cpus.append(num)
# Only query the numa information when something is missing.
# This is effectively the first time and when any disabled CPUs
# are enabled
if not num in cpunodes.keys():
numacores, cpunodes = gen_numa()
for line in out.readlines():
vals = line.split()
irqnum = vals[0].rstrip(':')
# Optionally exclude rows that are not an IRQ number
if totals is None:
try:
num = int(irqnum)
except ValueError:
continue
irq = {}
irq['cpus'] = [int(x) for x in vals[1:len(cpus)+1]]
irq['oldcpus'] = (irqs[irqnum]['cpus'] if irqnum in irqs
else [0] * len(cpus))
irq['name'] = ' '.join(vals[len(cpus)+1:])
irq['oldsum'] = irqs[irqnum]['sum'] if irqnum in irqs else 0
irq['sum'] = sum(irq['cpus'])
irq['num'] = irqnum
for node in numacores.keys():
oldkey = 'oldsum' + node
key = 'sum' + node
irq[oldkey] = (irqs[irqnum][key] if irqnum in irqs
and key in irqs[irqnum] else 0)
irq[key] = 0
for idx, val in enumerate(irq['cpus']):
key = 'sum' + cpunodes[cpus[idx]]
irq[key] = irq[key] + val if key in irq else val
# save old
irqs[irqnum] = irq
def sort_func(val):
"""Sort output"""
sortnum = -1
try:
sortnum = int(sort)
except ValueError:
pass
if sortnum >= 0:
for node in numacores.keys():
if sortnum == int(node):
return val['sum' + node] - val['oldsum' + node]
if sort == 't':
return val['sum'] - val['oldsum']
if sort == 'i':
return int(val['num'])
if sort == 'n':
return val['name']
# reverse sort all IRQ count sorts
rev = sort not in ['i', 'n']
rows = sorted(irqs.values(), key=sort_func, reverse=rev)
# determine the width required for the count field
for idx, irq in enumerate(rows):
width = max(width, len(str(irq['sum'] - irq['oldsum'])))
print "" + '\r'
print "IRQs / " + str(seconds) + " second(s)" + '\r'
fmtstr = ('IRQ# %' + str(width) + 's') % 'TOTAL'
# node view header
if int(dispnode) >= 0:
node = 'NODE%s' % dispnode
fmtstr += (' %' + str(width) + 's ') % node
for idx, val in enumerate(irq['cpus']):
if cpunodes[cpus[idx]] == dispnode:
cpu = 'CPU%s' % cpus[idx]
fmtstr += (' %' + str(width) + 's ') % cpu
# top view header
else:
for node in sorted(numacores.keys()):
node = 'NODE%s' % node
fmtstr += (' %' + str(width) + 's ') % node
fmtstr += ' NAME'
print fmtstr + '\r'
displayed_rows = 0
for idx, irq in enumerate(rows):
if len(filters) and not filter_found(irq['name'], filters):
continue
total = irq['sum'] - irq['oldsum']
if zero and not total:
continue
# IRQ# TOTAL
fmtstr = ('%4s %' + str(width) + 'd') % (irq['num'], total)
# node view
if int(dispnode) >= 0:
oldnodesum = 'oldsum' + dispnode
nodesum = 'sum' + dispnode
nodecnt = irq[nodesum] - irq[oldnodesum]
if zero and not nodecnt:
continue
fmtstr += (' %' + str(width) + 's ') % str(nodecnt)
for cpu, val in enumerate(irq['cpus']):
if cpunodes[cpus[cpu]] == dispnode:
fmtstr += ((' %' + str(width) + 's ') %
str(irq['cpus'][cpu] - irq['oldcpus'][cpu]))
# top view
else:
for node in sorted(numacores.keys()):
oldnodesum = 'oldsum' + node
nodesum = 'sum' + node
nodecnt = irq[nodesum] - irq[oldnodesum]
fmtstr += ((' %' + str(width) + 's ') % str(nodecnt))
fmtstr += ' ' + irq['name']
print fmtstr + '\r'
displayed_rows += 1
if displayed_rows == rowcnt:
break
# Update field widths after the first iteration. Data changes
# significantly between the all-time stats and the interval stats, so
# this compresses the fields quite a bit. Updating every iteration
# is too jumpy.
if loops == 0:
width = len('NODEXX')
loops += 1
if loops == iterations:
break
# thread.interrupt_main() does not seem to interrupt a sleep, so break
# it into tenth-of-a-second sleeps to improve user response time on exit
for _ in range(0, seconds * 10):
sleep(.1)
def main(args):
"""Parse arguments, call main loop"""
parser = OptionParser(description=__doc__)
parser.add_option("-i", "--iterations", default='-1',
help="iterations to run")
parser.add_option("-n", "--node", default='-1',
help="view a single node")
parser.add_option("-r", "--rows", default='10',
help="rows to display (default 10)")
parser.add_option("-s", "--sort", default='t',
help="column to sort on ('t':total, 'n': name, "
"'i':IRQ number, '1':node1, etc) (default: 't')")
parser.add_option("-t", "--time", default='5',
help="update interval in seconds")
parser.add_option("-z", "--zero", action="store_true",
help="exclude inactive IRQs")
parser.add_option("--filter", default="",
help="filter IRQs based on name matching comma "
"separated filters")
parser.add_option("--totals", action="store_true",
help="include total rows")
options = parser.parse_args(args)[0]
if options.filter:
options.filter = options.filter.split(',')
else:
options.filter = []
# Set the terminal to unbuffered, to catch a single keypress
out = sys.stdin.fileno()
old_settings = termios.tcgetattr(out)
tty.setraw(sys.stdin.fileno())
# input thread
thread.start_new_thread(wait_for_input, tuple())
try:
display_itop(int(options.time), int(options.rows),
int(options.iterations), options.sort, options.totals,
options.node, options.zero, options.filter)
except (KeyboardInterrupt, SystemExit):
pass
finally:
termios.tcsetattr(out, termios.TCSADRAIN, old_settings)
if __name__ == "__main__":
sys.exit(main(sys.argv))