2 # SPDX-License-Identifier: GPL-2.0-only
4 # top-like utility for displaying kvm statistics
6 # Copyright 2006-2008 Qumranet Technologies
7 # Copyright 2008-2011 Red Hat, Inc.
10 # Avi Kivity <avi@redhat.com>
12 """The kvm_stat module outputs statistics about running KVM VMs
14 Three different ways of output formatting are available:
15 - as a top-like text ui
16 - in a key -> value format
17 - in an all keys, all values format
19 The data is sampled from the KVM's debugfs entries and its perf events.
21 from __future__ import print_function
36 from collections import defaultdict, namedtuple
37 from functools import reduce
38 from datetime import datetime
42 'EXTERNAL_INTERRUPT': 1,
46 'INTERRUPT_WINDOW': 7,
72 'MWAIT_INSTRUCTION': 36,
73 'MONITOR_TRAP_FLAG': 37,
74 'MONITOR_INSTRUCTION': 39,
75 'PAUSE_INSTRUCTION': 40,
76 'MCE_DURING_VMENTRY': 41,
77 'TPR_BELOW_THRESHOLD': 43,
86 'PREEMPTION_TIMER': 52,
139 'CR0_SEL_WRITE': 0x065,
163 'TASK_SWITCH': 0x07d,
164 'FERR_FREEZE': 0x07e,
181 'EFER_WRITE_TRAP': 0x08f,
182 'CR0_WRITE_TRAP': 0x090,
183 'CR1_WRITE_TRAP': 0x091,
184 'CR2_WRITE_TRAP': 0x092,
185 'CR3_WRITE_TRAP': 0x093,
186 'CR4_WRITE_TRAP': 0x094,
187 'CR5_WRITE_TRAP': 0x095,
188 'CR6_WRITE_TRAP': 0x096,
189 'CR7_WRITE_TRAP': 0x097,
190 'CR8_WRITE_TRAP': 0x098,
191 'CR9_WRITE_TRAP': 0x099,
192 'CR10_WRITE_TRAP': 0x09a,
193 'CR11_WRITE_TRAP': 0x09b,
194 'CR12_WRITE_TRAP': 0x09c,
195 'CR13_WRITE_TRAP': 0x09d,
196 'CR14_WRITE_TRAP': 0x09e,
197 'CR15_WRITE_TRAP': 0x09f,
200 'AVIC_INCOMPLETE_IPI': 0x401,
201 'AVIC_UNACCELERATED_ACCESS': 0x402,
205 # EC definition of HSR (from arch/arm64/include/asm/esr.h)
206 AARCH64_EXIT_REASONS = {
251 # From include/uapi/linux/kvm.h, KVM_EXIT_xxx
252 USERSPACE_EXIT_REASONS = {
260 'IRQ_WINDOW_OPEN': 7,
270 'INTERNAL_ERROR': 17,
284 'DIRTY_RING_FULL': 31,
294 'SET_FILTER': 0x40082406,
295 'ENABLE': 0x00002400,
296 'DISABLE': 0x00002401,
300 signal_received = False
302 ENCODING = locale.getpreferredencoding(False)
303 TRACE_FILTER = re.compile(r'^[^\(]*$')
307 """Encapsulates global architecture specific data.
309 Contains the performance event open syscall and ioctl numbers, as
310 well as the VM exit reasons for the architecture it runs on.
315 machine = os.uname()[4]
317 if machine.startswith('ppc'):
319 elif machine.startswith('aarch64'):
321 elif machine.startswith('s390'):
325 for line in open('/proc/cpuinfo'):
326 if not line.startswith('flags'):
331 return ArchX86(VMX_EXIT_REASONS)
333 return ArchX86(SVM_EXIT_REASONS)
336 def tracepoint_is_child(self, field):
337 if (TRACE_FILTER.match(field)):
339 return field.split('(', 1)[0]
343 def __init__(self, exit_reasons):
344 self.sc_perf_evt_open = 298
345 self.ioctl_numbers = IOCTL_NUMBERS
346 self.exit_reason_field = 'exit_reason'
347 self.exit_reasons = exit_reasons
349 def debugfs_is_child(self, field):
350 """ Returns name of parent if 'field' is a child, None otherwise """
356 self.sc_perf_evt_open = 319
357 self.ioctl_numbers = IOCTL_NUMBERS
358 self.ioctl_numbers['ENABLE'] = 0x20002400
359 self.ioctl_numbers['DISABLE'] = 0x20002401
360 self.ioctl_numbers['RESET'] = 0x20002403
362 # PPC comes in 32 and 64 bit and some generated ioctl
363 # numbers depend on the wordsize.
364 char_ptr_size = ctypes.sizeof(ctypes.c_char_p)
365 self.ioctl_numbers['SET_FILTER'] = 0x80002406 | char_ptr_size << 16
366 self.exit_reason_field = 'exit_nr'
367 self.exit_reasons = {}
369 def debugfs_is_child(self, field):
370 """ Returns name of parent if 'field' is a child, None otherwise """
376 self.sc_perf_evt_open = 241
377 self.ioctl_numbers = IOCTL_NUMBERS
378 self.exit_reason_field = 'esr_ec'
379 self.exit_reasons = AARCH64_EXIT_REASONS
381 def debugfs_is_child(self, field):
382 """ Returns name of parent if 'field' is a child, None otherwise """
386 class ArchS390(Arch):
388 self.sc_perf_evt_open = 331
389 self.ioctl_numbers = IOCTL_NUMBERS
390 self.exit_reason_field = None
391 self.exit_reasons = None
393 def debugfs_is_child(self, field):
394 """ Returns name of parent if 'field' is a child, None otherwise """
395 if field.startswith('instruction_'):
396 return 'exit_instruction'
399 ARCH = Arch.get_arch()
402 class perf_event_attr(ctypes.Structure):
403 """Struct that holds the necessary data to set up a trace event.
405 For an extensive explanation see perf_event_open(2) and
406 include/uapi/linux/perf_event.h, struct perf_event_attr
408 All fields that are not initialized in the constructor are 0.
411 _fields_ = [('type', ctypes.c_uint32),
412 ('size', ctypes.c_uint32),
413 ('config', ctypes.c_uint64),
414 ('sample_freq', ctypes.c_uint64),
415 ('sample_type', ctypes.c_uint64),
416 ('read_format', ctypes.c_uint64),
417 ('flags', ctypes.c_uint64),
418 ('wakeup_events', ctypes.c_uint32),
419 ('bp_type', ctypes.c_uint32),
420 ('bp_addr', ctypes.c_uint64),
421 ('bp_len', ctypes.c_uint64),
425 super(self.__class__, self).__init__()
426 self.type = PERF_TYPE_TRACEPOINT
427 self.size = ctypes.sizeof(self)
428 self.read_format = PERF_FORMAT_GROUP
431 PERF_TYPE_TRACEPOINT = 2
432 PERF_FORMAT_GROUP = 1 << 3
436 """Represents a perf event group."""
441 def add_event(self, event):
442 self.events.append(event)
445 """Returns a dict with 'event name: value' for all events in the
448 Values are read by reading from the file descriptor of the
449 event that is the group leader. See perf_event_open(2) for
452 Read format for the used event configuration is:
454 u64 nr; /* The number of events */
456 u64 value; /* The value of the event */
461 length = 8 * (1 + len(self.events))
462 read_format = 'xxxxxxxx' + 'Q' * len(self.events)
463 return dict(zip([event.name for event in self.events],
464 struct.unpack(read_format,
465 os.read(self.events[0].fd, length))))
469 """Represents a performance event and manages its life cycle."""
470 def __init__(self, name, group, trace_cpu, trace_pid, trace_point,
471 trace_filter, trace_set='kvm'):
472 self.libc = ctypes.CDLL('libc.so.6', use_errno=True)
473 self.syscall = self.libc.syscall
476 self._setup_event(group, trace_cpu, trace_pid, trace_point,
477 trace_filter, trace_set)
480 """Closes the event's file descriptor.
482 As no python file object was created for the file descriptor,
483 python will not reference count the descriptor and will not
484 close it itself automatically, so we do it.
490 def _perf_event_open(self, attr, pid, cpu, group_fd, flags):
491 """Wrapper for the sys_perf_evt_open() syscall.
493 Used to set up performance events, returns a file descriptor or -1
498 - struct perf_event_attr *
499 - pid or -1 to monitor all pids
500 - cpu number or -1 to monitor all cpus
501 - The file descriptor of the group leader or -1 to create a group.
505 return self.syscall(ARCH.sc_perf_evt_open, ctypes.pointer(attr),
506 ctypes.c_int(pid), ctypes.c_int(cpu),
507 ctypes.c_int(group_fd), ctypes.c_long(flags))
509 def _setup_event_attribute(self, trace_set, trace_point):
510 """Returns an initialized ctype perf_event_attr struct."""
512 id_path = os.path.join(PATH_DEBUGFS_TRACING, 'events', trace_set,
515 event_attr = perf_event_attr()
516 event_attr.config = int(open(id_path).read())
519 def _setup_event(self, group, trace_cpu, trace_pid, trace_point,
520 trace_filter, trace_set):
521 """Sets up the perf event in Linux.
523 Issues the syscall to register the event in the kernel and
524 then sets the optional filter.
528 event_attr = self._setup_event_attribute(trace_set, trace_point)
530 # First event will be group leader.
533 # All others have to pass the leader's descriptor instead.
535 group_leader = group.events[0].fd
537 fd = self._perf_event_open(event_attr, trace_pid,
538 trace_cpu, group_leader, 0)
540 err = ctypes.get_errno()
541 raise OSError(err, os.strerror(err),
542 'while calling sys_perf_event_open().')
545 fcntl.ioctl(fd, ARCH.ioctl_numbers['SET_FILTER'],
551 """Enables the trace event in the kernel.
553 Enabling the group leader makes reading counters from it and the
554 events under it possible.
557 fcntl.ioctl(self.fd, ARCH.ioctl_numbers['ENABLE'], 0)
560 """Disables the trace event in the kernel.
562 Disabling the group leader makes reading all counters under it
566 fcntl.ioctl(self.fd, ARCH.ioctl_numbers['DISABLE'], 0)
569 """Resets the count of the trace event in the kernel."""
570 fcntl.ioctl(self.fd, ARCH.ioctl_numbers['RESET'], 0)
573 class Provider(object):
574 """Encapsulates functionalities used by all providers."""
575 def __init__(self, pid):
576 self.child_events = False
580 def is_field_wanted(fields_filter, field):
581 """Indicate whether field is valid according to fields_filter."""
582 if not fields_filter:
584 return re.match(fields_filter, field) is not None
588 """Returns os.walk() data for specified directory.
590 As it is only a wrapper it returns the same 3-tuple of (dirpath,
591 dirnames, filenames).
593 return next(os.walk(path))
596 class TracepointProvider(Provider):
597 """Data provider for the stats class.
599 Manages the events/groups from which it acquires its data.
602 def __init__(self, pid, fields_filter):
603 self.group_leaders = []
604 self.filters = self._get_filters()
605 self.update_fields(fields_filter)
606 super(TracepointProvider, self).__init__(pid)
610 """Returns a dict of trace events, their filter ids and
611 the values that can be filtered.
613 Trace events can be filtered for special values by setting a
614 filter string via an ioctl. The string normally has the format
615 identifier==value. For each filter a new event will be created, to
616 be able to distinguish the events.
620 filters['kvm_userspace_exit'] = ('reason', USERSPACE_EXIT_REASONS)
621 if ARCH.exit_reason_field and ARCH.exit_reasons:
622 filters['kvm_exit'] = (ARCH.exit_reason_field, ARCH.exit_reasons)
625 def _get_available_fields(self):
626 """Returns a list of available events of format 'event name(filter
629 All available events have directories under
630 /sys/kernel/tracing/events/ which export information
631 about the specific event. Therefore, listing the dirs gives us
632 a list of all available events.
634 Some events like the vm exit reasons can be filtered for
635 specific values. To take account for that, the routine below
636 creates special fields with the following format:
637 event name(filter name)
640 path = os.path.join(PATH_DEBUGFS_TRACING, 'events', 'kvm')
641 fields = self.walkdir(path)[1]
644 if field in self.filters:
645 filter_name_, filter_dicts = self.filters[field]
646 for name in filter_dicts:
647 extra.append(field + '(' + name + ')')
651 def update_fields(self, fields_filter):
652 """Refresh fields, applying fields_filter"""
653 self.fields = [field for field in self._get_available_fields()
654 if self.is_field_wanted(fields_filter, field)]
655 # add parents for child fields - otherwise we won't see any output!
656 for field in self._fields:
657 parent = ARCH.tracepoint_is_child(field)
658 if (parent and parent not in self._fields):
659 self.fields.append(parent)
662 def _get_online_cpus():
663 """Returns a list of cpu id integers."""
664 def parse_int_list(list_string):
665 """Returns an int list from a string of comma separated integers and
668 members = list_string.split(',')
670 for member in members:
671 if '-' not in member:
672 integers.append(int(member))
674 int_range = member.split('-')
675 integers.extend(range(int(int_range[0]),
676 int(int_range[1]) + 1))
680 with open('/sys/devices/system/cpu/online') as cpu_list:
681 cpu_string = cpu_list.readline()
682 return parse_int_list(cpu_string)
684 def _setup_traces(self):
685 """Creates all event and group objects needed to be able to retrieve
687 fields = self._get_available_fields()
689 # Fetch list of all threads of the monitored pid, as qemu
690 # starts a thread for each vcpu.
691 path = os.path.join('/proc', str(self._pid), 'task')
692 groupids = self.walkdir(path)[1]
694 groupids = self._get_online_cpus()
696 # The constant is needed as a buffer for python libs, std
697 # streams and other files that the script opens.
698 newlim = len(groupids) * len(fields) + 50
700 softlim_, hardlim = resource.getrlimit(resource.RLIMIT_NOFILE)
703 # Now we need CAP_SYS_RESOURCE, to increase the hard limit.
704 resource.setrlimit(resource.RLIMIT_NOFILE, (newlim, newlim))
706 # Raising the soft limit is sufficient.
707 resource.setrlimit(resource.RLIMIT_NOFILE, (newlim, hardlim))
710 sys.exit("NOFILE rlimit could not be raised to {0}".format(newlim))
712 for groupid in groupids:
717 match = re.match(r'(.*)\((.*)\)', name)
719 tracepoint, sub = match.groups()
720 tracefilter = ('%s==%d\0' %
721 (self.filters[tracepoint][0],
722 self.filters[tracepoint][1][sub]))
724 # From perf_event_open(2):
725 # pid > 0 and cpu == -1
726 # This measures the specified process/thread on any CPU.
728 # pid == -1 and cpu >= 0
729 # This measures all processes/threads on the specified CPU.
730 trace_cpu = groupid if self._pid == 0 else -1
731 trace_pid = int(groupid) if self._pid != 0 else -1
733 group.add_event(Event(name=name,
737 trace_point=tracepoint,
738 trace_filter=tracefilter))
740 self.group_leaders.append(group)
747 def fields(self, fields):
748 """Enables/disables the (un)wanted events"""
749 self._fields = fields
750 for group in self.group_leaders:
751 for index, event in enumerate(group.events):
752 if event.name in fields:
756 # Do not disable the group leader.
757 # It would disable all of its events.
767 """Changes the monitored pid by setting new traces."""
769 # The garbage collector will get rid of all Event/Group
770 # objects and open files after removing the references.
771 self.group_leaders = []
773 self.fields = self._fields
775 def read(self, by_guest=0):
776 """Returns 'event name: current value' for all enabled events."""
777 ret = defaultdict(int)
778 for group in self.group_leaders:
779 for name, val in group.read().items():
780 if name not in self._fields:
782 parent = ARCH.tracepoint_is_child(name)
789 """Reset all field counters"""
790 for group in self.group_leaders:
791 for event in group.events:
795 class DebugfsProvider(Provider):
796 """Provides data from the files that KVM creates in the kvm debugfs
798 def __init__(self, pid, fields_filter, include_past):
799 self.update_fields(fields_filter)
803 super(DebugfsProvider, self).__init__(pid)
807 def _get_available_fields(self):
808 """"Returns a list of available fields.
810 The fields are all available KVM debugfs files
813 exempt_list = ['halt_poll_fail_ns', 'halt_poll_success_ns', 'halt_wait_ns']
814 fields = [field for field in self.walkdir(PATH_DEBUGFS_KVM)[2]
815 if field not in exempt_list]
819 def update_fields(self, fields_filter):
820 """Refresh fields, applying fields_filter"""
821 self._fields = [field for field in self._get_available_fields()
822 if self.is_field_wanted(fields_filter, field)]
823 # add parents for child fields - otherwise we won't see any output!
824 for field in self._fields:
825 parent = ARCH.debugfs_is_child(field)
826 if (parent and parent not in self._fields):
827 self.fields.append(parent)
834 def fields(self, fields):
835 self._fields = fields
846 vms = self.walkdir(PATH_DEBUGFS_KVM)[1]
850 self.paths = list(filter(lambda x: "{}-".format(pid) in x, vms))
856 def _verify_paths(self):
857 """Remove invalid paths"""
858 for path in self.paths:
859 if not os.path.exists(os.path.join(PATH_DEBUGFS_KVM, path)):
860 self.paths.remove(path)
863 def read(self, reset=0, by_guest=0):
864 """Returns a dict with format:'file name / field -> current value'.
868 1 reset field counts to 0
869 2 restore the original field counts
874 # If no debugfs filtering support is available, then don't read.
882 for entry in os.walk(PATH_DEBUGFS_KVM):
886 for field in self._fields:
887 value = self._read_field(field, path)
890 self._baseline[key] = value
892 self._baseline[key] = 0
893 if self._baseline.get(key, -1) == -1:
894 self._baseline[key] = value
895 parent = ARCH.debugfs_is_child(field)
897 field = field + ' ' + parent
900 field = key.split('-')[0] # set 'field' to 'pid'
901 increment = value - self._baseline.get(key, 0)
903 results[field] += increment
905 results[field] = increment
909 def _read_field(self, field, path):
910 """Returns the value of a single field from a specific VM."""
912 return int(open(os.path.join(PATH_DEBUGFS_KVM,
920 """Reset field counters"""
925 """Reset field counters"""
930 EventStat = namedtuple('EventStat', ['value', 'delta'])
934 """Manages the data providers and the data they provide.
936 It is used to set filters on the provider's data and collect all
940 def __init__(self, options):
941 self.providers = self._get_providers(options)
942 self._pid_filter = options.pid
943 self._fields_filter = options.fields
945 self._child_events = False
947 def _get_providers(self, options):
948 """Returns a list of data providers depending on the passed options."""
952 providers.append(DebugfsProvider(options.pid, options.fields,
953 options.debugfs_include_past))
954 if options.tracepoints or not providers:
955 providers.append(TracepointProvider(options.pid, options.fields))
959 def _update_provider_filters(self):
960 """Propagates fields filters to providers."""
961 # As we reset the counters when updating the fields we can
962 # also clear the cache of old values.
964 for provider in self.providers:
965 provider.update_fields(self._fields_filter)
969 for provider in self.providers:
973 def fields_filter(self):
974 return self._fields_filter
976 @fields_filter.setter
977 def fields_filter(self, fields_filter):
978 if fields_filter != self._fields_filter:
979 self._fields_filter = fields_filter
980 self._update_provider_filters()
983 def pid_filter(self):
984 return self._pid_filter
987 def pid_filter(self, pid):
988 if pid != self._pid_filter:
989 self._pid_filter = pid
991 for provider in self.providers:
992 provider.pid = self._pid_filter
995 def child_events(self):
996 return self._child_events
999 def child_events(self, val):
1000 self._child_events = val
1001 for provider in self.providers:
1002 provider.child_events = val
1004 def get(self, by_guest=0):
1005 """Returns a dict with field -> (value, delta to last value) of all
1008 * plain: 'key' is event name
1009 * child-parent: 'key' is in format '<child> <parent>'
1010 * pid: 'key' is the pid of the guest, and the record contains the
1011 aggregated event data
1012 These formats are generated by the providers, and handled in class TUI.
1014 for provider in self.providers:
1015 new = provider.read(by_guest=by_guest)
1017 oldval = self.values.get(key, EventStat(0, 0)).value
1018 newval = new.get(key, 0)
1019 newdelta = newval - oldval
1020 self.values[key] = EventStat(newval, newdelta)
1023 def toggle_display_guests(self, to_pid):
1024 """Toggle between collection of stats by individual event and by
1027 Events reported by DebugfsProvider change when switching to/from
1028 reading by guest values. Hence we have to remove the excess event
1029 names from self.values.
1032 if any(isinstance(ins, TracepointProvider) for ins in self.providers):
1035 for provider in self.providers:
1036 if isinstance(provider, DebugfsProvider):
1037 for key in provider.fields:
1038 if key in self.values.keys():
1039 del self.values[key]
1041 oldvals = self.values.copy()
1044 del self.values[key]
1045 # Update oldval (see get())
1051 MAX_GUEST_NAME_LEN = 48
1059 """Instruments curses to draw a nice text ui."""
1060 def __init__(self, stats, opts):
1063 self._delay_initial = 0.25
1064 self._delay_regular = opts.set_delay
1065 self._sorting = SORT_DEFAULT
1066 self._display_guests = 0
1068 def __enter__(self):
1069 """Initialises curses for later use. Based on curses.wrapper
1070 implementation from the Python standard library."""
1071 self.screen = curses.initscr()
1075 # The try/catch works around a minor bit of
1076 # over-conscientiousness in the curses module, the error
1077 # return from C start_color() is ignorable.
1079 curses.start_color()
1080 except curses.error:
1083 # Hide cursor in extra statement as some monochrome terminals
1084 # might support hiding but not colors.
1087 except curses.error:
1090 curses.use_default_colors()
1093 def __exit__(self, *exception):
1094 """Resets the terminal to its normal state. Based on curses.wrapper
1095 implementation from the Python standard library."""
1097 self.screen.keypad(0)
1103 def get_all_gnames():
1104 """Returns a list of (pid, gname) tuples of all running guests"""
1107 child = subprocess.Popen(['ps', '-A', '--format', 'pid,args'],
1108 stdout=subprocess.PIPE)
1111 for line in child.stdout:
1112 line = line.decode(ENCODING).lstrip().split(' ', 1)
1113 # perform a sanity check before calling the more expensive
1114 # function to possibly extract the guest name
1115 if ' -name ' in line[1]:
1116 res.append((line[0], Tui.get_gname_from_pid(line[0])))
1117 child.stdout.close()
1121 def _print_all_gnames(self, row):
1122 """Print a list of all running guests along with their pids."""
1123 self.screen.addstr(row, 2, '%8s %-60s' %
1124 ('Pid', 'Guest Name (fuzzy list, might be '
1129 for line in self.get_all_gnames():
1130 self.screen.addstr(row, 2, '%8s %-60s' % (line[0], line[1]))
1132 if row >= self.screen.getmaxyx()[0]:
1135 self.screen.addstr(row + 1, 2, 'Not available')
1138 def get_pid_from_gname(gname):
1139 """Fuzzy function to convert guest name to QEMU process pid.
1141 Returns a list of potential pids, can be empty if no match found.
1142 Throws an exception on processing errors.
1146 for line in Tui.get_all_gnames():
1147 if gname == line[1]:
1148 pids.append(int(line[0]))
1153 def get_gname_from_pid(pid):
1154 """Returns the guest name for a QEMU process pid.
1156 Extracts the guest name from the QEMU comma line by processing the
1157 '-name' option. Will also handle names specified out of sequence.
1162 line = open('/proc/{}/cmdline'
1163 .format(pid), 'r').read().split('\0')
1164 parms = line[line.index('-name') + 1].split(',')
1166 # commas are escaped (i.e. ',,'), hence e.g. 'foo,bar' results
1167 # in # ['foo', '', 'bar'], which we revert here
1168 idx = parms.index('')
1169 parms[idx - 1] += ',' + parms[idx + 1]
1170 del parms[idx:idx+2]
1171 # the '-name' switch allows for two ways to specify the guest name,
1172 # where the plain name overrides the name specified via 'guest='
1177 if arg[:6] == 'guest=':
1179 except (ValueError, IOError, IndexError):
1184 def _update_pid(self, pid):
1185 """Propagates pid selection to stats object."""
1186 self.screen.addstr(4, 1, 'Updating pid filter...')
1187 self.screen.refresh()
1188 self.stats.pid_filter = pid
1190 def _refresh_header(self, pid=None):
1191 """Refreshes the header."""
1193 pid = self.stats.pid_filter
1195 gname = self.get_gname_from_pid(pid)
1198 gname = ('({})'.format(gname[:MAX_GUEST_NAME_LEN] + '...'
1199 if len(gname) > MAX_GUEST_NAME_LEN
1202 self._headline = 'kvm statistics - pid {0} {1}'.format(pid, gname)
1204 self._headline = 'kvm statistics - summary'
1205 self.screen.addstr(0, 0, self._headline, curses.A_BOLD)
1206 if self.stats.fields_filter:
1207 regex = self.stats.fields_filter
1208 if len(regex) > MAX_REGEX_LEN:
1209 regex = regex[:MAX_REGEX_LEN] + '...'
1210 self.screen.addstr(1, 17, 'regex filter: {0}'.format(regex))
1211 if self._display_guests:
1212 col_name = 'Guest Name'
1215 self.screen.addstr(2, 1, '%-40s %10s%7s %8s' %
1216 (col_name, 'Total', '%Total', 'CurAvg/s'),
1218 self.screen.addstr(4, 1, 'Collecting data...')
1219 self.screen.refresh()
1221 def _refresh_body(self, sleeptime):
1222 def insert_child(sorted_items, child, values, parent):
1223 num = len(sorted_items)
1224 for i in range(0, num):
1225 # only add child if parent is present
1226 if parent.startswith(sorted_items[i][0]):
1227 sorted_items.insert(i + 1, (' ' + child, values))
1229 def get_sorted_events(self, stats):
1230 """ separate parent and child events """
1231 if self._sorting == SORT_DEFAULT:
1233 # sort by (delta value, overall value)
1235 return (v.delta, v.value)
1238 # sort by overall value
1244 # we can't rule out child events to appear prior to parents even
1245 # when sorted - separate out all children first, and add in later
1246 for key, values in sorted(stats.items(), key=sortkey,
1248 if values == (0, 0):
1250 if key.find(' ') != -1:
1251 if not self.stats.child_events:
1253 childs.insert(0, (key, values))
1255 sorted_items.append((key, values))
1256 if self.stats.child_events:
1257 for key, values in childs:
1258 (child, parent) = key.split(' ')
1259 insert_child(sorted_items, child, values, parent)
1263 if not self._is_running_guest(self.stats.pid_filter):
1265 try: # ...to identify the guest by name in case it's back
1266 pids = self.get_pid_from_gname(self._gname)
1268 self._refresh_header(pids[0])
1269 self._update_pid(pids[0])
1273 self._display_guest_dead()
1274 # leave final data on screen
1277 self.screen.move(row, 0)
1278 self.screen.clrtobot()
1279 stats = self.stats.get(self._display_guests)
1282 for key, values in stats.items():
1283 if self._display_guests:
1284 if self.get_gname_from_pid(key):
1285 total += values.value
1287 if not key.find(' ') != -1:
1288 total += values.value
1290 ctotal += values.value
1292 # we don't have any fields, or all non-child events are filtered
1298 guest_removed = False
1299 for key, values in get_sorted_events(self, stats):
1300 if row >= self.screen.getmaxyx()[0] - 1 or values == (0, 0):
1302 if self._display_guests:
1303 key = self.get_gname_from_pid(key)
1306 cur = int(round(values.delta / sleeptime)) if values.delta else 0
1308 guest_removed = True
1312 tcur += values.delta
1313 ptotal = values.value
1317 self.screen.addstr(row, 1, '%-40s %10d%7.1f %8s' % (key,
1319 values.value * 100 / float(ltotal), cur))
1323 self.screen.addstr(4, 1, 'Guest removed, updating...')
1325 self.screen.addstr(4, 1, 'No matching events reported yet')
1327 tavg = int(round(tcur / sleeptime)) if tcur > 0 else ''
1328 self.screen.addstr(row, 1, '%-40s %10d %8s' %
1329 ('Total', total, tavg), curses.A_BOLD)
1330 self.screen.refresh()
1332 def _display_guest_dead(self):
1333 marker = ' Guest is DEAD '
1334 y = min(len(self._headline), 80 - len(marker))
1335 self.screen.addstr(0, y, marker, curses.A_BLINK | curses.A_STANDOUT)
1337 def _show_msg(self, text):
1338 """Display message centered text and exit on key press"""
1339 hint = 'Press any key to continue'
1342 (x, term_width) = self.screen.getmaxyx()
1345 start = (term_width - len(line)) // 2
1346 self.screen.addstr(row, start, line)
1348 self.screen.addstr(row + 1, (term_width - len(hint)) // 2, hint,
1350 self.screen.getkey()
1352 def _show_help_interactive(self):
1353 """Display help with list of interactive commands"""
1354 msg = (' b toggle events by guests (debugfs only, honors'
1357 ' f filter by regular expression',
1358 ' g filter by guest name/PID',
1359 ' h display interactive commands reference',
1360 ' o toggle sorting order (Total vs CurAvg/s)',
1361 ' p filter by guest name/PID',
1364 ' s set delay between refreshs (value range: '
1365 '%s-%s secs)' % (MIN_DELAY, MAX_DELAY),
1366 ' x toggle reporting of stats for individual child trace'
1368 'Any other key refreshes statistics immediately')
1371 self.screen.addstr(0, 0, "Interactive commands reference",
1373 self.screen.addstr(2, 0, "Press any key to exit", curses.A_STANDOUT)
1376 self.screen.addstr(row, 0, line)
1378 self.screen.getkey()
1379 self._refresh_header()
1381 def _show_filter_selection(self):
1382 """Draws filter selection mask.
1384 Asks for a valid regex and sets the fields filter accordingly.
1390 self.screen.addstr(0, 0,
1391 "Show statistics for events matching a regex.",
1393 self.screen.addstr(2, 0,
1394 "Current regex: {0}"
1395 .format(self.stats.fields_filter))
1396 self.screen.addstr(5, 0, msg)
1397 self.screen.addstr(3, 0, "New regex: ")
1399 regex = self.screen.getstr().decode(ENCODING)
1402 self.stats.fields_filter = ''
1403 self._refresh_header()
1407 self.stats.fields_filter = regex
1408 self._refresh_header()
1411 msg = '"' + regex + '": Not a valid regular expression'
1414 def _show_set_update_interval(self):
1415 """Draws update interval selection mask."""
1419 self.screen.addstr(0, 0, 'Set update interval (defaults to %.1fs).'
1420 % DELAY_DEFAULT, curses.A_BOLD)
1421 self.screen.addstr(4, 0, msg)
1422 self.screen.addstr(2, 0, 'Change delay from %.1fs to ' %
1423 self._delay_regular)
1425 val = self.screen.getstr().decode(ENCODING)
1431 err = is_delay_valid(delay)
1436 delay = DELAY_DEFAULT
1437 self._delay_regular = delay
1441 msg = '"' + str(val) + '": Invalid value'
1442 self._refresh_header()
1444 def _is_running_guest(self, pid):
1445 """Check if pid is still a running process."""
1448 return os.path.isdir(os.path.join('/proc/', str(pid)))
1450 def _show_vm_selection_by_guest(self):
1451 """Draws guest selection mask.
1453 Asks for a guest name or pid until a valid guest name or '' is entered.
1459 self.screen.addstr(0, 0,
1460 'Show statistics for specific guest or pid.',
1462 self.screen.addstr(1, 0,
1463 'This might limit the shown data to the trace '
1465 self.screen.addstr(5, 0, msg)
1466 self._print_all_gnames(7)
1469 self.screen.addstr(3, 0, "Guest or pid [ENTER exits]: ")
1470 guest = self.screen.getstr().decode(ENCODING)
1474 if not guest or guest == '0':
1477 if not self._is_running_guest(guest):
1478 msg = '"' + guest + '": Not a running process'
1484 pids = self.get_pid_from_gname(guest)
1486 msg = '"' + guest + '": Internal error while searching, ' \
1487 'use pid filter instead'
1490 msg = '"' + guest + '": Not an active guest'
1493 msg = '"' + guest + '": Multiple matches found, use pid ' \
1499 self._refresh_header(pid)
1500 self._update_pid(pid)
1502 def show_stats(self):
1503 """Refreshes the screen and processes user input."""
1504 sleeptime = self._delay_initial
1505 self._refresh_header()
1506 start = 0.0 # result based on init value never appears on screen
1508 self._refresh_body(time.time() - start)
1509 curses.halfdelay(int(sleeptime * 10))
1511 sleeptime = self._delay_regular
1513 char = self.screen.getkey()
1515 self._display_guests = not self._display_guests
1516 if self.stats.toggle_display_guests(self._display_guests):
1517 self._show_msg(['Command not available with '
1518 'tracepoints enabled', 'Restart with '
1519 'debugfs only (see option \'-d\') and '
1521 self._display_guests = not self._display_guests
1522 self._refresh_header()
1524 self.stats.fields_filter = ''
1525 self._refresh_header(0)
1529 self._show_filter_selection()
1531 sleeptime = self._delay_initial
1532 if char == 'g' or char == 'p':
1533 self._show_vm_selection_by_guest()
1534 sleeptime = self._delay_initial
1536 self._show_help_interactive()
1538 self._sorting = not self._sorting
1545 self._show_set_update_interval()
1547 sleeptime = self._delay_initial
1549 self.stats.child_events = not self.stats.child_events
1550 except KeyboardInterrupt:
1552 except curses.error:
1557 """Prints statistics in a key, value format."""
1562 for key, values in sorted(s.items()):
1563 print('%-42s%10d%10d' % (key.split(' ')[0], values.value,
1565 except KeyboardInterrupt:
1569 class StdFormat(object):
1570 def __init__(self, keys):
1573 self._banner += key.split(' ')[0] + ' '
1575 def get_banner(self):
1578 def get_statline(self, keys, s):
1581 res += ' %9d' % s[key].delta
1585 class CSVFormat(object):
1586 def __init__(self, keys):
1587 self._banner = 'timestamp'
1588 self._banner += reduce(lambda res, key: "{},{!s}".format(res,
1589 key.split(' ')[0]), keys, '')
1591 def get_banner(self):
1594 def get_statline(self, keys, s):
1595 return reduce(lambda res, key: "{},{!s}".format(res, s[key].delta),
1599 def log(stats, opts, frmt, keys):
1600 """Prints statistics as reiterating key block, multiple value blocks."""
1601 global signal_received
1606 def do_banner(opts):
1608 if opts.log_to_file:
1611 f = open(opts.log_to_file, 'a')
1612 except (IOError, OSError):
1613 sys.exit("Error: Could not open file: %s" %
1615 if isinstance(frmt, CSVFormat) and f.tell() != 0:
1617 print(frmt.get_banner(), file=f or sys.stdout)
1619 def do_statline(opts, values):
1620 statline = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + \
1621 frmt.get_statline(keys, values)
1622 print(statline, file=f or sys.stdout)
1625 banner_printed = True
1628 time.sleep(opts.set_delay)
1630 banner_printed = True
1634 signal_received = False
1635 if (line % banner_repeat == 0 and not banner_printed and
1636 not (opts.log_to_file and isinstance(frmt, CSVFormat))):
1638 banner_printed = True
1639 values = stats.get()
1640 if (not opts.skip_zero_records or
1641 any(values[k].delta != 0 for k in keys)):
1642 do_statline(opts, values)
1644 banner_printed = False
1645 except KeyboardInterrupt:
1648 if opts.log_to_file:
1652 def handle_signal(sig, frame):
1653 global signal_received
1655 signal_received = True
1660 def is_delay_valid(delay):
1661 """Verify delay is in valid value range."""
1663 if delay < MIN_DELAY:
1664 msg = '"' + str(delay) + '": Delay must be >=%s' % MIN_DELAY
1665 if delay > MAX_DELAY:
1666 msg = '"' + str(delay) + '": Delay must be <=%s' % MAX_DELAY
1671 """Returns processed program arguments."""
1672 description_text = """
1673 This script displays various statistics about VMs running under KVM.
1674 The statistics are gathered from the KVM debugfs entries and / or the
1675 currently available perf traces.
1677 The monitoring takes additional cpu cycles and might affect the VM's
1685 - /proc/sys/kernel/perf_event_paranoid < 1 if user has no
1686 CAP_SYS_ADMIN and perf events are used.
1687 - CAP_SYS_RESOURCE if the hard limit is not high enough to allow
1688 the large number of files that are possibly opened.
1690 Interactive Commands:
1691 b toggle events by guests (debugfs only, honors filters)
1693 f filter by regular expression
1694 g filter by guest name
1695 h display interactive commands reference
1696 o toggle sorting order (Total vs CurAvg/s)
1700 s set update interval (value range: 0.1-25.5 secs)
1701 x toggle reporting of stats for individual child trace events
1702 Press any other key to refresh statistics immediately.
1703 """ % (PATH_DEBUGFS_KVM, PATH_DEBUGFS_TRACING)
1705 class Guest_to_pid(argparse.Action):
1706 def __call__(self, parser, namespace, values, option_string=None):
1708 pids = Tui.get_pid_from_gname(values)
1710 sys.exit('Error while searching for guest "{}". Use "-p" to '
1711 'specify a pid instead?'.format(values))
1713 sys.exit('Error: No guest by the name "{}" found'
1716 sys.exit('Error: Multiple processes found (pids: {}). Use "-p"'
1717 ' to specify the desired pid'
1718 .format(" ".join(map(str, pids))))
1719 namespace.pid = pids[0]
1721 argparser = argparse.ArgumentParser(description=description_text,
1722 formatter_class=argparse
1723 .RawTextHelpFormatter)
1724 argparser.add_argument('-1', '--once', '--batch',
1725 action='store_true',
1727 help='run in batch mode for one second',
1729 argparser.add_argument('-c', '--csv',
1730 action='store_true',
1732 help='log in csv format - requires option -l/-L',
1734 argparser.add_argument('-d', '--debugfs',
1735 action='store_true',
1737 help='retrieve statistics from debugfs',
1739 argparser.add_argument('-f', '--fields',
1741 help='''fields to display (regex)
1742 "-f help" for a list of available events''',
1744 argparser.add_argument('-g', '--guest',
1746 help='restrict statistics to guest by name',
1747 action=Guest_to_pid,
1749 argparser.add_argument('-i', '--debugfs-include-past',
1750 action='store_true',
1752 help='include all available data on past events for'
1755 argparser.add_argument('-l', '--log',
1756 action='store_true',
1758 help='run in logging mode (like vmstat)',
1760 argparser.add_argument('-L', '--log-to-file',
1763 help="like '--log', but logging to a file"
1765 argparser.add_argument('-p', '--pid',
1768 help='restrict statistics to pid',
1770 argparser.add_argument('-s', '--set-delay',
1772 default=DELAY_DEFAULT,
1774 help='set delay between refreshs (value range: '
1775 '%s-%s secs)' % (MIN_DELAY, MAX_DELAY),
1777 argparser.add_argument('-t', '--tracepoints',
1778 action='store_true',
1780 help='retrieve statistics from tracepoints',
1782 argparser.add_argument('-z', '--skip-zero-records',
1783 action='store_true',
1785 help='omit records with all zeros in logging mode',
1787 options = argparser.parse_args()
1788 if options.csv and not (options.log or options.log_to_file):
1789 sys.exit('Error: Option -c/--csv requires -l/--log')
1790 if options.skip_zero_records and not (options.log or options.log_to_file):
1791 sys.exit('Error: Option -z/--skip-zero-records requires -l/-L')
1793 # verify that we were passed a valid regex up front
1794 re.compile(options.fields)
1796 sys.exit('Error: "' + options.fields + '" is not a valid regular '
1802 def check_access(options):
1803 """Exits if the current user can't access all needed directories."""
1804 if not os.path.exists(PATH_DEBUGFS_TRACING) and (options.tracepoints or
1805 not options.debugfs):
1806 sys.stderr.write("Please enable CONFIG_TRACING in your kernel "
1807 "when using the option -t (default).\n"
1808 "If it is enabled, make {0} readable by the "
1810 .format(PATH_DEBUGFS_TRACING))
1811 if options.tracepoints:
1814 sys.stderr.write("Falling back to debugfs statistics!\n")
1815 options.debugfs = True
1821 def assign_globals():
1822 global PATH_DEBUGFS_KVM
1823 global PATH_DEBUGFS_TRACING
1826 for line in open('/proc/mounts'):
1827 if line.split(' ')[2] == 'debugfs':
1828 debugfs = line.split(' ')[1]
1831 sys.stderr.write("Please make sure that CONFIG_DEBUG_FS is enabled in "
1832 "your kernel, mounted and\nreadable by the current "
1834 "('mount -t debugfs debugfs /sys/kernel/debug')\n")
1837 PATH_DEBUGFS_KVM = os.path.join(debugfs, 'kvm')
1838 PATH_DEBUGFS_TRACING = os.path.join(debugfs, 'tracing')
1840 if not os.path.exists(PATH_DEBUGFS_KVM):
1841 sys.stderr.write("Please make sure that CONFIG_KVM is enabled in "
1842 "your kernel and that the modules are loaded.\n")
1848 options = get_options()
1849 options = check_access(options)
1851 if (options.pid > 0 and
1852 not os.path.isdir(os.path.join('/proc/',
1853 str(options.pid)))):
1854 sys.stderr.write('Did you use a (unsupported) tid instead of a pid?\n')
1855 sys.exit('Specified pid does not exist.')
1857 err = is_delay_valid(options.set_delay)
1859 sys.exit('Error: ' + err)
1861 stats = Stats(options)
1863 if options.fields == 'help':
1864 stats.fields_filter = None
1866 for key in stats.get().keys():
1867 event_list.append(key.split('(', 1)[0])
1868 sys.stdout.write(' ' + '\n '.join(sorted(set(event_list))) + '\n')
1871 if options.log or options.log_to_file:
1872 if options.log_to_file:
1873 signal.signal(signal.SIGHUP, handle_signal)
1874 keys = sorted(stats.get().keys())
1876 frmt = CSVFormat(keys)
1878 frmt = StdFormat(keys)
1879 log(stats, options, frmt, keys)
1880 elif not options.once:
1881 with Tui(stats, options) as tui:
1887 if __name__ == "__main__":