2 # SPDX-License-Identifier: GPL-2.0-only
4 # top-like utility for displaying kvm statistics
6 # Copyright 2006-2008 Qumranet Technologies
7 # Copyright 2008-2011 Red Hat, Inc.
10 # Avi Kivity <avi@redhat.com>
12 """The kvm_stat module outputs statistics about running KVM VMs
14 Three different ways of output formatting are available:
15 - as a top-like text ui
16 - in a key -> value format
17 - in an all keys, all values format
19 The data is sampled from the KVM's debugfs entries and its perf events.
21 from __future__ import print_function
36 from collections import defaultdict, namedtuple
37 from functools import reduce
38 from datetime import datetime
42 'EXTERNAL_INTERRUPT': 1,
44 'PENDING_INTERRUPT': 7,
68 'MWAIT_INSTRUCTION': 36,
69 'MONITOR_INSTRUCTION': 39,
70 'PAUSE_INSTRUCTION': 40,
71 'MCE_DURING_VMENTRY': 41,
72 'TPR_BELOW_THRESHOLD': 43,
113 'CR0_SEL_WRITE': 0x065,
137 'TASK_SWITCH': 0x07d,
138 'FERR_FREEZE': 0x07e,
157 # EC definition of HSR (from arch/arm64/include/asm/kvm_arm.h)
158 AARCH64_EXIT_REASONS = {
196 # From include/uapi/linux/kvm.h, KVM_EXIT_xxx
197 USERSPACE_EXIT_REASONS = {
205 'IRQ_WINDOW_OPEN': 7,
215 'INTERNAL_ERROR': 17,
226 'SET_FILTER': 0x40082406,
227 'ENABLE': 0x00002400,
228 'DISABLE': 0x00002401,
232 signal_received = False
234 ENCODING = locale.getpreferredencoding(False)
235 TRACE_FILTER = re.compile(r'^[^\(]*$')
239 """Encapsulates global architecture specific data.
241 Contains the performance event open syscall and ioctl numbers, as
242 well as the VM exit reasons for the architecture it runs on.
247 machine = os.uname()[4]
249 if machine.startswith('ppc'):
251 elif machine.startswith('aarch64'):
253 elif machine.startswith('s390'):
257 for line in open('/proc/cpuinfo'):
258 if not line.startswith('flags'):
263 return ArchX86(VMX_EXIT_REASONS)
265 return ArchX86(SVM_EXIT_REASONS)
268 def tracepoint_is_child(self, field):
269 if (TRACE_FILTER.match(field)):
271 return field.split('(', 1)[0]
275 def __init__(self, exit_reasons):
276 self.sc_perf_evt_open = 298
277 self.ioctl_numbers = IOCTL_NUMBERS
278 self.exit_reason_field = 'exit_reason'
279 self.exit_reasons = exit_reasons
281 def debugfs_is_child(self, field):
282 """ Returns name of parent if 'field' is a child, None otherwise """
288 self.sc_perf_evt_open = 319
289 self.ioctl_numbers = IOCTL_NUMBERS
290 self.ioctl_numbers['ENABLE'] = 0x20002400
291 self.ioctl_numbers['DISABLE'] = 0x20002401
292 self.ioctl_numbers['RESET'] = 0x20002403
294 # PPC comes in 32 and 64 bit and some generated ioctl
295 # numbers depend on the wordsize.
296 char_ptr_size = ctypes.sizeof(ctypes.c_char_p)
297 self.ioctl_numbers['SET_FILTER'] = 0x80002406 | char_ptr_size << 16
298 self.exit_reason_field = 'exit_nr'
299 self.exit_reasons = {}
301 def debugfs_is_child(self, field):
302 """ Returns name of parent if 'field' is a child, None otherwise """
308 self.sc_perf_evt_open = 241
309 self.ioctl_numbers = IOCTL_NUMBERS
310 self.exit_reason_field = 'esr_ec'
311 self.exit_reasons = AARCH64_EXIT_REASONS
313 def debugfs_is_child(self, field):
314 """ Returns name of parent if 'field' is a child, None otherwise """
318 class ArchS390(Arch):
320 self.sc_perf_evt_open = 331
321 self.ioctl_numbers = IOCTL_NUMBERS
322 self.exit_reason_field = None
323 self.exit_reasons = None
325 def debugfs_is_child(self, field):
326 """ Returns name of parent if 'field' is a child, None otherwise """
327 if field.startswith('instruction_'):
328 return 'exit_instruction'
331 ARCH = Arch.get_arch()
334 class perf_event_attr(ctypes.Structure):
335 """Struct that holds the necessary data to set up a trace event.
337 For an extensive explanation see perf_event_open(2) and
338 include/uapi/linux/perf_event.h, struct perf_event_attr
340 All fields that are not initialized in the constructor are 0.
343 _fields_ = [('type', ctypes.c_uint32),
344 ('size', ctypes.c_uint32),
345 ('config', ctypes.c_uint64),
346 ('sample_freq', ctypes.c_uint64),
347 ('sample_type', ctypes.c_uint64),
348 ('read_format', ctypes.c_uint64),
349 ('flags', ctypes.c_uint64),
350 ('wakeup_events', ctypes.c_uint32),
351 ('bp_type', ctypes.c_uint32),
352 ('bp_addr', ctypes.c_uint64),
353 ('bp_len', ctypes.c_uint64),
357 super(self.__class__, self).__init__()
358 self.type = PERF_TYPE_TRACEPOINT
359 self.size = ctypes.sizeof(self)
360 self.read_format = PERF_FORMAT_GROUP
363 PERF_TYPE_TRACEPOINT = 2
364 PERF_FORMAT_GROUP = 1 << 3
368 """Represents a perf event group."""
373 def add_event(self, event):
374 self.events.append(event)
377 """Returns a dict with 'event name: value' for all events in the
380 Values are read by reading from the file descriptor of the
381 event that is the group leader. See perf_event_open(2) for
384 Read format for the used event configuration is:
386 u64 nr; /* The number of events */
388 u64 value; /* The value of the event */
393 length = 8 * (1 + len(self.events))
394 read_format = 'xxxxxxxx' + 'Q' * len(self.events)
395 return dict(zip([event.name for event in self.events],
396 struct.unpack(read_format,
397 os.read(self.events[0].fd, length))))
401 """Represents a performance event and manages its life cycle."""
402 def __init__(self, name, group, trace_cpu, trace_pid, trace_point,
403 trace_filter, trace_set='kvm'):
404 self.libc = ctypes.CDLL('libc.so.6', use_errno=True)
405 self.syscall = self.libc.syscall
408 self._setup_event(group, trace_cpu, trace_pid, trace_point,
409 trace_filter, trace_set)
412 """Closes the event's file descriptor.
414 As no python file object was created for the file descriptor,
415 python will not reference count the descriptor and will not
416 close it itself automatically, so we do it.
422 def _perf_event_open(self, attr, pid, cpu, group_fd, flags):
423 """Wrapper for the sys_perf_evt_open() syscall.
425 Used to set up performance events, returns a file descriptor or -1
430 - struct perf_event_attr *
431 - pid or -1 to monitor all pids
432 - cpu number or -1 to monitor all cpus
433 - The file descriptor of the group leader or -1 to create a group.
437 return self.syscall(ARCH.sc_perf_evt_open, ctypes.pointer(attr),
438 ctypes.c_int(pid), ctypes.c_int(cpu),
439 ctypes.c_int(group_fd), ctypes.c_long(flags))
441 def _setup_event_attribute(self, trace_set, trace_point):
442 """Returns an initialized ctype perf_event_attr struct."""
444 id_path = os.path.join(PATH_DEBUGFS_TRACING, 'events', trace_set,
447 event_attr = perf_event_attr()
448 event_attr.config = int(open(id_path).read())
451 def _setup_event(self, group, trace_cpu, trace_pid, trace_point,
452 trace_filter, trace_set):
453 """Sets up the perf event in Linux.
455 Issues the syscall to register the event in the kernel and
456 then sets the optional filter.
460 event_attr = self._setup_event_attribute(trace_set, trace_point)
462 # First event will be group leader.
465 # All others have to pass the leader's descriptor instead.
467 group_leader = group.events[0].fd
469 fd = self._perf_event_open(event_attr, trace_pid,
470 trace_cpu, group_leader, 0)
472 err = ctypes.get_errno()
473 raise OSError(err, os.strerror(err),
474 'while calling sys_perf_event_open().')
477 fcntl.ioctl(fd, ARCH.ioctl_numbers['SET_FILTER'],
483 """Enables the trace event in the kernel.
485 Enabling the group leader makes reading counters from it and the
486 events under it possible.
489 fcntl.ioctl(self.fd, ARCH.ioctl_numbers['ENABLE'], 0)
492 """Disables the trace event in the kernel.
494 Disabling the group leader makes reading all counters under it
498 fcntl.ioctl(self.fd, ARCH.ioctl_numbers['DISABLE'], 0)
501 """Resets the count of the trace event in the kernel."""
502 fcntl.ioctl(self.fd, ARCH.ioctl_numbers['RESET'], 0)
505 class Provider(object):
506 """Encapsulates functionalities used by all providers."""
507 def __init__(self, pid):
508 self.child_events = False
512 def is_field_wanted(fields_filter, field):
513 """Indicate whether field is valid according to fields_filter."""
514 if not fields_filter:
516 return re.match(fields_filter, field) is not None
520 """Returns os.walk() data for specified directory.
522 As it is only a wrapper it returns the same 3-tuple of (dirpath,
523 dirnames, filenames).
525 return next(os.walk(path))
528 class TracepointProvider(Provider):
529 """Data provider for the stats class.
531 Manages the events/groups from which it acquires its data.
534 def __init__(self, pid, fields_filter):
535 self.group_leaders = []
536 self.filters = self._get_filters()
537 self.update_fields(fields_filter)
538 super(TracepointProvider, self).__init__(pid)
542 """Returns a dict of trace events, their filter ids and
543 the values that can be filtered.
545 Trace events can be filtered for special values by setting a
546 filter string via an ioctl. The string normally has the format
547 identifier==value. For each filter a new event will be created, to
548 be able to distinguish the events.
552 filters['kvm_userspace_exit'] = ('reason', USERSPACE_EXIT_REASONS)
553 if ARCH.exit_reason_field and ARCH.exit_reasons:
554 filters['kvm_exit'] = (ARCH.exit_reason_field, ARCH.exit_reasons)
557 def _get_available_fields(self):
558 """Returns a list of available events of format 'event name(filter
561 All available events have directories under
562 /sys/kernel/debug/tracing/events/ which export information
563 about the specific event. Therefore, listing the dirs gives us
564 a list of all available events.
566 Some events like the vm exit reasons can be filtered for
567 specific values. To take account for that, the routine below
568 creates special fields with the following format:
569 event name(filter name)
572 path = os.path.join(PATH_DEBUGFS_TRACING, 'events', 'kvm')
573 fields = self.walkdir(path)[1]
576 if field in self.filters:
577 filter_name_, filter_dicts = self.filters[field]
578 for name in filter_dicts:
579 extra.append(field + '(' + name + ')')
583 def update_fields(self, fields_filter):
584 """Refresh fields, applying fields_filter"""
585 self.fields = [field for field in self._get_available_fields()
586 if self.is_field_wanted(fields_filter, field)]
587 # add parents for child fields - otherwise we won't see any output!
588 for field in self._fields:
589 parent = ARCH.tracepoint_is_child(field)
590 if (parent and parent not in self._fields):
591 self.fields.append(parent)
594 def _get_online_cpus():
595 """Returns a list of cpu id integers."""
596 def parse_int_list(list_string):
597 """Returns an int list from a string of comma separated integers and
600 members = list_string.split(',')
602 for member in members:
603 if '-' not in member:
604 integers.append(int(member))
606 int_range = member.split('-')
607 integers.extend(range(int(int_range[0]),
608 int(int_range[1]) + 1))
612 with open('/sys/devices/system/cpu/online') as cpu_list:
613 cpu_string = cpu_list.readline()
614 return parse_int_list(cpu_string)
616 def _setup_traces(self):
617 """Creates all event and group objects needed to be able to retrieve
619 fields = self._get_available_fields()
621 # Fetch list of all threads of the monitored pid, as qemu
622 # starts a thread for each vcpu.
623 path = os.path.join('/proc', str(self._pid), 'task')
624 groupids = self.walkdir(path)[1]
626 groupids = self._get_online_cpus()
628 # The constant is needed as a buffer for python libs, std
629 # streams and other files that the script opens.
630 newlim = len(groupids) * len(fields) + 50
632 softlim_, hardlim = resource.getrlimit(resource.RLIMIT_NOFILE)
635 # Now we need CAP_SYS_RESOURCE, to increase the hard limit.
636 resource.setrlimit(resource.RLIMIT_NOFILE, (newlim, newlim))
638 # Raising the soft limit is sufficient.
639 resource.setrlimit(resource.RLIMIT_NOFILE, (newlim, hardlim))
642 sys.exit("NOFILE rlimit could not be raised to {0}".format(newlim))
644 for groupid in groupids:
649 match = re.match(r'(.*)\((.*)\)', name)
651 tracepoint, sub = match.groups()
652 tracefilter = ('%s==%d\0' %
653 (self.filters[tracepoint][0],
654 self.filters[tracepoint][1][sub]))
656 # From perf_event_open(2):
657 # pid > 0 and cpu == -1
658 # This measures the specified process/thread on any CPU.
660 # pid == -1 and cpu >= 0
661 # This measures all processes/threads on the specified CPU.
662 trace_cpu = groupid if self._pid == 0 else -1
663 trace_pid = int(groupid) if self._pid != 0 else -1
665 group.add_event(Event(name=name,
669 trace_point=tracepoint,
670 trace_filter=tracefilter))
672 self.group_leaders.append(group)
679 def fields(self, fields):
680 """Enables/disables the (un)wanted events"""
681 self._fields = fields
682 for group in self.group_leaders:
683 for index, event in enumerate(group.events):
684 if event.name in fields:
688 # Do not disable the group leader.
689 # It would disable all of its events.
699 """Changes the monitored pid by setting new traces."""
701 # The garbage collector will get rid of all Event/Group
702 # objects and open files after removing the references.
703 self.group_leaders = []
705 self.fields = self._fields
707 def read(self, by_guest=0):
708 """Returns 'event name: current value' for all enabled events."""
709 ret = defaultdict(int)
710 for group in self.group_leaders:
711 for name, val in group.read().items():
712 if name not in self._fields:
714 parent = ARCH.tracepoint_is_child(name)
721 """Reset all field counters"""
722 for group in self.group_leaders:
723 for event in group.events:
727 class DebugfsProvider(Provider):
728 """Provides data from the files that KVM creates in the kvm debugfs
730 def __init__(self, pid, fields_filter, include_past):
731 self.update_fields(fields_filter)
735 super(DebugfsProvider, self).__init__(pid)
739 def _get_available_fields(self):
740 """"Returns a list of available fields.
742 The fields are all available KVM debugfs files
745 exempt_list = ['halt_poll_fail_ns', 'halt_poll_success_ns']
746 fields = [field for field in self.walkdir(PATH_DEBUGFS_KVM)[2]
747 if field not in exempt_list]
751 def update_fields(self, fields_filter):
752 """Refresh fields, applying fields_filter"""
753 self._fields = [field for field in self._get_available_fields()
754 if self.is_field_wanted(fields_filter, field)]
755 # add parents for child fields - otherwise we won't see any output!
756 for field in self._fields:
757 parent = ARCH.debugfs_is_child(field)
758 if (parent and parent not in self._fields):
759 self.fields.append(parent)
766 def fields(self, fields):
767 self._fields = fields
778 vms = self.walkdir(PATH_DEBUGFS_KVM)[1]
782 self.paths = list(filter(lambda x: "{}-".format(pid) in x, vms))
788 def _verify_paths(self):
789 """Remove invalid paths"""
790 for path in self.paths:
791 if not os.path.exists(os.path.join(PATH_DEBUGFS_KVM, path)):
792 self.paths.remove(path)
795 def read(self, reset=0, by_guest=0):
796 """Returns a dict with format:'file name / field -> current value'.
800 1 reset field counts to 0
801 2 restore the original field counts
806 # If no debugfs filtering support is available, then don't read.
814 for entry in os.walk(PATH_DEBUGFS_KVM):
818 for field in self._fields:
819 value = self._read_field(field, path)
822 self._baseline[key] = value
824 self._baseline[key] = 0
825 if self._baseline.get(key, -1) == -1:
826 self._baseline[key] = value
827 parent = ARCH.debugfs_is_child(field)
829 field = field + ' ' + parent
832 field = key.split('-')[0] # set 'field' to 'pid'
833 increment = value - self._baseline.get(key, 0)
835 results[field] += increment
837 results[field] = increment
841 def _read_field(self, field, path):
842 """Returns the value of a single field from a specific VM."""
844 return int(open(os.path.join(PATH_DEBUGFS_KVM,
852 """Reset field counters"""
857 """Reset field counters"""
862 EventStat = namedtuple('EventStat', ['value', 'delta'])
866 """Manages the data providers and the data they provide.
868 It is used to set filters on the provider's data and collect all
872 def __init__(self, options):
873 self.providers = self._get_providers(options)
874 self._pid_filter = options.pid
875 self._fields_filter = options.fields
877 self._child_events = False
879 def _get_providers(self, options):
880 """Returns a list of data providers depending on the passed options."""
884 providers.append(DebugfsProvider(options.pid, options.fields,
885 options.debugfs_include_past))
886 if options.tracepoints or not providers:
887 providers.append(TracepointProvider(options.pid, options.fields))
891 def _update_provider_filters(self):
892 """Propagates fields filters to providers."""
893 # As we reset the counters when updating the fields we can
894 # also clear the cache of old values.
896 for provider in self.providers:
897 provider.update_fields(self._fields_filter)
901 for provider in self.providers:
905 def fields_filter(self):
906 return self._fields_filter
908 @fields_filter.setter
909 def fields_filter(self, fields_filter):
910 if fields_filter != self._fields_filter:
911 self._fields_filter = fields_filter
912 self._update_provider_filters()
915 def pid_filter(self):
916 return self._pid_filter
919 def pid_filter(self, pid):
920 if pid != self._pid_filter:
921 self._pid_filter = pid
923 for provider in self.providers:
924 provider.pid = self._pid_filter
927 def child_events(self):
928 return self._child_events
931 def child_events(self, val):
932 self._child_events = val
933 for provider in self.providers:
934 provider.child_events = val
936 def get(self, by_guest=0):
937 """Returns a dict with field -> (value, delta to last value) of all
940 * plain: 'key' is event name
941 * child-parent: 'key' is in format '<child> <parent>'
942 * pid: 'key' is the pid of the guest, and the record contains the
943 aggregated event data
944 These formats are generated by the providers, and handled in class TUI.
946 for provider in self.providers:
947 new = provider.read(by_guest=by_guest)
949 oldval = self.values.get(key, EventStat(0, 0)).value
950 newval = new.get(key, 0)
951 newdelta = newval - oldval
952 self.values[key] = EventStat(newval, newdelta)
955 def toggle_display_guests(self, to_pid):
956 """Toggle between collection of stats by individual event and by
959 Events reported by DebugfsProvider change when switching to/from
960 reading by guest values. Hence we have to remove the excess event
961 names from self.values.
964 if any(isinstance(ins, TracepointProvider) for ins in self.providers):
967 for provider in self.providers:
968 if isinstance(provider, DebugfsProvider):
969 for key in provider.fields:
970 if key in self.values.keys():
973 oldvals = self.values.copy()
977 # Update oldval (see get())
983 MAX_GUEST_NAME_LEN = 48
991 """Instruments curses to draw a nice text ui."""
992 def __init__(self, stats, opts):
995 self._delay_initial = 0.25
996 self._delay_regular = opts.set_delay
997 self._sorting = SORT_DEFAULT
998 self._display_guests = 0
1000 def __enter__(self):
1001 """Initialises curses for later use. Based on curses.wrapper
1002 implementation from the Python standard library."""
1003 self.screen = curses.initscr()
1007 # The try/catch works around a minor bit of
1008 # over-conscientiousness in the curses module, the error
1009 # return from C start_color() is ignorable.
1011 curses.start_color()
1012 except curses.error:
1015 # Hide cursor in extra statement as some monochrome terminals
1016 # might support hiding but not colors.
1019 except curses.error:
1022 curses.use_default_colors()
1025 def __exit__(self, *exception):
1026 """Resets the terminal to its normal state. Based on curses.wrapper
1027 implementation from the Python standard library."""
1029 self.screen.keypad(0)
1035 def get_all_gnames():
1036 """Returns a list of (pid, gname) tuples of all running guests"""
1039 child = subprocess.Popen(['ps', '-A', '--format', 'pid,args'],
1040 stdout=subprocess.PIPE)
1043 for line in child.stdout:
1044 line = line.decode(ENCODING).lstrip().split(' ', 1)
1045 # perform a sanity check before calling the more expensive
1046 # function to possibly extract the guest name
1047 if ' -name ' in line[1]:
1048 res.append((line[0], Tui.get_gname_from_pid(line[0])))
1049 child.stdout.close()
1053 def _print_all_gnames(self, row):
1054 """Print a list of all running guests along with their pids."""
1055 self.screen.addstr(row, 2, '%8s %-60s' %
1056 ('Pid', 'Guest Name (fuzzy list, might be '
1061 for line in self.get_all_gnames():
1062 self.screen.addstr(row, 2, '%8s %-60s' % (line[0], line[1]))
1064 if row >= self.screen.getmaxyx()[0]:
1067 self.screen.addstr(row + 1, 2, 'Not available')
1070 def get_pid_from_gname(gname):
1071 """Fuzzy function to convert guest name to QEMU process pid.
1073 Returns a list of potential pids, can be empty if no match found.
1074 Throws an exception on processing errors.
1078 for line in Tui.get_all_gnames():
1079 if gname == line[1]:
1080 pids.append(int(line[0]))
1085 def get_gname_from_pid(pid):
1086 """Returns the guest name for a QEMU process pid.
1088 Extracts the guest name from the QEMU comma line by processing the
1089 '-name' option. Will also handle names specified out of sequence.
1094 line = open('/proc/{}/cmdline'
1095 .format(pid), 'r').read().split('\0')
1096 parms = line[line.index('-name') + 1].split(',')
1098 # commas are escaped (i.e. ',,'), hence e.g. 'foo,bar' results
1099 # in # ['foo', '', 'bar'], which we revert here
1100 idx = parms.index('')
1101 parms[idx - 1] += ',' + parms[idx + 1]
1102 del parms[idx:idx+2]
1103 # the '-name' switch allows for two ways to specify the guest name,
1104 # where the plain name overrides the name specified via 'guest='
1109 if arg[:6] == 'guest=':
1111 except (ValueError, IOError, IndexError):
1116 def _update_pid(self, pid):
1117 """Propagates pid selection to stats object."""
1118 self.screen.addstr(4, 1, 'Updating pid filter...')
1119 self.screen.refresh()
1120 self.stats.pid_filter = pid
1122 def _refresh_header(self, pid=None):
1123 """Refreshes the header."""
1125 pid = self.stats.pid_filter
1127 gname = self.get_gname_from_pid(pid)
1130 gname = ('({})'.format(gname[:MAX_GUEST_NAME_LEN] + '...'
1131 if len(gname) > MAX_GUEST_NAME_LEN
1134 self._headline = 'kvm statistics - pid {0} {1}'.format(pid, gname)
1136 self._headline = 'kvm statistics - summary'
1137 self.screen.addstr(0, 0, self._headline, curses.A_BOLD)
1138 if self.stats.fields_filter:
1139 regex = self.stats.fields_filter
1140 if len(regex) > MAX_REGEX_LEN:
1141 regex = regex[:MAX_REGEX_LEN] + '...'
1142 self.screen.addstr(1, 17, 'regex filter: {0}'.format(regex))
1143 if self._display_guests:
1144 col_name = 'Guest Name'
1147 self.screen.addstr(2, 1, '%-40s %10s%7s %8s' %
1148 (col_name, 'Total', '%Total', 'CurAvg/s'),
1150 self.screen.addstr(4, 1, 'Collecting data...')
1151 self.screen.refresh()
1153 def _refresh_body(self, sleeptime):
1154 def insert_child(sorted_items, child, values, parent):
1155 num = len(sorted_items)
1156 for i in range(0, num):
1157 # only add child if parent is present
1158 if parent.startswith(sorted_items[i][0]):
1159 sorted_items.insert(i + 1, (' ' + child, values))
1161 def get_sorted_events(self, stats):
1162 """ separate parent and child events """
1163 if self._sorting == SORT_DEFAULT:
1165 # sort by (delta value, overall value)
1167 return (v.delta, v.value)
1170 # sort by overall value
1176 # we can't rule out child events to appear prior to parents even
1177 # when sorted - separate out all children first, and add in later
1178 for key, values in sorted(stats.items(), key=sortkey,
1180 if values == (0, 0):
1182 if key.find(' ') != -1:
1183 if not self.stats.child_events:
1185 childs.insert(0, (key, values))
1187 sorted_items.append((key, values))
1188 if self.stats.child_events:
1189 for key, values in childs:
1190 (child, parent) = key.split(' ')
1191 insert_child(sorted_items, child, values, parent)
1195 if not self._is_running_guest(self.stats.pid_filter):
1197 try: # ...to identify the guest by name in case it's back
1198 pids = self.get_pid_from_gname(self._gname)
1200 self._refresh_header(pids[0])
1201 self._update_pid(pids[0])
1205 self._display_guest_dead()
1206 # leave final data on screen
1209 self.screen.move(row, 0)
1210 self.screen.clrtobot()
1211 stats = self.stats.get(self._display_guests)
1214 for key, values in stats.items():
1215 if self._display_guests:
1216 if self.get_gname_from_pid(key):
1217 total += values.value
1219 if not key.find(' ') != -1:
1220 total += values.value
1222 ctotal += values.value
1224 # we don't have any fields, or all non-child events are filtered
1230 guest_removed = False
1231 for key, values in get_sorted_events(self, stats):
1232 if row >= self.screen.getmaxyx()[0] - 1 or values == (0, 0):
1234 if self._display_guests:
1235 key = self.get_gname_from_pid(key)
1238 cur = int(round(values.delta / sleeptime)) if values.delta else 0
1240 guest_removed = True
1244 tcur += values.delta
1245 ptotal = values.value
1249 self.screen.addstr(row, 1, '%-40s %10d%7.1f %8s' % (key,
1251 values.value * 100 / float(ltotal), cur))
1255 self.screen.addstr(4, 1, 'Guest removed, updating...')
1257 self.screen.addstr(4, 1, 'No matching events reported yet')
1259 tavg = int(round(tcur / sleeptime)) if tcur > 0 else ''
1260 self.screen.addstr(row, 1, '%-40s %10d %8s' %
1261 ('Total', total, tavg), curses.A_BOLD)
1262 self.screen.refresh()
1264 def _display_guest_dead(self):
1265 marker = ' Guest is DEAD '
1266 y = min(len(self._headline), 80 - len(marker))
1267 self.screen.addstr(0, y, marker, curses.A_BLINK | curses.A_STANDOUT)
1269 def _show_msg(self, text):
1270 """Display message centered text and exit on key press"""
1271 hint = 'Press any key to continue'
1274 (x, term_width) = self.screen.getmaxyx()
1277 start = (term_width - len(line)) // 2
1278 self.screen.addstr(row, start, line)
1280 self.screen.addstr(row + 1, (term_width - len(hint)) // 2, hint,
1282 self.screen.getkey()
1284 def _show_help_interactive(self):
1285 """Display help with list of interactive commands"""
1286 msg = (' b toggle events by guests (debugfs only, honors'
1289 ' f filter by regular expression',
1290 ' g filter by guest name/PID',
1291 ' h display interactive commands reference',
1292 ' o toggle sorting order (Total vs CurAvg/s)',
1293 ' p filter by guest name/PID',
1296 ' s set delay between refreshs (value range: '
1297 '%s-%s secs)' % (MIN_DELAY, MAX_DELAY),
1298 ' x toggle reporting of stats for individual child trace'
1300 'Any other key refreshes statistics immediately')
1303 self.screen.addstr(0, 0, "Interactive commands reference",
1305 self.screen.addstr(2, 0, "Press any key to exit", curses.A_STANDOUT)
1308 self.screen.addstr(row, 0, line)
1310 self.screen.getkey()
1311 self._refresh_header()
1313 def _show_filter_selection(self):
1314 """Draws filter selection mask.
1316 Asks for a valid regex and sets the fields filter accordingly.
1322 self.screen.addstr(0, 0,
1323 "Show statistics for events matching a regex.",
1325 self.screen.addstr(2, 0,
1326 "Current regex: {0}"
1327 .format(self.stats.fields_filter))
1328 self.screen.addstr(5, 0, msg)
1329 self.screen.addstr(3, 0, "New regex: ")
1331 regex = self.screen.getstr().decode(ENCODING)
1334 self.stats.fields_filter = ''
1335 self._refresh_header()
1339 self.stats.fields_filter = regex
1340 self._refresh_header()
1343 msg = '"' + regex + '": Not a valid regular expression'
1346 def _show_set_update_interval(self):
1347 """Draws update interval selection mask."""
1351 self.screen.addstr(0, 0, 'Set update interval (defaults to %.1fs).'
1352 % DELAY_DEFAULT, curses.A_BOLD)
1353 self.screen.addstr(4, 0, msg)
1354 self.screen.addstr(2, 0, 'Change delay from %.1fs to ' %
1355 self._delay_regular)
1357 val = self.screen.getstr().decode(ENCODING)
1363 err = is_delay_valid(delay)
1368 delay = DELAY_DEFAULT
1369 self._delay_regular = delay
1373 msg = '"' + str(val) + '": Invalid value'
1374 self._refresh_header()
1376 def _is_running_guest(self, pid):
1377 """Check if pid is still a running process."""
1380 return os.path.isdir(os.path.join('/proc/', str(pid)))
1382 def _show_vm_selection_by_guest(self):
1383 """Draws guest selection mask.
1385 Asks for a guest name or pid until a valid guest name or '' is entered.
1391 self.screen.addstr(0, 0,
1392 'Show statistics for specific guest or pid.',
1394 self.screen.addstr(1, 0,
1395 'This might limit the shown data to the trace '
1397 self.screen.addstr(5, 0, msg)
1398 self._print_all_gnames(7)
1401 self.screen.addstr(3, 0, "Guest or pid [ENTER exits]: ")
1402 guest = self.screen.getstr().decode(ENCODING)
1406 if not guest or guest == '0':
1409 if not self._is_running_guest(guest):
1410 msg = '"' + guest + '": Not a running process'
1416 pids = self.get_pid_from_gname(guest)
1418 msg = '"' + guest + '": Internal error while searching, ' \
1419 'use pid filter instead'
1422 msg = '"' + guest + '": Not an active guest'
1425 msg = '"' + guest + '": Multiple matches found, use pid ' \
1431 self._refresh_header(pid)
1432 self._update_pid(pid)
1434 def show_stats(self):
1435 """Refreshes the screen and processes user input."""
1436 sleeptime = self._delay_initial
1437 self._refresh_header()
1438 start = 0.0 # result based on init value never appears on screen
1440 self._refresh_body(time.time() - start)
1441 curses.halfdelay(int(sleeptime * 10))
1443 sleeptime = self._delay_regular
1445 char = self.screen.getkey()
1447 self._display_guests = not self._display_guests
1448 if self.stats.toggle_display_guests(self._display_guests):
1449 self._show_msg(['Command not available with '
1450 'tracepoints enabled', 'Restart with '
1451 'debugfs only (see option \'-d\') and '
1453 self._display_guests = not self._display_guests
1454 self._refresh_header()
1456 self.stats.fields_filter = ''
1457 self._refresh_header(0)
1461 self._show_filter_selection()
1463 sleeptime = self._delay_initial
1464 if char == 'g' or char == 'p':
1465 self._show_vm_selection_by_guest()
1466 sleeptime = self._delay_initial
1468 self._show_help_interactive()
1470 self._sorting = not self._sorting
1477 self._show_set_update_interval()
1479 sleeptime = self._delay_initial
1481 self.stats.child_events = not self.stats.child_events
1482 except KeyboardInterrupt:
1484 except curses.error:
1489 """Prints statistics in a key, value format."""
1494 for key, values in sorted(s.items()):
1495 print('%-42s%10d%10d' % (key.split(' ')[0], values.value,
1497 except KeyboardInterrupt:
1501 class StdFormat(object):
1502 def __init__(self, keys):
1505 self._banner += key.split(' ')[0] + ' '
1507 def get_banner(self):
1510 def get_statline(self, keys, s):
1513 res += ' %9d' % s[key].delta
1517 class CSVFormat(object):
1518 def __init__(self, keys):
1519 self._banner = 'timestamp'
1520 self._banner += reduce(lambda res, key: "{},{!s}".format(res,
1521 key.split(' ')[0]), keys, '')
1523 def get_banner(self):
1526 def get_statline(self, keys, s):
1527 return reduce(lambda res, key: "{},{!s}".format(res, s[key].delta),
1531 def log(stats, opts, frmt, keys):
1532 """Prints statistics as reiterating key block, multiple value blocks."""
1533 global signal_received
1538 def do_banner(opts):
1540 if opts.log_to_file:
1543 f = open(opts.log_to_file, 'a')
1544 except (IOError, OSError):
1545 sys.exit("Error: Could not open file: %s" %
1547 if isinstance(frmt, CSVFormat) and f.tell() != 0:
1549 print(frmt.get_banner(), file=f or sys.stdout)
1551 def do_statline(opts, values):
1552 statline = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + \
1553 frmt.get_statline(keys, values)
1554 print(statline, file=f or sys.stdout)
1557 banner_printed = True
1560 time.sleep(opts.set_delay)
1562 banner_printed = True
1566 signal_received = False
1567 if (line % banner_repeat == 0 and not banner_printed and
1568 not (opts.log_to_file and isinstance(frmt, CSVFormat))):
1570 banner_printed = True
1571 values = stats.get()
1572 if (not opts.skip_zero_records or
1573 any(values[k].delta != 0 for k in keys)):
1574 do_statline(opts, values)
1576 banner_printed = False
1577 except KeyboardInterrupt:
1580 if opts.log_to_file:
1584 def handle_signal(sig, frame):
1585 global signal_received
1587 signal_received = True
1592 def is_delay_valid(delay):
1593 """Verify delay is in valid value range."""
1595 if delay < MIN_DELAY:
1596 msg = '"' + str(delay) + '": Delay must be >=%s' % MIN_DELAY
1597 if delay > MAX_DELAY:
1598 msg = '"' + str(delay) + '": Delay must be <=%s' % MAX_DELAY
1603 """Returns processed program arguments."""
1604 description_text = """
1605 This script displays various statistics about VMs running under KVM.
1606 The statistics are gathered from the KVM debugfs entries and / or the
1607 currently available perf traces.
1609 The monitoring takes additional cpu cycles and might affect the VM's
1617 - /proc/sys/kernel/perf_event_paranoid < 1 if user has no
1618 CAP_SYS_ADMIN and perf events are used.
1619 - CAP_SYS_RESOURCE if the hard limit is not high enough to allow
1620 the large number of files that are possibly opened.
1622 Interactive Commands:
1623 b toggle events by guests (debugfs only, honors filters)
1625 f filter by regular expression
1626 g filter by guest name
1627 h display interactive commands reference
1628 o toggle sorting order (Total vs CurAvg/s)
1632 s set update interval (value range: 0.1-25.5 secs)
1633 x toggle reporting of stats for individual child trace events
1634 Press any other key to refresh statistics immediately.
1635 """ % (PATH_DEBUGFS_KVM, PATH_DEBUGFS_TRACING)
1637 class Guest_to_pid(argparse.Action):
1638 def __call__(self, parser, namespace, values, option_string=None):
1640 pids = Tui.get_pid_from_gname(values)
1642 sys.exit('Error while searching for guest "{}". Use "-p" to '
1643 'specify a pid instead?'.format(values))
1645 sys.exit('Error: No guest by the name "{}" found'
1648 sys.exit('Error: Multiple processes found (pids: {}). Use "-p"'
1649 ' to specify the desired pid'
1650 .format(" ".join(map(str, pids))))
1651 namespace.pid = pids[0]
1653 argparser = argparse.ArgumentParser(description=description_text,
1654 formatter_class=argparse
1655 .RawTextHelpFormatter)
1656 argparser.add_argument('-1', '--once', '--batch',
1657 action='store_true',
1659 help='run in batch mode for one second',
1661 argparser.add_argument('-c', '--csv',
1662 action='store_true',
1664 help='log in csv format - requires option -l/-L',
1666 argparser.add_argument('-d', '--debugfs',
1667 action='store_true',
1669 help='retrieve statistics from debugfs',
1671 argparser.add_argument('-f', '--fields',
1673 help='''fields to display (regex)
1674 "-f help" for a list of available events''',
1676 argparser.add_argument('-g', '--guest',
1678 help='restrict statistics to guest by name',
1679 action=Guest_to_pid,
1681 argparser.add_argument('-i', '--debugfs-include-past',
1682 action='store_true',
1684 help='include all available data on past events for'
1687 argparser.add_argument('-l', '--log',
1688 action='store_true',
1690 help='run in logging mode (like vmstat)',
1692 argparser.add_argument('-L', '--log-to-file',
1695 help="like '--log', but logging to a file"
1697 argparser.add_argument('-p', '--pid',
1700 help='restrict statistics to pid',
1702 argparser.add_argument('-s', '--set-delay',
1704 default=DELAY_DEFAULT,
1706 help='set delay between refreshs (value range: '
1707 '%s-%s secs)' % (MIN_DELAY, MAX_DELAY),
1709 argparser.add_argument('-t', '--tracepoints',
1710 action='store_true',
1712 help='retrieve statistics from tracepoints',
1714 argparser.add_argument('-z', '--skip-zero-records',
1715 action='store_true',
1717 help='omit records with all zeros in logging mode',
1719 options = argparser.parse_args()
1720 if options.csv and not (options.log or options.log_to_file):
1721 sys.exit('Error: Option -c/--csv requires -l/--log')
1722 if options.skip_zero_records and not (options.log or options.log_to_file):
1723 sys.exit('Error: Option -z/--skip-zero-records requires -l/-L')
1725 # verify that we were passed a valid regex up front
1726 re.compile(options.fields)
1728 sys.exit('Error: "' + options.fields + '" is not a valid regular '
1734 def check_access(options):
1735 """Exits if the current user can't access all needed directories."""
1736 if not os.path.exists(PATH_DEBUGFS_TRACING) and (options.tracepoints or
1737 not options.debugfs):
1738 sys.stderr.write("Please enable CONFIG_TRACING in your kernel "
1739 "when using the option -t (default).\n"
1740 "If it is enabled, make {0} readable by the "
1742 .format(PATH_DEBUGFS_TRACING))
1743 if options.tracepoints:
1746 sys.stderr.write("Falling back to debugfs statistics!\n")
1747 options.debugfs = True
1753 def assign_globals():
1754 global PATH_DEBUGFS_KVM
1755 global PATH_DEBUGFS_TRACING
1758 for line in open('/proc/mounts'):
1759 if line.split(' ')[0] == 'debugfs':
1760 debugfs = line.split(' ')[1]
1763 sys.stderr.write("Please make sure that CONFIG_DEBUG_FS is enabled in "
1764 "your kernel, mounted and\nreadable by the current "
1766 "('mount -t debugfs debugfs /sys/kernel/debug')\n")
1769 PATH_DEBUGFS_KVM = os.path.join(debugfs, 'kvm')
1770 PATH_DEBUGFS_TRACING = os.path.join(debugfs, 'tracing')
1772 if not os.path.exists(PATH_DEBUGFS_KVM):
1773 sys.stderr.write("Please make sure that CONFIG_KVM is enabled in "
1774 "your kernel and that the modules are loaded.\n")
1780 options = get_options()
1781 options = check_access(options)
1783 if (options.pid > 0 and
1784 not os.path.isdir(os.path.join('/proc/',
1785 str(options.pid)))):
1786 sys.stderr.write('Did you use a (unsupported) tid instead of a pid?\n')
1787 sys.exit('Specified pid does not exist.')
1789 err = is_delay_valid(options.set_delay)
1791 sys.exit('Error: ' + err)
1793 stats = Stats(options)
1795 if options.fields == 'help':
1796 stats.fields_filter = None
1798 for key in stats.get().keys():
1799 event_list.append(key.split('(', 1)[0])
1800 sys.stdout.write(' ' + '\n '.join(sorted(set(event_list))) + '\n')
1803 if options.log or options.log_to_file:
1804 if options.log_to_file:
1805 signal.signal(signal.SIGHUP, handle_signal)
1806 keys = sorted(stats.get().keys())
1808 frmt = CSVFormat(keys)
1810 frmt = StdFormat(keys)
1811 log(stats, options, frmt, keys)
1812 elif not options.once:
1813 with Tui(stats, options) as tui:
1819 if __name__ == "__main__":