From: Fabiano Rosas Date: Wed, 27 Nov 2024 18:28:47 +0000 (-0300) Subject: tests/migration: Disambiguate guestperf vs. a-b X-Git-Url: http://git.maquefel.me/?a=commitdiff_plain;h=212c19331b0c53ab299ae3d646409fad2da90602;p=qemu.git tests/migration: Disambiguate guestperf vs. a-b The current build structure for migration tests is confusing. There is the tests/migration directory, which contains two different guest code implementations, one for the qtests (a-b-{bootblock|kernel}.S) and another for the guestperf script (stress.c). One uses a Makefile, while the other uses meson. The next patches will add a new qtests/migration/ directory to hold qtest code which will make the situation even more confusing. Move the guest code used by qtests into a new qtests/migration/ directory and rename the old one to tests/migration-stress. Reviewed-by: Peter Xu Signed-off-by: Fabiano Rosas --- diff --git a/MAINTAINERS b/MAINTAINERS index aaf0505a21..e8605d394e 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -118,7 +118,7 @@ F: pc-bios/s390-ccw.img F: target/s390x/ F: docs/system/target-s390x.rst F: docs/system/s390x/ -F: tests/migration/s390x/ +F: tests/qtest/migration/s390x/ K: ^Subject:.*(?i)s390x? L: qemu-s390x@nongnu.org @@ -3423,10 +3423,11 @@ F: include/qemu/userfaultfd.h F: migration/ F: scripts/vmstate-static-checker.py F: tests/vmstate-static-checker-data/ +F: tests/qtest/migration/ F: tests/qtest/migration-* F: docs/devel/migration/ F: qapi/migration.json -F: tests/migration/ +F: tests/migration-stress/ F: util/userfaultfd.c X: migration/rdma* diff --git a/tests/meson.build b/tests/meson.build index 907a4c1c98..f96c1be574 100644 --- a/tests/meson.build +++ b/tests/meson.build @@ -84,5 +84,5 @@ endif subdir('unit') subdir('qapi-schema') subdir('qtest') -subdir('migration') +subdir('migration-stress') subdir('functional') diff --git a/tests/migration-stress/guestperf-batch.py b/tests/migration-stress/guestperf-batch.py new file mode 100755 index 0000000000..9485eefe49 --- /dev/null +++ b/tests/migration-stress/guestperf-batch.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +# +# Migration test batch comparison invocation +# +# Copyright (c) 2016 Red Hat, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, see . +# + +import sys + +from guestperf.shell import BatchShell + +shell = BatchShell() +sys.exit(shell.run(sys.argv[1:])) diff --git a/tests/migration-stress/guestperf-plot.py b/tests/migration-stress/guestperf-plot.py new file mode 100755 index 0000000000..32977b4bf6 --- /dev/null +++ b/tests/migration-stress/guestperf-plot.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +# +# Migration test graph plotting command +# +# Copyright (c) 2016 Red Hat, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, see . +# + +import sys + +from guestperf.shell import PlotShell + +shell = PlotShell() +sys.exit(shell.run(sys.argv[1:])) diff --git a/tests/migration-stress/guestperf.py b/tests/migration-stress/guestperf.py new file mode 100755 index 0000000000..07182f211e --- /dev/null +++ b/tests/migration-stress/guestperf.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +# +# Migration test direct invocation command +# +# Copyright (c) 2016 Red Hat, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, see . +# + + +import sys + +from guestperf.shell import Shell + +shell = Shell() +sys.exit(shell.run(sys.argv[1:])) diff --git a/tests/migration-stress/guestperf/__init__.py b/tests/migration-stress/guestperf/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/migration-stress/guestperf/comparison.py b/tests/migration-stress/guestperf/comparison.py new file mode 100644 index 0000000000..42cc0372d1 --- /dev/null +++ b/tests/migration-stress/guestperf/comparison.py @@ -0,0 +1,161 @@ +# +# Migration test scenario comparison mapping +# +# Copyright (c) 2016 Red Hat, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, see . +# + +from guestperf.scenario import Scenario + +class Comparison(object): + def __init__(self, name, scenarios): + self._name = name + self._scenarios = scenarios + +COMPARISONS = [ + # Looking at effect of pausing guest during migration + # at various stages of iteration over RAM + Comparison("pause-iters", scenarios = [ + Scenario("pause-iters-0", + pause=True, pause_iters=0), + Scenario("pause-iters-1", + pause=True, pause_iters=1), + Scenario("pause-iters-5", + pause=True, pause_iters=5), + Scenario("pause-iters-20", + pause=True, pause_iters=20), + ]), + + + # Looking at use of post-copy in relation to bandwidth + # available for migration + Comparison("post-copy-bandwidth", scenarios = [ + Scenario("post-copy-bw-100mbs", + post_copy=True, bandwidth=12), + Scenario("post-copy-bw-300mbs", + post_copy=True, bandwidth=37), + Scenario("post-copy-bw-1gbs", + post_copy=True, bandwidth=125), + Scenario("post-copy-bw-10gbs", + post_copy=True, bandwidth=1250), + Scenario("post-copy-bw-100gbs", + post_copy=True, bandwidth=12500), + ]), + + + # Looking at effect of starting post-copy at different + # stages of the migration + Comparison("post-copy-iters", scenarios = [ + Scenario("post-copy-iters-0", + post_copy=True, post_copy_iters=0), + Scenario("post-copy-iters-1", + post_copy=True, post_copy_iters=1), + Scenario("post-copy-iters-5", + post_copy=True, post_copy_iters=5), + Scenario("post-copy-iters-20", + post_copy=True, post_copy_iters=20), + ]), + + + # Looking at effect of auto-converge with different + # throttling percentage step rates + Comparison("auto-converge-iters", scenarios = [ + Scenario("auto-converge-step-5", + auto_converge=True, auto_converge_step=5), + Scenario("auto-converge-step-10", + auto_converge=True, auto_converge_step=10), + Scenario("auto-converge-step-20", + auto_converge=True, auto_converge_step=20), + ]), + + + # Looking at use of auto-converge in relation to bandwidth + # available for migration + Comparison("auto-converge-bandwidth", scenarios = [ + Scenario("auto-converge-bw-100mbs", + auto_converge=True, bandwidth=12), + Scenario("auto-converge-bw-300mbs", + auto_converge=True, bandwidth=37), + Scenario("auto-converge-bw-1gbs", + auto_converge=True, bandwidth=125), + Scenario("auto-converge-bw-10gbs", + auto_converge=True, bandwidth=1250), + Scenario("auto-converge-bw-100gbs", + auto_converge=True, bandwidth=12500), + ]), + + + # Looking at effect of multi-thread compression with + # varying numbers of threads + Comparison("compr-mt", scenarios = [ + Scenario("compr-mt-threads-1", + compression_mt=True, compression_mt_threads=1), + Scenario("compr-mt-threads-2", + compression_mt=True, compression_mt_threads=2), + Scenario("compr-mt-threads-4", + compression_mt=True, compression_mt_threads=4), + ]), + + + # Looking at effect of xbzrle compression with varying + # cache sizes + Comparison("compr-xbzrle", scenarios = [ + Scenario("compr-xbzrle-cache-5", + compression_xbzrle=True, compression_xbzrle_cache=5), + Scenario("compr-xbzrle-cache-10", + compression_xbzrle=True, compression_xbzrle_cache=10), + Scenario("compr-xbzrle-cache-20", + compression_xbzrle=True, compression_xbzrle_cache=10), + Scenario("compr-xbzrle-cache-50", + compression_xbzrle=True, compression_xbzrle_cache=50), + ]), + + + # Looking at effect of multifd with + # varying numbers of channels + Comparison("compr-multifd", scenarios = [ + Scenario("compr-multifd-channels-4", + multifd=True, multifd_channels=2), + Scenario("compr-multifd-channels-8", + multifd=True, multifd_channels=8), + Scenario("compr-multifd-channels-32", + multifd=True, multifd_channels=32), + Scenario("compr-multifd-channels-64", + multifd=True, multifd_channels=64), + ]), + + # Looking at effect of dirty-limit with + # varying x_vcpu_dirty_limit_period + Comparison("compr-dirty-limit-period", scenarios = [ + Scenario("compr-dirty-limit-period-500", + dirty_limit=True, x_vcpu_dirty_limit_period=500), + Scenario("compr-dirty-limit-period-800", + dirty_limit=True, x_vcpu_dirty_limit_period=800), + Scenario("compr-dirty-limit-period-1000", + dirty_limit=True, x_vcpu_dirty_limit_period=1000), + ]), + + + # Looking at effect of dirty-limit with + # varying vcpu_dirty_limit + Comparison("compr-dirty-limit", scenarios = [ + Scenario("compr-dirty-limit-10MB", + dirty_limit=True, vcpu_dirty_limit=10), + Scenario("compr-dirty-limit-20MB", + dirty_limit=True, vcpu_dirty_limit=20), + Scenario("compr-dirty-limit-50MB", + dirty_limit=True, vcpu_dirty_limit=50), + ]), +] diff --git a/tests/migration-stress/guestperf/engine.py b/tests/migration-stress/guestperf/engine.py new file mode 100644 index 0000000000..608d7270f6 --- /dev/null +++ b/tests/migration-stress/guestperf/engine.py @@ -0,0 +1,505 @@ +# +# Migration test main engine +# +# Copyright (c) 2016 Red Hat, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, see . +# + + +import os +import re +import sys +import time + +from guestperf.progress import Progress, ProgressStats +from guestperf.report import Report +from guestperf.timings import TimingRecord, Timings + +sys.path.append(os.path.join(os.path.dirname(__file__), + '..', '..', '..', 'python')) +from qemu.machine import QEMUMachine + + +class Engine(object): + + def __init__(self, binary, dst_host, kernel, initrd, transport="tcp", + sleep=15, verbose=False, debug=False): + + self._binary = binary # Path to QEMU binary + self._dst_host = dst_host # Hostname of target host + self._kernel = kernel # Path to kernel image + self._initrd = initrd # Path to stress initrd + self._transport = transport # 'unix' or 'tcp' or 'rdma' + self._sleep = sleep + self._verbose = verbose + self._debug = debug + + if debug: + self._verbose = debug + + def _vcpu_timing(self, pid, tid_list): + records = [] + now = time.time() + + jiffies_per_sec = os.sysconf(os.sysconf_names['SC_CLK_TCK']) + for tid in tid_list: + statfile = "/proc/%d/task/%d/stat" % (pid, tid) + with open(statfile, "r") as fh: + stat = fh.readline() + fields = stat.split(" ") + stime = int(fields[13]) + utime = int(fields[14]) + records.append(TimingRecord(tid, now, 1000 * (stime + utime) / jiffies_per_sec)) + return records + + def _cpu_timing(self, pid): + now = time.time() + + jiffies_per_sec = os.sysconf(os.sysconf_names['SC_CLK_TCK']) + statfile = "/proc/%d/stat" % pid + with open(statfile, "r") as fh: + stat = fh.readline() + fields = stat.split(" ") + stime = int(fields[13]) + utime = int(fields[14]) + return TimingRecord(pid, now, 1000 * (stime + utime) / jiffies_per_sec) + + def _migrate_progress(self, vm): + info = vm.cmd("query-migrate") + + if "ram" not in info: + info["ram"] = {} + + return Progress( + info.get("status", "active"), + ProgressStats( + info["ram"].get("transferred", 0), + info["ram"].get("remaining", 0), + info["ram"].get("total", 0), + info["ram"].get("duplicate", 0), + info["ram"].get("skipped", 0), + info["ram"].get("normal", 0), + info["ram"].get("normal-bytes", 0), + info["ram"].get("dirty-pages-rate", 0), + info["ram"].get("mbps", 0), + info["ram"].get("dirty-sync-count", 0) + ), + time.time(), + info.get("total-time", 0), + info.get("downtime", 0), + info.get("expected-downtime", 0), + info.get("setup-time", 0), + info.get("cpu-throttle-percentage", 0), + info.get("dirty-limit-throttle-time-per-round", 0), + info.get("dirty-limit-ring-full-time", 0), + ) + + def _migrate(self, hardware, scenario, src, dst, connect_uri): + src_qemu_time = [] + src_vcpu_time = [] + src_pid = src.get_pid() + + vcpus = src.cmd("query-cpus-fast") + src_threads = [] + for vcpu in vcpus: + src_threads.append(vcpu["thread-id"]) + + # XXX how to get dst timings on remote host ? + + if self._verbose: + print("Sleeping %d seconds for initial guest workload run" % self._sleep) + sleep_secs = self._sleep + while sleep_secs > 1: + src_qemu_time.append(self._cpu_timing(src_pid)) + src_vcpu_time.extend(self._vcpu_timing(src_pid, src_threads)) + time.sleep(1) + sleep_secs -= 1 + + if self._verbose: + print("Starting migration") + if scenario._auto_converge: + resp = src.cmd("migrate-set-capabilities", + capabilities = [ + { "capability": "auto-converge", + "state": True } + ]) + resp = src.cmd("migrate-set-parameters", + cpu_throttle_increment=scenario._auto_converge_step) + + if scenario._post_copy: + resp = src.cmd("migrate-set-capabilities", + capabilities = [ + { "capability": "postcopy-ram", + "state": True } + ]) + resp = dst.cmd("migrate-set-capabilities", + capabilities = [ + { "capability": "postcopy-ram", + "state": True } + ]) + + resp = src.cmd("migrate-set-parameters", + max_bandwidth=scenario._bandwidth * 1024 * 1024) + + resp = src.cmd("migrate-set-parameters", + downtime_limit=scenario._downtime) + + if scenario._compression_mt: + resp = src.cmd("migrate-set-capabilities", + capabilities = [ + { "capability": "compress", + "state": True } + ]) + resp = src.cmd("migrate-set-parameters", + compress_threads=scenario._compression_mt_threads) + resp = dst.cmd("migrate-set-capabilities", + capabilities = [ + { "capability": "compress", + "state": True } + ]) + resp = dst.cmd("migrate-set-parameters", + decompress_threads=scenario._compression_mt_threads) + + if scenario._compression_xbzrle: + resp = src.cmd("migrate-set-capabilities", + capabilities = [ + { "capability": "xbzrle", + "state": True } + ]) + resp = dst.cmd("migrate-set-capabilities", + capabilities = [ + { "capability": "xbzrle", + "state": True } + ]) + resp = src.cmd("migrate-set-parameters", + xbzrle_cache_size=( + hardware._mem * + 1024 * 1024 * 1024 / 100 * + scenario._compression_xbzrle_cache)) + + if scenario._multifd: + resp = src.cmd("migrate-set-capabilities", + capabilities = [ + { "capability": "multifd", + "state": True } + ]) + resp = src.cmd("migrate-set-parameters", + multifd_channels=scenario._multifd_channels) + resp = dst.cmd("migrate-set-capabilities", + capabilities = [ + { "capability": "multifd", + "state": True } + ]) + resp = dst.cmd("migrate-set-parameters", + multifd_channels=scenario._multifd_channels) + + if scenario._dirty_limit: + if not hardware._dirty_ring_size: + raise Exception("dirty ring size must be configured when " + "testing dirty limit migration") + + resp = src.cmd("migrate-set-capabilities", + capabilities = [ + { "capability": "dirty-limit", + "state": True } + ]) + resp = src.cmd("migrate-set-parameters", + x_vcpu_dirty_limit_period=scenario._x_vcpu_dirty_limit_period) + resp = src.cmd("migrate-set-parameters", + vcpu_dirty_limit=scenario._vcpu_dirty_limit) + + resp = src.cmd("migrate", uri=connect_uri) + + post_copy = False + paused = False + + progress_history = [] + + start = time.time() + loop = 0 + while True: + loop = loop + 1 + time.sleep(0.05) + + progress = self._migrate_progress(src) + if (loop % 20) == 0: + src_qemu_time.append(self._cpu_timing(src_pid)) + src_vcpu_time.extend(self._vcpu_timing(src_pid, src_threads)) + + if (len(progress_history) == 0 or + (progress_history[-1]._ram._iterations < + progress._ram._iterations)): + progress_history.append(progress) + + if progress._status in ("completed", "failed", "cancelled"): + if progress._status == "completed" and paused: + dst.cmd("cont") + if progress_history[-1] != progress: + progress_history.append(progress) + + if progress._status == "completed": + if self._verbose: + print("Sleeping %d seconds for final guest workload run" % self._sleep) + sleep_secs = self._sleep + while sleep_secs > 1: + time.sleep(1) + src_qemu_time.append(self._cpu_timing(src_pid)) + src_vcpu_time.extend(self._vcpu_timing(src_pid, src_threads)) + sleep_secs -= 1 + + return [progress_history, src_qemu_time, src_vcpu_time] + + if self._verbose and (loop % 20) == 0: + print("Iter %d: remain %5dMB of %5dMB (total %5dMB @ %5dMb/sec)" % ( + progress._ram._iterations, + progress._ram._remaining_bytes / (1024 * 1024), + progress._ram._total_bytes / (1024 * 1024), + progress._ram._transferred_bytes / (1024 * 1024), + progress._ram._transfer_rate_mbs, + )) + + if progress._ram._iterations > scenario._max_iters: + if self._verbose: + print("No completion after %d iterations over RAM" % scenario._max_iters) + src.cmd("migrate_cancel") + continue + + if time.time() > (start + scenario._max_time): + if self._verbose: + print("No completion after %d seconds" % scenario._max_time) + src.cmd("migrate_cancel") + continue + + if (scenario._post_copy and + progress._ram._iterations >= scenario._post_copy_iters and + not post_copy): + if self._verbose: + print("Switching to post-copy after %d iterations" % scenario._post_copy_iters) + resp = src.cmd("migrate-start-postcopy") + post_copy = True + + if (scenario._pause and + progress._ram._iterations >= scenario._pause_iters and + not paused): + if self._verbose: + print("Pausing VM after %d iterations" % scenario._pause_iters) + resp = src.cmd("stop") + paused = True + + def _is_ppc64le(self): + _, _, _, _, machine = os.uname() + if machine == "ppc64le": + return True + return False + + def _get_guest_console_args(self): + if self._is_ppc64le(): + return "console=hvc0" + else: + return "console=ttyS0" + + def _get_qemu_serial_args(self): + if self._is_ppc64le(): + return ["-chardev", "stdio,id=cdev0", + "-device", "spapr-vty,chardev=cdev0"] + else: + return ["-chardev", "stdio,id=cdev0", + "-device", "isa-serial,chardev=cdev0"] + + def _get_common_args(self, hardware, tunnelled=False): + args = [ + "noapic", + "edd=off", + "printk.time=1", + "noreplace-smp", + "cgroup_disable=memory", + "pci=noearly", + ] + + args.append(self._get_guest_console_args()) + + if self._debug: + args.append("debug") + else: + args.append("quiet") + + args.append("ramsize=%s" % hardware._mem) + + cmdline = " ".join(args) + if tunnelled: + cmdline = "'" + cmdline + "'" + + argv = [ + "-cpu", "host", + "-kernel", self._kernel, + "-initrd", self._initrd, + "-append", cmdline, + "-m", str((hardware._mem * 1024) + 512), + "-smp", str(hardware._cpus), + ] + if hardware._dirty_ring_size: + argv.extend(["-accel", "kvm,dirty-ring-size=%s" % + hardware._dirty_ring_size]) + else: + argv.extend(["-accel", "kvm"]) + + argv.extend(self._get_qemu_serial_args()) + + if self._debug: + argv.extend(["-machine", "graphics=off"]) + + if hardware._prealloc_pages: + argv_source += ["-mem-path", "/dev/shm", + "-mem-prealloc"] + if hardware._locked_pages: + argv_source += ["-overcommit", "mem-lock=on"] + if hardware._huge_pages: + pass + + return argv + + def _get_src_args(self, hardware): + return self._get_common_args(hardware) + + def _get_dst_args(self, hardware, uri): + tunnelled = False + if self._dst_host != "localhost": + tunnelled = True + argv = self._get_common_args(hardware, tunnelled) + return argv + ["-incoming", uri] + + @staticmethod + def _get_common_wrapper(cpu_bind, mem_bind): + wrapper = [] + if len(cpu_bind) > 0 or len(mem_bind) > 0: + wrapper.append("numactl") + if cpu_bind: + wrapper.append("--physcpubind=%s" % ",".join(cpu_bind)) + if mem_bind: + wrapper.append("--membind=%s" % ",".join(mem_bind)) + + return wrapper + + def _get_src_wrapper(self, hardware): + return self._get_common_wrapper(hardware._src_cpu_bind, hardware._src_mem_bind) + + def _get_dst_wrapper(self, hardware): + wrapper = self._get_common_wrapper(hardware._dst_cpu_bind, hardware._dst_mem_bind) + if self._dst_host != "localhost": + return ["ssh", + "-R", "9001:localhost:9001", + self._dst_host] + wrapper + else: + return wrapper + + def _get_timings(self, vm): + log = vm.get_log() + if not log: + return [] + if self._debug: + print(log) + + regex = r"[^\s]+\s\((\d+)\):\sINFO:\s(\d+)ms\scopied\s\d+\sGB\sin\s(\d+)ms" + matcher = re.compile(regex) + records = [] + for line in log.split("\n"): + match = matcher.match(line) + if match: + records.append(TimingRecord(int(match.group(1)), + int(match.group(2)) / 1000.0, + int(match.group(3)))) + return records + + def run(self, hardware, scenario, result_dir=os.getcwd()): + abs_result_dir = os.path.join(result_dir, scenario._name) + + if self._transport == "tcp": + uri = "tcp:%s:9000" % self._dst_host + elif self._transport == "rdma": + uri = "rdma:%s:9000" % self._dst_host + elif self._transport == "unix": + if self._dst_host != "localhost": + raise Exception("Running use unix migration transport for non-local host") + uri = "unix:/var/tmp/qemu-migrate-%d.migrate" % os.getpid() + try: + os.remove(uri[5:]) + os.remove(monaddr) + except: + pass + + if self._dst_host != "localhost": + dstmonaddr = ("localhost", 9001) + else: + dstmonaddr = "/var/tmp/qemu-dst-%d-monitor.sock" % os.getpid() + srcmonaddr = "/var/tmp/qemu-src-%d-monitor.sock" % os.getpid() + + src = QEMUMachine(self._binary, + args=self._get_src_args(hardware), + wrapper=self._get_src_wrapper(hardware), + name="qemu-src-%d" % os.getpid(), + monitor_address=srcmonaddr) + + dst = QEMUMachine(self._binary, + args=self._get_dst_args(hardware, uri), + wrapper=self._get_dst_wrapper(hardware), + name="qemu-dst-%d" % os.getpid(), + monitor_address=dstmonaddr) + + try: + src.launch() + dst.launch() + + ret = self._migrate(hardware, scenario, src, dst, uri) + progress_history = ret[0] + qemu_timings = ret[1] + vcpu_timings = ret[2] + if uri[0:5] == "unix:" and os.path.exists(uri[5:]): + os.remove(uri[5:]) + + if os.path.exists(srcmonaddr): + os.remove(srcmonaddr) + + if self._dst_host == "localhost" and os.path.exists(dstmonaddr): + os.remove(dstmonaddr) + + if self._verbose: + print("Finished migration") + + src.shutdown() + dst.shutdown() + + return Report(hardware, scenario, progress_history, + Timings(self._get_timings(src) + self._get_timings(dst)), + Timings(qemu_timings), + Timings(vcpu_timings), + self._binary, self._dst_host, self._kernel, + self._initrd, self._transport, self._sleep) + except Exception as e: + if self._debug: + print("Failed: %s" % str(e)) + try: + src.shutdown() + except: + pass + try: + dst.shutdown() + except: + pass + + if self._debug: + print(src.get_log()) + print(dst.get_log()) + raise + diff --git a/tests/migration-stress/guestperf/hardware.py b/tests/migration-stress/guestperf/hardware.py new file mode 100644 index 0000000000..f779cc050b --- /dev/null +++ b/tests/migration-stress/guestperf/hardware.py @@ -0,0 +1,66 @@ +# +# Migration test hardware configuration description +# +# Copyright (c) 2016 Red Hat, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, see . +# + + +class Hardware(object): + def __init__(self, cpus=1, mem=1, + src_cpu_bind=None, src_mem_bind=None, + dst_cpu_bind=None, dst_mem_bind=None, + prealloc_pages = False, + huge_pages=False, locked_pages=False, + dirty_ring_size=0): + self._cpus = cpus + self._mem = mem # GiB + self._src_mem_bind = src_mem_bind # List of NUMA nodes + self._src_cpu_bind = src_cpu_bind # List of pCPUs + self._dst_mem_bind = dst_mem_bind # List of NUMA nodes + self._dst_cpu_bind = dst_cpu_bind # List of pCPUs + self._prealloc_pages = prealloc_pages + self._huge_pages = huge_pages + self._locked_pages = locked_pages + self._dirty_ring_size = dirty_ring_size + + + def serialize(self): + return { + "cpus": self._cpus, + "mem": self._mem, + "src_mem_bind": self._src_mem_bind, + "dst_mem_bind": self._dst_mem_bind, + "src_cpu_bind": self._src_cpu_bind, + "dst_cpu_bind": self._dst_cpu_bind, + "prealloc_pages": self._prealloc_pages, + "huge_pages": self._huge_pages, + "locked_pages": self._locked_pages, + "dirty_ring_size": self._dirty_ring_size, + } + + @classmethod + def deserialize(cls, data): + return cls( + data["cpus"], + data["mem"], + data["src_cpu_bind"], + data["src_mem_bind"], + data["dst_cpu_bind"], + data["dst_mem_bind"], + data["prealloc_pages"], + data["huge_pages"], + data["locked_pages"], + data["dirty_ring_size"]) diff --git a/tests/migration-stress/guestperf/plot.py b/tests/migration-stress/guestperf/plot.py new file mode 100644 index 0000000000..30b3f668d0 --- /dev/null +++ b/tests/migration-stress/guestperf/plot.py @@ -0,0 +1,623 @@ +# +# Migration test graph plotting +# +# Copyright (c) 2016 Red Hat, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, see . +# + +import sys + + +class Plot(object): + + # Generated using + # http://tools.medialab.sciences-po.fr/iwanthue/ + COLORS = ["#CD54D0", + "#79D94C", + "#7470CD", + "#D2D251", + "#863D79", + "#76DDA6", + "#D4467B", + "#61923D", + "#CB9CCA", + "#D98F36", + "#8CC8DA", + "#CE4831", + "#5E7693", + "#9B803F", + "#412F4C", + "#CECBA6", + "#6D3229", + "#598B73", + "#C8827C", + "#394427"] + + def __init__(self, + reports, + migration_iters, + total_guest_cpu, + split_guest_cpu, + qemu_cpu, + vcpu_cpu): + + self._reports = reports + self._migration_iters = migration_iters + self._total_guest_cpu = total_guest_cpu + self._split_guest_cpu = split_guest_cpu + self._qemu_cpu = qemu_cpu + self._vcpu_cpu = vcpu_cpu + self._color_idx = 0 + + def _next_color(self): + color = self.COLORS[self._color_idx] + self._color_idx += 1 + if self._color_idx >= len(self.COLORS): + self._color_idx = 0 + return color + + def _get_progress_label(self, progress): + if progress: + return "\n\n" + "\n".join( + ["Status: %s" % progress._status, + "Iteration: %d" % progress._ram._iterations, + "Throttle: %02d%%" % progress._throttle_pcent, + "Dirty rate: %dMB/s" % (progress._ram._dirty_rate_pps * 4 / 1024.0)]) + else: + return "\n\n" + "\n".join( + ["Status: %s" % "none", + "Iteration: %d" % 0]) + + def _find_start_time(self, report): + startqemu = report._qemu_timings._records[0]._timestamp + startguest = report._guest_timings._records[0]._timestamp + if startqemu < startguest: + return startqemu + else: + return stasrtguest + + def _get_guest_max_value(self, report): + maxvalue = 0 + for record in report._guest_timings._records: + if record._value > maxvalue: + maxvalue = record._value + return maxvalue + + def _get_qemu_max_value(self, report): + maxvalue = 0 + oldvalue = None + oldtime = None + for record in report._qemu_timings._records: + if oldvalue is not None: + cpudelta = (record._value - oldvalue) / 1000.0 + timedelta = record._timestamp - oldtime + if timedelta == 0: + continue + util = cpudelta / timedelta * 100.0 + else: + util = 0 + oldvalue = record._value + oldtime = record._timestamp + + if util > maxvalue: + maxvalue = util + return maxvalue + + def _get_total_guest_cpu_graph(self, report, starttime): + xaxis = [] + yaxis = [] + labels = [] + progress_idx = -1 + for record in report._guest_timings._records: + while ((progress_idx + 1) < len(report._progress_history) and + report._progress_history[progress_idx + 1]._now < record._timestamp): + progress_idx = progress_idx + 1 + + if progress_idx >= 0: + progress = report._progress_history[progress_idx] + else: + progress = None + + xaxis.append(record._timestamp - starttime) + yaxis.append(record._value) + labels.append(self._get_progress_label(progress)) + + from plotly import graph_objs as go + return go.Scatter(x=xaxis, + y=yaxis, + name="Guest PIDs: %s" % report._scenario._name, + mode='lines', + line={ + "dash": "solid", + "color": self._next_color(), + "shape": "linear", + "width": 1 + }, + text=labels) + + def _get_split_guest_cpu_graphs(self, report, starttime): + threads = {} + for record in report._guest_timings._records: + if record._tid in threads: + continue + threads[record._tid] = { + "xaxis": [], + "yaxis": [], + "labels": [], + } + + progress_idx = -1 + for record in report._guest_timings._records: + while ((progress_idx + 1) < len(report._progress_history) and + report._progress_history[progress_idx + 1]._now < record._timestamp): + progress_idx = progress_idx + 1 + + if progress_idx >= 0: + progress = report._progress_history[progress_idx] + else: + progress = None + + threads[record._tid]["xaxis"].append(record._timestamp - starttime) + threads[record._tid]["yaxis"].append(record._value) + threads[record._tid]["labels"].append(self._get_progress_label(progress)) + + + graphs = [] + from plotly import graph_objs as go + for tid in threads.keys(): + graphs.append( + go.Scatter(x=threads[tid]["xaxis"], + y=threads[tid]["yaxis"], + name="PID %s: %s" % (tid, report._scenario._name), + mode="lines", + line={ + "dash": "solid", + "color": self._next_color(), + "shape": "linear", + "width": 1 + }, + text=threads[tid]["labels"])) + return graphs + + def _get_migration_iters_graph(self, report, starttime): + xaxis = [] + yaxis = [] + labels = [] + for progress in report._progress_history: + xaxis.append(progress._now - starttime) + yaxis.append(0) + labels.append(self._get_progress_label(progress)) + + from plotly import graph_objs as go + return go.Scatter(x=xaxis, + y=yaxis, + text=labels, + name="Migration iterations", + mode="markers", + marker={ + "color": self._next_color(), + "symbol": "star", + "size": 5 + }) + + def _get_qemu_cpu_graph(self, report, starttime): + xaxis = [] + yaxis = [] + labels = [] + progress_idx = -1 + + first = report._qemu_timings._records[0] + abstimestamps = [first._timestamp] + absvalues = [first._value] + + for record in report._qemu_timings._records[1:]: + while ((progress_idx + 1) < len(report._progress_history) and + report._progress_history[progress_idx + 1]._now < record._timestamp): + progress_idx = progress_idx + 1 + + if progress_idx >= 0: + progress = report._progress_history[progress_idx] + else: + progress = None + + oldvalue = absvalues[-1] + oldtime = abstimestamps[-1] + + cpudelta = (record._value - oldvalue) / 1000.0 + timedelta = record._timestamp - oldtime + if timedelta == 0: + continue + util = cpudelta / timedelta * 100.0 + + abstimestamps.append(record._timestamp) + absvalues.append(record._value) + + xaxis.append(record._timestamp - starttime) + yaxis.append(util) + labels.append(self._get_progress_label(progress)) + + from plotly import graph_objs as go + return go.Scatter(x=xaxis, + y=yaxis, + yaxis="y2", + name="QEMU: %s" % report._scenario._name, + mode='lines', + line={ + "dash": "solid", + "color": self._next_color(), + "shape": "linear", + "width": 1 + }, + text=labels) + + def _get_vcpu_cpu_graphs(self, report, starttime): + threads = {} + for record in report._vcpu_timings._records: + if record._tid in threads: + continue + threads[record._tid] = { + "xaxis": [], + "yaxis": [], + "labels": [], + "absvalue": [record._value], + "abstime": [record._timestamp], + } + + progress_idx = -1 + for record in report._vcpu_timings._records: + while ((progress_idx + 1) < len(report._progress_history) and + report._progress_history[progress_idx + 1]._now < record._timestamp): + progress_idx = progress_idx + 1 + + if progress_idx >= 0: + progress = report._progress_history[progress_idx] + else: + progress = None + + oldvalue = threads[record._tid]["absvalue"][-1] + oldtime = threads[record._tid]["abstime"][-1] + + cpudelta = (record._value - oldvalue) / 1000.0 + timedelta = record._timestamp - oldtime + if timedelta == 0: + continue + util = cpudelta / timedelta * 100.0 + if util > 100: + util = 100 + + threads[record._tid]["absvalue"].append(record._value) + threads[record._tid]["abstime"].append(record._timestamp) + + threads[record._tid]["xaxis"].append(record._timestamp - starttime) + threads[record._tid]["yaxis"].append(util) + threads[record._tid]["labels"].append(self._get_progress_label(progress)) + + + graphs = [] + from plotly import graph_objs as go + for tid in threads.keys(): + graphs.append( + go.Scatter(x=threads[tid]["xaxis"], + y=threads[tid]["yaxis"], + yaxis="y2", + name="VCPU %s: %s" % (tid, report._scenario._name), + mode="lines", + line={ + "dash": "solid", + "color": self._next_color(), + "shape": "linear", + "width": 1 + }, + text=threads[tid]["labels"])) + return graphs + + def _generate_chart_report(self, report): + graphs = [] + starttime = self._find_start_time(report) + if self._total_guest_cpu: + graphs.append(self._get_total_guest_cpu_graph(report, starttime)) + if self._split_guest_cpu: + graphs.extend(self._get_split_guest_cpu_graphs(report, starttime)) + if self._qemu_cpu: + graphs.append(self._get_qemu_cpu_graph(report, starttime)) + if self._vcpu_cpu: + graphs.extend(self._get_vcpu_cpu_graphs(report, starttime)) + if self._migration_iters: + graphs.append(self._get_migration_iters_graph(report, starttime)) + return graphs + + def _generate_annotation(self, starttime, progress): + return { + "text": progress._status, + "x": progress._now - starttime, + "y": 10, + } + + def _generate_annotations(self, report): + starttime = self._find_start_time(report) + annotations = {} + started = False + for progress in report._progress_history: + if progress._status == "setup": + continue + if progress._status not in annotations: + annotations[progress._status] = self._generate_annotation(starttime, progress) + + return annotations.values() + + def _generate_chart(self): + from plotly.offline import plot + from plotly import graph_objs as go + + graphs = [] + yaxismax = 0 + yaxismax2 = 0 + for report in self._reports: + graphs.extend(self._generate_chart_report(report)) + + maxvalue = self._get_guest_max_value(report) + if maxvalue > yaxismax: + yaxismax = maxvalue + + maxvalue = self._get_qemu_max_value(report) + if maxvalue > yaxismax2: + yaxismax2 = maxvalue + + yaxismax += 100 + if not self._qemu_cpu: + yaxismax2 = 110 + yaxismax2 += 10 + + annotations = [] + if self._migration_iters: + for report in self._reports: + annotations.extend(self._generate_annotations(report)) + + layout = go.Layout(title="Migration comparison", + xaxis={ + "title": "Wallclock time (secs)", + "showgrid": False, + }, + yaxis={ + "title": "Memory update speed (ms/GB)", + "showgrid": False, + "range": [0, yaxismax], + }, + yaxis2={ + "title": "Hostutilization (%)", + "overlaying": "y", + "side": "right", + "range": [0, yaxismax2], + "showgrid": False, + }, + annotations=annotations) + + figure = go.Figure(data=graphs, layout=layout) + + return plot(figure, + show_link=False, + include_plotlyjs=False, + output_type="div") + + + def _generate_report(self): + pieces = [] + for report in self._reports: + pieces.append(""" +

Report %s

+ +""" % report._scenario._name) + + pieces.append(""" + + + + + + + + + + + + + + + + + + + + + + + +""" % (report._binary, report._kernel, + report._initrd, report._transport, report._dst_host)) + + hardware = report._hardware + pieces.append(""" + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +""" % (hardware._cpus, hardware._mem, + ",".join(hardware._src_cpu_bind), + ",".join(hardware._src_mem_bind), + ",".join(hardware._dst_cpu_bind), + ",".join(hardware._dst_mem_bind), + "yes" if hardware._prealloc_pages else "no", + "yes" if hardware._locked_pages else "no", + "yes" if hardware._huge_pages else "no")) + + scenario = report._scenario + pieces.append(""" + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +""" % (scenario._downtime, scenario._bandwidth, + scenario._max_iters, scenario._max_time, + "yes" if scenario._pause else "no", scenario._pause_iters, + "yes" if scenario._post_copy else "no", scenario._post_copy_iters, + "yes" if scenario._auto_converge else "no", scenario._auto_converge_step, + "yes" if scenario._compression_mt else "no", scenario._compression_mt_threads, + "yes" if scenario._compression_xbzrle else "no", scenario._compression_xbzrle_cache)) + + pieces.append(""" +
Test config
Emulator:%s
Kernel:%s
Ramdisk:%s
Transport:%s
Host:%s
Hardware config
CPUs:%d
RAM:%d GB
Source CPU bind:%s
Source RAM bind:%s
Dest CPU bind:%s
Dest RAM bind:%s
Preallocate RAM:%s
Locked RAM:%s
Huge pages:%s
Scenario config
Max downtime:%d milli-sec
Max bandwidth:%d MB/sec
Max iters:%d
Max time:%d secs
Pause:%s
Pause iters:%d
Post-copy:%s
Post-copy iters:%d
Auto-converge:%s
Auto-converge iters:%d
MT compression:%s
MT compression threads:%d
XBZRLE compression:%s
XBZRLE compression cache:%d%% of RAM
+""") + + return "\n".join(pieces) + + def _generate_style(self): + return """ +#report table tr th { + text-align: right; +} +#report table tr td { + text-align: left; +} +#report table tr.subhead th { + background: rgb(192, 192, 192); + text-align: center; +} + +""" + + def generate_html(self, fh): + print(""" + + + + Migration report + + +

Migration report

+

Chart summary

+
+""" % self._generate_style(), file=fh) + print(self._generate_chart(), file=fh) + print(""" +
+

Report details

+
+""", file=fh) + print(self._generate_report(), file=fh) + print(""" +
+ + +""", file=fh) + + def generate(self, filename): + if filename is None: + self.generate_html(sys.stdout) + else: + with open(filename, "w") as fh: + self.generate_html(fh) diff --git a/tests/migration-stress/guestperf/progress.py b/tests/migration-stress/guestperf/progress.py new file mode 100644 index 0000000000..d490584217 --- /dev/null +++ b/tests/migration-stress/guestperf/progress.py @@ -0,0 +1,129 @@ +# +# Migration test migration operation progress +# +# Copyright (c) 2016 Red Hat, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, see . +# + + +class ProgressStats(object): + + def __init__(self, + transferred_bytes, + remaining_bytes, + total_bytes, + duplicate_pages, + skipped_pages, + normal_pages, + normal_bytes, + dirty_rate_pps, + transfer_rate_mbs, + iterations): + self._transferred_bytes = transferred_bytes + self._remaining_bytes = remaining_bytes + self._total_bytes = total_bytes + self._duplicate_pages = duplicate_pages + self._skipped_pages = skipped_pages + self._normal_pages = normal_pages + self._normal_bytes = normal_bytes + self._dirty_rate_pps = dirty_rate_pps + self._transfer_rate_mbs = transfer_rate_mbs + self._iterations = iterations + + def serialize(self): + return { + "transferred_bytes": self._transferred_bytes, + "remaining_bytes": self._remaining_bytes, + "total_bytes": self._total_bytes, + "duplicate_pages": self._duplicate_pages, + "skipped_pages": self._skipped_pages, + "normal_pages": self._normal_pages, + "normal_bytes": self._normal_bytes, + "dirty_rate_pps": self._dirty_rate_pps, + "transfer_rate_mbs": self._transfer_rate_mbs, + "iterations": self._iterations, + } + + @classmethod + def deserialize(cls, data): + return cls( + data["transferred_bytes"], + data["remaining_bytes"], + data["total_bytes"], + data["duplicate_pages"], + data["skipped_pages"], + data["normal_pages"], + data["normal_bytes"], + data["dirty_rate_pps"], + data["transfer_rate_mbs"], + data["iterations"]) + + +class Progress(object): + + def __init__(self, + status, + ram, + now, + duration, + downtime, + downtime_expected, + setup_time, + throttle_pcent, + dirty_limit_throttle_time_per_round, + dirty_limit_ring_full_time): + + self._status = status + self._ram = ram + self._now = now + self._duration = duration + self._downtime = downtime + self._downtime_expected = downtime_expected + self._setup_time = setup_time + self._throttle_pcent = throttle_pcent + self._dirty_limit_throttle_time_per_round = \ + dirty_limit_throttle_time_per_round + self._dirty_limit_ring_full_time = \ + dirty_limit_ring_full_time + + def serialize(self): + return { + "status": self._status, + "ram": self._ram.serialize(), + "now": self._now, + "duration": self._duration, + "downtime": self._downtime, + "downtime_expected": self._downtime_expected, + "setup_time": self._setup_time, + "throttle_pcent": self._throttle_pcent, + "dirty_limit_throttle_time_per_round": + self._dirty_limit_throttle_time_per_round, + "dirty_limit_ring_full_time": + self._dirty_limit_ring_full_time, + } + + @classmethod + def deserialize(cls, data): + return cls( + data["status"], + ProgressStats.deserialize(data["ram"]), + data["now"], + data["duration"], + data["downtime"], + data["downtime_expected"], + data["setup_time"], + data["throttle_pcent"], + data["dirty_limit_throttle_time_per_round"], + data["dirty_limit_ring_full_time"]) diff --git a/tests/migration-stress/guestperf/report.py b/tests/migration-stress/guestperf/report.py new file mode 100644 index 0000000000..1efd40c868 --- /dev/null +++ b/tests/migration-stress/guestperf/report.py @@ -0,0 +1,98 @@ +# +# Migration test output result reporting +# +# Copyright (c) 2016 Red Hat, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, see . +# + +import json + +from guestperf.hardware import Hardware +from guestperf.scenario import Scenario +from guestperf.progress import Progress +from guestperf.timings import Timings + +class Report(object): + + def __init__(self, + hardware, + scenario, + progress_history, + guest_timings, + qemu_timings, + vcpu_timings, + binary, + dst_host, + kernel, + initrd, + transport, + sleep): + + self._hardware = hardware + self._scenario = scenario + self._progress_history = progress_history + self._guest_timings = guest_timings + self._qemu_timings = qemu_timings + self._vcpu_timings = vcpu_timings + self._binary = binary + self._dst_host = dst_host + self._kernel = kernel + self._initrd = initrd + self._transport = transport + self._sleep = sleep + + def serialize(self): + return { + "hardware": self._hardware.serialize(), + "scenario": self._scenario.serialize(), + "progress_history": [progress.serialize() for progress in self._progress_history], + "guest_timings": self._guest_timings.serialize(), + "qemu_timings": self._qemu_timings.serialize(), + "vcpu_timings": self._vcpu_timings.serialize(), + "binary": self._binary, + "dst_host": self._dst_host, + "kernel": self._kernel, + "initrd": self._initrd, + "transport": self._transport, + "sleep": self._sleep, + } + + @classmethod + def deserialize(cls, data): + return cls( + Hardware.deserialize(data["hardware"]), + Scenario.deserialize(data["scenario"]), + [Progress.deserialize(record) for record in data["progress_history"]], + Timings.deserialize(data["guest_timings"]), + Timings.deserialize(data["qemu_timings"]), + Timings.deserialize(data["vcpu_timings"]), + data["binary"], + data["dst_host"], + data["kernel"], + data["initrd"], + data["transport"], + data["sleep"]) + + def to_json(self): + return json.dumps(self.serialize(), indent=4) + + @classmethod + def from_json(cls, data): + return cls.deserialize(json.loads(data)) + + @classmethod + def from_json_file(cls, filename): + with open(filename, "r") as fh: + return cls.deserialize(json.load(fh)) diff --git a/tests/migration-stress/guestperf/scenario.py b/tests/migration-stress/guestperf/scenario.py new file mode 100644 index 0000000000..154c4f5d5f --- /dev/null +++ b/tests/migration-stress/guestperf/scenario.py @@ -0,0 +1,112 @@ +# +# Migration test scenario parameter description +# +# Copyright (c) 2016 Red Hat, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, see . +# + + +class Scenario(object): + + def __init__(self, name, + downtime=500, + bandwidth=125000, # 1000 gig-e, effectively unlimited + max_iters=30, + max_time=300, + pause=False, pause_iters=5, + post_copy=False, post_copy_iters=5, + auto_converge=False, auto_converge_step=10, + compression_mt=False, compression_mt_threads=1, + compression_xbzrle=False, compression_xbzrle_cache=10, + multifd=False, multifd_channels=2, + dirty_limit=False, x_vcpu_dirty_limit_period=500, + vcpu_dirty_limit=1): + + self._name = name + + # General migration tunables + self._downtime = downtime # milliseconds + self._bandwidth = bandwidth # MiB per second + self._max_iters = max_iters + self._max_time = max_time # seconds + + + # Strategies for ensuring completion + self._pause = pause + self._pause_iters = pause_iters + + self._post_copy = post_copy + self._post_copy_iters = post_copy_iters + + self._auto_converge = auto_converge + self._auto_converge_step = auto_converge_step # percentage CPU time + + self._compression_mt = compression_mt + self._compression_mt_threads = compression_mt_threads + + self._compression_xbzrle = compression_xbzrle + self._compression_xbzrle_cache = compression_xbzrle_cache # percentage of guest RAM + + self._multifd = multifd + self._multifd_channels = multifd_channels + + self._dirty_limit = dirty_limit + self._x_vcpu_dirty_limit_period = x_vcpu_dirty_limit_period + self._vcpu_dirty_limit = vcpu_dirty_limit + + def serialize(self): + return { + "name": self._name, + "downtime": self._downtime, + "bandwidth": self._bandwidth, + "max_iters": self._max_iters, + "max_time": self._max_time, + "pause": self._pause, + "pause_iters": self._pause_iters, + "post_copy": self._post_copy, + "post_copy_iters": self._post_copy_iters, + "auto_converge": self._auto_converge, + "auto_converge_step": self._auto_converge_step, + "compression_mt": self._compression_mt, + "compression_mt_threads": self._compression_mt_threads, + "compression_xbzrle": self._compression_xbzrle, + "compression_xbzrle_cache": self._compression_xbzrle_cache, + "multifd": self._multifd, + "multifd_channels": self._multifd_channels, + "dirty_limit": self._dirty_limit, + "x_vcpu_dirty_limit_period": self._x_vcpu_dirty_limit_period, + "vcpu_dirty_limit": self._vcpu_dirty_limit, + } + + @classmethod + def deserialize(cls, data): + return cls( + data["name"], + data["downtime"], + data["bandwidth"], + data["max_iters"], + data["max_time"], + data["pause"], + data["pause_iters"], + data["post_copy"], + data["post_copy_iters"], + data["auto_converge"], + data["auto_converge_step"], + data["compression_mt"], + data["compression_mt_threads"], + data["compression_xbzrle"], + data["compression_xbzrle_cache"], + data["multifd"], + data["multifd_channels"]) diff --git a/tests/migration-stress/guestperf/shell.py b/tests/migration-stress/guestperf/shell.py new file mode 100644 index 0000000000..046afeb84e --- /dev/null +++ b/tests/migration-stress/guestperf/shell.py @@ -0,0 +1,297 @@ +# +# Migration test command line shell integration +# +# Copyright (c) 2016 Red Hat, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, see . +# + + +import argparse +import fnmatch +import os +import os.path +import platform +import sys +import logging + +from guestperf.hardware import Hardware +from guestperf.engine import Engine +from guestperf.scenario import Scenario +from guestperf.comparison import COMPARISONS +from guestperf.plot import Plot +from guestperf.report import Report + + +class BaseShell(object): + + def __init__(self): + parser = argparse.ArgumentParser(description="Migration Test Tool") + + # Test args + parser.add_argument("--debug", dest="debug", default=False, action="store_true") + parser.add_argument("--verbose", dest="verbose", default=False, action="store_true") + parser.add_argument("--sleep", dest="sleep", default=15, type=int) + parser.add_argument("--binary", dest="binary", default="/usr/bin/qemu-system-x86_64") + parser.add_argument("--dst-host", dest="dst_host", default="localhost") + parser.add_argument("--kernel", dest="kernel", default="/boot/vmlinuz-%s" % platform.release()) + parser.add_argument("--initrd", dest="initrd", + default="tests/migration-stress/initrd-stress.img") + parser.add_argument("--transport", dest="transport", default="unix") + + + # Hardware args + parser.add_argument("--cpus", dest="cpus", default=1, type=int) + parser.add_argument("--mem", dest="mem", default=1, type=int) + parser.add_argument("--src-cpu-bind", dest="src_cpu_bind", default="") + parser.add_argument("--src-mem-bind", dest="src_mem_bind", default="") + parser.add_argument("--dst-cpu-bind", dest="dst_cpu_bind", default="") + parser.add_argument("--dst-mem-bind", dest="dst_mem_bind", default="") + parser.add_argument("--prealloc-pages", dest="prealloc_pages", default=False) + parser.add_argument("--huge-pages", dest="huge_pages", default=False) + parser.add_argument("--locked-pages", dest="locked_pages", default=False) + parser.add_argument("--dirty-ring-size", dest="dirty_ring_size", + default=0, type=int) + + self._parser = parser + + def get_engine(self, args): + return Engine(binary=args.binary, + dst_host=args.dst_host, + kernel=args.kernel, + initrd=args.initrd, + transport=args.transport, + sleep=args.sleep, + debug=args.debug, + verbose=args.verbose) + + def get_hardware(self, args): + def split_map(value): + if value == "": + return [] + return value.split(",") + + return Hardware(cpus=args.cpus, + mem=args.mem, + + src_cpu_bind=split_map(args.src_cpu_bind), + src_mem_bind=split_map(args.src_mem_bind), + dst_cpu_bind=split_map(args.dst_cpu_bind), + dst_mem_bind=split_map(args.dst_mem_bind), + + locked_pages=args.locked_pages, + huge_pages=args.huge_pages, + prealloc_pages=args.prealloc_pages, + + dirty_ring_size=args.dirty_ring_size) + + +class Shell(BaseShell): + + def __init__(self): + super(Shell, self).__init__() + + parser = self._parser + + parser.add_argument("--output", dest="output", default=None) + + # Scenario args + parser.add_argument("--max-iters", dest="max_iters", default=30, type=int) + parser.add_argument("--max-time", dest="max_time", default=300, type=int) + parser.add_argument("--bandwidth", dest="bandwidth", default=125000, type=int) + parser.add_argument("--downtime", dest="downtime", default=500, type=int) + + parser.add_argument("--pause", dest="pause", default=False, action="store_true") + parser.add_argument("--pause-iters", dest="pause_iters", default=5, type=int) + + parser.add_argument("--post-copy", dest="post_copy", default=False, action="store_true") + parser.add_argument("--post-copy-iters", dest="post_copy_iters", default=5, type=int) + + parser.add_argument("--auto-converge", dest="auto_converge", default=False, action="store_true") + parser.add_argument("--auto-converge-step", dest="auto_converge_step", default=10, type=int) + + parser.add_argument("--compression-mt", dest="compression_mt", default=False, action="store_true") + parser.add_argument("--compression-mt-threads", dest="compression_mt_threads", default=1, type=int) + + parser.add_argument("--compression-xbzrle", dest="compression_xbzrle", default=False, action="store_true") + parser.add_argument("--compression-xbzrle-cache", dest="compression_xbzrle_cache", default=10, type=int) + + parser.add_argument("--multifd", dest="multifd", default=False, + action="store_true") + parser.add_argument("--multifd-channels", dest="multifd_channels", + default=2, type=int) + + parser.add_argument("--dirty-limit", dest="dirty_limit", default=False, + action="store_true") + + parser.add_argument("--x-vcpu-dirty-limit-period", + dest="x_vcpu_dirty_limit_period", + default=500, type=int) + + parser.add_argument("--vcpu-dirty-limit", + dest="vcpu_dirty_limit", + default=1, type=int) + + def get_scenario(self, args): + return Scenario(name="perfreport", + downtime=args.downtime, + bandwidth=args.bandwidth, + max_iters=args.max_iters, + max_time=args.max_time, + + pause=args.pause, + pause_iters=args.pause_iters, + + post_copy=args.post_copy, + post_copy_iters=args.post_copy_iters, + + auto_converge=args.auto_converge, + auto_converge_step=args.auto_converge_step, + + compression_mt=args.compression_mt, + compression_mt_threads=args.compression_mt_threads, + + compression_xbzrle=args.compression_xbzrle, + compression_xbzrle_cache=args.compression_xbzrle_cache, + + multifd=args.multifd, + multifd_channels=args.multifd_channels, + + dirty_limit=args.dirty_limit, + x_vcpu_dirty_limit_period=\ + args.x_vcpu_dirty_limit_period, + vcpu_dirty_limit=args.vcpu_dirty_limit) + + def run(self, argv): + args = self._parser.parse_args(argv) + logging.basicConfig(level=(logging.DEBUG if args.debug else + logging.INFO if args.verbose else + logging.WARN)) + + + engine = self.get_engine(args) + hardware = self.get_hardware(args) + scenario = self.get_scenario(args) + + try: + report = engine.run(hardware, scenario) + if args.output is None: + print(report.to_json()) + else: + with open(args.output, "w") as fh: + print(report.to_json(), file=fh) + return 0 + except Exception as e: + print("Error: %s" % str(e), file=sys.stderr) + if args.debug: + raise + return 1 + + +class BatchShell(BaseShell): + + def __init__(self): + super(BatchShell, self).__init__() + + parser = self._parser + + parser.add_argument("--filter", dest="filter", default="*") + parser.add_argument("--output", dest="output", default=os.getcwd()) + + def run(self, argv): + args = self._parser.parse_args(argv) + logging.basicConfig(level=(logging.DEBUG if args.debug else + logging.INFO if args.verbose else + logging.WARN)) + + + engine = self.get_engine(args) + hardware = self.get_hardware(args) + + try: + for comparison in COMPARISONS: + compdir = os.path.join(args.output, comparison._name) + for scenario in comparison._scenarios: + name = os.path.join(comparison._name, scenario._name) + if not fnmatch.fnmatch(name, args.filter): + if args.verbose: + print("Skipping %s" % name) + continue + + if args.verbose: + print("Running %s" % name) + + dirname = os.path.join(args.output, comparison._name) + filename = os.path.join(dirname, scenario._name + ".json") + if not os.path.exists(dirname): + os.makedirs(dirname) + report = engine.run(hardware, scenario) + with open(filename, "w") as fh: + print(report.to_json(), file=fh) + except Exception as e: + print("Error: %s" % str(e), file=sys.stderr) + if args.debug: + raise + + +class PlotShell(object): + + def __init__(self): + super(PlotShell, self).__init__() + + self._parser = argparse.ArgumentParser(description="Migration Test Tool") + + self._parser.add_argument("--output", dest="output", default=None) + + self._parser.add_argument("--debug", dest="debug", default=False, action="store_true") + self._parser.add_argument("--verbose", dest="verbose", default=False, action="store_true") + + self._parser.add_argument("--migration-iters", dest="migration_iters", default=False, action="store_true") + self._parser.add_argument("--total-guest-cpu", dest="total_guest_cpu", default=False, action="store_true") + self._parser.add_argument("--split-guest-cpu", dest="split_guest_cpu", default=False, action="store_true") + self._parser.add_argument("--qemu-cpu", dest="qemu_cpu", default=False, action="store_true") + self._parser.add_argument("--vcpu-cpu", dest="vcpu_cpu", default=False, action="store_true") + + self._parser.add_argument("reports", nargs='*') + + def run(self, argv): + args = self._parser.parse_args(argv) + logging.basicConfig(level=(logging.DEBUG if args.debug else + logging.INFO if args.verbose else + logging.WARN)) + + + if len(args.reports) == 0: + print("At least one report required", file=sys.stderr) + return 1 + + if not (args.qemu_cpu or + args.vcpu_cpu or + args.total_guest_cpu or + args.split_guest_cpu): + print("At least one chart type is required", file=sys.stderr) + return 1 + + reports = [] + for report in args.reports: + reports.append(Report.from_json_file(report)) + + plot = Plot(reports, + args.migration_iters, + args.total_guest_cpu, + args.split_guest_cpu, + args.qemu_cpu, + args.vcpu_cpu) + + plot.generate(args.output) diff --git a/tests/migration-stress/guestperf/timings.py b/tests/migration-stress/guestperf/timings.py new file mode 100644 index 0000000000..2374010c6c --- /dev/null +++ b/tests/migration-stress/guestperf/timings.py @@ -0,0 +1,55 @@ +# +# Migration test timing records +# +# Copyright (c) 2016 Red Hat, Inc. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 2.1 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library; if not, see . +# + + +class TimingRecord(object): + + def __init__(self, tid, timestamp, value): + + self._tid = tid + self._timestamp = timestamp + self._value = value + + def serialize(self): + return { + "tid": self._tid, + "timestamp": self._timestamp, + "value": self._value + } + + @classmethod + def deserialize(cls, data): + return cls( + data["tid"], + data["timestamp"], + data["value"]) + + +class Timings(object): + + def __init__(self, records): + + self._records = records + + def serialize(self): + return [record.serialize() for record in self._records] + + @classmethod + def deserialize(cls, data): + return Timings([TimingRecord.deserialize(record) for record in data]) diff --git a/tests/migration-stress/initrd-stress.sh b/tests/migration-stress/initrd-stress.sh new file mode 100755 index 0000000000..0f20ac29a6 --- /dev/null +++ b/tests/migration-stress/initrd-stress.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +INITRD="$1" +STRESS="$2" + +INITRD_DIR=$(mktemp -d -p '' "initrd-stress.XXXXXX") +trap 'rm -rf $INITRD_DIR' EXIT + +cp "$STRESS" "$INITRD_DIR/init" +(cd "$INITRD_DIR" && (find | cpio --quiet -o -H newc | gzip -9)) > "$INITRD" diff --git a/tests/migration-stress/meson.build b/tests/migration-stress/meson.build new file mode 100644 index 0000000000..a91aa61c65 --- /dev/null +++ b/tests/migration-stress/meson.build @@ -0,0 +1,18 @@ +sysprof = dependency('sysprof-capture-4', method: 'pkg-config', required: false) +glib_static = dependency('glib-2.0', version: glib_req_ver, required: false, + method: 'pkg-config', static: true) + +stress = executable( + 'stress', + files('stress.c'), + dependencies: [glib_static, sysprof], + link_args: ['-static'], + build_by_default: false, +) + +custom_target( + 'initrd-stress.img', + output: 'initrd-stress.img', + input: stress, + command: [find_program('initrd-stress.sh'), '@OUTPUT@', '@INPUT@'] +) diff --git a/tests/migration-stress/stress.c b/tests/migration-stress/stress.c new file mode 100644 index 0000000000..88acf8dc25 --- /dev/null +++ b/tests/migration-stress/stress.c @@ -0,0 +1,328 @@ +/* + * Migration stress workload + * + * Copyright (c) 2016 Red Hat, Inc. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see . + */ + +#include "qemu/osdep.h" +#include +#include +#include +#include +#include +#include + +const char *argv0; + +#define RAM_PAGE_SIZE 4096 + +#ifndef CONFIG_GETTID +static int gettid(void) +{ + return syscall(SYS_gettid); +} +#endif + +static __attribute__((noreturn)) void exit_failure(void) +{ + if (getpid() == 1) { + sync(); + reboot(RB_POWER_OFF); + fprintf(stderr, "%s (%05d): ERROR: cannot reboot: %s\n", + argv0, gettid(), strerror(errno)); + abort(); + } else { + exit(1); + } +} + +static int get_command_arg_str(const char *name, + char **val) +{ + static char line[1024]; + FILE *fp = fopen("/proc/cmdline", "r"); + char *start, *end; + + if (fp == NULL) { + fprintf(stderr, "%s (%05d): ERROR: cannot open /proc/cmdline: %s\n", + argv0, gettid(), strerror(errno)); + return -1; + } + + if (!fgets(line, sizeof line, fp)) { + fprintf(stderr, "%s (%05d): ERROR: cannot read /proc/cmdline: %s\n", + argv0, gettid(), strerror(errno)); + fclose(fp); + return -1; + } + fclose(fp); + + start = strstr(line, name); + if (!start) + return 0; + + start += strlen(name); + + if (*start != '=') { + fprintf(stderr, "%s (%05d): ERROR: no value provided for '%s' in /proc/cmdline\n", + argv0, gettid(), name); + } + start++; + + end = strstr(start, " "); + if (!end) + end = strstr(start, "\n"); + + if (end == start) { + fprintf(stderr, "%s (%05d): ERROR: no value provided for '%s' in /proc/cmdline\n", + argv0, gettid(), name); + return -1; + } + + if (end) + *val = g_strndup(start, end - start); + else + *val = g_strdup(start); + return 1; +} + + +static int get_command_arg_ull(const char *name, + unsigned long long *val) +{ + char *valstr; + char *end; + + int ret = get_command_arg_str(name, &valstr); + if (ret <= 0) + return ret; + + errno = 0; + *val = strtoll(valstr, &end, 10); + if (errno || *end) { + fprintf(stderr, "%s (%05d): ERROR: cannot parse %s value %s\n", + argv0, gettid(), name, valstr); + g_free(valstr); + return -1; + } + g_free(valstr); + return 0; +} + + +static int random_bytes(char *buf, size_t len) +{ + int fd; + + fd = open("/dev/urandom", O_RDONLY); + if (fd < 0) { + fprintf(stderr, "%s (%05d): ERROR: cannot open /dev/urandom: %s\n", + argv0, gettid(), strerror(errno)); + return -1; + } + + if (read(fd, buf, len) != len) { + fprintf(stderr, "%s (%05d): ERROR: cannot read /dev/urandom: %s\n", + argv0, gettid(), strerror(errno)); + close(fd); + return -1; + } + + close(fd); + + return 0; +} + + +static unsigned long long now(void) +{ + struct timeval tv; + + gettimeofday(&tv, NULL); + + return (tv.tv_sec * 1000ull) + (tv.tv_usec / 1000ull); +} + +static void stressone(unsigned long long ramsizeMB) +{ + size_t pagesPerMB = 1024 * 1024 / RAM_PAGE_SIZE; + g_autofree char *ram = g_malloc(ramsizeMB * 1024 * 1024); + char *ramptr; + size_t i, j, k; + g_autofree char *data = g_malloc(RAM_PAGE_SIZE); + char *dataptr; + size_t nMB = 0; + unsigned long long before, after; + + /* We don't care about initial state, but we do want + * to fault it all into RAM, otherwise the first iter + * of the loop below will be quite slow. We can't use + * 0x0 as the byte as gcc optimizes that away into a + * calloc instead :-) */ + memset(ram, 0xfe, ramsizeMB * 1024 * 1024); + + if (random_bytes(data, RAM_PAGE_SIZE) < 0) { + return; + } + + before = now(); + + while (1) { + + ramptr = ram; + for (i = 0; i < ramsizeMB; i++, nMB++) { + for (j = 0; j < pagesPerMB; j++) { + dataptr = data; + for (k = 0; k < RAM_PAGE_SIZE; k += sizeof(long long)) { + ramptr += sizeof(long long); + dataptr += sizeof(long long); + *(unsigned long long *)ramptr ^= *(unsigned long long *)dataptr; + } + } + + if (nMB == 1024) { + after = now(); + fprintf(stderr, "%s (%05d): INFO: %06llums copied 1 GB in %05llums\n", + argv0, gettid(), after, after - before); + before = now(); + nMB = 0; + } + } + } +} + + +static void *stressthread(void *arg) +{ + unsigned long long ramsizeMB = *(unsigned long long *)arg; + + stressone(ramsizeMB); + + return NULL; +} + +static void stress(unsigned long long ramsizeGB, int ncpus) +{ + size_t i; + unsigned long long ramsizeMB = ramsizeGB * 1024 / ncpus; + ncpus--; + + for (i = 0; i < ncpus; i++) { + pthread_t thr; + pthread_create(&thr, NULL, + stressthread, &ramsizeMB); + } + + stressone(ramsizeMB); +} + + +static int mount_misc(const char *fstype, const char *dir) +{ + if (g_mkdir_with_parents(dir, 0755) < 0 && errno != EEXIST) { + fprintf(stderr, "%s (%05d): ERROR: cannot create %s: %s\n", + argv0, gettid(), dir, strerror(errno)); + return -1; + } + + if (mount("none", dir, fstype, 0, NULL) < 0) { + fprintf(stderr, "%s (%05d): ERROR: cannot mount %s: %s\n", + argv0, gettid(), dir, strerror(errno)); + return -1; + } + + return 0; +} + +static int mount_all(void) +{ + if (mount_misc("proc", "/proc") < 0 || + mount_misc("sysfs", "/sys") < 0 || + mount_misc("tmpfs", "/dev") < 0) + return -1; + + mknod("/dev/urandom", 0777 | S_IFCHR, makedev(1, 9)); + mknod("/dev/random", 0777 | S_IFCHR, makedev(1, 8)); + + return 0; +} + +int main(int argc, char **argv) +{ + unsigned long long ramsizeGB = 1; + char *end; + int ch; + int opt_ind = 0; + const char *sopt = "hr:c:"; + struct option lopt[] = { + { "help", no_argument, NULL, 'h' }, + { "ramsize", required_argument, NULL, 'r' }, + { "cpus", required_argument, NULL, 'c' }, + { NULL, 0, NULL, 0 } + }; + int ret; + int ncpus = 0; + + argv0 = argv[0]; + + while ((ch = getopt_long(argc, argv, sopt, lopt, &opt_ind)) != -1) { + switch (ch) { + case 'r': + errno = 0; + ramsizeGB = strtoll(optarg, &end, 10); + if (errno != 0 || *end) { + fprintf(stderr, "%s (%05d): ERROR: Cannot parse RAM size %s\n", + argv0, gettid(), optarg); + exit_failure(); + } + break; + + case 'c': + errno = 0; + ncpus = strtoll(optarg, &end, 10); + if (errno != 0 || *end) { + fprintf(stderr, "%s (%05d): ERROR: Cannot parse CPU count %s\n", + argv0, gettid(), optarg); + exit_failure(); + } + break; + + case '?': + case 'h': + fprintf(stderr, "%s: [--help][--ramsize GB][--cpus N]\n", argv0); + exit_failure(); + } + } + + if (getpid() == 1) { + if (mount_all() < 0) + exit_failure(); + + ret = get_command_arg_ull("ramsize", &ramsizeGB); + if (ret < 0) + exit_failure(); + } + + if (ncpus == 0) + ncpus = sysconf(_SC_NPROCESSORS_ONLN); + + fprintf(stdout, "%s (%05d): INFO: RAM %llu GiB across %d CPUs\n", + argv0, gettid(), ramsizeGB, ncpus); + + stress(ramsizeGB, ncpus); + + exit_failure(); +} diff --git a/tests/migration/Makefile b/tests/migration/Makefile deleted file mode 100644 index 2c5ee287ec..0000000000 --- a/tests/migration/Makefile +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright (c) 2018 Red Hat, Inc. and/or its affiliates -# -# This work is licensed under the terms of the GNU GPL, version 2 or later. -# See the COPYING file in the top-level directory. -# - -TARGET_LIST = i386 aarch64 s390x ppc64 - -SRC_PATH = ../.. - -.PHONY: help $(TARGET_LIST) -help: - @echo "Create migration guest includes. We generate a binary." - @echo "And then convert that binary to an include file that can be" - @echo "run in a guest." - @echo "Possible operations are:" - @echo - @echo " $(MAKE) clean Remove all intermediate files" - @echo " $(MAKE) target Generate for that target" - @echo " $(MAKE) CROSS_PREFIX=... target" - @echo " Cross-compile than target" - @echo " Possible targets are: $(TARGET_LIST)" - -override define __note -/* This file is automatically generated from the assembly file in - * tests/migration/$@. Edit that file and then run "make all" - * inside tests/migration to update, and then remember to send both - * the header and the assembler differences in your patch submission. - */ -endef -export __note - -$(TARGET_LIST): - $(MAKE) CROSS_PREFIX=$(CROSS_PREFIX) -C $@ - -clean: - for target in $(TARGET_LIST); do \ - $(MAKE) -C $$target clean; \ - done diff --git a/tests/migration/aarch64/Makefile b/tests/migration/aarch64/Makefile deleted file mode 100644 index 9c4fa18e76..0000000000 --- a/tests/migration/aarch64/Makefile +++ /dev/null @@ -1,18 +0,0 @@ -# To specify cross compiler prefix, use CROSS_PREFIX= -# $ make CROSS_PREFIX=aarch64-linux-gnu- - -.PHONY: all clean -all: a-b-kernel.h - -a-b-kernel.h: aarch64.kernel - echo "$$__note" > $@ - xxd -i $< | sed -e 's/.*int.*//' >> $@ - -aarch64.kernel: aarch64.elf - $(CROSS_PREFIX)objcopy -O binary $< $@ - -aarch64.elf: a-b-kernel.S - $(CROSS_PREFIX)gcc -o $@ -nostdlib -Wl,--build-id=none $< - -clean: - $(RM) *.kernel *.elf diff --git a/tests/migration/aarch64/a-b-kernel.S b/tests/migration/aarch64/a-b-kernel.S deleted file mode 100644 index a4103ecb71..0000000000 --- a/tests/migration/aarch64/a-b-kernel.S +++ /dev/null @@ -1,74 +0,0 @@ -# -# Copyright (c) 2018 Red Hat, Inc. and/or its affiliates -# -# Author: -# Wei Huang -# -# This work is licensed under the terms of the GNU GPL, version 2 or later. -# See the COPYING file in the top-level directory. -# -# Note: Please make sure the compiler compiles the assembly code below with -# pc-relative address. Also the branch instructions should use relative -# addresses only. - -#include "../migration-test.h" - -.section .text - - .globl _start - -_start: - /* disable MMU to use phys mem address */ - mrs x0, sctlr_el1 - bic x0, x0, #(1<<0) - msr sctlr_el1, x0 - isb - - /* traverse test memory region */ - mov x0, #ARM_TEST_MEM_START - mov x1, #ARM_TEST_MEM_END - - /* output char 'A' to PL011 */ - mov w3, 'A' - mov x2, #ARM_MACH_VIRT_UART - strb w3, [x2] - - /* clean up memory */ - mov w3, #0 - mov x4, x0 -clean: - strb w3, [x4] - add x4, x4, #TEST_MEM_PAGE_SIZE - cmp x4, x1 - ble clean - - /* w5 keeps a counter so we can limit the output speed */ - mov w5, #0 - - /* main body */ -mainloop: - mov x4, x0 - -innerloop: - /* increment the first byte of each page by 1 */ - ldrb w3, [x4] - add w3, w3, #1 - strb w3, [x4] - - /* make sure QEMU user space can see consistent data as MMU is off */ - dc civac, x4 - - add x4, x4, #TEST_MEM_PAGE_SIZE - cmp x4, x1 - blt innerloop - - add w5, w5, #1 - and w5, w5, #0x1f - cmp w5, #0 - bne mainloop - - /* output char 'B' to PL011 */ - mov w3, 'B' - strb w3, [x2] - - b mainloop diff --git a/tests/migration/aarch64/a-b-kernel.h b/tests/migration/aarch64/a-b-kernel.h deleted file mode 100644 index 34e518d061..0000000000 --- a/tests/migration/aarch64/a-b-kernel.h +++ /dev/null @@ -1,18 +0,0 @@ -/* This file is automatically generated from the assembly file in - * tests/migration/aarch64. Edit that file and then run "make all" - * inside tests/migration to update, and then remember to send both - * the header and the assembler differences in your patch submission. - */ -unsigned char aarch64_kernel[] = { - 0x00, 0x10, 0x38, 0xd5, 0x00, 0xf8, 0x7f, 0x92, 0x00, 0x10, 0x18, 0xd5, - 0xdf, 0x3f, 0x03, 0xd5, 0x00, 0x02, 0xa8, 0xd2, 0x01, 0xc8, 0xa8, 0xd2, - 0x23, 0x08, 0x80, 0x52, 0x02, 0x20, 0xa1, 0xd2, 0x43, 0x00, 0x00, 0x39, - 0x03, 0x00, 0x80, 0x52, 0xe4, 0x03, 0x00, 0xaa, 0x83, 0x00, 0x00, 0x39, - 0x84, 0x04, 0x40, 0x91, 0x9f, 0x00, 0x01, 0xeb, 0xad, 0xff, 0xff, 0x54, - 0x05, 0x00, 0x80, 0x52, 0xe4, 0x03, 0x00, 0xaa, 0x83, 0x00, 0x40, 0x39, - 0x63, 0x04, 0x00, 0x11, 0x83, 0x00, 0x00, 0x39, 0x24, 0x7e, 0x0b, 0xd5, - 0x84, 0x04, 0x40, 0x91, 0x9f, 0x00, 0x01, 0xeb, 0x4b, 0xff, 0xff, 0x54, - 0xa5, 0x04, 0x00, 0x11, 0xa5, 0x10, 0x00, 0x12, 0xbf, 0x00, 0x00, 0x71, - 0xa1, 0xfe, 0xff, 0x54, 0x43, 0x08, 0x80, 0x52, 0x43, 0x00, 0x00, 0x39, - 0xf2, 0xff, 0xff, 0x17 -}; diff --git a/tests/migration/guestperf-batch.py b/tests/migration/guestperf-batch.py deleted file mode 100755 index 9485eefe49..0000000000 --- a/tests/migration/guestperf-batch.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python3 -# -# Migration test batch comparison invocation -# -# Copyright (c) 2016 Red Hat, Inc. -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, see . -# - -import sys - -from guestperf.shell import BatchShell - -shell = BatchShell() -sys.exit(shell.run(sys.argv[1:])) diff --git a/tests/migration/guestperf-plot.py b/tests/migration/guestperf-plot.py deleted file mode 100755 index 32977b4bf6..0000000000 --- a/tests/migration/guestperf-plot.py +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env python3 -# -# Migration test graph plotting command -# -# Copyright (c) 2016 Red Hat, Inc. -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, see . -# - -import sys - -from guestperf.shell import PlotShell - -shell = PlotShell() -sys.exit(shell.run(sys.argv[1:])) diff --git a/tests/migration/guestperf.py b/tests/migration/guestperf.py deleted file mode 100755 index 07182f211e..0000000000 --- a/tests/migration/guestperf.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python3 -# -# Migration test direct invocation command -# -# Copyright (c) 2016 Red Hat, Inc. -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, see . -# - - -import sys - -from guestperf.shell import Shell - -shell = Shell() -sys.exit(shell.run(sys.argv[1:])) diff --git a/tests/migration/guestperf/__init__.py b/tests/migration/guestperf/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tests/migration/guestperf/comparison.py b/tests/migration/guestperf/comparison.py deleted file mode 100644 index 42cc0372d1..0000000000 --- a/tests/migration/guestperf/comparison.py +++ /dev/null @@ -1,161 +0,0 @@ -# -# Migration test scenario comparison mapping -# -# Copyright (c) 2016 Red Hat, Inc. -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, see . -# - -from guestperf.scenario import Scenario - -class Comparison(object): - def __init__(self, name, scenarios): - self._name = name - self._scenarios = scenarios - -COMPARISONS = [ - # Looking at effect of pausing guest during migration - # at various stages of iteration over RAM - Comparison("pause-iters", scenarios = [ - Scenario("pause-iters-0", - pause=True, pause_iters=0), - Scenario("pause-iters-1", - pause=True, pause_iters=1), - Scenario("pause-iters-5", - pause=True, pause_iters=5), - Scenario("pause-iters-20", - pause=True, pause_iters=20), - ]), - - - # Looking at use of post-copy in relation to bandwidth - # available for migration - Comparison("post-copy-bandwidth", scenarios = [ - Scenario("post-copy-bw-100mbs", - post_copy=True, bandwidth=12), - Scenario("post-copy-bw-300mbs", - post_copy=True, bandwidth=37), - Scenario("post-copy-bw-1gbs", - post_copy=True, bandwidth=125), - Scenario("post-copy-bw-10gbs", - post_copy=True, bandwidth=1250), - Scenario("post-copy-bw-100gbs", - post_copy=True, bandwidth=12500), - ]), - - - # Looking at effect of starting post-copy at different - # stages of the migration - Comparison("post-copy-iters", scenarios = [ - Scenario("post-copy-iters-0", - post_copy=True, post_copy_iters=0), - Scenario("post-copy-iters-1", - post_copy=True, post_copy_iters=1), - Scenario("post-copy-iters-5", - post_copy=True, post_copy_iters=5), - Scenario("post-copy-iters-20", - post_copy=True, post_copy_iters=20), - ]), - - - # Looking at effect of auto-converge with different - # throttling percentage step rates - Comparison("auto-converge-iters", scenarios = [ - Scenario("auto-converge-step-5", - auto_converge=True, auto_converge_step=5), - Scenario("auto-converge-step-10", - auto_converge=True, auto_converge_step=10), - Scenario("auto-converge-step-20", - auto_converge=True, auto_converge_step=20), - ]), - - - # Looking at use of auto-converge in relation to bandwidth - # available for migration - Comparison("auto-converge-bandwidth", scenarios = [ - Scenario("auto-converge-bw-100mbs", - auto_converge=True, bandwidth=12), - Scenario("auto-converge-bw-300mbs", - auto_converge=True, bandwidth=37), - Scenario("auto-converge-bw-1gbs", - auto_converge=True, bandwidth=125), - Scenario("auto-converge-bw-10gbs", - auto_converge=True, bandwidth=1250), - Scenario("auto-converge-bw-100gbs", - auto_converge=True, bandwidth=12500), - ]), - - - # Looking at effect of multi-thread compression with - # varying numbers of threads - Comparison("compr-mt", scenarios = [ - Scenario("compr-mt-threads-1", - compression_mt=True, compression_mt_threads=1), - Scenario("compr-mt-threads-2", - compression_mt=True, compression_mt_threads=2), - Scenario("compr-mt-threads-4", - compression_mt=True, compression_mt_threads=4), - ]), - - - # Looking at effect of xbzrle compression with varying - # cache sizes - Comparison("compr-xbzrle", scenarios = [ - Scenario("compr-xbzrle-cache-5", - compression_xbzrle=True, compression_xbzrle_cache=5), - Scenario("compr-xbzrle-cache-10", - compression_xbzrle=True, compression_xbzrle_cache=10), - Scenario("compr-xbzrle-cache-20", - compression_xbzrle=True, compression_xbzrle_cache=10), - Scenario("compr-xbzrle-cache-50", - compression_xbzrle=True, compression_xbzrle_cache=50), - ]), - - - # Looking at effect of multifd with - # varying numbers of channels - Comparison("compr-multifd", scenarios = [ - Scenario("compr-multifd-channels-4", - multifd=True, multifd_channels=2), - Scenario("compr-multifd-channels-8", - multifd=True, multifd_channels=8), - Scenario("compr-multifd-channels-32", - multifd=True, multifd_channels=32), - Scenario("compr-multifd-channels-64", - multifd=True, multifd_channels=64), - ]), - - # Looking at effect of dirty-limit with - # varying x_vcpu_dirty_limit_period - Comparison("compr-dirty-limit-period", scenarios = [ - Scenario("compr-dirty-limit-period-500", - dirty_limit=True, x_vcpu_dirty_limit_period=500), - Scenario("compr-dirty-limit-period-800", - dirty_limit=True, x_vcpu_dirty_limit_period=800), - Scenario("compr-dirty-limit-period-1000", - dirty_limit=True, x_vcpu_dirty_limit_period=1000), - ]), - - - # Looking at effect of dirty-limit with - # varying vcpu_dirty_limit - Comparison("compr-dirty-limit", scenarios = [ - Scenario("compr-dirty-limit-10MB", - dirty_limit=True, vcpu_dirty_limit=10), - Scenario("compr-dirty-limit-20MB", - dirty_limit=True, vcpu_dirty_limit=20), - Scenario("compr-dirty-limit-50MB", - dirty_limit=True, vcpu_dirty_limit=50), - ]), -] diff --git a/tests/migration/guestperf/engine.py b/tests/migration/guestperf/engine.py deleted file mode 100644 index 608d7270f6..0000000000 --- a/tests/migration/guestperf/engine.py +++ /dev/null @@ -1,505 +0,0 @@ -# -# Migration test main engine -# -# Copyright (c) 2016 Red Hat, Inc. -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, see . -# - - -import os -import re -import sys -import time - -from guestperf.progress import Progress, ProgressStats -from guestperf.report import Report -from guestperf.timings import TimingRecord, Timings - -sys.path.append(os.path.join(os.path.dirname(__file__), - '..', '..', '..', 'python')) -from qemu.machine import QEMUMachine - - -class Engine(object): - - def __init__(self, binary, dst_host, kernel, initrd, transport="tcp", - sleep=15, verbose=False, debug=False): - - self._binary = binary # Path to QEMU binary - self._dst_host = dst_host # Hostname of target host - self._kernel = kernel # Path to kernel image - self._initrd = initrd # Path to stress initrd - self._transport = transport # 'unix' or 'tcp' or 'rdma' - self._sleep = sleep - self._verbose = verbose - self._debug = debug - - if debug: - self._verbose = debug - - def _vcpu_timing(self, pid, tid_list): - records = [] - now = time.time() - - jiffies_per_sec = os.sysconf(os.sysconf_names['SC_CLK_TCK']) - for tid in tid_list: - statfile = "/proc/%d/task/%d/stat" % (pid, tid) - with open(statfile, "r") as fh: - stat = fh.readline() - fields = stat.split(" ") - stime = int(fields[13]) - utime = int(fields[14]) - records.append(TimingRecord(tid, now, 1000 * (stime + utime) / jiffies_per_sec)) - return records - - def _cpu_timing(self, pid): - now = time.time() - - jiffies_per_sec = os.sysconf(os.sysconf_names['SC_CLK_TCK']) - statfile = "/proc/%d/stat" % pid - with open(statfile, "r") as fh: - stat = fh.readline() - fields = stat.split(" ") - stime = int(fields[13]) - utime = int(fields[14]) - return TimingRecord(pid, now, 1000 * (stime + utime) / jiffies_per_sec) - - def _migrate_progress(self, vm): - info = vm.cmd("query-migrate") - - if "ram" not in info: - info["ram"] = {} - - return Progress( - info.get("status", "active"), - ProgressStats( - info["ram"].get("transferred", 0), - info["ram"].get("remaining", 0), - info["ram"].get("total", 0), - info["ram"].get("duplicate", 0), - info["ram"].get("skipped", 0), - info["ram"].get("normal", 0), - info["ram"].get("normal-bytes", 0), - info["ram"].get("dirty-pages-rate", 0), - info["ram"].get("mbps", 0), - info["ram"].get("dirty-sync-count", 0) - ), - time.time(), - info.get("total-time", 0), - info.get("downtime", 0), - info.get("expected-downtime", 0), - info.get("setup-time", 0), - info.get("cpu-throttle-percentage", 0), - info.get("dirty-limit-throttle-time-per-round", 0), - info.get("dirty-limit-ring-full-time", 0), - ) - - def _migrate(self, hardware, scenario, src, dst, connect_uri): - src_qemu_time = [] - src_vcpu_time = [] - src_pid = src.get_pid() - - vcpus = src.cmd("query-cpus-fast") - src_threads = [] - for vcpu in vcpus: - src_threads.append(vcpu["thread-id"]) - - # XXX how to get dst timings on remote host ? - - if self._verbose: - print("Sleeping %d seconds for initial guest workload run" % self._sleep) - sleep_secs = self._sleep - while sleep_secs > 1: - src_qemu_time.append(self._cpu_timing(src_pid)) - src_vcpu_time.extend(self._vcpu_timing(src_pid, src_threads)) - time.sleep(1) - sleep_secs -= 1 - - if self._verbose: - print("Starting migration") - if scenario._auto_converge: - resp = src.cmd("migrate-set-capabilities", - capabilities = [ - { "capability": "auto-converge", - "state": True } - ]) - resp = src.cmd("migrate-set-parameters", - cpu_throttle_increment=scenario._auto_converge_step) - - if scenario._post_copy: - resp = src.cmd("migrate-set-capabilities", - capabilities = [ - { "capability": "postcopy-ram", - "state": True } - ]) - resp = dst.cmd("migrate-set-capabilities", - capabilities = [ - { "capability": "postcopy-ram", - "state": True } - ]) - - resp = src.cmd("migrate-set-parameters", - max_bandwidth=scenario._bandwidth * 1024 * 1024) - - resp = src.cmd("migrate-set-parameters", - downtime_limit=scenario._downtime) - - if scenario._compression_mt: - resp = src.cmd("migrate-set-capabilities", - capabilities = [ - { "capability": "compress", - "state": True } - ]) - resp = src.cmd("migrate-set-parameters", - compress_threads=scenario._compression_mt_threads) - resp = dst.cmd("migrate-set-capabilities", - capabilities = [ - { "capability": "compress", - "state": True } - ]) - resp = dst.cmd("migrate-set-parameters", - decompress_threads=scenario._compression_mt_threads) - - if scenario._compression_xbzrle: - resp = src.cmd("migrate-set-capabilities", - capabilities = [ - { "capability": "xbzrle", - "state": True } - ]) - resp = dst.cmd("migrate-set-capabilities", - capabilities = [ - { "capability": "xbzrle", - "state": True } - ]) - resp = src.cmd("migrate-set-parameters", - xbzrle_cache_size=( - hardware._mem * - 1024 * 1024 * 1024 / 100 * - scenario._compression_xbzrle_cache)) - - if scenario._multifd: - resp = src.cmd("migrate-set-capabilities", - capabilities = [ - { "capability": "multifd", - "state": True } - ]) - resp = src.cmd("migrate-set-parameters", - multifd_channels=scenario._multifd_channels) - resp = dst.cmd("migrate-set-capabilities", - capabilities = [ - { "capability": "multifd", - "state": True } - ]) - resp = dst.cmd("migrate-set-parameters", - multifd_channels=scenario._multifd_channels) - - if scenario._dirty_limit: - if not hardware._dirty_ring_size: - raise Exception("dirty ring size must be configured when " - "testing dirty limit migration") - - resp = src.cmd("migrate-set-capabilities", - capabilities = [ - { "capability": "dirty-limit", - "state": True } - ]) - resp = src.cmd("migrate-set-parameters", - x_vcpu_dirty_limit_period=scenario._x_vcpu_dirty_limit_period) - resp = src.cmd("migrate-set-parameters", - vcpu_dirty_limit=scenario._vcpu_dirty_limit) - - resp = src.cmd("migrate", uri=connect_uri) - - post_copy = False - paused = False - - progress_history = [] - - start = time.time() - loop = 0 - while True: - loop = loop + 1 - time.sleep(0.05) - - progress = self._migrate_progress(src) - if (loop % 20) == 0: - src_qemu_time.append(self._cpu_timing(src_pid)) - src_vcpu_time.extend(self._vcpu_timing(src_pid, src_threads)) - - if (len(progress_history) == 0 or - (progress_history[-1]._ram._iterations < - progress._ram._iterations)): - progress_history.append(progress) - - if progress._status in ("completed", "failed", "cancelled"): - if progress._status == "completed" and paused: - dst.cmd("cont") - if progress_history[-1] != progress: - progress_history.append(progress) - - if progress._status == "completed": - if self._verbose: - print("Sleeping %d seconds for final guest workload run" % self._sleep) - sleep_secs = self._sleep - while sleep_secs > 1: - time.sleep(1) - src_qemu_time.append(self._cpu_timing(src_pid)) - src_vcpu_time.extend(self._vcpu_timing(src_pid, src_threads)) - sleep_secs -= 1 - - return [progress_history, src_qemu_time, src_vcpu_time] - - if self._verbose and (loop % 20) == 0: - print("Iter %d: remain %5dMB of %5dMB (total %5dMB @ %5dMb/sec)" % ( - progress._ram._iterations, - progress._ram._remaining_bytes / (1024 * 1024), - progress._ram._total_bytes / (1024 * 1024), - progress._ram._transferred_bytes / (1024 * 1024), - progress._ram._transfer_rate_mbs, - )) - - if progress._ram._iterations > scenario._max_iters: - if self._verbose: - print("No completion after %d iterations over RAM" % scenario._max_iters) - src.cmd("migrate_cancel") - continue - - if time.time() > (start + scenario._max_time): - if self._verbose: - print("No completion after %d seconds" % scenario._max_time) - src.cmd("migrate_cancel") - continue - - if (scenario._post_copy and - progress._ram._iterations >= scenario._post_copy_iters and - not post_copy): - if self._verbose: - print("Switching to post-copy after %d iterations" % scenario._post_copy_iters) - resp = src.cmd("migrate-start-postcopy") - post_copy = True - - if (scenario._pause and - progress._ram._iterations >= scenario._pause_iters and - not paused): - if self._verbose: - print("Pausing VM after %d iterations" % scenario._pause_iters) - resp = src.cmd("stop") - paused = True - - def _is_ppc64le(self): - _, _, _, _, machine = os.uname() - if machine == "ppc64le": - return True - return False - - def _get_guest_console_args(self): - if self._is_ppc64le(): - return "console=hvc0" - else: - return "console=ttyS0" - - def _get_qemu_serial_args(self): - if self._is_ppc64le(): - return ["-chardev", "stdio,id=cdev0", - "-device", "spapr-vty,chardev=cdev0"] - else: - return ["-chardev", "stdio,id=cdev0", - "-device", "isa-serial,chardev=cdev0"] - - def _get_common_args(self, hardware, tunnelled=False): - args = [ - "noapic", - "edd=off", - "printk.time=1", - "noreplace-smp", - "cgroup_disable=memory", - "pci=noearly", - ] - - args.append(self._get_guest_console_args()) - - if self._debug: - args.append("debug") - else: - args.append("quiet") - - args.append("ramsize=%s" % hardware._mem) - - cmdline = " ".join(args) - if tunnelled: - cmdline = "'" + cmdline + "'" - - argv = [ - "-cpu", "host", - "-kernel", self._kernel, - "-initrd", self._initrd, - "-append", cmdline, - "-m", str((hardware._mem * 1024) + 512), - "-smp", str(hardware._cpus), - ] - if hardware._dirty_ring_size: - argv.extend(["-accel", "kvm,dirty-ring-size=%s" % - hardware._dirty_ring_size]) - else: - argv.extend(["-accel", "kvm"]) - - argv.extend(self._get_qemu_serial_args()) - - if self._debug: - argv.extend(["-machine", "graphics=off"]) - - if hardware._prealloc_pages: - argv_source += ["-mem-path", "/dev/shm", - "-mem-prealloc"] - if hardware._locked_pages: - argv_source += ["-overcommit", "mem-lock=on"] - if hardware._huge_pages: - pass - - return argv - - def _get_src_args(self, hardware): - return self._get_common_args(hardware) - - def _get_dst_args(self, hardware, uri): - tunnelled = False - if self._dst_host != "localhost": - tunnelled = True - argv = self._get_common_args(hardware, tunnelled) - return argv + ["-incoming", uri] - - @staticmethod - def _get_common_wrapper(cpu_bind, mem_bind): - wrapper = [] - if len(cpu_bind) > 0 or len(mem_bind) > 0: - wrapper.append("numactl") - if cpu_bind: - wrapper.append("--physcpubind=%s" % ",".join(cpu_bind)) - if mem_bind: - wrapper.append("--membind=%s" % ",".join(mem_bind)) - - return wrapper - - def _get_src_wrapper(self, hardware): - return self._get_common_wrapper(hardware._src_cpu_bind, hardware._src_mem_bind) - - def _get_dst_wrapper(self, hardware): - wrapper = self._get_common_wrapper(hardware._dst_cpu_bind, hardware._dst_mem_bind) - if self._dst_host != "localhost": - return ["ssh", - "-R", "9001:localhost:9001", - self._dst_host] + wrapper - else: - return wrapper - - def _get_timings(self, vm): - log = vm.get_log() - if not log: - return [] - if self._debug: - print(log) - - regex = r"[^\s]+\s\((\d+)\):\sINFO:\s(\d+)ms\scopied\s\d+\sGB\sin\s(\d+)ms" - matcher = re.compile(regex) - records = [] - for line in log.split("\n"): - match = matcher.match(line) - if match: - records.append(TimingRecord(int(match.group(1)), - int(match.group(2)) / 1000.0, - int(match.group(3)))) - return records - - def run(self, hardware, scenario, result_dir=os.getcwd()): - abs_result_dir = os.path.join(result_dir, scenario._name) - - if self._transport == "tcp": - uri = "tcp:%s:9000" % self._dst_host - elif self._transport == "rdma": - uri = "rdma:%s:9000" % self._dst_host - elif self._transport == "unix": - if self._dst_host != "localhost": - raise Exception("Running use unix migration transport for non-local host") - uri = "unix:/var/tmp/qemu-migrate-%d.migrate" % os.getpid() - try: - os.remove(uri[5:]) - os.remove(monaddr) - except: - pass - - if self._dst_host != "localhost": - dstmonaddr = ("localhost", 9001) - else: - dstmonaddr = "/var/tmp/qemu-dst-%d-monitor.sock" % os.getpid() - srcmonaddr = "/var/tmp/qemu-src-%d-monitor.sock" % os.getpid() - - src = QEMUMachine(self._binary, - args=self._get_src_args(hardware), - wrapper=self._get_src_wrapper(hardware), - name="qemu-src-%d" % os.getpid(), - monitor_address=srcmonaddr) - - dst = QEMUMachine(self._binary, - args=self._get_dst_args(hardware, uri), - wrapper=self._get_dst_wrapper(hardware), - name="qemu-dst-%d" % os.getpid(), - monitor_address=dstmonaddr) - - try: - src.launch() - dst.launch() - - ret = self._migrate(hardware, scenario, src, dst, uri) - progress_history = ret[0] - qemu_timings = ret[1] - vcpu_timings = ret[2] - if uri[0:5] == "unix:" and os.path.exists(uri[5:]): - os.remove(uri[5:]) - - if os.path.exists(srcmonaddr): - os.remove(srcmonaddr) - - if self._dst_host == "localhost" and os.path.exists(dstmonaddr): - os.remove(dstmonaddr) - - if self._verbose: - print("Finished migration") - - src.shutdown() - dst.shutdown() - - return Report(hardware, scenario, progress_history, - Timings(self._get_timings(src) + self._get_timings(dst)), - Timings(qemu_timings), - Timings(vcpu_timings), - self._binary, self._dst_host, self._kernel, - self._initrd, self._transport, self._sleep) - except Exception as e: - if self._debug: - print("Failed: %s" % str(e)) - try: - src.shutdown() - except: - pass - try: - dst.shutdown() - except: - pass - - if self._debug: - print(src.get_log()) - print(dst.get_log()) - raise - diff --git a/tests/migration/guestperf/hardware.py b/tests/migration/guestperf/hardware.py deleted file mode 100644 index f779cc050b..0000000000 --- a/tests/migration/guestperf/hardware.py +++ /dev/null @@ -1,66 +0,0 @@ -# -# Migration test hardware configuration description -# -# Copyright (c) 2016 Red Hat, Inc. -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, see . -# - - -class Hardware(object): - def __init__(self, cpus=1, mem=1, - src_cpu_bind=None, src_mem_bind=None, - dst_cpu_bind=None, dst_mem_bind=None, - prealloc_pages = False, - huge_pages=False, locked_pages=False, - dirty_ring_size=0): - self._cpus = cpus - self._mem = mem # GiB - self._src_mem_bind = src_mem_bind # List of NUMA nodes - self._src_cpu_bind = src_cpu_bind # List of pCPUs - self._dst_mem_bind = dst_mem_bind # List of NUMA nodes - self._dst_cpu_bind = dst_cpu_bind # List of pCPUs - self._prealloc_pages = prealloc_pages - self._huge_pages = huge_pages - self._locked_pages = locked_pages - self._dirty_ring_size = dirty_ring_size - - - def serialize(self): - return { - "cpus": self._cpus, - "mem": self._mem, - "src_mem_bind": self._src_mem_bind, - "dst_mem_bind": self._dst_mem_bind, - "src_cpu_bind": self._src_cpu_bind, - "dst_cpu_bind": self._dst_cpu_bind, - "prealloc_pages": self._prealloc_pages, - "huge_pages": self._huge_pages, - "locked_pages": self._locked_pages, - "dirty_ring_size": self._dirty_ring_size, - } - - @classmethod - def deserialize(cls, data): - return cls( - data["cpus"], - data["mem"], - data["src_cpu_bind"], - data["src_mem_bind"], - data["dst_cpu_bind"], - data["dst_mem_bind"], - data["prealloc_pages"], - data["huge_pages"], - data["locked_pages"], - data["dirty_ring_size"]) diff --git a/tests/migration/guestperf/plot.py b/tests/migration/guestperf/plot.py deleted file mode 100644 index 30b3f668d0..0000000000 --- a/tests/migration/guestperf/plot.py +++ /dev/null @@ -1,623 +0,0 @@ -# -# Migration test graph plotting -# -# Copyright (c) 2016 Red Hat, Inc. -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, see . -# - -import sys - - -class Plot(object): - - # Generated using - # http://tools.medialab.sciences-po.fr/iwanthue/ - COLORS = ["#CD54D0", - "#79D94C", - "#7470CD", - "#D2D251", - "#863D79", - "#76DDA6", - "#D4467B", - "#61923D", - "#CB9CCA", - "#D98F36", - "#8CC8DA", - "#CE4831", - "#5E7693", - "#9B803F", - "#412F4C", - "#CECBA6", - "#6D3229", - "#598B73", - "#C8827C", - "#394427"] - - def __init__(self, - reports, - migration_iters, - total_guest_cpu, - split_guest_cpu, - qemu_cpu, - vcpu_cpu): - - self._reports = reports - self._migration_iters = migration_iters - self._total_guest_cpu = total_guest_cpu - self._split_guest_cpu = split_guest_cpu - self._qemu_cpu = qemu_cpu - self._vcpu_cpu = vcpu_cpu - self._color_idx = 0 - - def _next_color(self): - color = self.COLORS[self._color_idx] - self._color_idx += 1 - if self._color_idx >= len(self.COLORS): - self._color_idx = 0 - return color - - def _get_progress_label(self, progress): - if progress: - return "\n\n" + "\n".join( - ["Status: %s" % progress._status, - "Iteration: %d" % progress._ram._iterations, - "Throttle: %02d%%" % progress._throttle_pcent, - "Dirty rate: %dMB/s" % (progress._ram._dirty_rate_pps * 4 / 1024.0)]) - else: - return "\n\n" + "\n".join( - ["Status: %s" % "none", - "Iteration: %d" % 0]) - - def _find_start_time(self, report): - startqemu = report._qemu_timings._records[0]._timestamp - startguest = report._guest_timings._records[0]._timestamp - if startqemu < startguest: - return startqemu - else: - return stasrtguest - - def _get_guest_max_value(self, report): - maxvalue = 0 - for record in report._guest_timings._records: - if record._value > maxvalue: - maxvalue = record._value - return maxvalue - - def _get_qemu_max_value(self, report): - maxvalue = 0 - oldvalue = None - oldtime = None - for record in report._qemu_timings._records: - if oldvalue is not None: - cpudelta = (record._value - oldvalue) / 1000.0 - timedelta = record._timestamp - oldtime - if timedelta == 0: - continue - util = cpudelta / timedelta * 100.0 - else: - util = 0 - oldvalue = record._value - oldtime = record._timestamp - - if util > maxvalue: - maxvalue = util - return maxvalue - - def _get_total_guest_cpu_graph(self, report, starttime): - xaxis = [] - yaxis = [] - labels = [] - progress_idx = -1 - for record in report._guest_timings._records: - while ((progress_idx + 1) < len(report._progress_history) and - report._progress_history[progress_idx + 1]._now < record._timestamp): - progress_idx = progress_idx + 1 - - if progress_idx >= 0: - progress = report._progress_history[progress_idx] - else: - progress = None - - xaxis.append(record._timestamp - starttime) - yaxis.append(record._value) - labels.append(self._get_progress_label(progress)) - - from plotly import graph_objs as go - return go.Scatter(x=xaxis, - y=yaxis, - name="Guest PIDs: %s" % report._scenario._name, - mode='lines', - line={ - "dash": "solid", - "color": self._next_color(), - "shape": "linear", - "width": 1 - }, - text=labels) - - def _get_split_guest_cpu_graphs(self, report, starttime): - threads = {} - for record in report._guest_timings._records: - if record._tid in threads: - continue - threads[record._tid] = { - "xaxis": [], - "yaxis": [], - "labels": [], - } - - progress_idx = -1 - for record in report._guest_timings._records: - while ((progress_idx + 1) < len(report._progress_history) and - report._progress_history[progress_idx + 1]._now < record._timestamp): - progress_idx = progress_idx + 1 - - if progress_idx >= 0: - progress = report._progress_history[progress_idx] - else: - progress = None - - threads[record._tid]["xaxis"].append(record._timestamp - starttime) - threads[record._tid]["yaxis"].append(record._value) - threads[record._tid]["labels"].append(self._get_progress_label(progress)) - - - graphs = [] - from plotly import graph_objs as go - for tid in threads.keys(): - graphs.append( - go.Scatter(x=threads[tid]["xaxis"], - y=threads[tid]["yaxis"], - name="PID %s: %s" % (tid, report._scenario._name), - mode="lines", - line={ - "dash": "solid", - "color": self._next_color(), - "shape": "linear", - "width": 1 - }, - text=threads[tid]["labels"])) - return graphs - - def _get_migration_iters_graph(self, report, starttime): - xaxis = [] - yaxis = [] - labels = [] - for progress in report._progress_history: - xaxis.append(progress._now - starttime) - yaxis.append(0) - labels.append(self._get_progress_label(progress)) - - from plotly import graph_objs as go - return go.Scatter(x=xaxis, - y=yaxis, - text=labels, - name="Migration iterations", - mode="markers", - marker={ - "color": self._next_color(), - "symbol": "star", - "size": 5 - }) - - def _get_qemu_cpu_graph(self, report, starttime): - xaxis = [] - yaxis = [] - labels = [] - progress_idx = -1 - - first = report._qemu_timings._records[0] - abstimestamps = [first._timestamp] - absvalues = [first._value] - - for record in report._qemu_timings._records[1:]: - while ((progress_idx + 1) < len(report._progress_history) and - report._progress_history[progress_idx + 1]._now < record._timestamp): - progress_idx = progress_idx + 1 - - if progress_idx >= 0: - progress = report._progress_history[progress_idx] - else: - progress = None - - oldvalue = absvalues[-1] - oldtime = abstimestamps[-1] - - cpudelta = (record._value - oldvalue) / 1000.0 - timedelta = record._timestamp - oldtime - if timedelta == 0: - continue - util = cpudelta / timedelta * 100.0 - - abstimestamps.append(record._timestamp) - absvalues.append(record._value) - - xaxis.append(record._timestamp - starttime) - yaxis.append(util) - labels.append(self._get_progress_label(progress)) - - from plotly import graph_objs as go - return go.Scatter(x=xaxis, - y=yaxis, - yaxis="y2", - name="QEMU: %s" % report._scenario._name, - mode='lines', - line={ - "dash": "solid", - "color": self._next_color(), - "shape": "linear", - "width": 1 - }, - text=labels) - - def _get_vcpu_cpu_graphs(self, report, starttime): - threads = {} - for record in report._vcpu_timings._records: - if record._tid in threads: - continue - threads[record._tid] = { - "xaxis": [], - "yaxis": [], - "labels": [], - "absvalue": [record._value], - "abstime": [record._timestamp], - } - - progress_idx = -1 - for record in report._vcpu_timings._records: - while ((progress_idx + 1) < len(report._progress_history) and - report._progress_history[progress_idx + 1]._now < record._timestamp): - progress_idx = progress_idx + 1 - - if progress_idx >= 0: - progress = report._progress_history[progress_idx] - else: - progress = None - - oldvalue = threads[record._tid]["absvalue"][-1] - oldtime = threads[record._tid]["abstime"][-1] - - cpudelta = (record._value - oldvalue) / 1000.0 - timedelta = record._timestamp - oldtime - if timedelta == 0: - continue - util = cpudelta / timedelta * 100.0 - if util > 100: - util = 100 - - threads[record._tid]["absvalue"].append(record._value) - threads[record._tid]["abstime"].append(record._timestamp) - - threads[record._tid]["xaxis"].append(record._timestamp - starttime) - threads[record._tid]["yaxis"].append(util) - threads[record._tid]["labels"].append(self._get_progress_label(progress)) - - - graphs = [] - from plotly import graph_objs as go - for tid in threads.keys(): - graphs.append( - go.Scatter(x=threads[tid]["xaxis"], - y=threads[tid]["yaxis"], - yaxis="y2", - name="VCPU %s: %s" % (tid, report._scenario._name), - mode="lines", - line={ - "dash": "solid", - "color": self._next_color(), - "shape": "linear", - "width": 1 - }, - text=threads[tid]["labels"])) - return graphs - - def _generate_chart_report(self, report): - graphs = [] - starttime = self._find_start_time(report) - if self._total_guest_cpu: - graphs.append(self._get_total_guest_cpu_graph(report, starttime)) - if self._split_guest_cpu: - graphs.extend(self._get_split_guest_cpu_graphs(report, starttime)) - if self._qemu_cpu: - graphs.append(self._get_qemu_cpu_graph(report, starttime)) - if self._vcpu_cpu: - graphs.extend(self._get_vcpu_cpu_graphs(report, starttime)) - if self._migration_iters: - graphs.append(self._get_migration_iters_graph(report, starttime)) - return graphs - - def _generate_annotation(self, starttime, progress): - return { - "text": progress._status, - "x": progress._now - starttime, - "y": 10, - } - - def _generate_annotations(self, report): - starttime = self._find_start_time(report) - annotations = {} - started = False - for progress in report._progress_history: - if progress._status == "setup": - continue - if progress._status not in annotations: - annotations[progress._status] = self._generate_annotation(starttime, progress) - - return annotations.values() - - def _generate_chart(self): - from plotly.offline import plot - from plotly import graph_objs as go - - graphs = [] - yaxismax = 0 - yaxismax2 = 0 - for report in self._reports: - graphs.extend(self._generate_chart_report(report)) - - maxvalue = self._get_guest_max_value(report) - if maxvalue > yaxismax: - yaxismax = maxvalue - - maxvalue = self._get_qemu_max_value(report) - if maxvalue > yaxismax2: - yaxismax2 = maxvalue - - yaxismax += 100 - if not self._qemu_cpu: - yaxismax2 = 110 - yaxismax2 += 10 - - annotations = [] - if self._migration_iters: - for report in self._reports: - annotations.extend(self._generate_annotations(report)) - - layout = go.Layout(title="Migration comparison", - xaxis={ - "title": "Wallclock time (secs)", - "showgrid": False, - }, - yaxis={ - "title": "Memory update speed (ms/GB)", - "showgrid": False, - "range": [0, yaxismax], - }, - yaxis2={ - "title": "Hostutilization (%)", - "overlaying": "y", - "side": "right", - "range": [0, yaxismax2], - "showgrid": False, - }, - annotations=annotations) - - figure = go.Figure(data=graphs, layout=layout) - - return plot(figure, - show_link=False, - include_plotlyjs=False, - output_type="div") - - - def _generate_report(self): - pieces = [] - for report in self._reports: - pieces.append(""" -

Report %s

- -""" % report._scenario._name) - - pieces.append(""" - - - - - - - - - - - - - - - - - - - - - - - -""" % (report._binary, report._kernel, - report._initrd, report._transport, report._dst_host)) - - hardware = report._hardware - pieces.append(""" - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -""" % (hardware._cpus, hardware._mem, - ",".join(hardware._src_cpu_bind), - ",".join(hardware._src_mem_bind), - ",".join(hardware._dst_cpu_bind), - ",".join(hardware._dst_mem_bind), - "yes" if hardware._prealloc_pages else "no", - "yes" if hardware._locked_pages else "no", - "yes" if hardware._huge_pages else "no")) - - scenario = report._scenario - pieces.append(""" - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -""" % (scenario._downtime, scenario._bandwidth, - scenario._max_iters, scenario._max_time, - "yes" if scenario._pause else "no", scenario._pause_iters, - "yes" if scenario._post_copy else "no", scenario._post_copy_iters, - "yes" if scenario._auto_converge else "no", scenario._auto_converge_step, - "yes" if scenario._compression_mt else "no", scenario._compression_mt_threads, - "yes" if scenario._compression_xbzrle else "no", scenario._compression_xbzrle_cache)) - - pieces.append(""" -
Test config
Emulator:%s
Kernel:%s
Ramdisk:%s
Transport:%s
Host:%s
Hardware config
CPUs:%d
RAM:%d GB
Source CPU bind:%s
Source RAM bind:%s
Dest CPU bind:%s
Dest RAM bind:%s
Preallocate RAM:%s
Locked RAM:%s
Huge pages:%s
Scenario config
Max downtime:%d milli-sec
Max bandwidth:%d MB/sec
Max iters:%d
Max time:%d secs
Pause:%s
Pause iters:%d
Post-copy:%s
Post-copy iters:%d
Auto-converge:%s
Auto-converge iters:%d
MT compression:%s
MT compression threads:%d
XBZRLE compression:%s
XBZRLE compression cache:%d%% of RAM
-""") - - return "\n".join(pieces) - - def _generate_style(self): - return """ -#report table tr th { - text-align: right; -} -#report table tr td { - text-align: left; -} -#report table tr.subhead th { - background: rgb(192, 192, 192); - text-align: center; -} - -""" - - def generate_html(self, fh): - print(""" - - - - Migration report - - -

Migration report

-

Chart summary

-
-""" % self._generate_style(), file=fh) - print(self._generate_chart(), file=fh) - print(""" -
-

Report details

-
-""", file=fh) - print(self._generate_report(), file=fh) - print(""" -
- - -""", file=fh) - - def generate(self, filename): - if filename is None: - self.generate_html(sys.stdout) - else: - with open(filename, "w") as fh: - self.generate_html(fh) diff --git a/tests/migration/guestperf/progress.py b/tests/migration/guestperf/progress.py deleted file mode 100644 index d490584217..0000000000 --- a/tests/migration/guestperf/progress.py +++ /dev/null @@ -1,129 +0,0 @@ -# -# Migration test migration operation progress -# -# Copyright (c) 2016 Red Hat, Inc. -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, see . -# - - -class ProgressStats(object): - - def __init__(self, - transferred_bytes, - remaining_bytes, - total_bytes, - duplicate_pages, - skipped_pages, - normal_pages, - normal_bytes, - dirty_rate_pps, - transfer_rate_mbs, - iterations): - self._transferred_bytes = transferred_bytes - self._remaining_bytes = remaining_bytes - self._total_bytes = total_bytes - self._duplicate_pages = duplicate_pages - self._skipped_pages = skipped_pages - self._normal_pages = normal_pages - self._normal_bytes = normal_bytes - self._dirty_rate_pps = dirty_rate_pps - self._transfer_rate_mbs = transfer_rate_mbs - self._iterations = iterations - - def serialize(self): - return { - "transferred_bytes": self._transferred_bytes, - "remaining_bytes": self._remaining_bytes, - "total_bytes": self._total_bytes, - "duplicate_pages": self._duplicate_pages, - "skipped_pages": self._skipped_pages, - "normal_pages": self._normal_pages, - "normal_bytes": self._normal_bytes, - "dirty_rate_pps": self._dirty_rate_pps, - "transfer_rate_mbs": self._transfer_rate_mbs, - "iterations": self._iterations, - } - - @classmethod - def deserialize(cls, data): - return cls( - data["transferred_bytes"], - data["remaining_bytes"], - data["total_bytes"], - data["duplicate_pages"], - data["skipped_pages"], - data["normal_pages"], - data["normal_bytes"], - data["dirty_rate_pps"], - data["transfer_rate_mbs"], - data["iterations"]) - - -class Progress(object): - - def __init__(self, - status, - ram, - now, - duration, - downtime, - downtime_expected, - setup_time, - throttle_pcent, - dirty_limit_throttle_time_per_round, - dirty_limit_ring_full_time): - - self._status = status - self._ram = ram - self._now = now - self._duration = duration - self._downtime = downtime - self._downtime_expected = downtime_expected - self._setup_time = setup_time - self._throttle_pcent = throttle_pcent - self._dirty_limit_throttle_time_per_round = \ - dirty_limit_throttle_time_per_round - self._dirty_limit_ring_full_time = \ - dirty_limit_ring_full_time - - def serialize(self): - return { - "status": self._status, - "ram": self._ram.serialize(), - "now": self._now, - "duration": self._duration, - "downtime": self._downtime, - "downtime_expected": self._downtime_expected, - "setup_time": self._setup_time, - "throttle_pcent": self._throttle_pcent, - "dirty_limit_throttle_time_per_round": - self._dirty_limit_throttle_time_per_round, - "dirty_limit_ring_full_time": - self._dirty_limit_ring_full_time, - } - - @classmethod - def deserialize(cls, data): - return cls( - data["status"], - ProgressStats.deserialize(data["ram"]), - data["now"], - data["duration"], - data["downtime"], - data["downtime_expected"], - data["setup_time"], - data["throttle_pcent"], - data["dirty_limit_throttle_time_per_round"], - data["dirty_limit_ring_full_time"]) diff --git a/tests/migration/guestperf/report.py b/tests/migration/guestperf/report.py deleted file mode 100644 index 1efd40c868..0000000000 --- a/tests/migration/guestperf/report.py +++ /dev/null @@ -1,98 +0,0 @@ -# -# Migration test output result reporting -# -# Copyright (c) 2016 Red Hat, Inc. -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, see . -# - -import json - -from guestperf.hardware import Hardware -from guestperf.scenario import Scenario -from guestperf.progress import Progress -from guestperf.timings import Timings - -class Report(object): - - def __init__(self, - hardware, - scenario, - progress_history, - guest_timings, - qemu_timings, - vcpu_timings, - binary, - dst_host, - kernel, - initrd, - transport, - sleep): - - self._hardware = hardware - self._scenario = scenario - self._progress_history = progress_history - self._guest_timings = guest_timings - self._qemu_timings = qemu_timings - self._vcpu_timings = vcpu_timings - self._binary = binary - self._dst_host = dst_host - self._kernel = kernel - self._initrd = initrd - self._transport = transport - self._sleep = sleep - - def serialize(self): - return { - "hardware": self._hardware.serialize(), - "scenario": self._scenario.serialize(), - "progress_history": [progress.serialize() for progress in self._progress_history], - "guest_timings": self._guest_timings.serialize(), - "qemu_timings": self._qemu_timings.serialize(), - "vcpu_timings": self._vcpu_timings.serialize(), - "binary": self._binary, - "dst_host": self._dst_host, - "kernel": self._kernel, - "initrd": self._initrd, - "transport": self._transport, - "sleep": self._sleep, - } - - @classmethod - def deserialize(cls, data): - return cls( - Hardware.deserialize(data["hardware"]), - Scenario.deserialize(data["scenario"]), - [Progress.deserialize(record) for record in data["progress_history"]], - Timings.deserialize(data["guest_timings"]), - Timings.deserialize(data["qemu_timings"]), - Timings.deserialize(data["vcpu_timings"]), - data["binary"], - data["dst_host"], - data["kernel"], - data["initrd"], - data["transport"], - data["sleep"]) - - def to_json(self): - return json.dumps(self.serialize(), indent=4) - - @classmethod - def from_json(cls, data): - return cls.deserialize(json.loads(data)) - - @classmethod - def from_json_file(cls, filename): - with open(filename, "r") as fh: - return cls.deserialize(json.load(fh)) diff --git a/tests/migration/guestperf/scenario.py b/tests/migration/guestperf/scenario.py deleted file mode 100644 index 154c4f5d5f..0000000000 --- a/tests/migration/guestperf/scenario.py +++ /dev/null @@ -1,112 +0,0 @@ -# -# Migration test scenario parameter description -# -# Copyright (c) 2016 Red Hat, Inc. -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, see . -# - - -class Scenario(object): - - def __init__(self, name, - downtime=500, - bandwidth=125000, # 1000 gig-e, effectively unlimited - max_iters=30, - max_time=300, - pause=False, pause_iters=5, - post_copy=False, post_copy_iters=5, - auto_converge=False, auto_converge_step=10, - compression_mt=False, compression_mt_threads=1, - compression_xbzrle=False, compression_xbzrle_cache=10, - multifd=False, multifd_channels=2, - dirty_limit=False, x_vcpu_dirty_limit_period=500, - vcpu_dirty_limit=1): - - self._name = name - - # General migration tunables - self._downtime = downtime # milliseconds - self._bandwidth = bandwidth # MiB per second - self._max_iters = max_iters - self._max_time = max_time # seconds - - - # Strategies for ensuring completion - self._pause = pause - self._pause_iters = pause_iters - - self._post_copy = post_copy - self._post_copy_iters = post_copy_iters - - self._auto_converge = auto_converge - self._auto_converge_step = auto_converge_step # percentage CPU time - - self._compression_mt = compression_mt - self._compression_mt_threads = compression_mt_threads - - self._compression_xbzrle = compression_xbzrle - self._compression_xbzrle_cache = compression_xbzrle_cache # percentage of guest RAM - - self._multifd = multifd - self._multifd_channels = multifd_channels - - self._dirty_limit = dirty_limit - self._x_vcpu_dirty_limit_period = x_vcpu_dirty_limit_period - self._vcpu_dirty_limit = vcpu_dirty_limit - - def serialize(self): - return { - "name": self._name, - "downtime": self._downtime, - "bandwidth": self._bandwidth, - "max_iters": self._max_iters, - "max_time": self._max_time, - "pause": self._pause, - "pause_iters": self._pause_iters, - "post_copy": self._post_copy, - "post_copy_iters": self._post_copy_iters, - "auto_converge": self._auto_converge, - "auto_converge_step": self._auto_converge_step, - "compression_mt": self._compression_mt, - "compression_mt_threads": self._compression_mt_threads, - "compression_xbzrle": self._compression_xbzrle, - "compression_xbzrle_cache": self._compression_xbzrle_cache, - "multifd": self._multifd, - "multifd_channels": self._multifd_channels, - "dirty_limit": self._dirty_limit, - "x_vcpu_dirty_limit_period": self._x_vcpu_dirty_limit_period, - "vcpu_dirty_limit": self._vcpu_dirty_limit, - } - - @classmethod - def deserialize(cls, data): - return cls( - data["name"], - data["downtime"], - data["bandwidth"], - data["max_iters"], - data["max_time"], - data["pause"], - data["pause_iters"], - data["post_copy"], - data["post_copy_iters"], - data["auto_converge"], - data["auto_converge_step"], - data["compression_mt"], - data["compression_mt_threads"], - data["compression_xbzrle"], - data["compression_xbzrle_cache"], - data["multifd"], - data["multifd_channels"]) diff --git a/tests/migration/guestperf/shell.py b/tests/migration/guestperf/shell.py deleted file mode 100644 index c85d89efec..0000000000 --- a/tests/migration/guestperf/shell.py +++ /dev/null @@ -1,296 +0,0 @@ -# -# Migration test command line shell integration -# -# Copyright (c) 2016 Red Hat, Inc. -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, see . -# - - -import argparse -import fnmatch -import os -import os.path -import platform -import sys -import logging - -from guestperf.hardware import Hardware -from guestperf.engine import Engine -from guestperf.scenario import Scenario -from guestperf.comparison import COMPARISONS -from guestperf.plot import Plot -from guestperf.report import Report - - -class BaseShell(object): - - def __init__(self): - parser = argparse.ArgumentParser(description="Migration Test Tool") - - # Test args - parser.add_argument("--debug", dest="debug", default=False, action="store_true") - parser.add_argument("--verbose", dest="verbose", default=False, action="store_true") - parser.add_argument("--sleep", dest="sleep", default=15, type=int) - parser.add_argument("--binary", dest="binary", default="/usr/bin/qemu-system-x86_64") - parser.add_argument("--dst-host", dest="dst_host", default="localhost") - parser.add_argument("--kernel", dest="kernel", default="/boot/vmlinuz-%s" % platform.release()) - parser.add_argument("--initrd", dest="initrd", default="tests/migration/initrd-stress.img") - parser.add_argument("--transport", dest="transport", default="unix") - - - # Hardware args - parser.add_argument("--cpus", dest="cpus", default=1, type=int) - parser.add_argument("--mem", dest="mem", default=1, type=int) - parser.add_argument("--src-cpu-bind", dest="src_cpu_bind", default="") - parser.add_argument("--src-mem-bind", dest="src_mem_bind", default="") - parser.add_argument("--dst-cpu-bind", dest="dst_cpu_bind", default="") - parser.add_argument("--dst-mem-bind", dest="dst_mem_bind", default="") - parser.add_argument("--prealloc-pages", dest="prealloc_pages", default=False) - parser.add_argument("--huge-pages", dest="huge_pages", default=False) - parser.add_argument("--locked-pages", dest="locked_pages", default=False) - parser.add_argument("--dirty-ring-size", dest="dirty_ring_size", - default=0, type=int) - - self._parser = parser - - def get_engine(self, args): - return Engine(binary=args.binary, - dst_host=args.dst_host, - kernel=args.kernel, - initrd=args.initrd, - transport=args.transport, - sleep=args.sleep, - debug=args.debug, - verbose=args.verbose) - - def get_hardware(self, args): - def split_map(value): - if value == "": - return [] - return value.split(",") - - return Hardware(cpus=args.cpus, - mem=args.mem, - - src_cpu_bind=split_map(args.src_cpu_bind), - src_mem_bind=split_map(args.src_mem_bind), - dst_cpu_bind=split_map(args.dst_cpu_bind), - dst_mem_bind=split_map(args.dst_mem_bind), - - locked_pages=args.locked_pages, - huge_pages=args.huge_pages, - prealloc_pages=args.prealloc_pages, - - dirty_ring_size=args.dirty_ring_size) - - -class Shell(BaseShell): - - def __init__(self): - super(Shell, self).__init__() - - parser = self._parser - - parser.add_argument("--output", dest="output", default=None) - - # Scenario args - parser.add_argument("--max-iters", dest="max_iters", default=30, type=int) - parser.add_argument("--max-time", dest="max_time", default=300, type=int) - parser.add_argument("--bandwidth", dest="bandwidth", default=125000, type=int) - parser.add_argument("--downtime", dest="downtime", default=500, type=int) - - parser.add_argument("--pause", dest="pause", default=False, action="store_true") - parser.add_argument("--pause-iters", dest="pause_iters", default=5, type=int) - - parser.add_argument("--post-copy", dest="post_copy", default=False, action="store_true") - parser.add_argument("--post-copy-iters", dest="post_copy_iters", default=5, type=int) - - parser.add_argument("--auto-converge", dest="auto_converge", default=False, action="store_true") - parser.add_argument("--auto-converge-step", dest="auto_converge_step", default=10, type=int) - - parser.add_argument("--compression-mt", dest="compression_mt", default=False, action="store_true") - parser.add_argument("--compression-mt-threads", dest="compression_mt_threads", default=1, type=int) - - parser.add_argument("--compression-xbzrle", dest="compression_xbzrle", default=False, action="store_true") - parser.add_argument("--compression-xbzrle-cache", dest="compression_xbzrle_cache", default=10, type=int) - - parser.add_argument("--multifd", dest="multifd", default=False, - action="store_true") - parser.add_argument("--multifd-channels", dest="multifd_channels", - default=2, type=int) - - parser.add_argument("--dirty-limit", dest="dirty_limit", default=False, - action="store_true") - - parser.add_argument("--x-vcpu-dirty-limit-period", - dest="x_vcpu_dirty_limit_period", - default=500, type=int) - - parser.add_argument("--vcpu-dirty-limit", - dest="vcpu_dirty_limit", - default=1, type=int) - - def get_scenario(self, args): - return Scenario(name="perfreport", - downtime=args.downtime, - bandwidth=args.bandwidth, - max_iters=args.max_iters, - max_time=args.max_time, - - pause=args.pause, - pause_iters=args.pause_iters, - - post_copy=args.post_copy, - post_copy_iters=args.post_copy_iters, - - auto_converge=args.auto_converge, - auto_converge_step=args.auto_converge_step, - - compression_mt=args.compression_mt, - compression_mt_threads=args.compression_mt_threads, - - compression_xbzrle=args.compression_xbzrle, - compression_xbzrle_cache=args.compression_xbzrle_cache, - - multifd=args.multifd, - multifd_channels=args.multifd_channels, - - dirty_limit=args.dirty_limit, - x_vcpu_dirty_limit_period=\ - args.x_vcpu_dirty_limit_period, - vcpu_dirty_limit=args.vcpu_dirty_limit) - - def run(self, argv): - args = self._parser.parse_args(argv) - logging.basicConfig(level=(logging.DEBUG if args.debug else - logging.INFO if args.verbose else - logging.WARN)) - - - engine = self.get_engine(args) - hardware = self.get_hardware(args) - scenario = self.get_scenario(args) - - try: - report = engine.run(hardware, scenario) - if args.output is None: - print(report.to_json()) - else: - with open(args.output, "w") as fh: - print(report.to_json(), file=fh) - return 0 - except Exception as e: - print("Error: %s" % str(e), file=sys.stderr) - if args.debug: - raise - return 1 - - -class BatchShell(BaseShell): - - def __init__(self): - super(BatchShell, self).__init__() - - parser = self._parser - - parser.add_argument("--filter", dest="filter", default="*") - parser.add_argument("--output", dest="output", default=os.getcwd()) - - def run(self, argv): - args = self._parser.parse_args(argv) - logging.basicConfig(level=(logging.DEBUG if args.debug else - logging.INFO if args.verbose else - logging.WARN)) - - - engine = self.get_engine(args) - hardware = self.get_hardware(args) - - try: - for comparison in COMPARISONS: - compdir = os.path.join(args.output, comparison._name) - for scenario in comparison._scenarios: - name = os.path.join(comparison._name, scenario._name) - if not fnmatch.fnmatch(name, args.filter): - if args.verbose: - print("Skipping %s" % name) - continue - - if args.verbose: - print("Running %s" % name) - - dirname = os.path.join(args.output, comparison._name) - filename = os.path.join(dirname, scenario._name + ".json") - if not os.path.exists(dirname): - os.makedirs(dirname) - report = engine.run(hardware, scenario) - with open(filename, "w") as fh: - print(report.to_json(), file=fh) - except Exception as e: - print("Error: %s" % str(e), file=sys.stderr) - if args.debug: - raise - - -class PlotShell(object): - - def __init__(self): - super(PlotShell, self).__init__() - - self._parser = argparse.ArgumentParser(description="Migration Test Tool") - - self._parser.add_argument("--output", dest="output", default=None) - - self._parser.add_argument("--debug", dest="debug", default=False, action="store_true") - self._parser.add_argument("--verbose", dest="verbose", default=False, action="store_true") - - self._parser.add_argument("--migration-iters", dest="migration_iters", default=False, action="store_true") - self._parser.add_argument("--total-guest-cpu", dest="total_guest_cpu", default=False, action="store_true") - self._parser.add_argument("--split-guest-cpu", dest="split_guest_cpu", default=False, action="store_true") - self._parser.add_argument("--qemu-cpu", dest="qemu_cpu", default=False, action="store_true") - self._parser.add_argument("--vcpu-cpu", dest="vcpu_cpu", default=False, action="store_true") - - self._parser.add_argument("reports", nargs='*') - - def run(self, argv): - args = self._parser.parse_args(argv) - logging.basicConfig(level=(logging.DEBUG if args.debug else - logging.INFO if args.verbose else - logging.WARN)) - - - if len(args.reports) == 0: - print("At least one report required", file=sys.stderr) - return 1 - - if not (args.qemu_cpu or - args.vcpu_cpu or - args.total_guest_cpu or - args.split_guest_cpu): - print("At least one chart type is required", file=sys.stderr) - return 1 - - reports = [] - for report in args.reports: - reports.append(Report.from_json_file(report)) - - plot = Plot(reports, - args.migration_iters, - args.total_guest_cpu, - args.split_guest_cpu, - args.qemu_cpu, - args.vcpu_cpu) - - plot.generate(args.output) diff --git a/tests/migration/guestperf/timings.py b/tests/migration/guestperf/timings.py deleted file mode 100644 index 2374010c6c..0000000000 --- a/tests/migration/guestperf/timings.py +++ /dev/null @@ -1,55 +0,0 @@ -# -# Migration test timing records -# -# Copyright (c) 2016 Red Hat, Inc. -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2.1 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library; if not, see . -# - - -class TimingRecord(object): - - def __init__(self, tid, timestamp, value): - - self._tid = tid - self._timestamp = timestamp - self._value = value - - def serialize(self): - return { - "tid": self._tid, - "timestamp": self._timestamp, - "value": self._value - } - - @classmethod - def deserialize(cls, data): - return cls( - data["tid"], - data["timestamp"], - data["value"]) - - -class Timings(object): - - def __init__(self, records): - - self._records = records - - def serialize(self): - return [record.serialize() for record in self._records] - - @classmethod - def deserialize(cls, data): - return Timings([TimingRecord.deserialize(record) for record in data]) diff --git a/tests/migration/i386/Makefile b/tests/migration/i386/Makefile deleted file mode 100644 index 37a72ae353..0000000000 --- a/tests/migration/i386/Makefile +++ /dev/null @@ -1,23 +0,0 @@ -# To specify cross compiler prefix, use CROSS_PREFIX= -# $ make CROSS_PREFIX=x86_64-linux-gnu- - -.PHONY: all clean -all: a-b-bootblock.h - -a-b-bootblock.h: x86.bootsect x86.o - echo "$$__note" > header.tmp - xxd -i $< | sed -e 's/.*int.*//' >> header.tmp - nm x86.o | awk '{print "#define SYM_"$$3" 0x"$$1}' >> header.tmp - mv header.tmp $@ - -x86.bootsect: x86.boot - dd if=$< of=$@ bs=256 count=2 skip=124 - -x86.boot: x86.o - $(CROSS_PREFIX)objcopy -O binary $< $@ - -x86.o: a-b-bootblock.S - $(CROSS_PREFIX)gcc -I.. -m32 -march=i486 -c $< -o $@ - -clean: - @rm -rf *.boot *.o *.bootsect diff --git a/tests/migration/i386/a-b-bootblock.S b/tests/migration/i386/a-b-bootblock.S deleted file mode 100644 index 6f39eb6051..0000000000 --- a/tests/migration/i386/a-b-bootblock.S +++ /dev/null @@ -1,145 +0,0 @@ -# x86 bootblock used in migration test -# repeatedly increments the first byte of each page in a 100MB -# range. -# Outputs an initial 'A' on serial followed by repeated 'B's -# -# Copyright (c) 2016 Red Hat, Inc. and/or its affiliates -# This work is licensed under the terms of the GNU GPL, version 2 or later. -# See the COPYING file in the top-level directory. -# -# Author: dgilbert@redhat.com - -#include "migration-test.h" - -#define ACPI_ENABLE 0xf1 -#define ACPI_PORT_SMI_CMD 0xb2 -#define ACPI_PM_BASE 0x600 -#define PM1A_CNT_OFFSET 4 - -#define ACPI_SCI_ENABLE 0x0001 -#define ACPI_SLEEP_TYPE 0x0400 -#define ACPI_SLEEP_ENABLE 0x2000 -#define SLEEP (ACPI_SCI_ENABLE + ACPI_SLEEP_TYPE + ACPI_SLEEP_ENABLE) - -#define LOW_ADDR X86_TEST_MEM_START -#define HIGH_ADDR X86_TEST_MEM_END - -/* Save the suspended status at an address that is not written in the loop. */ -#define suspended (X86_TEST_MEM_START + 4) - -.code16 -.org 0x7c00 - .file "fill.s" - .text - .globl start - .type start, @function -start: # at 0x7c00 ? - cli - lgdt gdtdesc - mov $1,%eax - mov %eax,%cr0 # Protected mode enable - data32 ljmp $8,$0x7c20 - -.org 0x7c20 -.code32 - # A20 enable - not sure I actually need this - inb $0x92,%al - or $2,%al - outb %al, $0x92 - - # set up DS for the whole of RAM (needed on KVM) - mov $16,%eax - mov %eax,%ds - -# Start from 1MB -.set TEST_MEM_START, X86_TEST_MEM_START -.set TEST_MEM_END, X86_TEST_MEM_END - - mov $65,%ax - mov $0x3f8,%dx - outb %al,%dx - - # bl keeps a counter so we limit the output speed - mov $0, %bl - -pre_zero: - mov $TEST_MEM_START,%eax -do_zero: - movb $0, (%eax) - add $4096,%eax - cmp $TEST_MEM_END,%eax - jl do_zero - -mainloop: - mov $TEST_MEM_START,%eax -innerloop: - incb (%eax) - add $4096,%eax - cmp $TEST_MEM_END,%eax - jl innerloop - - inc %bl - andb $0x3f,%bl - jnz mainloop - - mov $66,%ax - mov $0x3f8,%dx - outb %al,%dx - - # should this test suspend? - mov (suspend_me),%eax - cmp $0,%eax - je mainloop - - # are we waking after suspend? do not suspend again. - mov $suspended,%eax - mov (%eax),%eax - cmp $1,%eax - je mainloop - - # enable acpi - mov $ACPI_ENABLE,%al - outb %al,$ACPI_PORT_SMI_CMD - - # suspend to ram - mov $suspended,%eax - movl $1,(%eax) - mov $SLEEP,%ax - mov $(ACPI_PM_BASE + PM1A_CNT_OFFSET),%dx - outw %ax,%dx - # not reached. The wakeup causes reset and restart at 0x7c00, and we - # do not save and restore registers as a real kernel would do. - - - # GDT magic from old (GPLv2) Grub startup.S - .p2align 2 /* force 4-byte alignment */ -gdt: - .word 0, 0 - .byte 0, 0, 0, 0 - - /* -- code segment -- - * base = 0x00000000, limit = 0xFFFFF (4 KiB Granularity), present - * type = 32bit code execute/read, DPL = 0 - */ - .word 0xFFFF, 0 - .byte 0, 0x9A, 0xCF, 0 - - /* -- data segment -- - * base = 0x00000000, limit 0xFFFFF (4 KiB Granularity), present - * type = 32 bit data read/write, DPL = 0 - */ - .word 0xFFFF, 0 - .byte 0, 0x92, 0xCF, 0 - -gdtdesc: - .word 0x27 /* limit */ - .long gdt /* addr */ - - /* test launcher can poke a 1 here to exercise suspend */ -suspend_me: - .int 0 - -/* I'm a bootable disk */ -.org 0x7dfe - .byte 0x55 - .byte 0xAA diff --git a/tests/migration/i386/a-b-bootblock.h b/tests/migration/i386/a-b-bootblock.h deleted file mode 100644 index c83f8711db..0000000000 --- a/tests/migration/i386/a-b-bootblock.h +++ /dev/null @@ -1,61 +0,0 @@ -/* This file is automatically generated from the assembly file in - * tests/migration/i386. Edit that file and then run "make all" - * inside tests/migration to update, and then remember to send both - * the header and the assembler differences in your patch submission. - */ -unsigned char x86_bootsect[] = { - 0xfa, 0x0f, 0x01, 0x16, 0xb8, 0x7c, 0x66, 0xb8, 0x01, 0x00, 0x00, 0x00, - 0x0f, 0x22, 0xc0, 0x66, 0xea, 0x20, 0x7c, 0x00, 0x00, 0x08, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe4, 0x92, 0x0c, 0x02, - 0xe6, 0x92, 0xb8, 0x10, 0x00, 0x00, 0x00, 0x8e, 0xd8, 0x66, 0xb8, 0x41, - 0x00, 0x66, 0xba, 0xf8, 0x03, 0xee, 0xb3, 0x00, 0xb8, 0x00, 0x00, 0x10, - 0x00, 0xc6, 0x00, 0x00, 0x05, 0x00, 0x10, 0x00, 0x00, 0x3d, 0x00, 0x00, - 0x40, 0x06, 0x7c, 0xf1, 0xb8, 0x00, 0x00, 0x10, 0x00, 0xfe, 0x00, 0x05, - 0x00, 0x10, 0x00, 0x00, 0x3d, 0x00, 0x00, 0x40, 0x06, 0x7c, 0xf2, 0xfe, - 0xc3, 0x80, 0xe3, 0x3f, 0x75, 0xe6, 0x66, 0xb8, 0x42, 0x00, 0x66, 0xba, - 0xf8, 0x03, 0xee, 0xa1, 0xbe, 0x7c, 0x00, 0x00, 0x83, 0xf8, 0x00, 0x74, - 0xd3, 0xb8, 0x04, 0x00, 0x10, 0x00, 0x8b, 0x00, 0x83, 0xf8, 0x01, 0x74, - 0xc7, 0xb0, 0xf1, 0xe6, 0xb2, 0xb8, 0x04, 0x00, 0x10, 0x00, 0xc7, 0x00, - 0x01, 0x00, 0x00, 0x00, 0x66, 0xb8, 0x01, 0x24, 0x66, 0xba, 0x04, 0x06, - 0x66, 0xef, 0x66, 0x90, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0xff, 0xff, 0x00, 0x00, 0x00, 0x9a, 0xcf, 0x00, 0xff, 0xff, 0x00, 0x00, - 0x00, 0x92, 0xcf, 0x00, 0x27, 0x00, 0xa0, 0x7c, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0xaa -}; - -#define SYM_do_zero 0x00007c3d -#define SYM_gdt 0x00007ca0 -#define SYM_gdtdesc 0x00007cb8 -#define SYM_innerloop 0x00007c51 -#define SYM_mainloop 0x00007c4c -#define SYM_pre_zero 0x00007c38 -#define SYM_start 0x00007c00 -#define SYM_suspend_me 0x00007cbe -#define SYM_TEST_MEM_END 0x06400000 -#define SYM_TEST_MEM_START 0x00100000 diff --git a/tests/migration/initrd-stress.sh b/tests/migration/initrd-stress.sh deleted file mode 100755 index 0f20ac29a6..0000000000 --- a/tests/migration/initrd-stress.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -INITRD="$1" -STRESS="$2" - -INITRD_DIR=$(mktemp -d -p '' "initrd-stress.XXXXXX") -trap 'rm -rf $INITRD_DIR' EXIT - -cp "$STRESS" "$INITRD_DIR/init" -(cd "$INITRD_DIR" && (find | cpio --quiet -o -H newc | gzip -9)) > "$INITRD" diff --git a/tests/migration/meson.build b/tests/migration/meson.build deleted file mode 100644 index a91aa61c65..0000000000 --- a/tests/migration/meson.build +++ /dev/null @@ -1,18 +0,0 @@ -sysprof = dependency('sysprof-capture-4', method: 'pkg-config', required: false) -glib_static = dependency('glib-2.0', version: glib_req_ver, required: false, - method: 'pkg-config', static: true) - -stress = executable( - 'stress', - files('stress.c'), - dependencies: [glib_static, sysprof], - link_args: ['-static'], - build_by_default: false, -) - -custom_target( - 'initrd-stress.img', - output: 'initrd-stress.img', - input: stress, - command: [find_program('initrd-stress.sh'), '@OUTPUT@', '@INPUT@'] -) diff --git a/tests/migration/migration-test.h b/tests/migration/migration-test.h deleted file mode 100644 index 194df7df6f..0000000000 --- a/tests/migration/migration-test.h +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright (c) 2018 Red Hat, Inc. and/or its affiliates - * - * This work is licensed under the terms of the GNU GPL, version 2 or later. - * See the COPYING file in the top-level directory. - */ - -#ifndef MIGRATION_TEST_H -#define MIGRATION_TEST_H - -/* Common */ -#define TEST_MEM_PAGE_SIZE 4096 - -/* x86 */ -#define X86_TEST_MEM_START (1 * 1024 * 1024) -#define X86_TEST_MEM_END (100 * 1024 * 1024) - -/* S390 */ -#define S390_TEST_MEM_START (1 * 1024 * 1024) -#define S390_TEST_MEM_END (100 * 1024 * 1024) - -/* PPC */ -#define PPC_TEST_MEM_START (1 * 1024 * 1024) -#define PPC_TEST_MEM_END (100 * 1024 * 1024) -#define PPC_H_PUT_TERM_CHAR 0x58 - -/* ARM */ -#define ARM_TEST_MEM_START (0x40000000 + 1 * 1024 * 1024) -#define ARM_TEST_MEM_END (0x40000000 + 100 * 1024 * 1024) -#define ARM_MACH_VIRT_UART 0x09000000 -/* AArch64 kernel load address is 0x40080000, and the test memory starts at - * 0x40100000. So the maximum allowable kernel size is 512KB. - */ -#define ARM_TEST_MAX_KERNEL_SIZE (512 * 1024) - -#endif /* MIGRATION_TEST_H */ diff --git a/tests/migration/ppc64/Makefile b/tests/migration/ppc64/Makefile deleted file mode 100644 index a3a2d98ac8..0000000000 --- a/tests/migration/ppc64/Makefile +++ /dev/null @@ -1,15 +0,0 @@ -.PHONY: all clean -all: a-b-kernel.h - -a-b-kernel.h: ppc64.kernel - echo "$$__note" > $@ - xxd -i $< | sed -e 's/.*int.*//' >> $@ - -ppc64.kernel: ppc64.elf - $(CROSS_PREFIX)objcopy -O binary -S $< $@ - -ppc64.elf: a-b-kernel.S - $(CROSS_PREFIX)gcc -static -o $@ -nostdlib -Wl,--build-id=none $< - -clean: - $(RM) *.kernel *.elf diff --git a/tests/migration/ppc64/a-b-kernel.S b/tests/migration/ppc64/a-b-kernel.S deleted file mode 100644 index 0613a8d18e..0000000000 --- a/tests/migration/ppc64/a-b-kernel.S +++ /dev/null @@ -1,66 +0,0 @@ -# -# Copyright (c) 2024 IBM, Inc -# -# This work is licensed under the terms of the GNU GPL, version 2 or later. -# See the COPYING file in the top-level directory. - -#include "../migration-test.h" - -.section .text - -.macro print ch - li %r3,PPC_H_PUT_TERM_CHAR - li %r4,0 - li %r5,1 - li %r6,\ch - sldi %r6,%r6,56 - sc 1 -.endm - - .globl _start -_start: -. = 0x100 - /* - * Enter 64-bit mode. Not necessary because the test uses 32-bit - * addresses, but those constants could easily be changed and break - * in 32-bit mode. - */ - mfmsr %r9 - li %r10,-1 - rldimi %r9,%r10,63,0 - mtmsrd %r9 - - /* - * Set up test memory region. Non-volatiles are used because the - * hcall can clobber regs. - * r20 - start address - * r21 - number of pages - */ - lis %r20,PPC_TEST_MEM_START@h - ori %r20,%r20,PPC_TEST_MEM_START@l - lis %r9,PPC_TEST_MEM_END@h - ori %r9,%r9,PPC_TEST_MEM_END@l - subf %r21,%r20,%r9 - li %r10,TEST_MEM_PAGE_SIZE - divd %r21,%r21,%r10 - - print 'A' - - li %r3,0 - mr %r9,%r20 - mtctr %r21 -1: stb %r3,0(%r9) - addi %r9,%r9,TEST_MEM_PAGE_SIZE - bdnz 1b - -loop: - mr %r9,%r20 - mtctr %r21 -1: lbz %r3,0(%r9) - addi %r3,%r3,1 - stb %r3,0(%r9) - addi %r9,%r9,TEST_MEM_PAGE_SIZE - bdnz 1b - - print 'B' - b loop diff --git a/tests/migration/ppc64/a-b-kernel.h b/tests/migration/ppc64/a-b-kernel.h deleted file mode 100644 index 673317efdb..0000000000 --- a/tests/migration/ppc64/a-b-kernel.h +++ /dev/null @@ -1,42 +0,0 @@ -/* This file is automatically generated from the assembly file in - * tests/migration/ppc64. Edit that file and then run "make all" - * inside tests/migration to update, and then remember to send both - * the header and the assembler differences in your patch submission. - */ -unsigned char ppc64_kernel[] = { - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x7d, 0x20, 0x00, 0xa6, 0x39, 0x40, 0xff, 0xff, - 0x79, 0x49, 0xf8, 0x0e, 0x7d, 0x20, 0x01, 0x64, 0x3e, 0x80, 0x00, 0x10, - 0x62, 0x94, 0x00, 0x00, 0x3d, 0x20, 0x06, 0x40, 0x61, 0x29, 0x00, 0x00, - 0x7e, 0xb4, 0x48, 0x50, 0x39, 0x40, 0x10, 0x00, 0x7e, 0xb5, 0x53, 0xd2, - 0x38, 0x60, 0x00, 0x58, 0x38, 0x80, 0x00, 0x00, 0x38, 0xa0, 0x00, 0x01, - 0x38, 0xc0, 0x00, 0x41, 0x78, 0xc6, 0xc1, 0xc6, 0x44, 0x00, 0x00, 0x22, - 0x38, 0x60, 0x00, 0x00, 0x7e, 0x89, 0xa3, 0x78, 0x7e, 0xa9, 0x03, 0xa6, - 0x98, 0x69, 0x00, 0x00, 0x39, 0x29, 0x10, 0x00, 0x42, 0x00, 0xff, 0xf8, - 0x7e, 0x89, 0xa3, 0x78, 0x7e, 0xa9, 0x03, 0xa6, 0x88, 0x69, 0x00, 0x00, - 0x38, 0x63, 0x00, 0x01, 0x98, 0x69, 0x00, 0x00, 0x39, 0x29, 0x10, 0x00, - 0x42, 0x00, 0xff, 0xf0, 0x38, 0x60, 0x00, 0x58, 0x38, 0x80, 0x00, 0x00, - 0x38, 0xa0, 0x00, 0x01, 0x38, 0xc0, 0x00, 0x42, 0x78, 0xc6, 0xc1, 0xc6, - 0x44, 0x00, 0x00, 0x22, 0x4b, 0xff, 0xff, 0xcc -}; - diff --git a/tests/migration/s390x/Makefile b/tests/migration/s390x/Makefile deleted file mode 100644 index 6671de2efc..0000000000 --- a/tests/migration/s390x/Makefile +++ /dev/null @@ -1,24 +0,0 @@ -# To specify cross compiler prefix, use CROSS_PREFIX= -# $ make CROSS_PREFIX=s390x-linux-gnu- - -.PHONY: all clean -all: a-b-bios.h -fwdir=../../../pc-bios/s390-ccw - -CFLAGS+=-ffreestanding -fno-delete-null-pointer-checks -fPIE -Os \ - -msoft-float -march=z900 -fno-asynchronous-unwind-tables \ - -fno-stack-protector -Wl,-pie -Wl,--build-id=none -nostdlib - -a-b-bios.h: s390x.elf - echo "$$__note" > header.tmp - xxd -i $< | sed -e 's/.*int.*//' >> header.tmp - mv header.tmp $@ - -# We use common-page-size=16 to avoid big padding in the ELF file -s390x.elf: a-b-bios.c - $(CROSS_PREFIX)gcc $(CFLAGS) -I$(fwdir) $(fwdir)/start.S \ - $(fwdir)/sclp.c -Wl,-zcommon-page-size=16 -o $@ $< - $(CROSS_PREFIX)strip $@ - -clean: - @rm -rf *.elf *.o diff --git a/tests/migration/s390x/a-b-bios.c b/tests/migration/s390x/a-b-bios.c deleted file mode 100644 index ff99a3ef57..0000000000 --- a/tests/migration/s390x/a-b-bios.c +++ /dev/null @@ -1,44 +0,0 @@ -/* - * S390 guest code used in migration tests - * - * Copyright 2018 Thomas Huth, Red Hat Inc. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License as published by the - * Free Software Foundation; either version 2 of the License, or (at your - * option) any later version. - */ - -#define LOADPARM_LEN 8 /* Needed for sclp.h */ - -#include -#include -#include - -char stack[0x8000] __attribute__((aligned(4096))); - -#define START_ADDRESS (1024 * 1024) -#define END_ADDRESS (100 * 1024 * 1024) - -void main(void) -{ - unsigned long addr; - - sclp_setup(); - sclp_print("A"); - - /* - * Make sure all of the pages have consistent contents before incrementing - * the first byte below. - */ - for (addr = START_ADDRESS; addr < END_ADDRESS; addr += 4096) { - *(volatile char *)addr = 0; - } - - while (1) { - for (addr = START_ADDRESS; addr < END_ADDRESS; addr += 4096) { - *(volatile char *)addr += 1; /* Change pages */ - } - sclp_print("B"); - } -} diff --git a/tests/migration/s390x/a-b-bios.h b/tests/migration/s390x/a-b-bios.h deleted file mode 100644 index 96103dadbb..0000000000 --- a/tests/migration/s390x/a-b-bios.h +++ /dev/null @@ -1,279 +0,0 @@ -/* This file is automatically generated from the a-b-bios.c file in - * tests/migration/s390x. Edit that file and then run "make all" - * inside tests/migration to update, and then remember to send both - * the header and the assembler differences in your patch submission. - */ -unsigned char s390x_elf[] = { - 0x7f, 0x45, 0x4c, 0x46, 0x02, 0x02, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x16, 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xa8, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x80, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x38, 0x00, 0x07, 0x00, 0x40, - 0x00, 0x0d, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x04, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x88, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x01, 0x88, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, - 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x01, 0xc8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xc8, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xc8, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xac, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xac, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x06, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xb8, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x17, 0xb8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x17, 0xb8, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x38, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x01, 0x18, 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, - 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x07, 0xb8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x17, 0xb8, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x17, 0xb8, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x01, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x20, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x64, 0x74, 0xe5, 0x51, - 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x10, 0x64, 0x74, 0xe5, 0x52, 0x00, 0x00, 0x00, 0x04, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xb8, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x17, 0xb8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x17, 0xb8, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x38, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x01, 0x38, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - 0x2f, 0x6c, 0x69, 0x62, 0x2f, 0x6c, 0x64, 0x36, 0x34, 0x2e, 0x73, 0x6f, - 0x2e, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x02, 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x03, 0xa8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0xf0, 0xeb, 0xef, 0xf0, 0x70, - 0x00, 0x24, 0xa7, 0xfb, 0xff, 0x60, 0xc0, 0xe5, 0x00, 0x00, 0x01, 0x5f, - 0xc0, 0x20, 0x00, 0x00, 0x02, 0xa8, 0xc0, 0xe5, 0x00, 0x00, 0x01, 0x75, - 0xa5, 0x2e, 0x00, 0x10, 0xa7, 0x19, 0x63, 0x00, 0x92, 0x00, 0x20, 0x00, - 0xa7, 0x2b, 0x10, 0x00, 0xa7, 0x17, 0xff, 0xfc, 0xa5, 0x1e, 0x00, 0x10, - 0xa7, 0x29, 0x63, 0x00, 0xe3, 0x30, 0x10, 0x00, 0x00, 0x90, 0xa7, 0x3a, - 0x00, 0x01, 0x42, 0x30, 0x10, 0x00, 0xa7, 0x1b, 0x10, 0x00, 0xa7, 0x27, - 0xff, 0xf7, 0xc0, 0x20, 0x00, 0x00, 0x02, 0x8a, 0xc0, 0xe5, 0x00, 0x00, - 0x01, 0x56, 0xa7, 0xf4, 0xff, 0xeb, 0x07, 0x07, 0xc0, 0xf0, 0x00, 0x00, - 0x4e, 0x5c, 0xc0, 0x20, 0x00, 0x00, 0x00, 0x7d, 0xe3, 0x20, 0x20, 0x00, - 0x00, 0x04, 0xc0, 0x30, 0x00, 0x00, 0x96, 0xa3, 0xb9, 0x0b, 0x00, 0x32, - 0xb9, 0x02, 0x00, 0x33, 0xa7, 0x84, 0x00, 0x19, 0xa7, 0x3b, 0xff, 0xff, - 0xeb, 0x43, 0x00, 0x08, 0x00, 0x0c, 0xb9, 0x02, 0x00, 0x44, 0xb9, 0x04, - 0x00, 0x12, 0xa7, 0x84, 0x00, 0x09, 0xd7, 0xff, 0x10, 0x00, 0x10, 0x00, - 0x41, 0x10, 0x11, 0x00, 0xa7, 0x47, 0xff, 0xfb, 0xc0, 0x20, 0x00, 0x00, - 0x00, 0x0d, 0x44, 0x30, 0x20, 0x00, 0xc0, 0x20, 0x00, 0x00, 0x00, 0x5b, - 0xd2, 0x0f, 0x01, 0xd0, 0x20, 0x00, 0xa7, 0xf4, 0xff, 0xa1, 0xd7, 0x00, - 0x10, 0x00, 0x10, 0x00, 0xc0, 0x10, 0x00, 0x00, 0x00, 0x50, 0xb2, 0xb2, - 0x10, 0x00, 0xa7, 0xf4, 0x00, 0x00, 0xeb, 0x00, 0xf0, 0x00, 0x00, 0x25, - 0x96, 0x02, 0xf0, 0x06, 0xeb, 0x00, 0xf0, 0x00, 0x00, 0x2f, 0xc0, 0x10, - 0x00, 0x00, 0x00, 0x2a, 0xe3, 0x10, 0x01, 0xb8, 0x00, 0x24, 0xc0, 0x10, - 0x00, 0x00, 0x00, 0x4b, 0xd2, 0x07, 0x01, 0xb0, 0x10, 0x00, 0xc0, 0x10, - 0x00, 0x00, 0x00, 0x3d, 0xb2, 0xb2, 0x10, 0x00, 0xeb, 0x66, 0xf0, 0x00, - 0x00, 0x25, 0x96, 0xff, 0xf0, 0x04, 0xeb, 0x66, 0xf0, 0x00, 0x00, 0x2f, - 0xc0, 0x10, 0x00, 0x00, 0x00, 0x1a, 0xe3, 0x10, 0x01, 0xf8, 0x00, 0x24, - 0xc0, 0x10, 0x00, 0x00, 0x00, 0x36, 0xd2, 0x07, 0x01, 0xf0, 0x10, 0x00, - 0xc0, 0x10, 0x00, 0x00, 0x00, 0x24, 0xb2, 0xb2, 0x10, 0x00, 0xeb, 0x00, - 0xf0, 0x00, 0x00, 0x25, 0x94, 0xfd, 0xf0, 0x06, 0xeb, 0x00, 0xf0, 0x00, - 0x00, 0x2f, 0x07, 0xfe, 0xeb, 0x66, 0xf0, 0x00, 0x00, 0x25, 0x94, 0x00, - 0xf0, 0x04, 0xeb, 0x66, 0xf0, 0x00, 0x00, 0x2f, 0x07, 0xfe, 0x07, 0x07, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0xf0, 0x00, 0x02, 0x00, 0x01, - 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x03, 0x02, 0x00, 0x01, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x80, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x01, 0x80, 0x00, 0x00, 0x00, 0xeb, 0xbf, 0xf0, 0x58, - 0x00, 0x24, 0xc0, 0x10, 0x00, 0x00, 0x4e, 0x0d, 0xa7, 0xfb, 0xff, 0x60, - 0xb2, 0x20, 0x00, 0x21, 0xb2, 0x22, 0x00, 0xb0, 0x88, 0xb0, 0x00, 0x1c, - 0xc0, 0xe5, 0xff, 0xff, 0xff, 0x91, 0xa7, 0xbe, 0x00, 0x03, 0xa7, 0x84, - 0x00, 0x13, 0xa7, 0xbe, 0x00, 0x02, 0xa7, 0x28, 0x00, 0x00, 0xa7, 0x74, - 0x00, 0x04, 0xa7, 0x28, 0xff, 0xfe, 0xe3, 0x40, 0xf1, 0x10, 0x00, 0x04, - 0xb9, 0x14, 0x00, 0x22, 0xeb, 0xbf, 0xf0, 0xf8, 0x00, 0x04, 0x07, 0xf4, - 0xa7, 0x28, 0xff, 0xff, 0xa7, 0xf4, 0xff, 0xf5, 0x07, 0x07, 0x07, 0x07, - 0xeb, 0xbf, 0xf0, 0x58, 0x00, 0x24, 0xc0, 0xd0, 0x00, 0x00, 0x01, 0x25, - 0xa7, 0xfb, 0xff, 0x60, 0xa7, 0xb9, 0x00, 0x00, 0xa7, 0x19, 0x00, 0x00, - 0xc0, 0x40, 0x00, 0x00, 0x4d, 0xd8, 0xa7, 0x3b, 0x00, 0x01, 0xa7, 0x37, - 0x00, 0x23, 0xc0, 0x20, 0x00, 0x00, 0x4d, 0xd1, 0x18, 0x31, 0xa7, 0x1a, - 0x00, 0x06, 0x40, 0x10, 0x20, 0x08, 0xa7, 0x3a, 0x00, 0x0e, 0xa7, 0x18, - 0x1a, 0x00, 0x40, 0x30, 0x20, 0x00, 0x92, 0x00, 0x20, 0x02, 0x40, 0x10, - 0x20, 0x0a, 0xe3, 0x20, 0xd0, 0x00, 0x00, 0x04, 0xc0, 0xe5, 0xff, 0xff, - 0xff, 0xac, 0xe3, 0x40, 0xf1, 0x10, 0x00, 0x04, 0xb9, 0x04, 0x00, 0x2b, - 0xeb, 0xbf, 0xf0, 0xf8, 0x00, 0x04, 0x07, 0xf4, 0xb9, 0x04, 0x00, 0x51, - 0xa7, 0x5b, 0x00, 0x01, 0xa7, 0x09, 0x0f, 0xf7, 0xb9, 0x21, 0x00, 0x50, - 0xa7, 0x24, 0xff, 0xd7, 0x41, 0xeb, 0x20, 0x00, 0x95, 0x0a, 0xe0, 0x00, - 0xa7, 0x74, 0x00, 0x08, 0x41, 0x11, 0x40, 0x0e, 0x92, 0x0d, 0x10, 0x00, - 0xb9, 0x04, 0x00, 0x15, 0x43, 0x5b, 0x20, 0x00, 0x42, 0x51, 0x40, 0x0e, - 0xa7, 0xbb, 0x00, 0x01, 0x41, 0x10, 0x10, 0x01, 0xa7, 0xf4, 0xff, 0xbf, - 0xc0, 0x50, 0x00, 0x00, 0x00, 0xd8, 0xc0, 0x10, 0x00, 0x00, 0x4d, 0x8d, - 0xa7, 0x48, 0x00, 0x1c, 0x40, 0x40, 0x10, 0x00, 0x50, 0x20, 0x10, 0x0c, - 0xa7, 0x48, 0x00, 0x04, 0xe3, 0x20, 0x50, 0x00, 0x00, 0x04, 0x40, 0x40, - 0x10, 0x0a, 0x50, 0x30, 0x10, 0x10, 0xc0, 0xf4, 0xff, 0xff, 0xff, 0x6b, - 0xa7, 0x39, 0x00, 0x40, 0xa7, 0x29, 0x00, 0x00, 0xc0, 0xf4, 0xff, 0xff, - 0xff, 0xe4, 0x07, 0x07, 0xb9, 0x04, 0x00, 0x13, 0xa7, 0x2a, 0xff, 0xff, - 0xb9, 0x04, 0x00, 0x34, 0xa7, 0x48, 0x00, 0x01, 0x15, 0x24, 0xa7, 0x24, - 0x00, 0x07, 0xb9, 0x04, 0x00, 0x21, 0xc0, 0xf4, 0xff, 0xff, 0xff, 0x7f, - 0xa7, 0x29, 0xff, 0xff, 0x07, 0xfe, 0x07, 0x07, 0xa7, 0x39, 0x00, 0x00, - 0x41, 0x13, 0x20, 0x00, 0x95, 0x00, 0x10, 0x00, 0xa7, 0x74, 0x00, 0x05, - 0xc0, 0xf4, 0xff, 0xff, 0xff, 0x70, 0xa7, 0x3b, 0x00, 0x01, 0xa7, 0xf4, - 0xff, 0xf5, 0x07, 0x07, 0xeb, 0xbf, 0xf0, 0x58, 0x00, 0x24, 0xc0, 0xd0, - 0x00, 0x00, 0x00, 0x95, 0xa7, 0xfb, 0xff, 0x60, 0xb9, 0x04, 0x00, 0xb2, - 0xa7, 0x19, 0x00, 0x20, 0xc0, 0x20, 0x00, 0x00, 0x4d, 0x40, 0x92, 0x00, - 0x20, 0x00, 0xa7, 0x2b, 0x00, 0x01, 0xa7, 0x17, 0xff, 0xfc, 0xc0, 0x10, - 0x00, 0x00, 0x4d, 0x37, 0xa7, 0x28, 0x10, 0x00, 0x40, 0x20, 0x10, 0x00, - 0xe3, 0x20, 0xd0, 0x00, 0x00, 0x04, 0xc0, 0xe5, 0xff, 0xff, 0xff, 0x1d, - 0x12, 0x22, 0xa7, 0x74, 0x00, 0x19, 0xa7, 0x19, 0x00, 0x00, 0xc0, 0x40, - 0x00, 0x00, 0x00, 0x79, 0xa7, 0x39, 0x00, 0x08, 0xc0, 0x20, 0x00, 0x00, - 0x4d, 0x2c, 0x41, 0x21, 0x20, 0x00, 0xe3, 0x20, 0x20, 0x00, 0x00, 0x90, - 0x43, 0x22, 0x40, 0x00, 0x42, 0x21, 0xb0, 0x00, 0xa7, 0x1b, 0x00, 0x01, - 0xa7, 0x37, 0xff, 0xf2, 0xe3, 0x40, 0xf1, 0x10, 0x00, 0x04, 0xeb, 0xbf, - 0xf0, 0xf8, 0x00, 0x04, 0x07, 0xf4, 0x07, 0x07, 0xeb, 0xaf, 0xf0, 0x50, - 0x00, 0x24, 0xc0, 0xd0, 0x00, 0x00, 0x00, 0x55, 0xa7, 0xfb, 0xff, 0x60, - 0xa7, 0x19, 0x0f, 0xf8, 0xb9, 0x21, 0x00, 0x31, 0xb9, 0x04, 0x00, 0xa2, - 0xa7, 0xc4, 0x00, 0x2a, 0xa7, 0xb9, 0x0f, 0xf8, 0xc0, 0x10, 0x00, 0x00, - 0x4c, 0xf6, 0xa7, 0x28, 0x10, 0x00, 0x40, 0x20, 0x10, 0x00, 0x92, 0x00, - 0x10, 0x02, 0xe3, 0x20, 0xd0, 0x00, 0x00, 0x04, 0xc0, 0xe5, 0xff, 0xff, - 0xfe, 0xda, 0xa7, 0xbb, 0x00, 0x01, 0xa7, 0x19, 0x00, 0x00, 0xa7, 0xb7, - 0x00, 0x17, 0xc0, 0x10, 0x00, 0x00, 0x4c, 0xe1, 0xe3, 0x40, 0xf1, 0x10, - 0x00, 0x04, 0xe3, 0x20, 0x10, 0x08, 0x00, 0x91, 0xa7, 0x2a, 0xff, 0xf9, - 0xb9, 0x14, 0x00, 0x22, 0xeb, 0xaf, 0xf0, 0xf0, 0x00, 0x04, 0x07, 0xf4, - 0xb9, 0x04, 0x00, 0xb3, 0xa7, 0xf4, 0xff, 0xd8, 0xc0, 0x20, 0x00, 0x00, - 0x4c, 0xcc, 0x41, 0x31, 0xa0, 0x00, 0x41, 0x21, 0x20, 0x00, 0xa7, 0x1b, - 0x00, 0x01, 0xd2, 0x00, 0x30, 0x00, 0x20, 0x0f, 0xa7, 0xf4, 0xff, 0xdd, - 0x07, 0x07, 0x07, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x76, 0x00, 0x05, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x02, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x77, 0x00, 0x05, - 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, - 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, - 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, - 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, - 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, - 0x2e, 0x2e, 0x2e, 0x2e, 0x20, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, - 0x2e, 0x2e, 0x2e, 0x2e, 0x3c, 0x28, 0x2b, 0x7c, 0x26, 0x2e, 0x2e, 0x2e, - 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x21, 0x24, 0x2a, 0x29, 0x3b, 0x2e, - 0x2d, 0x2f, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2c, - 0x25, 0x5f, 0x3e, 0x3f, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, - 0x2e, 0x60, 0x3a, 0x23, 0x40, 0x27, 0x3d, 0x22, 0x2e, 0x61, 0x62, 0x63, - 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, - 0x2e, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x2e, 0x2e, - 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, - 0x79, 0x7a, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, - 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, - 0x2e, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x2e, 0x2e, - 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, - 0x51, 0x52, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x53, 0x54, - 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, - 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x2e, 0x2e, - 0x2e, 0x2e, 0x2e, 0x2e, 0x41, 0x00, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x6f, 0xff, 0xfe, 0xf5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xd8, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x02, 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xf8, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x15, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x30, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1e, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x6f, 0xff, 0xff, 0xfb, - 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, - 0x6f, 0xff, 0xff, 0xf9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x17, 0xb8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x43, 0x43, 0x3a, - 0x20, 0x28, 0x55, 0x62, 0x75, 0x6e, 0x74, 0x75, 0x20, 0x31, 0x31, 0x2e, - 0x34, 0x2e, 0x30, 0x2d, 0x31, 0x75, 0x62, 0x75, 0x6e, 0x74, 0x75, 0x31, - 0x7e, 0x32, 0x32, 0x2e, 0x30, 0x34, 0x29, 0x20, 0x31, 0x31, 0x2e, 0x34, - 0x2e, 0x30, 0x00, 0x00, 0x2e, 0x73, 0x68, 0x73, 0x74, 0x72, 0x74, 0x61, - 0x62, 0x00, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x00, 0x2e, 0x67, - 0x6e, 0x75, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x00, 0x2e, 0x64, 0x79, 0x6e, - 0x73, 0x79, 0x6d, 0x00, 0x2e, 0x64, 0x79, 0x6e, 0x73, 0x74, 0x72, 0x00, - 0x2e, 0x72, 0x65, 0x6c, 0x61, 0x2e, 0x64, 0x79, 0x6e, 0x00, 0x2e, 0x74, - 0x65, 0x78, 0x74, 0x00, 0x2e, 0x72, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x00, - 0x2e, 0x64, 0x79, 0x6e, 0x61, 0x6d, 0x69, 0x63, 0x00, 0x2e, 0x67, 0x6f, - 0x74, 0x00, 0x2e, 0x62, 0x73, 0x73, 0x00, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, - 0x65, 0x6e, 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xc8, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xc8, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 0x6f, 0xff, 0xff, 0xf6, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x01, 0xd8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xd8, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x03, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, - 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xf8, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x01, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, - 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, - 0x00, 0x00, 0x00, 0x25, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x28, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x28, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2d, 0x00, 0x00, 0x00, 0x04, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x02, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x30, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x03, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x37, - 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x48, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x02, 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x40, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x3d, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x88, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x88, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x01, 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x45, 0x00, 0x00, 0x00, 0x06, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x17, 0xb8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xb8, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x20, 0x00, 0x00, 0x00, 0x04, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x4e, - 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0xd8, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x08, 0xd8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x53, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0xf0, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x01, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0xf0, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x09, 0x1b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x61, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 -}; diff --git a/tests/migration/stress.c b/tests/migration/stress.c deleted file mode 100644 index 88acf8dc25..0000000000 --- a/tests/migration/stress.c +++ /dev/null @@ -1,328 +0,0 @@ -/* - * Migration stress workload - * - * Copyright (c) 2016 Red Hat, Inc. - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, see . - */ - -#include "qemu/osdep.h" -#include -#include -#include -#include -#include -#include - -const char *argv0; - -#define RAM_PAGE_SIZE 4096 - -#ifndef CONFIG_GETTID -static int gettid(void) -{ - return syscall(SYS_gettid); -} -#endif - -static __attribute__((noreturn)) void exit_failure(void) -{ - if (getpid() == 1) { - sync(); - reboot(RB_POWER_OFF); - fprintf(stderr, "%s (%05d): ERROR: cannot reboot: %s\n", - argv0, gettid(), strerror(errno)); - abort(); - } else { - exit(1); - } -} - -static int get_command_arg_str(const char *name, - char **val) -{ - static char line[1024]; - FILE *fp = fopen("/proc/cmdline", "r"); - char *start, *end; - - if (fp == NULL) { - fprintf(stderr, "%s (%05d): ERROR: cannot open /proc/cmdline: %s\n", - argv0, gettid(), strerror(errno)); - return -1; - } - - if (!fgets(line, sizeof line, fp)) { - fprintf(stderr, "%s (%05d): ERROR: cannot read /proc/cmdline: %s\n", - argv0, gettid(), strerror(errno)); - fclose(fp); - return -1; - } - fclose(fp); - - start = strstr(line, name); - if (!start) - return 0; - - start += strlen(name); - - if (*start != '=') { - fprintf(stderr, "%s (%05d): ERROR: no value provided for '%s' in /proc/cmdline\n", - argv0, gettid(), name); - } - start++; - - end = strstr(start, " "); - if (!end) - end = strstr(start, "\n"); - - if (end == start) { - fprintf(stderr, "%s (%05d): ERROR: no value provided for '%s' in /proc/cmdline\n", - argv0, gettid(), name); - return -1; - } - - if (end) - *val = g_strndup(start, end - start); - else - *val = g_strdup(start); - return 1; -} - - -static int get_command_arg_ull(const char *name, - unsigned long long *val) -{ - char *valstr; - char *end; - - int ret = get_command_arg_str(name, &valstr); - if (ret <= 0) - return ret; - - errno = 0; - *val = strtoll(valstr, &end, 10); - if (errno || *end) { - fprintf(stderr, "%s (%05d): ERROR: cannot parse %s value %s\n", - argv0, gettid(), name, valstr); - g_free(valstr); - return -1; - } - g_free(valstr); - return 0; -} - - -static int random_bytes(char *buf, size_t len) -{ - int fd; - - fd = open("/dev/urandom", O_RDONLY); - if (fd < 0) { - fprintf(stderr, "%s (%05d): ERROR: cannot open /dev/urandom: %s\n", - argv0, gettid(), strerror(errno)); - return -1; - } - - if (read(fd, buf, len) != len) { - fprintf(stderr, "%s (%05d): ERROR: cannot read /dev/urandom: %s\n", - argv0, gettid(), strerror(errno)); - close(fd); - return -1; - } - - close(fd); - - return 0; -} - - -static unsigned long long now(void) -{ - struct timeval tv; - - gettimeofday(&tv, NULL); - - return (tv.tv_sec * 1000ull) + (tv.tv_usec / 1000ull); -} - -static void stressone(unsigned long long ramsizeMB) -{ - size_t pagesPerMB = 1024 * 1024 / RAM_PAGE_SIZE; - g_autofree char *ram = g_malloc(ramsizeMB * 1024 * 1024); - char *ramptr; - size_t i, j, k; - g_autofree char *data = g_malloc(RAM_PAGE_SIZE); - char *dataptr; - size_t nMB = 0; - unsigned long long before, after; - - /* We don't care about initial state, but we do want - * to fault it all into RAM, otherwise the first iter - * of the loop below will be quite slow. We can't use - * 0x0 as the byte as gcc optimizes that away into a - * calloc instead :-) */ - memset(ram, 0xfe, ramsizeMB * 1024 * 1024); - - if (random_bytes(data, RAM_PAGE_SIZE) < 0) { - return; - } - - before = now(); - - while (1) { - - ramptr = ram; - for (i = 0; i < ramsizeMB; i++, nMB++) { - for (j = 0; j < pagesPerMB; j++) { - dataptr = data; - for (k = 0; k < RAM_PAGE_SIZE; k += sizeof(long long)) { - ramptr += sizeof(long long); - dataptr += sizeof(long long); - *(unsigned long long *)ramptr ^= *(unsigned long long *)dataptr; - } - } - - if (nMB == 1024) { - after = now(); - fprintf(stderr, "%s (%05d): INFO: %06llums copied 1 GB in %05llums\n", - argv0, gettid(), after, after - before); - before = now(); - nMB = 0; - } - } - } -} - - -static void *stressthread(void *arg) -{ - unsigned long long ramsizeMB = *(unsigned long long *)arg; - - stressone(ramsizeMB); - - return NULL; -} - -static void stress(unsigned long long ramsizeGB, int ncpus) -{ - size_t i; - unsigned long long ramsizeMB = ramsizeGB * 1024 / ncpus; - ncpus--; - - for (i = 0; i < ncpus; i++) { - pthread_t thr; - pthread_create(&thr, NULL, - stressthread, &ramsizeMB); - } - - stressone(ramsizeMB); -} - - -static int mount_misc(const char *fstype, const char *dir) -{ - if (g_mkdir_with_parents(dir, 0755) < 0 && errno != EEXIST) { - fprintf(stderr, "%s (%05d): ERROR: cannot create %s: %s\n", - argv0, gettid(), dir, strerror(errno)); - return -1; - } - - if (mount("none", dir, fstype, 0, NULL) < 0) { - fprintf(stderr, "%s (%05d): ERROR: cannot mount %s: %s\n", - argv0, gettid(), dir, strerror(errno)); - return -1; - } - - return 0; -} - -static int mount_all(void) -{ - if (mount_misc("proc", "/proc") < 0 || - mount_misc("sysfs", "/sys") < 0 || - mount_misc("tmpfs", "/dev") < 0) - return -1; - - mknod("/dev/urandom", 0777 | S_IFCHR, makedev(1, 9)); - mknod("/dev/random", 0777 | S_IFCHR, makedev(1, 8)); - - return 0; -} - -int main(int argc, char **argv) -{ - unsigned long long ramsizeGB = 1; - char *end; - int ch; - int opt_ind = 0; - const char *sopt = "hr:c:"; - struct option lopt[] = { - { "help", no_argument, NULL, 'h' }, - { "ramsize", required_argument, NULL, 'r' }, - { "cpus", required_argument, NULL, 'c' }, - { NULL, 0, NULL, 0 } - }; - int ret; - int ncpus = 0; - - argv0 = argv[0]; - - while ((ch = getopt_long(argc, argv, sopt, lopt, &opt_ind)) != -1) { - switch (ch) { - case 'r': - errno = 0; - ramsizeGB = strtoll(optarg, &end, 10); - if (errno != 0 || *end) { - fprintf(stderr, "%s (%05d): ERROR: Cannot parse RAM size %s\n", - argv0, gettid(), optarg); - exit_failure(); - } - break; - - case 'c': - errno = 0; - ncpus = strtoll(optarg, &end, 10); - if (errno != 0 || *end) { - fprintf(stderr, "%s (%05d): ERROR: Cannot parse CPU count %s\n", - argv0, gettid(), optarg); - exit_failure(); - } - break; - - case '?': - case 'h': - fprintf(stderr, "%s: [--help][--ramsize GB][--cpus N]\n", argv0); - exit_failure(); - } - } - - if (getpid() == 1) { - if (mount_all() < 0) - exit_failure(); - - ret = get_command_arg_ull("ramsize", &ramsizeGB); - if (ret < 0) - exit_failure(); - } - - if (ncpus == 0) - ncpus = sysconf(_SC_NPROCESSORS_ONLN); - - fprintf(stdout, "%s (%05d): INFO: RAM %llu GiB across %d CPUs\n", - argv0, gettid(), ramsizeGB, ncpus); - - stress(ramsizeGB, ncpus); - - exit_failure(); -} diff --git a/tests/qtest/migration-test.c b/tests/qtest/migration-test.c index 30bc965b28..82b9170e3c 100644 --- a/tests/qtest/migration-test.c +++ b/tests/qtest/migration-test.c @@ -24,7 +24,7 @@ #include "ppc-util.h" #include "migration-helpers.h" -#include "tests/migration/migration-test.h" +#include "migration/migration-test.h" #ifdef CONFIG_GNUTLS # include "tests/unit/crypto-tls-psk-helpers.h" # ifdef CONFIG_TASN1 @@ -138,10 +138,10 @@ static char *bootpath; /* The boot file modifies memory area in [start_address, end_address) * repeatedly. It outputs a 'B' at a fixed rate while it's still running. */ -#include "tests/migration/i386/a-b-bootblock.h" -#include "tests/migration/aarch64/a-b-kernel.h" -#include "tests/migration/ppc64/a-b-kernel.h" -#include "tests/migration/s390x/a-b-bios.h" +#include "migration/i386/a-b-bootblock.h" +#include "migration/aarch64/a-b-kernel.h" +#include "migration/ppc64/a-b-kernel.h" +#include "migration/s390x/a-b-bios.h" static void bootfile_delete(void) { diff --git a/tests/qtest/migration/Makefile b/tests/qtest/migration/Makefile new file mode 100644 index 0000000000..2c5ee287ec --- /dev/null +++ b/tests/qtest/migration/Makefile @@ -0,0 +1,40 @@ +# +# Copyright (c) 2018 Red Hat, Inc. and/or its affiliates +# +# This work is licensed under the terms of the GNU GPL, version 2 or later. +# See the COPYING file in the top-level directory. +# + +TARGET_LIST = i386 aarch64 s390x ppc64 + +SRC_PATH = ../.. + +.PHONY: help $(TARGET_LIST) +help: + @echo "Create migration guest includes. We generate a binary." + @echo "And then convert that binary to an include file that can be" + @echo "run in a guest." + @echo "Possible operations are:" + @echo + @echo " $(MAKE) clean Remove all intermediate files" + @echo " $(MAKE) target Generate for that target" + @echo " $(MAKE) CROSS_PREFIX=... target" + @echo " Cross-compile than target" + @echo " Possible targets are: $(TARGET_LIST)" + +override define __note +/* This file is automatically generated from the assembly file in + * tests/migration/$@. Edit that file and then run "make all" + * inside tests/migration to update, and then remember to send both + * the header and the assembler differences in your patch submission. + */ +endef +export __note + +$(TARGET_LIST): + $(MAKE) CROSS_PREFIX=$(CROSS_PREFIX) -C $@ + +clean: + for target in $(TARGET_LIST); do \ + $(MAKE) -C $$target clean; \ + done diff --git a/tests/qtest/migration/aarch64/Makefile b/tests/qtest/migration/aarch64/Makefile new file mode 100644 index 0000000000..9c4fa18e76 --- /dev/null +++ b/tests/qtest/migration/aarch64/Makefile @@ -0,0 +1,18 @@ +# To specify cross compiler prefix, use CROSS_PREFIX= +# $ make CROSS_PREFIX=aarch64-linux-gnu- + +.PHONY: all clean +all: a-b-kernel.h + +a-b-kernel.h: aarch64.kernel + echo "$$__note" > $@ + xxd -i $< | sed -e 's/.*int.*//' >> $@ + +aarch64.kernel: aarch64.elf + $(CROSS_PREFIX)objcopy -O binary $< $@ + +aarch64.elf: a-b-kernel.S + $(CROSS_PREFIX)gcc -o $@ -nostdlib -Wl,--build-id=none $< + +clean: + $(RM) *.kernel *.elf diff --git a/tests/qtest/migration/aarch64/a-b-kernel.S b/tests/qtest/migration/aarch64/a-b-kernel.S new file mode 100644 index 0000000000..a4103ecb71 --- /dev/null +++ b/tests/qtest/migration/aarch64/a-b-kernel.S @@ -0,0 +1,74 @@ +# +# Copyright (c) 2018 Red Hat, Inc. and/or its affiliates +# +# Author: +# Wei Huang +# +# This work is licensed under the terms of the GNU GPL, version 2 or later. +# See the COPYING file in the top-level directory. +# +# Note: Please make sure the compiler compiles the assembly code below with +# pc-relative address. Also the branch instructions should use relative +# addresses only. + +#include "../migration-test.h" + +.section .text + + .globl _start + +_start: + /* disable MMU to use phys mem address */ + mrs x0, sctlr_el1 + bic x0, x0, #(1<<0) + msr sctlr_el1, x0 + isb + + /* traverse test memory region */ + mov x0, #ARM_TEST_MEM_START + mov x1, #ARM_TEST_MEM_END + + /* output char 'A' to PL011 */ + mov w3, 'A' + mov x2, #ARM_MACH_VIRT_UART + strb w3, [x2] + + /* clean up memory */ + mov w3, #0 + mov x4, x0 +clean: + strb w3, [x4] + add x4, x4, #TEST_MEM_PAGE_SIZE + cmp x4, x1 + ble clean + + /* w5 keeps a counter so we can limit the output speed */ + mov w5, #0 + + /* main body */ +mainloop: + mov x4, x0 + +innerloop: + /* increment the first byte of each page by 1 */ + ldrb w3, [x4] + add w3, w3, #1 + strb w3, [x4] + + /* make sure QEMU user space can see consistent data as MMU is off */ + dc civac, x4 + + add x4, x4, #TEST_MEM_PAGE_SIZE + cmp x4, x1 + blt innerloop + + add w5, w5, #1 + and w5, w5, #0x1f + cmp w5, #0 + bne mainloop + + /* output char 'B' to PL011 */ + mov w3, 'B' + strb w3, [x2] + + b mainloop diff --git a/tests/qtest/migration/aarch64/a-b-kernel.h b/tests/qtest/migration/aarch64/a-b-kernel.h new file mode 100644 index 0000000000..34e518d061 --- /dev/null +++ b/tests/qtest/migration/aarch64/a-b-kernel.h @@ -0,0 +1,18 @@ +/* This file is automatically generated from the assembly file in + * tests/migration/aarch64. Edit that file and then run "make all" + * inside tests/migration to update, and then remember to send both + * the header and the assembler differences in your patch submission. + */ +unsigned char aarch64_kernel[] = { + 0x00, 0x10, 0x38, 0xd5, 0x00, 0xf8, 0x7f, 0x92, 0x00, 0x10, 0x18, 0xd5, + 0xdf, 0x3f, 0x03, 0xd5, 0x00, 0x02, 0xa8, 0xd2, 0x01, 0xc8, 0xa8, 0xd2, + 0x23, 0x08, 0x80, 0x52, 0x02, 0x20, 0xa1, 0xd2, 0x43, 0x00, 0x00, 0x39, + 0x03, 0x00, 0x80, 0x52, 0xe4, 0x03, 0x00, 0xaa, 0x83, 0x00, 0x00, 0x39, + 0x84, 0x04, 0x40, 0x91, 0x9f, 0x00, 0x01, 0xeb, 0xad, 0xff, 0xff, 0x54, + 0x05, 0x00, 0x80, 0x52, 0xe4, 0x03, 0x00, 0xaa, 0x83, 0x00, 0x40, 0x39, + 0x63, 0x04, 0x00, 0x11, 0x83, 0x00, 0x00, 0x39, 0x24, 0x7e, 0x0b, 0xd5, + 0x84, 0x04, 0x40, 0x91, 0x9f, 0x00, 0x01, 0xeb, 0x4b, 0xff, 0xff, 0x54, + 0xa5, 0x04, 0x00, 0x11, 0xa5, 0x10, 0x00, 0x12, 0xbf, 0x00, 0x00, 0x71, + 0xa1, 0xfe, 0xff, 0x54, 0x43, 0x08, 0x80, 0x52, 0x43, 0x00, 0x00, 0x39, + 0xf2, 0xff, 0xff, 0x17 +}; diff --git a/tests/qtest/migration/i386/Makefile b/tests/qtest/migration/i386/Makefile new file mode 100644 index 0000000000..37a72ae353 --- /dev/null +++ b/tests/qtest/migration/i386/Makefile @@ -0,0 +1,23 @@ +# To specify cross compiler prefix, use CROSS_PREFIX= +# $ make CROSS_PREFIX=x86_64-linux-gnu- + +.PHONY: all clean +all: a-b-bootblock.h + +a-b-bootblock.h: x86.bootsect x86.o + echo "$$__note" > header.tmp + xxd -i $< | sed -e 's/.*int.*//' >> header.tmp + nm x86.o | awk '{print "#define SYM_"$$3" 0x"$$1}' >> header.tmp + mv header.tmp $@ + +x86.bootsect: x86.boot + dd if=$< of=$@ bs=256 count=2 skip=124 + +x86.boot: x86.o + $(CROSS_PREFIX)objcopy -O binary $< $@ + +x86.o: a-b-bootblock.S + $(CROSS_PREFIX)gcc -I.. -m32 -march=i486 -c $< -o $@ + +clean: + @rm -rf *.boot *.o *.bootsect diff --git a/tests/qtest/migration/i386/a-b-bootblock.S b/tests/qtest/migration/i386/a-b-bootblock.S new file mode 100644 index 0000000000..6f39eb6051 --- /dev/null +++ b/tests/qtest/migration/i386/a-b-bootblock.S @@ -0,0 +1,145 @@ +# x86 bootblock used in migration test +# repeatedly increments the first byte of each page in a 100MB +# range. +# Outputs an initial 'A' on serial followed by repeated 'B's +# +# Copyright (c) 2016 Red Hat, Inc. and/or its affiliates +# This work is licensed under the terms of the GNU GPL, version 2 or later. +# See the COPYING file in the top-level directory. +# +# Author: dgilbert@redhat.com + +#include "migration-test.h" + +#define ACPI_ENABLE 0xf1 +#define ACPI_PORT_SMI_CMD 0xb2 +#define ACPI_PM_BASE 0x600 +#define PM1A_CNT_OFFSET 4 + +#define ACPI_SCI_ENABLE 0x0001 +#define ACPI_SLEEP_TYPE 0x0400 +#define ACPI_SLEEP_ENABLE 0x2000 +#define SLEEP (ACPI_SCI_ENABLE + ACPI_SLEEP_TYPE + ACPI_SLEEP_ENABLE) + +#define LOW_ADDR X86_TEST_MEM_START +#define HIGH_ADDR X86_TEST_MEM_END + +/* Save the suspended status at an address that is not written in the loop. */ +#define suspended (X86_TEST_MEM_START + 4) + +.code16 +.org 0x7c00 + .file "fill.s" + .text + .globl start + .type start, @function +start: # at 0x7c00 ? + cli + lgdt gdtdesc + mov $1,%eax + mov %eax,%cr0 # Protected mode enable + data32 ljmp $8,$0x7c20 + +.org 0x7c20 +.code32 + # A20 enable - not sure I actually need this + inb $0x92,%al + or $2,%al + outb %al, $0x92 + + # set up DS for the whole of RAM (needed on KVM) + mov $16,%eax + mov %eax,%ds + +# Start from 1MB +.set TEST_MEM_START, X86_TEST_MEM_START +.set TEST_MEM_END, X86_TEST_MEM_END + + mov $65,%ax + mov $0x3f8,%dx + outb %al,%dx + + # bl keeps a counter so we limit the output speed + mov $0, %bl + +pre_zero: + mov $TEST_MEM_START,%eax +do_zero: + movb $0, (%eax) + add $4096,%eax + cmp $TEST_MEM_END,%eax + jl do_zero + +mainloop: + mov $TEST_MEM_START,%eax +innerloop: + incb (%eax) + add $4096,%eax + cmp $TEST_MEM_END,%eax + jl innerloop + + inc %bl + andb $0x3f,%bl + jnz mainloop + + mov $66,%ax + mov $0x3f8,%dx + outb %al,%dx + + # should this test suspend? + mov (suspend_me),%eax + cmp $0,%eax + je mainloop + + # are we waking after suspend? do not suspend again. + mov $suspended,%eax + mov (%eax),%eax + cmp $1,%eax + je mainloop + + # enable acpi + mov $ACPI_ENABLE,%al + outb %al,$ACPI_PORT_SMI_CMD + + # suspend to ram + mov $suspended,%eax + movl $1,(%eax) + mov $SLEEP,%ax + mov $(ACPI_PM_BASE + PM1A_CNT_OFFSET),%dx + outw %ax,%dx + # not reached. The wakeup causes reset and restart at 0x7c00, and we + # do not save and restore registers as a real kernel would do. + + + # GDT magic from old (GPLv2) Grub startup.S + .p2align 2 /* force 4-byte alignment */ +gdt: + .word 0, 0 + .byte 0, 0, 0, 0 + + /* -- code segment -- + * base = 0x00000000, limit = 0xFFFFF (4 KiB Granularity), present + * type = 32bit code execute/read, DPL = 0 + */ + .word 0xFFFF, 0 + .byte 0, 0x9A, 0xCF, 0 + + /* -- data segment -- + * base = 0x00000000, limit 0xFFFFF (4 KiB Granularity), present + * type = 32 bit data read/write, DPL = 0 + */ + .word 0xFFFF, 0 + .byte 0, 0x92, 0xCF, 0 + +gdtdesc: + .word 0x27 /* limit */ + .long gdt /* addr */ + + /* test launcher can poke a 1 here to exercise suspend */ +suspend_me: + .int 0 + +/* I'm a bootable disk */ +.org 0x7dfe + .byte 0x55 + .byte 0xAA diff --git a/tests/qtest/migration/i386/a-b-bootblock.h b/tests/qtest/migration/i386/a-b-bootblock.h new file mode 100644 index 0000000000..c83f8711db --- /dev/null +++ b/tests/qtest/migration/i386/a-b-bootblock.h @@ -0,0 +1,61 @@ +/* This file is automatically generated from the assembly file in + * tests/migration/i386. Edit that file and then run "make all" + * inside tests/migration to update, and then remember to send both + * the header and the assembler differences in your patch submission. + */ +unsigned char x86_bootsect[] = { + 0xfa, 0x0f, 0x01, 0x16, 0xb8, 0x7c, 0x66, 0xb8, 0x01, 0x00, 0x00, 0x00, + 0x0f, 0x22, 0xc0, 0x66, 0xea, 0x20, 0x7c, 0x00, 0x00, 0x08, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xe4, 0x92, 0x0c, 0x02, + 0xe6, 0x92, 0xb8, 0x10, 0x00, 0x00, 0x00, 0x8e, 0xd8, 0x66, 0xb8, 0x41, + 0x00, 0x66, 0xba, 0xf8, 0x03, 0xee, 0xb3, 0x00, 0xb8, 0x00, 0x00, 0x10, + 0x00, 0xc6, 0x00, 0x00, 0x05, 0x00, 0x10, 0x00, 0x00, 0x3d, 0x00, 0x00, + 0x40, 0x06, 0x7c, 0xf1, 0xb8, 0x00, 0x00, 0x10, 0x00, 0xfe, 0x00, 0x05, + 0x00, 0x10, 0x00, 0x00, 0x3d, 0x00, 0x00, 0x40, 0x06, 0x7c, 0xf2, 0xfe, + 0xc3, 0x80, 0xe3, 0x3f, 0x75, 0xe6, 0x66, 0xb8, 0x42, 0x00, 0x66, 0xba, + 0xf8, 0x03, 0xee, 0xa1, 0xbe, 0x7c, 0x00, 0x00, 0x83, 0xf8, 0x00, 0x74, + 0xd3, 0xb8, 0x04, 0x00, 0x10, 0x00, 0x8b, 0x00, 0x83, 0xf8, 0x01, 0x74, + 0xc7, 0xb0, 0xf1, 0xe6, 0xb2, 0xb8, 0x04, 0x00, 0x10, 0x00, 0xc7, 0x00, + 0x01, 0x00, 0x00, 0x00, 0x66, 0xb8, 0x01, 0x24, 0x66, 0xba, 0x04, 0x06, + 0x66, 0xef, 0x66, 0x90, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x00, 0x00, 0x00, 0x9a, 0xcf, 0x00, 0xff, 0xff, 0x00, 0x00, + 0x00, 0x92, 0xcf, 0x00, 0x27, 0x00, 0xa0, 0x7c, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x55, 0xaa +}; + +#define SYM_do_zero 0x00007c3d +#define SYM_gdt 0x00007ca0 +#define SYM_gdtdesc 0x00007cb8 +#define SYM_innerloop 0x00007c51 +#define SYM_mainloop 0x00007c4c +#define SYM_pre_zero 0x00007c38 +#define SYM_start 0x00007c00 +#define SYM_suspend_me 0x00007cbe +#define SYM_TEST_MEM_END 0x06400000 +#define SYM_TEST_MEM_START 0x00100000 diff --git a/tests/qtest/migration/migration-test.h b/tests/qtest/migration/migration-test.h new file mode 100644 index 0000000000..194df7df6f --- /dev/null +++ b/tests/qtest/migration/migration-test.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2018 Red Hat, Inc. and/or its affiliates + * + * This work is licensed under the terms of the GNU GPL, version 2 or later. + * See the COPYING file in the top-level directory. + */ + +#ifndef MIGRATION_TEST_H +#define MIGRATION_TEST_H + +/* Common */ +#define TEST_MEM_PAGE_SIZE 4096 + +/* x86 */ +#define X86_TEST_MEM_START (1 * 1024 * 1024) +#define X86_TEST_MEM_END (100 * 1024 * 1024) + +/* S390 */ +#define S390_TEST_MEM_START (1 * 1024 * 1024) +#define S390_TEST_MEM_END (100 * 1024 * 1024) + +/* PPC */ +#define PPC_TEST_MEM_START (1 * 1024 * 1024) +#define PPC_TEST_MEM_END (100 * 1024 * 1024) +#define PPC_H_PUT_TERM_CHAR 0x58 + +/* ARM */ +#define ARM_TEST_MEM_START (0x40000000 + 1 * 1024 * 1024) +#define ARM_TEST_MEM_END (0x40000000 + 100 * 1024 * 1024) +#define ARM_MACH_VIRT_UART 0x09000000 +/* AArch64 kernel load address is 0x40080000, and the test memory starts at + * 0x40100000. So the maximum allowable kernel size is 512KB. + */ +#define ARM_TEST_MAX_KERNEL_SIZE (512 * 1024) + +#endif /* MIGRATION_TEST_H */ diff --git a/tests/qtest/migration/ppc64/Makefile b/tests/qtest/migration/ppc64/Makefile new file mode 100644 index 0000000000..a3a2d98ac8 --- /dev/null +++ b/tests/qtest/migration/ppc64/Makefile @@ -0,0 +1,15 @@ +.PHONY: all clean +all: a-b-kernel.h + +a-b-kernel.h: ppc64.kernel + echo "$$__note" > $@ + xxd -i $< | sed -e 's/.*int.*//' >> $@ + +ppc64.kernel: ppc64.elf + $(CROSS_PREFIX)objcopy -O binary -S $< $@ + +ppc64.elf: a-b-kernel.S + $(CROSS_PREFIX)gcc -static -o $@ -nostdlib -Wl,--build-id=none $< + +clean: + $(RM) *.kernel *.elf diff --git a/tests/qtest/migration/ppc64/a-b-kernel.S b/tests/qtest/migration/ppc64/a-b-kernel.S new file mode 100644 index 0000000000..0613a8d18e --- /dev/null +++ b/tests/qtest/migration/ppc64/a-b-kernel.S @@ -0,0 +1,66 @@ +# +# Copyright (c) 2024 IBM, Inc +# +# This work is licensed under the terms of the GNU GPL, version 2 or later. +# See the COPYING file in the top-level directory. + +#include "../migration-test.h" + +.section .text + +.macro print ch + li %r3,PPC_H_PUT_TERM_CHAR + li %r4,0 + li %r5,1 + li %r6,\ch + sldi %r6,%r6,56 + sc 1 +.endm + + .globl _start +_start: +. = 0x100 + /* + * Enter 64-bit mode. Not necessary because the test uses 32-bit + * addresses, but those constants could easily be changed and break + * in 32-bit mode. + */ + mfmsr %r9 + li %r10,-1 + rldimi %r9,%r10,63,0 + mtmsrd %r9 + + /* + * Set up test memory region. Non-volatiles are used because the + * hcall can clobber regs. + * r20 - start address + * r21 - number of pages + */ + lis %r20,PPC_TEST_MEM_START@h + ori %r20,%r20,PPC_TEST_MEM_START@l + lis %r9,PPC_TEST_MEM_END@h + ori %r9,%r9,PPC_TEST_MEM_END@l + subf %r21,%r20,%r9 + li %r10,TEST_MEM_PAGE_SIZE + divd %r21,%r21,%r10 + + print 'A' + + li %r3,0 + mr %r9,%r20 + mtctr %r21 +1: stb %r3,0(%r9) + addi %r9,%r9,TEST_MEM_PAGE_SIZE + bdnz 1b + +loop: + mr %r9,%r20 + mtctr %r21 +1: lbz %r3,0(%r9) + addi %r3,%r3,1 + stb %r3,0(%r9) + addi %r9,%r9,TEST_MEM_PAGE_SIZE + bdnz 1b + + print 'B' + b loop diff --git a/tests/qtest/migration/ppc64/a-b-kernel.h b/tests/qtest/migration/ppc64/a-b-kernel.h new file mode 100644 index 0000000000..673317efdb --- /dev/null +++ b/tests/qtest/migration/ppc64/a-b-kernel.h @@ -0,0 +1,42 @@ +/* This file is automatically generated from the assembly file in + * tests/migration/ppc64. Edit that file and then run "make all" + * inside tests/migration to update, and then remember to send both + * the header and the assembler differences in your patch submission. + */ +unsigned char ppc64_kernel[] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x7d, 0x20, 0x00, 0xa6, 0x39, 0x40, 0xff, 0xff, + 0x79, 0x49, 0xf8, 0x0e, 0x7d, 0x20, 0x01, 0x64, 0x3e, 0x80, 0x00, 0x10, + 0x62, 0x94, 0x00, 0x00, 0x3d, 0x20, 0x06, 0x40, 0x61, 0x29, 0x00, 0x00, + 0x7e, 0xb4, 0x48, 0x50, 0x39, 0x40, 0x10, 0x00, 0x7e, 0xb5, 0x53, 0xd2, + 0x38, 0x60, 0x00, 0x58, 0x38, 0x80, 0x00, 0x00, 0x38, 0xa0, 0x00, 0x01, + 0x38, 0xc0, 0x00, 0x41, 0x78, 0xc6, 0xc1, 0xc6, 0x44, 0x00, 0x00, 0x22, + 0x38, 0x60, 0x00, 0x00, 0x7e, 0x89, 0xa3, 0x78, 0x7e, 0xa9, 0x03, 0xa6, + 0x98, 0x69, 0x00, 0x00, 0x39, 0x29, 0x10, 0x00, 0x42, 0x00, 0xff, 0xf8, + 0x7e, 0x89, 0xa3, 0x78, 0x7e, 0xa9, 0x03, 0xa6, 0x88, 0x69, 0x00, 0x00, + 0x38, 0x63, 0x00, 0x01, 0x98, 0x69, 0x00, 0x00, 0x39, 0x29, 0x10, 0x00, + 0x42, 0x00, 0xff, 0xf0, 0x38, 0x60, 0x00, 0x58, 0x38, 0x80, 0x00, 0x00, + 0x38, 0xa0, 0x00, 0x01, 0x38, 0xc0, 0x00, 0x42, 0x78, 0xc6, 0xc1, 0xc6, + 0x44, 0x00, 0x00, 0x22, 0x4b, 0xff, 0xff, 0xcc +}; + diff --git a/tests/qtest/migration/s390x/Makefile b/tests/qtest/migration/s390x/Makefile new file mode 100644 index 0000000000..6671de2efc --- /dev/null +++ b/tests/qtest/migration/s390x/Makefile @@ -0,0 +1,24 @@ +# To specify cross compiler prefix, use CROSS_PREFIX= +# $ make CROSS_PREFIX=s390x-linux-gnu- + +.PHONY: all clean +all: a-b-bios.h +fwdir=../../../pc-bios/s390-ccw + +CFLAGS+=-ffreestanding -fno-delete-null-pointer-checks -fPIE -Os \ + -msoft-float -march=z900 -fno-asynchronous-unwind-tables \ + -fno-stack-protector -Wl,-pie -Wl,--build-id=none -nostdlib + +a-b-bios.h: s390x.elf + echo "$$__note" > header.tmp + xxd -i $< | sed -e 's/.*int.*//' >> header.tmp + mv header.tmp $@ + +# We use common-page-size=16 to avoid big padding in the ELF file +s390x.elf: a-b-bios.c + $(CROSS_PREFIX)gcc $(CFLAGS) -I$(fwdir) $(fwdir)/start.S \ + $(fwdir)/sclp.c -Wl,-zcommon-page-size=16 -o $@ $< + $(CROSS_PREFIX)strip $@ + +clean: + @rm -rf *.elf *.o diff --git a/tests/qtest/migration/s390x/a-b-bios.c b/tests/qtest/migration/s390x/a-b-bios.c new file mode 100644 index 0000000000..ff99a3ef57 --- /dev/null +++ b/tests/qtest/migration/s390x/a-b-bios.c @@ -0,0 +1,44 @@ +/* + * S390 guest code used in migration tests + * + * Copyright 2018 Thomas Huth, Red Hat Inc. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation; either version 2 of the License, or (at your + * option) any later version. + */ + +#define LOADPARM_LEN 8 /* Needed for sclp.h */ + +#include +#include +#include + +char stack[0x8000] __attribute__((aligned(4096))); + +#define START_ADDRESS (1024 * 1024) +#define END_ADDRESS (100 * 1024 * 1024) + +void main(void) +{ + unsigned long addr; + + sclp_setup(); + sclp_print("A"); + + /* + * Make sure all of the pages have consistent contents before incrementing + * the first byte below. + */ + for (addr = START_ADDRESS; addr < END_ADDRESS; addr += 4096) { + *(volatile char *)addr = 0; + } + + while (1) { + for (addr = START_ADDRESS; addr < END_ADDRESS; addr += 4096) { + *(volatile char *)addr += 1; /* Change pages */ + } + sclp_print("B"); + } +} diff --git a/tests/qtest/migration/s390x/a-b-bios.h b/tests/qtest/migration/s390x/a-b-bios.h new file mode 100644 index 0000000000..96103dadbb --- /dev/null +++ b/tests/qtest/migration/s390x/a-b-bios.h @@ -0,0 +1,279 @@ +/* This file is automatically generated from the a-b-bios.c file in + * tests/migration/s390x. Edit that file and then run "make all" + * inside tests/migration to update, and then remember to send both + * the header and the assembler differences in your patch submission. + */ +unsigned char s390x_elf[] = { + 0x7f, 0x45, 0x4c, 0x46, 0x02, 0x02, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x16, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xa8, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x38, 0x00, 0x07, 0x00, 0x40, + 0x00, 0x0d, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x88, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x88, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, + 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0xc8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xc8, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xc8, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xac, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xac, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x06, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xb8, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x17, 0xb8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x17, 0xb8, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x38, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x18, 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, + 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x07, 0xb8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x17, 0xb8, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x17, 0xb8, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x20, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x64, 0x74, 0xe5, 0x51, + 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x10, 0x64, 0x74, 0xe5, 0x52, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xb8, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x17, 0xb8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x17, 0xb8, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x38, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x38, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x2f, 0x6c, 0x69, 0x62, 0x2f, 0x6c, 0x64, 0x36, 0x34, 0x2e, 0x73, 0x6f, + 0x2e, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x06, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x02, 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x03, 0xa8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0xf0, 0xeb, 0xef, 0xf0, 0x70, + 0x00, 0x24, 0xa7, 0xfb, 0xff, 0x60, 0xc0, 0xe5, 0x00, 0x00, 0x01, 0x5f, + 0xc0, 0x20, 0x00, 0x00, 0x02, 0xa8, 0xc0, 0xe5, 0x00, 0x00, 0x01, 0x75, + 0xa5, 0x2e, 0x00, 0x10, 0xa7, 0x19, 0x63, 0x00, 0x92, 0x00, 0x20, 0x00, + 0xa7, 0x2b, 0x10, 0x00, 0xa7, 0x17, 0xff, 0xfc, 0xa5, 0x1e, 0x00, 0x10, + 0xa7, 0x29, 0x63, 0x00, 0xe3, 0x30, 0x10, 0x00, 0x00, 0x90, 0xa7, 0x3a, + 0x00, 0x01, 0x42, 0x30, 0x10, 0x00, 0xa7, 0x1b, 0x10, 0x00, 0xa7, 0x27, + 0xff, 0xf7, 0xc0, 0x20, 0x00, 0x00, 0x02, 0x8a, 0xc0, 0xe5, 0x00, 0x00, + 0x01, 0x56, 0xa7, 0xf4, 0xff, 0xeb, 0x07, 0x07, 0xc0, 0xf0, 0x00, 0x00, + 0x4e, 0x5c, 0xc0, 0x20, 0x00, 0x00, 0x00, 0x7d, 0xe3, 0x20, 0x20, 0x00, + 0x00, 0x04, 0xc0, 0x30, 0x00, 0x00, 0x96, 0xa3, 0xb9, 0x0b, 0x00, 0x32, + 0xb9, 0x02, 0x00, 0x33, 0xa7, 0x84, 0x00, 0x19, 0xa7, 0x3b, 0xff, 0xff, + 0xeb, 0x43, 0x00, 0x08, 0x00, 0x0c, 0xb9, 0x02, 0x00, 0x44, 0xb9, 0x04, + 0x00, 0x12, 0xa7, 0x84, 0x00, 0x09, 0xd7, 0xff, 0x10, 0x00, 0x10, 0x00, + 0x41, 0x10, 0x11, 0x00, 0xa7, 0x47, 0xff, 0xfb, 0xc0, 0x20, 0x00, 0x00, + 0x00, 0x0d, 0x44, 0x30, 0x20, 0x00, 0xc0, 0x20, 0x00, 0x00, 0x00, 0x5b, + 0xd2, 0x0f, 0x01, 0xd0, 0x20, 0x00, 0xa7, 0xf4, 0xff, 0xa1, 0xd7, 0x00, + 0x10, 0x00, 0x10, 0x00, 0xc0, 0x10, 0x00, 0x00, 0x00, 0x50, 0xb2, 0xb2, + 0x10, 0x00, 0xa7, 0xf4, 0x00, 0x00, 0xeb, 0x00, 0xf0, 0x00, 0x00, 0x25, + 0x96, 0x02, 0xf0, 0x06, 0xeb, 0x00, 0xf0, 0x00, 0x00, 0x2f, 0xc0, 0x10, + 0x00, 0x00, 0x00, 0x2a, 0xe3, 0x10, 0x01, 0xb8, 0x00, 0x24, 0xc0, 0x10, + 0x00, 0x00, 0x00, 0x4b, 0xd2, 0x07, 0x01, 0xb0, 0x10, 0x00, 0xc0, 0x10, + 0x00, 0x00, 0x00, 0x3d, 0xb2, 0xb2, 0x10, 0x00, 0xeb, 0x66, 0xf0, 0x00, + 0x00, 0x25, 0x96, 0xff, 0xf0, 0x04, 0xeb, 0x66, 0xf0, 0x00, 0x00, 0x2f, + 0xc0, 0x10, 0x00, 0x00, 0x00, 0x1a, 0xe3, 0x10, 0x01, 0xf8, 0x00, 0x24, + 0xc0, 0x10, 0x00, 0x00, 0x00, 0x36, 0xd2, 0x07, 0x01, 0xf0, 0x10, 0x00, + 0xc0, 0x10, 0x00, 0x00, 0x00, 0x24, 0xb2, 0xb2, 0x10, 0x00, 0xeb, 0x00, + 0xf0, 0x00, 0x00, 0x25, 0x94, 0xfd, 0xf0, 0x06, 0xeb, 0x00, 0xf0, 0x00, + 0x00, 0x2f, 0x07, 0xfe, 0xeb, 0x66, 0xf0, 0x00, 0x00, 0x25, 0x94, 0x00, + 0xf0, 0x04, 0xeb, 0x66, 0xf0, 0x00, 0x00, 0x2f, 0x07, 0xfe, 0x07, 0x07, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0xf0, 0x00, 0x02, 0x00, 0x01, + 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x03, 0x02, 0x00, 0x01, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x80, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x80, 0x00, 0x00, 0x00, 0xeb, 0xbf, 0xf0, 0x58, + 0x00, 0x24, 0xc0, 0x10, 0x00, 0x00, 0x4e, 0x0d, 0xa7, 0xfb, 0xff, 0x60, + 0xb2, 0x20, 0x00, 0x21, 0xb2, 0x22, 0x00, 0xb0, 0x88, 0xb0, 0x00, 0x1c, + 0xc0, 0xe5, 0xff, 0xff, 0xff, 0x91, 0xa7, 0xbe, 0x00, 0x03, 0xa7, 0x84, + 0x00, 0x13, 0xa7, 0xbe, 0x00, 0x02, 0xa7, 0x28, 0x00, 0x00, 0xa7, 0x74, + 0x00, 0x04, 0xa7, 0x28, 0xff, 0xfe, 0xe3, 0x40, 0xf1, 0x10, 0x00, 0x04, + 0xb9, 0x14, 0x00, 0x22, 0xeb, 0xbf, 0xf0, 0xf8, 0x00, 0x04, 0x07, 0xf4, + 0xa7, 0x28, 0xff, 0xff, 0xa7, 0xf4, 0xff, 0xf5, 0x07, 0x07, 0x07, 0x07, + 0xeb, 0xbf, 0xf0, 0x58, 0x00, 0x24, 0xc0, 0xd0, 0x00, 0x00, 0x01, 0x25, + 0xa7, 0xfb, 0xff, 0x60, 0xa7, 0xb9, 0x00, 0x00, 0xa7, 0x19, 0x00, 0x00, + 0xc0, 0x40, 0x00, 0x00, 0x4d, 0xd8, 0xa7, 0x3b, 0x00, 0x01, 0xa7, 0x37, + 0x00, 0x23, 0xc0, 0x20, 0x00, 0x00, 0x4d, 0xd1, 0x18, 0x31, 0xa7, 0x1a, + 0x00, 0x06, 0x40, 0x10, 0x20, 0x08, 0xa7, 0x3a, 0x00, 0x0e, 0xa7, 0x18, + 0x1a, 0x00, 0x40, 0x30, 0x20, 0x00, 0x92, 0x00, 0x20, 0x02, 0x40, 0x10, + 0x20, 0x0a, 0xe3, 0x20, 0xd0, 0x00, 0x00, 0x04, 0xc0, 0xe5, 0xff, 0xff, + 0xff, 0xac, 0xe3, 0x40, 0xf1, 0x10, 0x00, 0x04, 0xb9, 0x04, 0x00, 0x2b, + 0xeb, 0xbf, 0xf0, 0xf8, 0x00, 0x04, 0x07, 0xf4, 0xb9, 0x04, 0x00, 0x51, + 0xa7, 0x5b, 0x00, 0x01, 0xa7, 0x09, 0x0f, 0xf7, 0xb9, 0x21, 0x00, 0x50, + 0xa7, 0x24, 0xff, 0xd7, 0x41, 0xeb, 0x20, 0x00, 0x95, 0x0a, 0xe0, 0x00, + 0xa7, 0x74, 0x00, 0x08, 0x41, 0x11, 0x40, 0x0e, 0x92, 0x0d, 0x10, 0x00, + 0xb9, 0x04, 0x00, 0x15, 0x43, 0x5b, 0x20, 0x00, 0x42, 0x51, 0x40, 0x0e, + 0xa7, 0xbb, 0x00, 0x01, 0x41, 0x10, 0x10, 0x01, 0xa7, 0xf4, 0xff, 0xbf, + 0xc0, 0x50, 0x00, 0x00, 0x00, 0xd8, 0xc0, 0x10, 0x00, 0x00, 0x4d, 0x8d, + 0xa7, 0x48, 0x00, 0x1c, 0x40, 0x40, 0x10, 0x00, 0x50, 0x20, 0x10, 0x0c, + 0xa7, 0x48, 0x00, 0x04, 0xe3, 0x20, 0x50, 0x00, 0x00, 0x04, 0x40, 0x40, + 0x10, 0x0a, 0x50, 0x30, 0x10, 0x10, 0xc0, 0xf4, 0xff, 0xff, 0xff, 0x6b, + 0xa7, 0x39, 0x00, 0x40, 0xa7, 0x29, 0x00, 0x00, 0xc0, 0xf4, 0xff, 0xff, + 0xff, 0xe4, 0x07, 0x07, 0xb9, 0x04, 0x00, 0x13, 0xa7, 0x2a, 0xff, 0xff, + 0xb9, 0x04, 0x00, 0x34, 0xa7, 0x48, 0x00, 0x01, 0x15, 0x24, 0xa7, 0x24, + 0x00, 0x07, 0xb9, 0x04, 0x00, 0x21, 0xc0, 0xf4, 0xff, 0xff, 0xff, 0x7f, + 0xa7, 0x29, 0xff, 0xff, 0x07, 0xfe, 0x07, 0x07, 0xa7, 0x39, 0x00, 0x00, + 0x41, 0x13, 0x20, 0x00, 0x95, 0x00, 0x10, 0x00, 0xa7, 0x74, 0x00, 0x05, + 0xc0, 0xf4, 0xff, 0xff, 0xff, 0x70, 0xa7, 0x3b, 0x00, 0x01, 0xa7, 0xf4, + 0xff, 0xf5, 0x07, 0x07, 0xeb, 0xbf, 0xf0, 0x58, 0x00, 0x24, 0xc0, 0xd0, + 0x00, 0x00, 0x00, 0x95, 0xa7, 0xfb, 0xff, 0x60, 0xb9, 0x04, 0x00, 0xb2, + 0xa7, 0x19, 0x00, 0x20, 0xc0, 0x20, 0x00, 0x00, 0x4d, 0x40, 0x92, 0x00, + 0x20, 0x00, 0xa7, 0x2b, 0x00, 0x01, 0xa7, 0x17, 0xff, 0xfc, 0xc0, 0x10, + 0x00, 0x00, 0x4d, 0x37, 0xa7, 0x28, 0x10, 0x00, 0x40, 0x20, 0x10, 0x00, + 0xe3, 0x20, 0xd0, 0x00, 0x00, 0x04, 0xc0, 0xe5, 0xff, 0xff, 0xff, 0x1d, + 0x12, 0x22, 0xa7, 0x74, 0x00, 0x19, 0xa7, 0x19, 0x00, 0x00, 0xc0, 0x40, + 0x00, 0x00, 0x00, 0x79, 0xa7, 0x39, 0x00, 0x08, 0xc0, 0x20, 0x00, 0x00, + 0x4d, 0x2c, 0x41, 0x21, 0x20, 0x00, 0xe3, 0x20, 0x20, 0x00, 0x00, 0x90, + 0x43, 0x22, 0x40, 0x00, 0x42, 0x21, 0xb0, 0x00, 0xa7, 0x1b, 0x00, 0x01, + 0xa7, 0x37, 0xff, 0xf2, 0xe3, 0x40, 0xf1, 0x10, 0x00, 0x04, 0xeb, 0xbf, + 0xf0, 0xf8, 0x00, 0x04, 0x07, 0xf4, 0x07, 0x07, 0xeb, 0xaf, 0xf0, 0x50, + 0x00, 0x24, 0xc0, 0xd0, 0x00, 0x00, 0x00, 0x55, 0xa7, 0xfb, 0xff, 0x60, + 0xa7, 0x19, 0x0f, 0xf8, 0xb9, 0x21, 0x00, 0x31, 0xb9, 0x04, 0x00, 0xa2, + 0xa7, 0xc4, 0x00, 0x2a, 0xa7, 0xb9, 0x0f, 0xf8, 0xc0, 0x10, 0x00, 0x00, + 0x4c, 0xf6, 0xa7, 0x28, 0x10, 0x00, 0x40, 0x20, 0x10, 0x00, 0x92, 0x00, + 0x10, 0x02, 0xe3, 0x20, 0xd0, 0x00, 0x00, 0x04, 0xc0, 0xe5, 0xff, 0xff, + 0xfe, 0xda, 0xa7, 0xbb, 0x00, 0x01, 0xa7, 0x19, 0x00, 0x00, 0xa7, 0xb7, + 0x00, 0x17, 0xc0, 0x10, 0x00, 0x00, 0x4c, 0xe1, 0xe3, 0x40, 0xf1, 0x10, + 0x00, 0x04, 0xe3, 0x20, 0x10, 0x08, 0x00, 0x91, 0xa7, 0x2a, 0xff, 0xf9, + 0xb9, 0x14, 0x00, 0x22, 0xeb, 0xaf, 0xf0, 0xf0, 0x00, 0x04, 0x07, 0xf4, + 0xb9, 0x04, 0x00, 0xb3, 0xa7, 0xf4, 0xff, 0xd8, 0xc0, 0x20, 0x00, 0x00, + 0x4c, 0xcc, 0x41, 0x31, 0xa0, 0x00, 0x41, 0x21, 0x20, 0x00, 0xa7, 0x1b, + 0x00, 0x01, 0xd2, 0x00, 0x30, 0x00, 0x20, 0x0f, 0xa7, 0xf4, 0xff, 0xdd, + 0x07, 0x07, 0x07, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x76, 0x00, 0x05, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x02, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x77, 0x00, 0x05, + 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, + 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, + 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, + 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, + 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, + 0x2e, 0x2e, 0x2e, 0x2e, 0x20, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, + 0x2e, 0x2e, 0x2e, 0x2e, 0x3c, 0x28, 0x2b, 0x7c, 0x26, 0x2e, 0x2e, 0x2e, + 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x21, 0x24, 0x2a, 0x29, 0x3b, 0x2e, + 0x2d, 0x2f, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2c, + 0x25, 0x5f, 0x3e, 0x3f, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, + 0x2e, 0x60, 0x3a, 0x23, 0x40, 0x27, 0x3d, 0x22, 0x2e, 0x61, 0x62, 0x63, + 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, + 0x2e, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x2e, 0x2e, + 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, + 0x79, 0x7a, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, + 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, + 0x2e, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x2e, 0x2e, + 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x4a, 0x4b, 0x4c, 0x4d, 0x4e, 0x4f, 0x50, + 0x51, 0x52, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x53, 0x54, + 0x55, 0x56, 0x57, 0x58, 0x59, 0x5a, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, 0x2e, + 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38, 0x39, 0x2e, 0x2e, + 0x2e, 0x2e, 0x2e, 0x2e, 0x41, 0x00, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x6f, 0xff, 0xfe, 0xf5, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xd8, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x02, 0x28, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xf8, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x0a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x15, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x30, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x09, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1e, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x00, 0x6f, 0xff, 0xff, 0xfb, + 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x6f, 0xff, 0xff, 0xf9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x17, 0xb8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x47, 0x43, 0x43, 0x3a, + 0x20, 0x28, 0x55, 0x62, 0x75, 0x6e, 0x74, 0x75, 0x20, 0x31, 0x31, 0x2e, + 0x34, 0x2e, 0x30, 0x2d, 0x31, 0x75, 0x62, 0x75, 0x6e, 0x74, 0x75, 0x31, + 0x7e, 0x32, 0x32, 0x2e, 0x30, 0x34, 0x29, 0x20, 0x31, 0x31, 0x2e, 0x34, + 0x2e, 0x30, 0x00, 0x00, 0x2e, 0x73, 0x68, 0x73, 0x74, 0x72, 0x74, 0x61, + 0x62, 0x00, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x00, 0x2e, 0x67, + 0x6e, 0x75, 0x2e, 0x68, 0x61, 0x73, 0x68, 0x00, 0x2e, 0x64, 0x79, 0x6e, + 0x73, 0x79, 0x6d, 0x00, 0x2e, 0x64, 0x79, 0x6e, 0x73, 0x74, 0x72, 0x00, + 0x2e, 0x72, 0x65, 0x6c, 0x61, 0x2e, 0x64, 0x79, 0x6e, 0x00, 0x2e, 0x74, + 0x65, 0x78, 0x74, 0x00, 0x2e, 0x72, 0x6f, 0x64, 0x61, 0x74, 0x61, 0x00, + 0x2e, 0x64, 0x79, 0x6e, 0x61, 0x6d, 0x69, 0x63, 0x00, 0x2e, 0x67, 0x6f, + 0x74, 0x00, 0x2e, 0x62, 0x73, 0x73, 0x00, 0x2e, 0x63, 0x6f, 0x6d, 0x6d, + 0x65, 0x6e, 0x74, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xc8, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xc8, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 0x6f, 0xff, 0xff, 0xf6, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0xd8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xd8, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1d, + 0x00, 0x00, 0x00, 0x0b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0xf8, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, + 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, + 0x00, 0x00, 0x00, 0x25, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x28, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x28, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2d, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x02, 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x30, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0x00, 0x00, 0x00, 0x37, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x48, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x02, 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x40, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x3d, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x88, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x06, 0x88, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x24, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x45, 0x00, 0x00, 0x00, 0x06, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x17, 0xb8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x07, 0xb8, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x20, 0x00, 0x00, 0x00, 0x04, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x4e, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, 0xd8, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x08, 0xd8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x18, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x53, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x01, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x30, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x08, 0xf0, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2b, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, + 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x09, 0x1b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x61, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 +};