From 1dd2553dcd653a4eca7dd5b410918b3b70336df7 Mon Sep 17 00:00:00 2001 From: "James J. Nutaro" Date: Fri, 27 Jul 2018 15:16:50 -0400 Subject: [PATCH V10] Module for synchronizing with a simulator This patch adds an interface for pacing the execution of QEMU to match an external simulation clock. Its aim is to permit QEMU to be used as a module within a larger simulation system. The updates a prior patch by changing the synchronization channel from a UNIX domain socket to a shared memory segment with access control via semaphores. Signed-off-by: James J. Nutaro --- Makefile.target | 1 + accel/kvm/kvm-all.c | 10 +++ cpus.c | 8 ++ docs/external-sim.txt | 20 +++++ external_sim.c | 211 ++++++++++++++++++++++++++++++++++++++++++++++++++ external_sim.h | 29 +++++++ include/sysemu/cpus.h | 1 + qemu-options.hx | 8 ++ vl.c | 18 ++++- 9 files changed, 305 insertions(+), 1 deletion(-) create mode 100644 docs/external-sim.txt create mode 100644 external_sim.c create mode 100644 external_sim.h diff --git a/Makefile.target b/Makefile.target index f9a9da7..61bd0cb 100644 --- a/Makefile.target +++ b/Makefile.target @@ -91,6 +91,7 @@ all: $(PROGS) stap ######################################################### # cpu emulator library +obj-y += external_sim.o obj-y += exec.o obj-y += accel/ obj-$(CONFIG_TCG) += tcg/tcg.o tcg/tcg-op.o tcg/optimize.o diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c index f290f48..f42546a 100644 --- a/accel/kvm/kvm-all.c +++ b/accel/kvm/kvm-all.c @@ -40,6 +40,7 @@ #include "hw/irq.h" #include "hw/boards.h" +#include "external_sim.h" /* This check must be after config-host.h is included */ #ifdef CONFIG_EVENTFD @@ -1863,6 +1864,15 @@ int kvm_cpu_exec(CPUState *cpu) do { MemTxAttrs attrs; + if (external_sim_enabled()) { + /* Pause here while synchronizing with a simulation clock. + * We do not want to execute instructions past the synchronization + * deadline, but it is ok to update the states of other equipment + * like timers, i/o devices, etc. + */ + external_sim_sync(); + } + if (cpu->vcpu_dirty) { kvm_arch_put_registers(cpu, KVM_PUT_RUNTIME_STATE); cpu->vcpu_dirty = false; diff --git a/cpus.c b/cpus.c index 96bb688..222dea2 100644 --- a/cpus.c +++ b/cpus.c @@ -2068,3 +2068,11 @@ void dump_drift_info(FILE *f, fprintf_function cpu_fprintf) cpu_fprintf(f, "Max guest advance NA\n"); } } + +void kick_all_vcpus(void) +{ + CPUState *cpu; + CPU_FOREACH(cpu) { + qemu_cpu_kick(cpu); + } +} diff --git a/docs/external-sim.txt b/docs/external-sim.txt new file mode 100644 index 0000000..e161112 --- /dev/null +++ b/docs/external-sim.txt @@ -0,0 +1,20 @@ += Synchronizing the virtual clock with an external source = + +QEMU has a protocol for synchronizing its virtual clock +with the clock of a simulator in which QEMU is embedded +as a component. This options is enabled with the -external_sim +argument, and it should generally be accompanied by the +following additional command line arguments: + +-icount 1,sleep=off -rtc clock=vm + or +-enable-kvm -cpu kvm=off,-tsc,-kvmclock -rtc clock=vm + +You can the simulator side of the synchronization protocoal and +an example of its use in the adevs simulation package at + +http://sourceforge.net/projects/adevs/ + +The files src/qemu/qemu_sync.h and src/qemu/qemu_sync.cpp in that +package contain standalone implementations of the synchronization +protocol. diff --git a/external_sim.c b/external_sim.c new file mode 100644 index 0000000..f8e27b2 --- /dev/null +++ b/external_sim.c @@ -0,0 +1,211 @@ +#include "qemu/osdep.h" +#include "qemu/timer.h" +#include "sysemu/cpus.h" +#include "sysemu/kvm.h" +#include "qemu/error-report.h" +#include "external_sim.h" +#include +#include +#include +#include +#include +#include +#include +#include + +/* This is a Linux only feature */ + +#ifndef _WIN32 + +/* Semaphores and shared memory segments for coordinating + * communication with a simulator. + */ +#define NAME_SIZE 100 +static char name_str[NAME_SIZE]; +static sem_t *sem[3]; +static long* buf; +static const size_t memlen = 2*sizeof(long); +static const char* semname[3] = { + "/qemu_sem_a", + "/qemu_sem_b", + "/qemu_sem_c" +}; +static const char* memname = "/qemu_mem"; + +static void init_external_sim(void) { + int idx, mem_fd; + void *addr; + /* Get the shared memory segment that was created by the simulator */ + snprintf(name_str, NAME_SIZE, "%s_%d", memname, getppid()); + mem_fd = shm_open(name_str, O_RDWR, 0); + if (mem_fd == -1) { + error_report("shm_open failed"); + exit(0); + } + /* Remove the name because we don't need it anymore. Shared memory + * will be deleted by the OS when this process and the simulator exit. */ + shm_unlink(name_str); + /* Get the semaphores that were created by the simulator. */ + for (idx = 0; idx < 3; idx++) { + /* Get the semaphore */ + snprintf(name_str, NAME_SIZE, "%s_%d", semname[idx], getppid()); + sem[idx] = sem_open(name_str, O_RDWR); + if (sem[idx] == SEM_FAILED) { + error_report("sem_open failed"); + exit(0); + } + /* Remove the name because we don't need it anymore. Semaphore + * might be deleted by the OS when this process and the simulator + * exit. Simulator should try to delete it on exit. */ + sem_unlink(name_str); + } + /* Map the shared memory segment into our address space. This space + * has already been created by the simulator. */ + addr = mmap(NULL, memlen, PROT_READ|PROT_WRITE, MAP_SHARED, mem_fd, 0); + if (addr == (void*)-1) { + error_report("mmap failed:"); + exit(0); + } + /* We'll be writing and reading longs to the shared memory. */ + buf = (long*)addr; +} + +static void handshake_sim(void) { + /* The simulator is waiting for us to post here after initializing + * the shared semaphores and memory. This lets it know that we + * are done getting set up. */ + sem_post(sem[0]); +} + +static void run_sim(long *h) { + /* Wait for the simulator to write a time step to shared memory + * and then go get that time step value. */ + sem_wait(sem[1]); + *h = *(buf); +} + +static void sync_sim(long e, long h) { + /* Tell the simulator how large a time step we would like and + * how long a step we actually took when we ran. */ + *(buf) = h; + *(buf + 1) = e; + sem_post(sem[2]); +} + +static bool enabled = false, syncing = true; +static int64_t t; +static QEMUTimer *sync_timer; +static QemuMutex external_sim_mutex; +static QemuCond external_sim_cond; + +bool external_sim_enabled(void) +{ + return enabled; +} + +void external_sim_sync(void) +{ + /* kvm-all.c will call this function before running + * instructions with kvm. Because syncing will be + * true while external_sim is waiting for a new time advance + * from the simulation, no instructions will execute + * while the machine is supposed to be suspended in + * simulation time. + */ + qemu_mutex_lock(&external_sim_mutex); + while (syncing) { + qemu_cond_wait(&external_sim_cond, &external_sim_mutex); + } + qemu_mutex_unlock(&external_sim_mutex); +} + +static void start_emulator(void) +{ + if (kvm_enabled()) { + /* Setting syncing to false tells kvm-all that + * it can execute guest instructions. + */ + qemu_mutex_lock(&external_sim_mutex); + syncing = false; + qemu_mutex_unlock(&external_sim_mutex); + qemu_cond_signal(&external_sim_cond); + /* Restart the emulator clock */ + cpu_enable_ticks(); + } +} + +static void stop_emulator(void) +{ + if (kvm_enabled()) { + /* Tell the emulator that it is not allowed to + * execute guest instructions. + */ + qemu_mutex_lock(&external_sim_mutex); + syncing = true; + qemu_mutex_unlock(&external_sim_mutex); + /* Kick KVM off of the CPU and stop the emulator clock. */ + cpu_disable_ticks(); + kick_all_vcpus(); + } +} + +static void schedule_next_event(void) +{ + int64_t h, elapsed; + /* Report the actual elapsed time to the external simulator. */ + h = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL); + elapsed = h - t; + t = h; + /* Request a time advance and report the elapsed time */ + h = qemu_clock_deadline_ns_all(QEMU_CLOCK_VIRTUAL); + if (h < 0) { + h = LONG_MAX; + } + sync_sim(elapsed, h); + /* Get the allowed time advance. This will be less than or + * equal to the request. */ + run_sim(&h); + /* Schedule the next synchronization point */ + timer_mod(sync_timer, t + h); + /* Start advancing cpu ticks and the wall clock */ + start_emulator(); +} + +static void sync_func(void *data) +{ + /* Stop advancing cpu ticks and the wall clock */ + stop_emulator(); + /* Schedule the next event */ + schedule_next_event(); +} + +void setup_external_sim(void) +{ + /* The module has been enabled */ + enabled = true; + if (kvm_enabled()) { + qemu_mutex_init(&external_sim_mutex); + qemu_cond_init(&external_sim_cond); + } + /* Stop the clock while the simulation is initialized */ + stop_emulator(); + /* Setup the synchronization channel */ + init_external_sim(); + handshake_sim(); + /* Initialize the simulation clock */ + t = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL); + /* Start the timer to ensure time warps advance the clock */ + sync_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, sync_func, NULL); + /* Get the time advance that is requested by the simulation */ + schedule_next_event(); +} + +#else + +void setup_external_sim(void) +{ + error_report(stderr, "-external_sim is not supported on Windows, exiting\n"); + exit(0); +} + +#endif diff --git a/external_sim.h b/external_sim.h new file mode 100644 index 0000000..d619ebd --- /dev/null +++ b/external_sim.h @@ -0,0 +1,29 @@ +/* + * This work is licensed under the terms of the GNU GPL + * version 2. Seethe COPYING file in the top-level directory. + * + * A module for pacing the rate of advance of the computer + * clock in reference to an external simulation clock. The + * basic approach used here is adapted from QBox from Green + * Socs. The simulator uses shared memory to exchange timing + * data. The external simulator starts the exchange by forking a + * QEMU process and creating semaphores and shared memory with + * a well known name. The external simulator use these to get + * time advance requests from QEMU and supply time advance + * grants in return. The QEMU side of the protocol is implemented + * here. The simulator side of the protocol is available in + * the qemu_sync.cpp and qemu_sync.h files that are included in + * the adevs simulation package. + * + * Authors: + * James Nutaro + * + */ +#ifndef EXTERNAL_SIM_H +#define EXTERNAL_SIM_H + +void external_sim_sync(void); +bool external_sim_enabled(void); +void setup_external_sim(void); + +#endif diff --git a/include/sysemu/cpus.h b/include/sysemu/cpus.h index 731756d..7fde006 100644 --- a/include/sysemu/cpus.h +++ b/include/sysemu/cpus.h @@ -10,6 +10,7 @@ void resume_all_vcpus(void); void pause_all_vcpus(void); void cpu_stop_current(void); void cpu_ticks_init(void); +void kick_all_vcpus(void); void configure_icount(QemuOpts *opts, Error **errp); extern int use_icount; diff --git a/qemu-options.hx b/qemu-options.hx index 57f2c6a..bfb8f85 100644 --- a/qemu-options.hx +++ b/qemu-options.hx @@ -4473,6 +4473,14 @@ contents of @code{iv.b64} to the second secret ETEXI +DEF("external-sim", 0, QEMU_OPTION_external_sim, \ + "-external-sim enable synch with a simulation clock\n", QEMU_ARCH_ALL) +STEXI +@item -external-sim +@findex -external-sim +Qemu will synchronize its virtual clock with the simulation clock +in an external program. +ETEXI HXCOMM This is the last statement. Insert new options before this line! STEXI diff --git a/vl.c b/vl.c index 1ad1c04..af13959 100644 --- a/vl.c +++ b/vl.c @@ -129,9 +129,12 @@ int main(int argc, char **argv) #include "qapi/qmp/qerror.h" #include "sysemu/iothread.h" +#include "external_sim.h" + #define MAX_VIRTIO_CONSOLES 1 #define MAX_SCLP_CONSOLES 1 +int use_external_sim = 0; static const char *data_dir[16]; static int data_dir_idx; const char *bios_name = NULL; @@ -824,7 +827,7 @@ int qemu_timedate_diff(struct tm *tm) struct tm tmp = *tm; tmp.tm_isdst = -1; /* use timezone to figure it out */ seconds = mktime(&tmp); - } + } else seconds = mktimegm(tm) + rtc_date_offset; @@ -3498,6 +3501,9 @@ int main(int argc, char **argv, char **envp) case QEMU_OPTION_soundhw: select_soundhw (optarg); break; + case QEMU_OPTION_external_sim: + use_external_sim = 1; + break; case QEMU_OPTION_h: help(0); break; @@ -4605,6 +4611,16 @@ int main(int argc, char **argv, char **envp) /* spice needs the timers to be initialized by this point */ qemu_spice_init(); + if (use_external_sim) { + if (!(rtc_clock == QEMU_CLOCK_VIRTUAL && ( + (icount_opts && !qemu_opt_get_bool(icount_opts, "sleep", true)) || + kvm_enabled()))) { + error_report("-external_sim requires -rtc clock=vm and either " + "icount -1,sleep=off or -enable-kvm"); + exit(1); + } + setup_external_sim(); + } cpu_ticks_init(); if (icount_opts) { if (!tcg_enabled()) { -- 2.7.4