cpu: Convert cpu_index into a bitmap

Currently CPUState::cpu_index is monotonically increasing and a newly
created CPU always gets the next higher index. The next available
index is calculated by counting the existing number of CPUs. This is
fine as long as we only add CPUs, but there are architectures which
are starting to support CPU removal, too. For an architecture like PowerPC
which derives its CPU identifier (device tree ID) from cpu_index, the
existing logic of generating cpu_index values causes problems.

With the currently proposed method of handling vCPU removal by parking
the vCPU fd in QEMU
(Ref: http://lists.gnu.org/archive/html/qemu-devel/2015-02/msg02604.html),
generating cpu_index this way will not work for PowerPC.

This patch changes the way cpu_index is handed out by maintaining
a bit map of the CPUs that tracks both addition and removal of CPUs.

The CPU bitmap allocation logic is part of cpu_exec_init(), which is
called by instance_init routines of various CPU targets. Newly added
cpu_exec_exit() API handles the deallocation part and this routine is
called from generic CPU instance_finalize.

Note: This new CPU enumeration is for !CONFIG_USER_ONLY only.
CONFIG_USER_ONLY continues to have the old enumeration logic.

Backports commit b7bca7333411bd19c449147e8202ae6b0e4a8e09 from qemu

include/uc_priv.h

@@ -58,6 +58,7 @@ typedef void (*uc_args_void_t)(void*);
 
 typedef void (*uc_args_uc_t)(struct uc_struct*);
 typedef int (*uc_args_int_uc_t)(struct uc_struct*);
+typedef void (*uc_cpu_exec_exit_t)(CPUState*);
 
 typedef bool (*uc_args_tcg_enable_t)(struct uc_struct*);
 
@@ -162,6 +163,7 @@ struct uc_struct {
     uc_args_int_t stop_interrupt;   // check if the interrupt should stop emulation
 
     uc_args_uc_t init_arch, cpu_exec_init_all;
+    uc_cpu_exec_exit_t cpu_exec_exit;
     uc_args_int_uc_t vm_start;
     uc_args_tcg_enable_t tcg_enabled;
     uc_args_uc_long_t tcg_exec_init;

qemu/aarch64.h

@@ -276,6 +276,7 @@
 #define cpu_common_reset cpu_common_reset_aarch64
 #define cpu_dump_statistics cpu_dump_statistics_aarch64
 #define cpu_exec cpu_exec_aarch64
+#define cpu_exec_exit cpu_exec_exit_aarch64
 #define cpu_exec_init cpu_exec_init_aarch64
 #define cpu_exec_init_all cpu_exec_init_all_aarch64
 #define cpu_exec_step_atomic cpu_exec_step_atomic_aarch64

qemu/aarch64eb.h

@@ -276,6 +276,7 @@
 #define cpu_common_reset cpu_common_reset_aarch64eb
 #define cpu_dump_statistics cpu_dump_statistics_aarch64eb
 #define cpu_exec cpu_exec_aarch64eb
+#define cpu_exec_exit cpu_exec_exit_aarch64eb
 #define cpu_exec_init cpu_exec_init_aarch64eb
 #define cpu_exec_init_all cpu_exec_init_all_aarch64eb
 #define cpu_exec_step_atomic cpu_exec_step_atomic_aarch64eb

qemu/arm.h

@@ -276,6 +276,7 @@
 #define cpu_common_reset cpu_common_reset_arm
 #define cpu_dump_statistics cpu_dump_statistics_arm
 #define cpu_exec cpu_exec_arm
+#define cpu_exec_exit cpu_exec_exit_arm
 #define cpu_exec_init cpu_exec_init_arm
 #define cpu_exec_init_all cpu_exec_init_all_arm
 #define cpu_exec_step_atomic cpu_exec_step_atomic_arm

qemu/armeb.h

@@ -276,6 +276,7 @@
 #define cpu_common_reset cpu_common_reset_armeb
 #define cpu_dump_statistics cpu_dump_statistics_armeb
 #define cpu_exec cpu_exec_armeb
+#define cpu_exec_exit cpu_exec_exit_armeb
 #define cpu_exec_init cpu_exec_init_armeb
 #define cpu_exec_init_all cpu_exec_init_all_armeb
 #define cpu_exec_step_atomic cpu_exec_step_atomic_armeb

qemu/exec.c

@@ -577,18 +577,78 @@ AddressSpace *cpu_get_address_space(CPUState *cpu, int asidx)
 }
 #endif
 
+#ifndef CONFIG_USER_ONLY
+static int cpu_get_free_index(Error **errp)
+{
+    // Unicorn: if'd out
+#if 0
+    int cpu = find_first_zero_bit(cpu_index_map, MAX_CPUMASK_BITS);
+
+    if (cpu >= MAX_CPUMASK_BITS) {
+        error_setg(errp, "Trying to use more CPUs than max of %d",
+                   MAX_CPUMASK_BITS);
+        return -1;
+    }
+
+    bitmap_set(cpu_index_map, cpu, 1);
+    return cpu;
+#endif
+    return 0;
+}
+
+void cpu_exec_exit(CPUState *cpu)
+{
+    if (cpu->cpu_index == -1) {
+        /* cpu_index was never allocated by this @cpu or was already freed. */
+        return;
+    }
+
+    // Unicorn: if'd out
+#if 0
+    bitmap_clear(cpu_index_map, cpu->cpu_index, 1);
+#endif
+    cpu->cpu_index = -1;
+}
+#else
+
+static int cpu_get_free_index(Error **errp)
+{
+    // Unicorn: if'd out
+#if 0
+    CPUState *some_cpu;
+    int cpu_index = 0;
+
+    CPU_FOREACH(some_cpu) {
+        cpu_index++;
+    }
+    return cpu_index;
+#endif
+    return 0;
+}
+
+void cpu_exec_exit(CPUState *cpu)
+{
+}
+#endif
+
 void cpu_exec_init(CPUState *cpu, Error **errp, void *opaque)
 {
     struct uc_struct *uc = opaque;
     CPUClass *cc = CPU_GET_CLASS(uc, cpu);
     CPUArchState *env = cpu->env_ptr;
+    Error *local_err = NULL;
 
     cpu->as = NULL;
-    cpu->cpu_index = 0;
     cpu->num_ases = 0;
     cpu->uc = uc;
     env->uc = uc;
 
+    cpu->cpu_index = cpu_get_free_index(&local_err);
+    if (local_err) {
+        error_propagate(errp, local_err);
+        return;
+    }
+
     // TODO: assert uc does not already have a cpu?
     uc->cpu = cpu;
 

qemu/header_gen.py

@@ -282,6 +282,7 @@ symbols = (
     'cpu_common_reset',
     'cpu_dump_statistics',
     'cpu_exec',
+    'cpu_exec_exit',
     'cpu_exec_init',
     'cpu_exec_init_all',
     'cpu_exec_step_atomic',

qemu/include/qom/cpu.h

@@ -807,6 +807,7 @@ static inline bool cpu_breakpoint_test(CPUState *cpu, vaddr pc, int mask)
 
 void QEMU_NORETURN cpu_abort(CPUState *cpu, const char *fmt, ...)
     GCC_FMT_ATTR(2, 3);
+void cpu_exec_exit(CPUState *cpu);
 
 void cpu_register_types(struct uc_struct *uc);
 

qemu/m68k.h

@@ -276,6 +276,7 @@
 #define cpu_common_reset cpu_common_reset_m68k
 #define cpu_dump_statistics cpu_dump_statistics_m68k
 #define cpu_exec cpu_exec_m68k
+#define cpu_exec_exit cpu_exec_exit_m68k
 #define cpu_exec_init cpu_exec_init_m68k
 #define cpu_exec_init_all cpu_exec_init_all_m68k
 #define cpu_exec_step_atomic cpu_exec_step_atomic_m68k

qemu/mips.h

@@ -276,6 +276,7 @@
 #define cpu_common_reset cpu_common_reset_mips
 #define cpu_dump_statistics cpu_dump_statistics_mips
 #define cpu_exec cpu_exec_mips
+#define cpu_exec_exit cpu_exec_exit_mips
 #define cpu_exec_init cpu_exec_init_mips
 #define cpu_exec_init_all cpu_exec_init_all_mips
 #define cpu_exec_step_atomic cpu_exec_step_atomic_mips

qemu/mips64.h

@@ -276,6 +276,7 @@
 #define cpu_common_reset cpu_common_reset_mips64
 #define cpu_dump_statistics cpu_dump_statistics_mips64
 #define cpu_exec cpu_exec_mips64
+#define cpu_exec_exit cpu_exec_exit_mips64
 #define cpu_exec_init cpu_exec_init_mips64
 #define cpu_exec_init_all cpu_exec_init_all_mips64
 #define cpu_exec_step_atomic cpu_exec_step_atomic_mips64

qemu/mips64el.h

@@ -276,6 +276,7 @@
 #define cpu_common_reset cpu_common_reset_mips64el
 #define cpu_dump_statistics cpu_dump_statistics_mips64el
 #define cpu_exec cpu_exec_mips64el
+#define cpu_exec_exit cpu_exec_exit_mips64el
 #define cpu_exec_init cpu_exec_init_mips64el
 #define cpu_exec_init_all cpu_exec_init_all_mips64el
 #define cpu_exec_step_atomic cpu_exec_step_atomic_mips64el

qemu/mipsel.h

@@ -276,6 +276,7 @@
 #define cpu_common_reset cpu_common_reset_mipsel
 #define cpu_dump_statistics cpu_dump_statistics_mipsel
 #define cpu_exec cpu_exec_mipsel
+#define cpu_exec_exit cpu_exec_exit_mipsel
 #define cpu_exec_init cpu_exec_init_mipsel
 #define cpu_exec_init_all cpu_exec_init_all_mipsel
 #define cpu_exec_step_atomic cpu_exec_step_atomic_mipsel

qemu/powerpc.h

@@ -276,6 +276,7 @@
 #define cpu_common_reset cpu_common_reset_powerpc
 #define cpu_dump_statistics cpu_dump_statistics_powerpc
 #define cpu_exec cpu_exec_powerpc
+#define cpu_exec_exit cpu_exec_exit_powerpc
 #define cpu_exec_init cpu_exec_init_powerpc
 #define cpu_exec_init_all cpu_exec_init_all_powerpc
 #define cpu_exec_step_atomic cpu_exec_step_atomic_powerpc

qemu/qom/cpu.c

@@ -258,10 +258,16 @@ static void cpu_common_initfn(struct uc_struct *uc, Object *obj, void *opaque)
 {
     CPUState *cpu = CPU(obj);
 
+    cpu->cpu_index = -1;
     QTAILQ_INIT(&cpu->breakpoints);
     QTAILQ_INIT(&cpu->watchpoints);
 }
 
+static void cpu_common_finalize(struct uc_struct *uc, Object *obj, void *opaque)
+{
+    uc->cpu_exec_exit(CPU(obj));
+}
+
 static int64_t cpu_common_get_arch_id(CPUState *cpu)
 {
     return cpu->cpu_index;
@@ -308,7 +314,7 @@ static const TypeInfo cpu_type_info = {
 
     cpu_common_initfn,
     NULL,
-    NULL,
+    cpu_common_finalize,
 
     NULL,
 

qemu/sparc.h

@@ -276,6 +276,7 @@
 #define cpu_common_reset cpu_common_reset_sparc
 #define cpu_dump_statistics cpu_dump_statistics_sparc
 #define cpu_exec cpu_exec_sparc
+#define cpu_exec_exit cpu_exec_exit_sparc
 #define cpu_exec_init cpu_exec_init_sparc
 #define cpu_exec_init_all cpu_exec_init_all_sparc
 #define cpu_exec_step_atomic cpu_exec_step_atomic_sparc

qemu/sparc64.h

@@ -276,6 +276,7 @@
 #define cpu_common_reset cpu_common_reset_sparc64
 #define cpu_dump_statistics cpu_dump_statistics_sparc64
 #define cpu_exec cpu_exec_sparc64
+#define cpu_exec_exit cpu_exec_exit_sparc64
 #define cpu_exec_init cpu_exec_init_sparc64
 #define cpu_exec_init_all cpu_exec_init_all_sparc64
 #define cpu_exec_step_atomic cpu_exec_step_atomic_sparc64

qemu/unicorn_common.h

@@ -100,6 +100,7 @@ static inline void uc_common_init(struct uc_struct* uc)
     uc->tcg_enabled = tcg_enabled;
     uc->tcg_exec_init = tcg_exec_init;
     uc->cpu_exec_init_all = cpu_exec_init_all;
+    uc->cpu_exec_exit = cpu_exec_exit;
     uc->vm_start = vm_start;
     uc->memory_map = memory_map;
     uc->memory_map_ptr = memory_map_ptr;

qemu/x86_64.h

@@ -276,6 +276,7 @@
 #define cpu_common_reset cpu_common_reset_x86_64
 #define cpu_dump_statistics cpu_dump_statistics_x86_64
 #define cpu_exec cpu_exec_x86_64
+#define cpu_exec_exit cpu_exec_exit_x86_64
 #define cpu_exec_init cpu_exec_init_x86_64
 #define cpu_exec_init_all cpu_exec_init_all_x86_64
 #define cpu_exec_step_atomic cpu_exec_step_atomic_x86_64