@@ -43,4 +43,8 @@ int get_colo_mode(void);
int create_and_init_ram_cache(void);
void colo_flush_ram_cache(void);
void release_ram_cache(void);
+
+/* failover */
+void colo_do_failover(MigrationState *s);
+
#endif
@@ -25,5 +25,6 @@ typedef enum COLOFailoverStatus {
int failover_set_state(int old_state, int new_state);
int failover_get_state(void);
void failover_request_active(Error **errp);
+bool failover_request_is_active(void);
#endif
@@ -22,9 +22,17 @@ static COLOFailoverStatus failover_state;
static void colo_failover_bh(void *opaque)
{
+ int old_state;
+
qemu_bh_delete(failover_bh);
failover_bh = NULL;
- /*TODO: Do failover work */
+ old_state = failover_set_state(FAILOVER_STATUS_REQUEST,
+ FAILOVER_STATUS_HANDLING);
+ if (old_state != FAILOVER_STATUS_REQUEST) {
+ error_report(" Unkown error for failover, old_state=%d", old_state);
+ return;
+ }
+ colo_do_failover(NULL);
}
void failover_request_active(Error **errp)
@@ -54,6 +62,11 @@ int failover_get_state(void)
return atomic_read(&failover_state);
}
+bool failover_request_is_active(void)
+{
+ return ((failover_get_state() != FAILOVER_STATUS_NONE));
+}
+
void qmp_colo_lost_heartbeat(Error **errp)
{
if (get_colo_mode() == COLO_MODE_UNKNOWN) {
@@ -73,6 +73,83 @@ bool loadvm_in_colo_state(void)
return colo != NULL;
}
+static bool colo_runstate_is_stopped(void)
+{
+ return runstate_check(RUN_STATE_COLO) || !runstate_is_running();
+}
+
+/*
+ * there are two way to entry this function
+ * 1. From colo checkpoint incoming thread, in this case
+ * we should protect it by iothread lock
+ * 2. From user command, because hmp/qmp command
+ * was happened in main loop, iothread lock will cause a
+ * dead lock.
+ */
+static void secondary_vm_do_failover(void)
+{
+ int old_state;
+
+ /* It means that VM exit from COLO state */
+ colo = NULL;
+
+ if (!autostart) {
+ error_report("\"-S\" qemu option will be ignored in secondary side");
+ /* recover runstate to normal migration finish state */
+ autostart = true;
+ }
+
+ old_state = failover_set_state(FAILOVER_STATUS_HANDLING,
+ FAILOVER_STATUS_COMPLETED);
+ if (old_state != FAILOVER_STATUS_HANDLING) {
+ error_report("Serious error while do failover for secondary VM,"
+ "old_state: %d", old_state);
+ return;
+ }
+ /* For Secondary VM, jump to incoming co */
+ if (migration_incoming_co) {
+ qemu_coroutine_enter(migration_incoming_co, NULL);
+ }
+}
+
+static void primary_vm_do_failover(void)
+{
+ MigrationState *s = migrate_get_current();
+ int old_state;
+
+ if (!colo_runstate_is_stopped()) {
+ vm_stop_force_state(RUN_STATE_COLO);
+ }
+
+ if (s->state != MIGRATION_STATUS_FAILED) {
+ migrate_set_state(s, MIGRATION_STATUS_COLO, MIGRATION_STATUS_COMPLETED);
+ }
+
+ vm_start();
+
+ old_state = failover_set_state(FAILOVER_STATUS_HANDLING,
+ FAILOVER_STATUS_COMPLETED);
+ if (old_state != FAILOVER_STATUS_COMPLETED) {
+ error_report("Serious error while do failover for Primary VM,"
+ "old_state: %d", old_state);
+ return;
+ }
+}
+
+void colo_do_failover(MigrationState *s)
+{
+ /* Make sure vm stopped while failover */
+ if (!colo_runstate_is_stopped()) {
+ vm_stop_force_state(RUN_STATE_COLO);
+ }
+
+ if (get_colo_mode() == COLO_MODE_SECONDARY) {
+ secondary_vm_do_failover();
+ } else {
+ primary_vm_do_failover();
+ }
+}
+
/* colo checkpoint control helper */
static int colo_ctl_put(QEMUFile *f, uint64_t request)
{
@@ -144,11 +221,23 @@ static int colo_do_checkpoint_transaction(MigrationState *s, QEMUFile *control)
goto out;
}
+ if (failover_request_is_active()) {
+ ret = -1;
+ goto out;
+ }
/* suspend and save vm state to colo buffer */
qemu_mutex_lock_iothread();
vm_stop_force_state(RUN_STATE_COLO);
qemu_mutex_unlock_iothread();
trace_colo_vm_state_change("run", "stop");
+ /*
+ * failover request bh could be called after
+ * vm_stop_force_state so we check failover_request_is_active() again.
+ */
+ if (failover_request_is_active()) {
+ ret = -1;
+ goto out;
+ }
/* Disable block migration */
s->params.blk = 0;
@@ -209,7 +298,7 @@ static void *colo_thread(void *opaque)
{
MigrationState *s = opaque;
QEMUFile *colo_control = NULL;
- int ret;
+ int i, ret;
colo_control = qemu_fopen_socket(qemu_get_fd(s->file), "rb");
if (!colo_control) {
@@ -239,6 +328,11 @@ static void *colo_thread(void *opaque)
trace_colo_vm_state_change("stop", "run");
while (s->state == MIGRATION_STATUS_COLO) {
+ if (failover_request_is_active()) {
+ error_report("failover request");
+ goto out;
+ }
+
/* start a colo checkpoint */
if (colo_do_checkpoint_transaction(s, colo_control)) {
goto out;
@@ -246,7 +340,26 @@ static void *colo_thread(void *opaque)
}
out:
- migrate_set_state(s, MIGRATION_STATUS_COLO, MIGRATION_STATUS_COMPLETED);
+ error_report("colo: some error happens in colo_thread");
+ /* Give users time (2s) to get involved in this verdict */
+ for (i = 0; i < 10; i++) {
+ if (failover_request_is_active()) {
+ error_report("Primary VM will take over work");
+ break;
+ }
+ usleep(200 * 1000);
+ }
+ qemu_mutex_lock_iothread();
+ if (!failover_request_is_active()) {
+ error_report("Primary VM will take over work in default");
+ failover_request_active(NULL);
+ }
+ qemu_mutex_unlock_iothread();
+
+ while (failover_get_state() != FAILOVER_STATUS_COMPLETED) {
+ ;
+ }
+ failover_set_state(FAILOVER_STATUS_COMPLETED, FAILOVER_STATUS_NONE);
qsb_free(colo_buffer);
colo_buffer = NULL;
@@ -317,7 +430,7 @@ void *colo_process_incoming_checkpoints(void *opaque)
QEMUFile *f = colo_in->file;
int fd = qemu_get_fd(f);
QEMUFile *ctl = NULL, *fb = NULL;
- int ret;
+ int i, ret;
uint64_t total_size;
colo = qemu_coroutine_self();
@@ -363,6 +476,11 @@ void *colo_process_incoming_checkpoints(void *opaque)
}
}
+ if (failover_request_is_active()) {
+ error_report("failover request");
+ goto out;
+ }
+
/* suspend guest */
qemu_mutex_lock_iothread();
vm_stop_force_state(RUN_STATE_COLO);
@@ -431,7 +549,30 @@ void *colo_process_incoming_checkpoints(void *opaque)
}
out:
- colo = NULL;
+ error_report("Detect some error or get a failover request");
+ /* Give users time (2s) to get involved in this verdict */
+ for (i = 0; i < 10; i++) {
+ if (failover_request_is_active()) {
+ error_report("Secondary VM will take over work");
+ break;
+ }
+ usleep(200*1000);
+ }
+ /* check flag again*/
+ if (!failover_request_is_active()) {
+ /*
+ * We assume that Primary VM is still alive according to heartbeat,
+ * just kill Secondary VM
+ */
+ error_report("SVM is going to exit in default!");
+ exit(1);
+ } else {
+ /* if we went here, means Primary VM may dead, we are doing failover */
+ while (failover_get_state() != FAILOVER_STATUS_COMPLETED) {
+ ;
+ }
+ failover_set_state(FAILOVER_STATUS_COMPLETED, FAILOVER_STATUS_NONE);
+ }
if (fb) {
qemu_fclose(fb);
@@ -1476,6 +1476,7 @@ rdma_start_outgoing_migration_after_rdma_source_init(void) ""
colo_vm_state_change(const char *old, const char *new) "Change '%s' => '%s'"
colo_receive_message(const char *msg) "Receive '%s'"
failover_set_state(int new_state) "new state %d"
+colo_rcv_pkt(int result) "Result of net packets comparing is different: %d"
# kvm-all.c
kvm_ioctl(int type, void *arg) "type 0x%x, arg %p"