* [Qemu-devel] [PATCH V11 01/20] filter-rewriter: Add TCP state machine and fix memory leak in connection_track_table
2018-08-11 20:59 [Qemu-devel] [PATCH V11 00/20] COLO: integrate colo frame with block replication and COLO proxy Zhang Chen
@ 2018-08-11 20:59 ` Zhang Chen
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 02/20] colo-compare: implement the process of checkpoint Zhang Chen
` (19 subsequent siblings)
20 siblings, 0 replies; 35+ messages in thread
From: Zhang Chen @ 2018-08-11 20:59 UTC (permalink / raw)
To: qemu-devel, Paolo Bonzini, Juan Quintela, Dr . David Alan Gilbert,
Jason Wang, Eric Blake, Markus Armbruster
Cc: Zhang Chen, zhanghailiang, Li Zhijian, Zhang Chen
We add almost full TCP state machine in filter-rewriter, except
TCPS_LISTEN and some simplify in VM active close FIN states.
After a net connection is closed, we didn't clear its releated resources
in connection_track_table, which will lead to memory leak.
Let't track the state of net connection, if it is closed, its related
resources will be cleared up.
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Zhang Chen <zhangckid@gmail.com>
Signed-off-by: Zhang Chen <chen.zhang@intel.com>
---
net/colo.c | 2 +-
net/colo.h | 9 ++--
net/filter-rewriter.c | 105 ++++++++++++++++++++++++++++++++++++++----
3 files changed, 100 insertions(+), 16 deletions(-)
diff --git a/net/colo.c b/net/colo.c
index 6dda4ed66e..97c8fc928f 100644
--- a/net/colo.c
+++ b/net/colo.c
@@ -137,7 +137,7 @@ Connection *connection_new(ConnectionKey *key)
conn->ip_proto = key->ip_proto;
conn->processing = false;
conn->offset = 0;
- conn->syn_flag = 0;
+ conn->tcp_state = TCPS_CLOSED;
conn->pack = 0;
conn->sack = 0;
g_queue_init(&conn->primary_list);
diff --git a/net/colo.h b/net/colo.h
index da6c36dcf7..0277e0e9ba 100644
--- a/net/colo.h
+++ b/net/colo.h
@@ -18,6 +18,7 @@
#include "slirp/slirp.h"
#include "qemu/jhash.h"
#include "qemu/timer.h"
+#include "slirp/tcp.h"
#define HASHTABLE_MAX_SIZE 16384
@@ -81,11 +82,9 @@ typedef struct Connection {
uint32_t sack;
/* offset = secondary_seq - primary_seq */
tcp_seq offset;
- /*
- * we use this flag update offset func
- * run once in independent tcp connection
- */
- int syn_flag;
+
+ int tcp_state; /* TCP FSM state */
+ tcp_seq fin_ack_seq; /* the seq of 'fin=1,ack=1' */
} Connection;
uint32_t connection_key_hash(const void *opaque);
diff --git a/net/filter-rewriter.c b/net/filter-rewriter.c
index f584e4eba4..f18a71bf2e 100644
--- a/net/filter-rewriter.c
+++ b/net/filter-rewriter.c
@@ -59,9 +59,9 @@ static int is_tcp_packet(Packet *pkt)
}
/* handle tcp packet from primary guest */
-static int handle_primary_tcp_pkt(NetFilterState *nf,
+static int handle_primary_tcp_pkt(RewriterState *rf,
Connection *conn,
- Packet *pkt)
+ Packet *pkt, ConnectionKey *key)
{
struct tcphdr *tcp_pkt;
@@ -74,23 +74,28 @@ static int handle_primary_tcp_pkt(NetFilterState *nf,
trace_colo_filter_rewriter_conn_offset(conn->offset);
}
+ if (((tcp_pkt->th_flags & (TH_ACK | TH_SYN)) == (TH_ACK | TH_SYN)) &&
+ conn->tcp_state == TCPS_SYN_SENT) {
+ conn->tcp_state = TCPS_ESTABLISHED;
+ }
+
if (((tcp_pkt->th_flags & (TH_ACK | TH_SYN)) == TH_SYN)) {
/*
* we use this flag update offset func
* run once in independent tcp connection
*/
- conn->syn_flag = 1;
+ conn->tcp_state = TCPS_SYN_RECEIVED;
}
if (((tcp_pkt->th_flags & (TH_ACK | TH_SYN)) == TH_ACK)) {
- if (conn->syn_flag) {
+ if (conn->tcp_state == TCPS_SYN_RECEIVED) {
/*
* offset = secondary_seq - primary seq
* ack packet sent by guest from primary node,
* so we use th_ack - 1 get primary_seq
*/
conn->offset -= (ntohl(tcp_pkt->th_ack) - 1);
- conn->syn_flag = 0;
+ conn->tcp_state = TCPS_ESTABLISHED;
}
if (conn->offset) {
/* handle packets to the secondary from the primary */
@@ -99,15 +104,63 @@ static int handle_primary_tcp_pkt(NetFilterState *nf,
net_checksum_calculate((uint8_t *)pkt->data + pkt->vnet_hdr_len,
pkt->size - pkt->vnet_hdr_len);
}
+ /*
+ * Case 1:
+ * Step 3:
+ * The *server* side of this connect is VM, *client* tries to close
+ * the connection.
+ *
+ * We got 'ack=1' packets from client side, it acks 'fin=1, ack=1'
+ * packet from server side. From this point, we can ensure that there
+ * will be no packets in the connection, except that, some errors
+ * happen between the path of 'filter object' and vNIC, if this rare
+ * case really happen, we can still create a new connection,
+ * So it is safe to remove the connection from connection_track_table.
+ *
+ */
+ if ((conn->tcp_state == TCPS_LAST_ACK) &&
+ (ntohl(tcp_pkt->th_ack) == (conn->fin_ack_seq + 1))) {
+ conn->tcp_state = TCPS_CLOSED;
+ g_hash_table_remove(rf->connection_track_table, key);
+ }
+ }
+
+ if ((tcp_pkt->th_flags & TH_FIN) == TH_FIN) {
+ /*
+ * Case 1:
+ * Step 1:
+ * The *server* side of this connect is VM, *client* tries to close
+ * the connection. We will into CLOSE_WAIT status.
+ */
+ if (conn->tcp_state == TCPS_ESTABLISHED) {
+ conn->tcp_state = TCPS_CLOSE_WAIT;
+ }
+
+ /*
+ * Case 2:
+ * Step 2:
+ * The *server* side of this connect is VM, *server* tries to close
+ * the connection. We will into CLOSE_WAIT status.
+ * We simplify the TCPS_FIN_WAIT_2, TCPS_TIME_WAIT and CLOSING status.
+ */
+ if (conn->tcp_state == TCPS_FIN_WAIT_1) {
+ conn->tcp_state = TCPS_TIME_WAIT;
+ /*
+ * For simplify implementation, we needn't wait 2MSL time
+ * in filter rewriter.
+ */
+ conn->tcp_state = TCPS_CLOSED;
+ g_hash_table_remove(rf->connection_track_table, key);
+ }
}
return 0;
}
/* handle tcp packet from secondary guest */
-static int handle_secondary_tcp_pkt(NetFilterState *nf,
+static int handle_secondary_tcp_pkt(RewriterState *rf,
Connection *conn,
- Packet *pkt)
+ Packet *pkt, ConnectionKey *key)
{
struct tcphdr *tcp_pkt;
@@ -121,7 +174,8 @@ static int handle_secondary_tcp_pkt(NetFilterState *nf,
trace_colo_filter_rewriter_conn_offset(conn->offset);
}
- if (((tcp_pkt->th_flags & (TH_ACK | TH_SYN)) == (TH_ACK | TH_SYN))) {
+ if (conn->tcp_state == TCPS_SYN_RECEIVED &&
+ ((tcp_pkt->th_flags & (TH_ACK | TH_SYN)) == (TH_ACK | TH_SYN))) {
/*
* save offset = secondary_seq and then
* in handle_primary_tcp_pkt make offset
@@ -130,6 +184,12 @@ static int handle_secondary_tcp_pkt(NetFilterState *nf,
conn->offset = ntohl(tcp_pkt->th_seq);
}
+ /* VM active connect */
+ if (conn->tcp_state == TCPS_CLOSED &&
+ ((tcp_pkt->th_flags & (TH_ACK | TH_SYN)) == TH_SYN)) {
+ conn->tcp_state = TCPS_SYN_SENT;
+ }
+
if ((tcp_pkt->th_flags & (TH_ACK | TH_SYN)) == TH_ACK) {
/* Only need to adjust seq while offset is Non-zero */
if (conn->offset) {
@@ -140,6 +200,31 @@ static int handle_secondary_tcp_pkt(NetFilterState *nf,
pkt->size - pkt->vnet_hdr_len);
}
}
+ /*
+ * Case 1:
+ * Step 2:
+ * The *server* side of this connect is VM, *client* tries to close
+ * the connection. In this step we will into LAST_ACK status.
+ *
+ * We got 'fin=1, ack=1' packet from server side, we need to
+ * record the seq of 'fin=1, ack=1' packet.
+ */
+ if (conn->tcp_state == TCPS_CLOSE_WAIT &&
+ (tcp_pkt->th_flags & (TH_ACK | TH_FIN)) == (TH_ACK | TH_FIN)) {
+ conn->fin_ack_seq = ntohl(tcp_pkt->th_seq);
+ conn->tcp_state = TCPS_LAST_ACK;
+ }
+
+ /*
+ * Case 2:
+ * Step 1:
+ * The *server* side of this connect is VM, *server* tries to close
+ * the connection.
+ */
+ if (conn->tcp_state == TCPS_ESTABLISHED &&
+ (tcp_pkt->th_flags & (TH_ACK | TH_FIN)) == TH_FIN) {
+ conn->tcp_state = TCPS_FIN_WAIT_1;
+ }
return 0;
}
@@ -190,7 +275,7 @@ static ssize_t colo_rewriter_receive_iov(NetFilterState *nf,
if (sender == nf->netdev) {
/* NET_FILTER_DIRECTION_TX */
- if (!handle_primary_tcp_pkt(nf, conn, pkt)) {
+ if (!handle_primary_tcp_pkt(s, conn, pkt, &key)) {
qemu_net_queue_send(s->incoming_queue, sender, 0,
(const uint8_t *)pkt->data, pkt->size, NULL);
packet_destroy(pkt, NULL);
@@ -203,7 +288,7 @@ static ssize_t colo_rewriter_receive_iov(NetFilterState *nf,
}
} else {
/* NET_FILTER_DIRECTION_RX */
- if (!handle_secondary_tcp_pkt(nf, conn, pkt)) {
+ if (!handle_secondary_tcp_pkt(s, conn, pkt, &key)) {
qemu_net_queue_send(s->incoming_queue, sender, 0,
(const uint8_t *)pkt->data, pkt->size, NULL);
packet_destroy(pkt, NULL);
--
2.17.GIT
^ permalink raw reply related [flat|nested] 35+ messages in thread* [Qemu-devel] [PATCH V11 02/20] colo-compare: implement the process of checkpoint
2018-08-11 20:59 [Qemu-devel] [PATCH V11 00/20] COLO: integrate colo frame with block replication and COLO proxy Zhang Chen
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 01/20] filter-rewriter: Add TCP state machine and fix memory leak in connection_track_table Zhang Chen
@ 2018-08-11 20:59 ` Zhang Chen
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 03/20] colo-compare: use notifier to notify packets comparing result Zhang Chen
` (18 subsequent siblings)
20 siblings, 0 replies; 35+ messages in thread
From: Zhang Chen @ 2018-08-11 20:59 UTC (permalink / raw)
To: qemu-devel, Paolo Bonzini, Juan Quintela, Dr . David Alan Gilbert,
Jason Wang, Eric Blake, Markus Armbruster
Cc: Zhang Chen, zhanghailiang, Li Zhijian, Zhang Chen
While do checkpoint, we need to flush all the unhandled packets,
By using the filter notifier mechanism, we can easily to notify
every compare object to do this process, which runs inside
of compare threads as a coroutine.
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Zhang Chen <zhangckid@gmail.com>
Signed-off-by: Zhang Chen <chen.zhang@intel.com>
---
include/migration/colo.h | 6 ++++
net/colo-compare.c | 78 ++++++++++++++++++++++++++++++++++++++++
net/colo-compare.h | 22 ++++++++++++
3 files changed, 106 insertions(+)
create mode 100644 net/colo-compare.h
diff --git a/include/migration/colo.h b/include/migration/colo.h
index 2fe48ad353..fefb2fcf4c 100644
--- a/include/migration/colo.h
+++ b/include/migration/colo.h
@@ -16,6 +16,12 @@
#include "qemu-common.h"
#include "qapi/qapi-types-migration.h"
+enum colo_event {
+ COLO_EVENT_NONE,
+ COLO_EVENT_CHECKPOINT,
+ COLO_EVENT_FAILOVER,
+};
+
void colo_info_init(void);
void migrate_start_colo_process(MigrationState *s);
diff --git a/net/colo-compare.c b/net/colo-compare.c
index dd745a491b..80e6532e8b 100644
--- a/net/colo-compare.c
+++ b/net/colo-compare.c
@@ -27,11 +27,16 @@
#include "qemu/sockets.h"
#include "colo.h"
#include "sysemu/iothread.h"
+#include "net/colo-compare.h"
+#include "migration/colo.h"
#define TYPE_COLO_COMPARE "colo-compare"
#define COLO_COMPARE(obj) \
OBJECT_CHECK(CompareState, (obj), TYPE_COLO_COMPARE)
+static QTAILQ_HEAD(, CompareState) net_compares =
+ QTAILQ_HEAD_INITIALIZER(net_compares);
+
#define COMPARE_READ_LEN_MAX NET_BUFSIZE
#define MAX_QUEUE_SIZE 1024
@@ -41,6 +46,10 @@
/* TODO: Should be configurable */
#define REGULAR_PACKET_CHECK_MS 3000
+static QemuMutex event_mtx;
+static QemuCond event_complete_cond;
+static int event_unhandled_count;
+
/*
* + CompareState ++
* | |
@@ -87,6 +96,11 @@ typedef struct CompareState {
IOThread *iothread;
GMainContext *worker_context;
QEMUTimer *packet_check_timer;
+
+ QEMUBH *event_bh;
+ enum colo_event event;
+
+ QTAILQ_ENTRY(CompareState) next;
} CompareState;
typedef struct CompareClass {
@@ -736,6 +750,25 @@ static void check_old_packet_regular(void *opaque)
REGULAR_PACKET_CHECK_MS);
}
+/* Public API, Used for COLO frame to notify compare event */
+void colo_notify_compares_event(void *opaque, int event, Error **errp)
+{
+ CompareState *s;
+
+ qemu_mutex_lock(&event_mtx);
+ QTAILQ_FOREACH(s, &net_compares, next) {
+ s->event = event;
+ qemu_bh_schedule(s->event_bh);
+ event_unhandled_count++;
+ }
+ /* Wait all compare threads to finish handling this event */
+ while (event_unhandled_count > 0) {
+ qemu_cond_wait(&event_complete_cond, &event_mtx);
+ }
+
+ qemu_mutex_unlock(&event_mtx);
+}
+
static void colo_compare_timer_init(CompareState *s)
{
AioContext *ctx = iothread_get_aio_context(s->iothread);
@@ -756,6 +789,30 @@ static void colo_compare_timer_del(CompareState *s)
}
}
+static void colo_flush_packets(void *opaque, void *user_data);
+
+static void colo_compare_handle_event(void *opaque)
+{
+ CompareState *s = opaque;
+
+ switch (s->event) {
+ case COLO_EVENT_CHECKPOINT:
+ g_queue_foreach(&s->conn_list, colo_flush_packets, s);
+ break;
+ case COLO_EVENT_FAILOVER:
+ break;
+ default:
+ break;
+ }
+
+ assert(event_unhandled_count > 0);
+
+ qemu_mutex_lock(&event_mtx);
+ event_unhandled_count--;
+ qemu_cond_broadcast(&event_complete_cond);
+ qemu_mutex_unlock(&event_mtx);
+}
+
static void colo_compare_iothread(CompareState *s)
{
object_ref(OBJECT(s->iothread));
@@ -769,6 +826,7 @@ static void colo_compare_iothread(CompareState *s)
s, s->worker_context, true);
colo_compare_timer_init(s);
+ s->event_bh = qemu_bh_new(colo_compare_handle_event, s);
}
static char *compare_get_pri_indev(Object *obj, Error **errp)
@@ -926,8 +984,13 @@ static void colo_compare_complete(UserCreatable *uc, Error **errp)
net_socket_rs_init(&s->pri_rs, compare_pri_rs_finalize, s->vnet_hdr);
net_socket_rs_init(&s->sec_rs, compare_sec_rs_finalize, s->vnet_hdr);
+ QTAILQ_INSERT_TAIL(&net_compares, s, next);
+
g_queue_init(&s->conn_list);
+ qemu_mutex_init(&event_mtx);
+ qemu_cond_init(&event_complete_cond);
+
s->connection_track_table = g_hash_table_new_full(connection_key_hash,
connection_key_equal,
g_free,
@@ -990,6 +1053,7 @@ static void colo_compare_init(Object *obj)
static void colo_compare_finalize(Object *obj)
{
CompareState *s = COLO_COMPARE(obj);
+ CompareState *tmp = NULL;
qemu_chr_fe_deinit(&s->chr_pri_in, false);
qemu_chr_fe_deinit(&s->chr_sec_in, false);
@@ -997,6 +1061,16 @@ static void colo_compare_finalize(Object *obj)
if (s->iothread) {
colo_compare_timer_del(s);
}
+
+ qemu_bh_delete(s->event_bh);
+
+ QTAILQ_FOREACH(tmp, &net_compares, next) {
+ if (tmp == s) {
+ QTAILQ_REMOVE(&net_compares, s, next);
+ break;
+ }
+ }
+
/* Release all unhandled packets after compare thead exited */
g_queue_foreach(&s->conn_list, colo_flush_packets, s);
@@ -1009,6 +1083,10 @@ static void colo_compare_finalize(Object *obj)
if (s->iothread) {
object_unref(OBJECT(s->iothread));
}
+
+ qemu_mutex_destroy(&event_mtx);
+ qemu_cond_destroy(&event_complete_cond);
+
g_free(s->pri_indev);
g_free(s->sec_indev);
g_free(s->outdev);
diff --git a/net/colo-compare.h b/net/colo-compare.h
new file mode 100644
index 0000000000..1b1ce76aea
--- /dev/null
+++ b/net/colo-compare.h
@@ -0,0 +1,22 @@
+/*
+ * COarse-grain LOck-stepping Virtual Machines for Non-stop Service (COLO)
+ * (a.k.a. Fault Tolerance or Continuous Replication)
+ *
+ * Copyright (c) 2017 HUAWEI TECHNOLOGIES CO., LTD.
+ * Copyright (c) 2017 FUJITSU LIMITED
+ * Copyright (c) 2017 Intel Corporation
+ *
+ * Authors:
+ * zhanghailiang <zhang.zhanghailiang@huawei.com>
+ * Zhang Chen <zhangckid@gmail.com>
+ *
+ * This work is licensed under the terms of the GNU GPL, version 2 or
+ * later. See the COPYING file in the top-level directory.
+ */
+
+#ifndef QEMU_COLO_COMPARE_H
+#define QEMU_COLO_COMPARE_H
+
+void colo_notify_compares_event(void *opaque, int event, Error **errp);
+
+#endif /* QEMU_COLO_COMPARE_H */
--
2.17.GIT
^ permalink raw reply related [flat|nested] 35+ messages in thread* [Qemu-devel] [PATCH V11 03/20] colo-compare: use notifier to notify packets comparing result
2018-08-11 20:59 [Qemu-devel] [PATCH V11 00/20] COLO: integrate colo frame with block replication and COLO proxy Zhang Chen
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 01/20] filter-rewriter: Add TCP state machine and fix memory leak in connection_track_table Zhang Chen
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 02/20] colo-compare: implement the process of checkpoint Zhang Chen
@ 2018-08-11 20:59 ` Zhang Chen
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 04/20] COLO: integrate colo compare with colo frame Zhang Chen
` (17 subsequent siblings)
20 siblings, 0 replies; 35+ messages in thread
From: Zhang Chen @ 2018-08-11 20:59 UTC (permalink / raw)
To: qemu-devel, Paolo Bonzini, Juan Quintela, Dr . David Alan Gilbert,
Jason Wang, Eric Blake, Markus Armbruster
Cc: Zhang Chen, zhanghailiang, Li Zhijian, Zhang Chen
It's a good idea to use notifier to notify COLO frame of
inconsistent packets comparing.
Signed-off-by: Zhang Chen <zhangckid@gmail.com>
Signed-off-by: Zhang Chen <chen.zhang@intel.com>
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
---
net/colo-compare.c | 37 ++++++++++++++++++++++++++-----------
net/colo-compare.h | 2 ++
2 files changed, 28 insertions(+), 11 deletions(-)
diff --git a/net/colo-compare.c b/net/colo-compare.c
index 80e6532e8b..426eab5973 100644
--- a/net/colo-compare.c
+++ b/net/colo-compare.c
@@ -29,6 +29,7 @@
#include "sysemu/iothread.h"
#include "net/colo-compare.h"
#include "migration/colo.h"
+#include "migration/migration.h"
#define TYPE_COLO_COMPARE "colo-compare"
#define COLO_COMPARE(obj) \
@@ -37,6 +38,9 @@
static QTAILQ_HEAD(, CompareState) net_compares =
QTAILQ_HEAD_INITIALIZER(net_compares);
+static NotifierList colo_compare_notifiers =
+ NOTIFIER_LIST_INITIALIZER(colo_compare_notifiers);
+
#define COMPARE_READ_LEN_MAX NET_BUFSIZE
#define MAX_QUEUE_SIZE 1024
@@ -427,10 +431,7 @@ sec:
qemu_hexdump((char *)spkt->data, stderr,
"colo-compare spkt", spkt->size);
- /*
- * colo_compare_inconsistent_notify();
- * TODO: notice to checkpoint();
- */
+ colo_compare_inconsistency_notify();
}
}
@@ -561,8 +562,24 @@ static int colo_old_packet_check_one(Packet *pkt, int64_t *check_time)
}
}
+static void colo_compare_inconsistency_notify(void)
+{
+ notifier_list_notify(&colo_compare_notifiers,
+ migrate_get_current());
+}
+
+void colo_compare_register_notifier(Notifier *notify)
+{
+ notifier_list_add(&colo_compare_notifiers, notify);
+}
+
+void colo_compare_unregister_notifier(Notifier *notify)
+{
+ notifier_remove(notify);
+}
+
static int colo_old_packet_check_one_conn(Connection *conn,
- void *user_data)
+ void *user_data)
{
GList *result = NULL;
int64_t check_time = REGULAR_PACKET_CHECK_MS;
@@ -573,10 +590,7 @@ static int colo_old_packet_check_one_conn(Connection *conn,
if (result) {
/* Do checkpoint will flush old packet */
- /*
- * TODO: Notify colo frame to do checkpoint.
- * colo_compare_inconsistent_notify();
- */
+ colo_compare_inconsistency_notify();
return 0;
}
@@ -620,11 +634,12 @@ static void colo_compare_packet(CompareState *s, Connection *conn,
/*
* If one packet arrive late, the secondary_list or
* primary_list will be empty, so we can't compare it
- * until next comparison.
+ * until next comparison. If the packets in the list are
+ * timeout, it will trigger a checkpoint request.
*/
trace_colo_compare_main("packet different");
g_queue_push_head(&conn->primary_list, pkt);
- /* TODO: colo_notify_checkpoint();*/
+ colo_compare_inconsistency_notify();
break;
}
}
diff --git a/net/colo-compare.h b/net/colo-compare.h
index 1b1ce76aea..22ddd512e2 100644
--- a/net/colo-compare.h
+++ b/net/colo-compare.h
@@ -18,5 +18,7 @@
#define QEMU_COLO_COMPARE_H
void colo_notify_compares_event(void *opaque, int event, Error **errp);
+void colo_compare_register_notifier(Notifier *notify);
+void colo_compare_unregister_notifier(Notifier *notify);
#endif /* QEMU_COLO_COMPARE_H */
--
2.17.GIT
^ permalink raw reply related [flat|nested] 35+ messages in thread* [Qemu-devel] [PATCH V11 04/20] COLO: integrate colo compare with colo frame
2018-08-11 20:59 [Qemu-devel] [PATCH V11 00/20] COLO: integrate colo frame with block replication and COLO proxy Zhang Chen
` (2 preceding siblings ...)
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 03/20] colo-compare: use notifier to notify packets comparing result Zhang Chen
@ 2018-08-11 20:59 ` Zhang Chen
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 05/20] COLO: Add block replication into colo process Zhang Chen
` (16 subsequent siblings)
20 siblings, 0 replies; 35+ messages in thread
From: Zhang Chen @ 2018-08-11 20:59 UTC (permalink / raw)
To: qemu-devel, Paolo Bonzini, Juan Quintela, Dr . David Alan Gilbert,
Jason Wang, Eric Blake, Markus Armbruster
Cc: Zhang Chen, zhanghailiang, Li Zhijian, Zhang Chen
For COLO FT, both the PVM and SVM run at the same time,
only sync the state while it needs.
So here, let SVM runs while not doing checkpoint, change
DEFAULT_MIGRATE_X_CHECKPOINT_DELAY to 200*100.
Besides, we forgot to release colo_checkpoint_semd and
colo_delay_timer, fix them here.
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Zhang Chen <zhangckid@gmail.com>
Signed-off-by: Zhang Chen <chen.zhang@intel.com>
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
---
migration/colo.c | 42 ++++++++++++++++++++++++++++++++++++++++--
migration/migration.c | 6 ++----
2 files changed, 42 insertions(+), 6 deletions(-)
diff --git a/migration/colo.c b/migration/colo.c
index 4381067ed4..081df1835f 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -25,8 +25,11 @@
#include "qemu/error-report.h"
#include "migration/failover.h"
#include "replication.h"
+#include "net/colo-compare.h"
+#include "net/colo.h"
static bool vmstate_loading;
+static Notifier packets_compare_notifier;
#define COLO_BUFFER_BASE_SIZE (4 * 1024 * 1024)
@@ -343,6 +346,11 @@ static int colo_do_checkpoint_transaction(MigrationState *s,
goto out;
}
+ colo_notify_compares_event(NULL, COLO_EVENT_CHECKPOINT, &local_err);
+ if (local_err) {
+ goto out;
+ }
+
/* Disable block migration */
migrate_set_block_enabled(false, &local_err);
qemu_savevm_state_header(fb);
@@ -400,6 +408,11 @@ out:
return ret;
}
+static void colo_compare_notify_checkpoint(Notifier *notifier, void *data)
+{
+ colo_checkpoint_notify(data);
+}
+
static void colo_process_checkpoint(MigrationState *s)
{
QIOChannelBuffer *bioc;
@@ -416,6 +429,9 @@ static void colo_process_checkpoint(MigrationState *s)
goto out;
}
+ packets_compare_notifier.notify = colo_compare_notify_checkpoint;
+ colo_compare_register_notifier(&packets_compare_notifier);
+
/*
* Wait for Secondary finish loading VM states and enter COLO
* restore.
@@ -461,11 +477,21 @@ out:
qemu_fclose(fb);
}
- timer_del(s->colo_delay_timer);
-
/* Hope this not to be too long to wait here */
qemu_sem_wait(&s->colo_exit_sem);
qemu_sem_destroy(&s->colo_exit_sem);
+
+ /*
+ * It is safe to unregister notifier after failover finished.
+ * Besides, colo_delay_timer and colo_checkpoint_sem can't be
+ * released befor unregister notifier, or there will be use-after-free
+ * error.
+ */
+ colo_compare_unregister_notifier(&packets_compare_notifier);
+ timer_del(s->colo_delay_timer);
+ timer_free(s->colo_delay_timer);
+ qemu_sem_destroy(&s->colo_checkpoint_sem);
+
/*
* Must be called after failover BH is completed,
* Or the failover BH may shutdown the wrong fd that
@@ -558,6 +584,11 @@ void *colo_process_incoming_thread(void *opaque)
fb = qemu_fopen_channel_input(QIO_CHANNEL(bioc));
object_unref(OBJECT(bioc));
+ qemu_mutex_lock_iothread();
+ vm_start();
+ trace_colo_vm_state_change("stop", "run");
+ qemu_mutex_unlock_iothread();
+
colo_send_message(mis->to_src_file, COLO_MESSAGE_CHECKPOINT_READY,
&local_err);
if (local_err) {
@@ -577,6 +608,11 @@ void *colo_process_incoming_thread(void *opaque)
goto out;
}
+ qemu_mutex_lock_iothread();
+ vm_stop_force_state(RUN_STATE_COLO);
+ trace_colo_vm_state_change("run", "stop");
+ qemu_mutex_unlock_iothread();
+
/* FIXME: This is unnecessary for periodic checkpoint mode */
colo_send_message(mis->to_src_file, COLO_MESSAGE_CHECKPOINT_REPLY,
&local_err);
@@ -630,6 +666,8 @@ void *colo_process_incoming_thread(void *opaque)
}
vmstate_loading = false;
+ vm_start();
+ trace_colo_vm_state_change("stop", "run");
qemu_mutex_unlock_iothread();
if (failover_get_state() == FAILOVER_STATUS_RELAUNCH) {
diff --git a/migration/migration.c b/migration/migration.c
index b7d9854bda..33a2fd74c6 100644
--- a/migration/migration.c
+++ b/migration/migration.c
@@ -75,10 +75,8 @@
/* Migration XBZRLE default cache size */
#define DEFAULT_MIGRATE_XBZRLE_CACHE_SIZE (64 * 1024 * 1024)
-/* The delay time (in ms) between two COLO checkpoints
- * Note: Please change this default value to 10000 when we support hybrid mode.
- */
-#define DEFAULT_MIGRATE_X_CHECKPOINT_DELAY 200
+/* The delay time (in ms) between two COLO checkpoints */
+#define DEFAULT_MIGRATE_X_CHECKPOINT_DELAY (200 * 100)
#define DEFAULT_MIGRATE_MULTIFD_CHANNELS 2
#define DEFAULT_MIGRATE_MULTIFD_PAGE_COUNT 16
--
2.17.GIT
^ permalink raw reply related [flat|nested] 35+ messages in thread* [Qemu-devel] [PATCH V11 05/20] COLO: Add block replication into colo process
2018-08-11 20:59 [Qemu-devel] [PATCH V11 00/20] COLO: integrate colo frame with block replication and COLO proxy Zhang Chen
` (3 preceding siblings ...)
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 04/20] COLO: integrate colo compare with colo frame Zhang Chen
@ 2018-08-11 20:59 ` Zhang Chen
2018-08-17 11:04 ` Dr. David Alan Gilbert
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 06/20] COLO: Remove colo_state migration struct Zhang Chen
` (15 subsequent siblings)
20 siblings, 1 reply; 35+ messages in thread
From: Zhang Chen @ 2018-08-11 20:59 UTC (permalink / raw)
To: qemu-devel, Paolo Bonzini, Juan Quintela, Dr . David Alan Gilbert,
Jason Wang, Eric Blake, Markus Armbruster
Cc: Zhang Chen, zhanghailiang, Li Zhijian, Zhang Chen
Make sure master start block replication after slave's block
replication started.
Besides, we need to activate VM's blocks before goes into
COLO state.
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
Signed-off-by: Zhang Chen <zhangckid@gmail.com>
Signed-off-by: Zhang Chen <chen.zhang@intel.com>
---
migration/colo.c | 43 +++++++++++++++++++++++++++++++++++++++++++
migration/migration.c | 9 +++++++++
2 files changed, 52 insertions(+)
diff --git a/migration/colo.c b/migration/colo.c
index 081df1835f..e06640c3d6 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -27,6 +27,7 @@
#include "replication.h"
#include "net/colo-compare.h"
#include "net/colo.h"
+#include "block/block.h"
static bool vmstate_loading;
static Notifier packets_compare_notifier;
@@ -56,6 +57,7 @@ static void secondary_vm_do_failover(void)
{
int old_state;
MigrationIncomingState *mis = migration_incoming_get_current();
+ Error *local_err = NULL;
/* Can not do failover during the process of VM's loading VMstate, Or
* it will break the secondary VM.
@@ -73,6 +75,11 @@ static void secondary_vm_do_failover(void)
migrate_set_state(&mis->state, MIGRATION_STATUS_COLO,
MIGRATION_STATUS_COMPLETED);
+ replication_stop_all(true, &local_err);
+ if (local_err) {
+ error_report_err(local_err);
+ }
+
if (!autostart) {
error_report("\"-S\" qemu option will be ignored in secondary side");
/* recover runstate to normal migration finish state */
@@ -110,6 +117,7 @@ static void primary_vm_do_failover(void)
{
MigrationState *s = migrate_get_current();
int old_state;
+ Error *local_err = NULL;
migrate_set_state(&s->state, MIGRATION_STATUS_COLO,
MIGRATION_STATUS_COMPLETED);
@@ -133,6 +141,13 @@ static void primary_vm_do_failover(void)
FailoverStatus_str(old_state));
return;
}
+
+ replication_stop_all(true, &local_err);
+ if (local_err) {
+ error_report_err(local_err);
+ local_err = NULL;
+ }
+
/* Notify COLO thread that failover work is finished */
qemu_sem_post(&s->colo_exit_sem);
}
@@ -356,6 +371,11 @@ static int colo_do_checkpoint_transaction(MigrationState *s,
qemu_savevm_state_header(fb);
qemu_savevm_state_setup(fb);
qemu_mutex_lock_iothread();
+ replication_do_checkpoint_all(&local_err);
+ if (local_err) {
+ qemu_mutex_unlock_iothread();
+ goto out;
+ }
qemu_savevm_state_complete_precopy(fb, false, false);
qemu_mutex_unlock_iothread();
@@ -446,6 +466,12 @@ static void colo_process_checkpoint(MigrationState *s)
object_unref(OBJECT(bioc));
qemu_mutex_lock_iothread();
+ replication_start_all(REPLICATION_MODE_PRIMARY, &local_err);
+ if (local_err) {
+ qemu_mutex_unlock_iothread();
+ goto out;
+ }
+
vm_start();
qemu_mutex_unlock_iothread();
trace_colo_vm_state_change("stop", "run");
@@ -585,6 +611,11 @@ void *colo_process_incoming_thread(void *opaque)
object_unref(OBJECT(bioc));
qemu_mutex_lock_iothread();
+ replication_start_all(REPLICATION_MODE_SECONDARY, &local_err);
+ if (local_err) {
+ qemu_mutex_unlock_iothread();
+ goto out;
+ }
vm_start();
trace_colo_vm_state_change("stop", "run");
qemu_mutex_unlock_iothread();
@@ -665,6 +696,18 @@ void *colo_process_incoming_thread(void *opaque)
goto out;
}
+ replication_get_error_all(&local_err);
+ if (local_err) {
+ qemu_mutex_unlock_iothread();
+ goto out;
+ }
+ /* discard colo disk buffer */
+ replication_do_checkpoint_all(&local_err);
+ if (local_err) {
+ qemu_mutex_unlock_iothread();
+ goto out;
+ }
+
vmstate_loading = false;
vm_start();
trace_colo_vm_state_change("stop", "run");
diff --git a/migration/migration.c b/migration/migration.c
index 33a2fd74c6..71492c642d 100644
--- a/migration/migration.c
+++ b/migration/migration.c
@@ -385,6 +385,7 @@ static void process_incoming_migration_co(void *opaque)
MigrationIncomingState *mis = migration_incoming_get_current();
PostcopyState ps;
int ret;
+ Error *local_err = NULL;
assert(mis->from_src_file);
mis->largest_page_size = qemu_ram_pagesize_largest();
@@ -416,6 +417,14 @@ static void process_incoming_migration_co(void *opaque)
/* we get COLO info, and know if we are in COLO mode */
if (!ret && migration_incoming_enable_colo()) {
+ /* Make sure all file formats flush their mutable metadata */
+ bdrv_invalidate_cache_all(&local_err);
+ if (local_err) {
+ migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
+ MIGRATION_STATUS_FAILED);
+ error_report_err(local_err);
+ exit(EXIT_FAILURE);
+ }
mis->migration_incoming_co = qemu_coroutine_self();
qemu_thread_create(&mis->colo_incoming_thread, "COLO incoming",
colo_process_incoming_thread, mis, QEMU_THREAD_JOINABLE);
--
2.17.GIT
^ permalink raw reply related [flat|nested] 35+ messages in thread* Re: [Qemu-devel] [PATCH V11 05/20] COLO: Add block replication into colo process
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 05/20] COLO: Add block replication into colo process Zhang Chen
@ 2018-08-17 11:04 ` Dr. David Alan Gilbert
0 siblings, 0 replies; 35+ messages in thread
From: Dr. David Alan Gilbert @ 2018-08-17 11:04 UTC (permalink / raw)
To: Zhang Chen
Cc: qemu-devel, Paolo Bonzini, Juan Quintela, Jason Wang, Eric Blake,
Markus Armbruster, zhanghailiang, Li Zhijian, Zhang Chen
* Zhang Chen (zhangckid@gmail.com) wrote:
> Make sure master start block replication after slave's block
> replication started.
>
> Besides, we need to activate VM's blocks before goes into
> COLO state.
>
> Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
> Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
> Signed-off-by: Zhang Chen <zhangckid@gmail.com>
> Signed-off-by: Zhang Chen <chen.zhang@intel.com>
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
> ---
> migration/colo.c | 43 +++++++++++++++++++++++++++++++++++++++++++
> migration/migration.c | 9 +++++++++
> 2 files changed, 52 insertions(+)
>
> diff --git a/migration/colo.c b/migration/colo.c
> index 081df1835f..e06640c3d6 100644
> --- a/migration/colo.c
> +++ b/migration/colo.c
> @@ -27,6 +27,7 @@
> #include "replication.h"
> #include "net/colo-compare.h"
> #include "net/colo.h"
> +#include "block/block.h"
>
> static bool vmstate_loading;
> static Notifier packets_compare_notifier;
> @@ -56,6 +57,7 @@ static void secondary_vm_do_failover(void)
> {
> int old_state;
> MigrationIncomingState *mis = migration_incoming_get_current();
> + Error *local_err = NULL;
>
> /* Can not do failover during the process of VM's loading VMstate, Or
> * it will break the secondary VM.
> @@ -73,6 +75,11 @@ static void secondary_vm_do_failover(void)
> migrate_set_state(&mis->state, MIGRATION_STATUS_COLO,
> MIGRATION_STATUS_COMPLETED);
>
> + replication_stop_all(true, &local_err);
> + if (local_err) {
> + error_report_err(local_err);
> + }
> +
> if (!autostart) {
> error_report("\"-S\" qemu option will be ignored in secondary side");
> /* recover runstate to normal migration finish state */
> @@ -110,6 +117,7 @@ static void primary_vm_do_failover(void)
> {
> MigrationState *s = migrate_get_current();
> int old_state;
> + Error *local_err = NULL;
>
> migrate_set_state(&s->state, MIGRATION_STATUS_COLO,
> MIGRATION_STATUS_COMPLETED);
> @@ -133,6 +141,13 @@ static void primary_vm_do_failover(void)
> FailoverStatus_str(old_state));
> return;
> }
> +
> + replication_stop_all(true, &local_err);
> + if (local_err) {
> + error_report_err(local_err);
> + local_err = NULL;
> + }
> +
> /* Notify COLO thread that failover work is finished */
> qemu_sem_post(&s->colo_exit_sem);
> }
> @@ -356,6 +371,11 @@ static int colo_do_checkpoint_transaction(MigrationState *s,
> qemu_savevm_state_header(fb);
> qemu_savevm_state_setup(fb);
> qemu_mutex_lock_iothread();
> + replication_do_checkpoint_all(&local_err);
> + if (local_err) {
> + qemu_mutex_unlock_iothread();
> + goto out;
> + }
> qemu_savevm_state_complete_precopy(fb, false, false);
> qemu_mutex_unlock_iothread();
>
> @@ -446,6 +466,12 @@ static void colo_process_checkpoint(MigrationState *s)
> object_unref(OBJECT(bioc));
>
> qemu_mutex_lock_iothread();
> + replication_start_all(REPLICATION_MODE_PRIMARY, &local_err);
> + if (local_err) {
> + qemu_mutex_unlock_iothread();
> + goto out;
> + }
> +
> vm_start();
> qemu_mutex_unlock_iothread();
> trace_colo_vm_state_change("stop", "run");
> @@ -585,6 +611,11 @@ void *colo_process_incoming_thread(void *opaque)
> object_unref(OBJECT(bioc));
>
> qemu_mutex_lock_iothread();
> + replication_start_all(REPLICATION_MODE_SECONDARY, &local_err);
> + if (local_err) {
> + qemu_mutex_unlock_iothread();
> + goto out;
> + }
> vm_start();
> trace_colo_vm_state_change("stop", "run");
> qemu_mutex_unlock_iothread();
> @@ -665,6 +696,18 @@ void *colo_process_incoming_thread(void *opaque)
> goto out;
> }
>
> + replication_get_error_all(&local_err);
> + if (local_err) {
> + qemu_mutex_unlock_iothread();
> + goto out;
> + }
> + /* discard colo disk buffer */
> + replication_do_checkpoint_all(&local_err);
> + if (local_err) {
> + qemu_mutex_unlock_iothread();
> + goto out;
> + }
> +
> vmstate_loading = false;
> vm_start();
> trace_colo_vm_state_change("stop", "run");
> diff --git a/migration/migration.c b/migration/migration.c
> index 33a2fd74c6..71492c642d 100644
> --- a/migration/migration.c
> +++ b/migration/migration.c
> @@ -385,6 +385,7 @@ static void process_incoming_migration_co(void *opaque)
> MigrationIncomingState *mis = migration_incoming_get_current();
> PostcopyState ps;
> int ret;
> + Error *local_err = NULL;
>
> assert(mis->from_src_file);
> mis->largest_page_size = qemu_ram_pagesize_largest();
> @@ -416,6 +417,14 @@ static void process_incoming_migration_co(void *opaque)
>
> /* we get COLO info, and know if we are in COLO mode */
> if (!ret && migration_incoming_enable_colo()) {
> + /* Make sure all file formats flush their mutable metadata */
> + bdrv_invalidate_cache_all(&local_err);
> + if (local_err) {
> + migrate_set_state(&mis->state, MIGRATION_STATUS_ACTIVE,
> + MIGRATION_STATUS_FAILED);
> + error_report_err(local_err);
> + exit(EXIT_FAILURE);
> + }
> mis->migration_incoming_co = qemu_coroutine_self();
> qemu_thread_create(&mis->colo_incoming_thread, "COLO incoming",
> colo_process_incoming_thread, mis, QEMU_THREAD_JOINABLE);
> --
> 2.17.GIT
>
--
Dr. David Alan Gilbert / dgilbert@redhat.com / Manchester, UK
^ permalink raw reply [flat|nested] 35+ messages in thread
* [Qemu-devel] [PATCH V11 06/20] COLO: Remove colo_state migration struct
2018-08-11 20:59 [Qemu-devel] [PATCH V11 00/20] COLO: integrate colo frame with block replication and COLO proxy Zhang Chen
` (4 preceding siblings ...)
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 05/20] COLO: Add block replication into colo process Zhang Chen
@ 2018-08-11 20:59 ` Zhang Chen
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 07/20] COLO: Load dirty pages into SVM's RAM cache firstly Zhang Chen
` (14 subsequent siblings)
20 siblings, 0 replies; 35+ messages in thread
From: Zhang Chen @ 2018-08-11 20:59 UTC (permalink / raw)
To: qemu-devel, Paolo Bonzini, Juan Quintela, Dr . David Alan Gilbert,
Jason Wang, Eric Blake, Markus Armbruster
Cc: Zhang Chen, zhanghailiang, Li Zhijian, Zhang Chen
We need to know if migration is going into COLO state for
incoming side before start normal migration.
Instead by using the VMStateDescription to send colo_state
from source side to destination side, we use MIG_CMD_ENABLE_COLO
to indicate whether COLO is enabled or not.
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Zhang Chen <zhangckid@gmail.com>
Signed-off-by: Zhang Chen <chen.zhang@intel.com>
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
---
include/migration/colo.h | 5 +--
migration/Makefile.objs | 2 +-
migration/colo-comm.c | 76 ----------------------------------------
migration/colo.c | 13 ++++++-
migration/migration.c | 23 +++++++++++-
migration/savevm.c | 17 +++++++++
migration/savevm.h | 1 +
migration/trace-events | 1 +
vl.c | 2 --
9 files changed, 57 insertions(+), 83 deletions(-)
delete mode 100644 migration/colo-comm.c
diff --git a/include/migration/colo.h b/include/migration/colo.h
index fefb2fcf4c..99ce17aca7 100644
--- a/include/migration/colo.h
+++ b/include/migration/colo.h
@@ -28,8 +28,9 @@ void migrate_start_colo_process(MigrationState *s);
bool migration_in_colo_state(void);
/* loadvm */
-bool migration_incoming_enable_colo(void);
-void migration_incoming_exit_colo(void);
+void migration_incoming_enable_colo(void);
+void migration_incoming_disable_colo(void);
+bool migration_incoming_colo_enabled(void);
void *colo_process_incoming_thread(void *opaque);
bool migration_incoming_in_colo_state(void);
diff --git a/migration/Makefile.objs b/migration/Makefile.objs
index c83ec47ba8..a4f3bafd86 100644
--- a/migration/Makefile.objs
+++ b/migration/Makefile.objs
@@ -1,6 +1,6 @@
common-obj-y += migration.o socket.o fd.o exec.o
common-obj-y += tls.o channel.o savevm.o
-common-obj-y += colo-comm.o colo.o colo-failover.o
+common-obj-y += colo.o colo-failover.o
common-obj-y += vmstate.o vmstate-types.o page_cache.o
common-obj-y += qemu-file.o global_state.o
common-obj-y += qemu-file-channel.o
diff --git a/migration/colo-comm.c b/migration/colo-comm.c
deleted file mode 100644
index df26e4dfe7..0000000000
--- a/migration/colo-comm.c
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * COarse-grain LOck-stepping Virtual Machines for Non-stop Service (COLO)
- * (a.k.a. Fault Tolerance or Continuous Replication)
- *
- * Copyright (c) 2016 HUAWEI TECHNOLOGIES CO., LTD.
- * Copyright (c) 2016 FUJITSU LIMITED
- * Copyright (c) 2016 Intel Corporation
- *
- * This work is licensed under the terms of the GNU GPL, version 2 or
- * later. See the COPYING file in the top-level directory.
- *
- */
-
-#include "qemu/osdep.h"
-#include "migration.h"
-#include "migration/colo.h"
-#include "migration/vmstate.h"
-#include "trace.h"
-
-typedef struct {
- bool colo_requested;
-} COLOInfo;
-
-static COLOInfo colo_info;
-
-COLOMode get_colo_mode(void)
-{
- if (migration_in_colo_state()) {
- return COLO_MODE_PRIMARY;
- } else if (migration_incoming_in_colo_state()) {
- return COLO_MODE_SECONDARY;
- } else {
- return COLO_MODE_UNKNOWN;
- }
-}
-
-static int colo_info_pre_save(void *opaque)
-{
- COLOInfo *s = opaque;
-
- s->colo_requested = migrate_colo_enabled();
-
- return 0;
-}
-
-static bool colo_info_need(void *opaque)
-{
- return migrate_colo_enabled();
-}
-
-static const VMStateDescription colo_state = {
- .name = "COLOState",
- .version_id = 1,
- .minimum_version_id = 1,
- .pre_save = colo_info_pre_save,
- .needed = colo_info_need,
- .fields = (VMStateField[]) {
- VMSTATE_BOOL(colo_requested, COLOInfo),
- VMSTATE_END_OF_LIST()
- },
-};
-
-void colo_info_init(void)
-{
- vmstate_register(NULL, 0, &colo_state, &colo_info);
-}
-
-bool migration_incoming_enable_colo(void)
-{
- return colo_info.colo_requested;
-}
-
-void migration_incoming_exit_colo(void)
-{
- colo_info.colo_requested = false;
-}
diff --git a/migration/colo.c b/migration/colo.c
index e06640c3d6..c083d3696f 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -152,6 +152,17 @@ static void primary_vm_do_failover(void)
qemu_sem_post(&s->colo_exit_sem);
}
+COLOMode get_colo_mode(void)
+{
+ if (migration_in_colo_state()) {
+ return COLO_MODE_PRIMARY;
+ } else if (migration_incoming_in_colo_state()) {
+ return COLO_MODE_SECONDARY;
+ } else {
+ return COLO_MODE_UNKNOWN;
+ }
+}
+
void colo_do_failover(MigrationState *s)
{
/* Make sure VM stopped while failover happened. */
@@ -745,7 +756,7 @@ out:
if (mis->to_src_file) {
qemu_fclose(mis->to_src_file);
}
- migration_incoming_exit_colo();
+ migration_incoming_disable_colo();
return NULL;
}
diff --git a/migration/migration.c b/migration/migration.c
index 71492c642d..e5330dbede 100644
--- a/migration/migration.c
+++ b/migration/migration.c
@@ -295,6 +295,22 @@ int migrate_send_rp_req_pages(MigrationIncomingState *mis, const char *rbname,
return migrate_send_rp_message(mis, msg_type, msglen, bufc);
}
+static bool migration_colo_enabled;
+bool migration_incoming_colo_enabled(void)
+{
+ return migration_colo_enabled;
+}
+
+void migration_incoming_disable_colo(void)
+{
+ migration_colo_enabled = false;
+}
+
+void migration_incoming_enable_colo(void)
+{
+ migration_colo_enabled = true;
+}
+
void qemu_start_incoming_migration(const char *uri, Error **errp)
{
const char *p;
@@ -416,7 +432,7 @@ static void process_incoming_migration_co(void *opaque)
}
/* we get COLO info, and know if we are in COLO mode */
- if (!ret && migration_incoming_enable_colo()) {
+ if (!ret && migration_incoming_colo_enabled()) {
/* Make sure all file formats flush their mutable metadata */
bdrv_invalidate_cache_all(&local_err);
if (local_err) {
@@ -2970,6 +2986,11 @@ static void *migration_thread(void *opaque)
qemu_savevm_send_postcopy_advise(s->to_dst_file);
}
+ if (migrate_colo_enabled()) {
+ /* Notify migration destination that we enable COLO */
+ qemu_savevm_send_colo_enable(s->to_dst_file);
+ }
+
qemu_savevm_state_setup(s->to_dst_file);
s->setup_time = qemu_clock_get_ms(QEMU_CLOCK_HOST) - setup_start;
diff --git a/migration/savevm.c b/migration/savevm.c
index 7f92567a10..437308877a 100644
--- a/migration/savevm.c
+++ b/migration/savevm.c
@@ -56,6 +56,7 @@
#include "io/channel-file.h"
#include "sysemu/replay.h"
#include "qjson.h"
+#include "migration/colo.h"
#ifndef ETH_P_RARP
#define ETH_P_RARP 0x8035
@@ -82,6 +83,7 @@ enum qemu_vm_cmd {
were previously sent during
precopy but are dirty. */
MIG_CMD_PACKAGED, /* Send a wrapped stream within this stream */
+ MIG_CMD_ENABLE_COLO, /* Enable COLO */
MIG_CMD_POSTCOPY_RESUME, /* resume postcopy on dest */
MIG_CMD_RECV_BITMAP, /* Request for recved bitmap on dst */
MIG_CMD_MAX
@@ -841,6 +843,12 @@ static void qemu_savevm_command_send(QEMUFile *f,
qemu_fflush(f);
}
+void qemu_savevm_send_colo_enable(QEMUFile *f)
+{
+ trace_savevm_send_colo_enable();
+ qemu_savevm_command_send(f, MIG_CMD_ENABLE_COLO, 0, NULL);
+}
+
void qemu_savevm_send_ping(QEMUFile *f, uint32_t value)
{
uint32_t buf;
@@ -1918,6 +1926,12 @@ static int loadvm_handle_recv_bitmap(MigrationIncomingState *mis,
return 0;
}
+static int loadvm_process_enable_colo(MigrationIncomingState *mis)
+{
+ migration_incoming_enable_colo();
+ return 0;
+}
+
/*
* Process an incoming 'QEMU_VM_COMMAND'
* 0 just a normal return
@@ -1997,6 +2011,9 @@ static int loadvm_process_command(QEMUFile *f)
case MIG_CMD_RECV_BITMAP:
return loadvm_handle_recv_bitmap(mis, len);
+
+ case MIG_CMD_ENABLE_COLO:
+ return loadvm_process_enable_colo(mis);
}
return 0;
diff --git a/migration/savevm.h b/migration/savevm.h
index a5e65b8ae3..8373c2f6bd 100644
--- a/migration/savevm.h
+++ b/migration/savevm.h
@@ -55,6 +55,7 @@ void qemu_savevm_send_postcopy_ram_discard(QEMUFile *f, const char *name,
uint16_t len,
uint64_t *start_list,
uint64_t *length_list);
+void qemu_savevm_send_colo_enable(QEMUFile *f);
int qemu_loadvm_state(QEMUFile *f);
void qemu_loadvm_state_cleanup(void);
diff --git a/migration/trace-events b/migration/trace-events
index 9430f3cbe0..fa0ff3f3bf 100644
--- a/migration/trace-events
+++ b/migration/trace-events
@@ -37,6 +37,7 @@ savevm_send_ping(uint32_t val) "0x%x"
savevm_send_postcopy_listen(void) ""
savevm_send_postcopy_run(void) ""
savevm_send_postcopy_resume(void) ""
+savevm_send_colo_enable(void) ""
savevm_send_recv_bitmap(char *name) "%s"
savevm_state_setup(void) ""
savevm_state_resume_prepare(void) ""
diff --git a/vl.c b/vl.c
index 16b913f9d5..a4e877e2ff 100644
--- a/vl.c
+++ b/vl.c
@@ -4392,8 +4392,6 @@ int main(int argc, char **argv, char **envp)
#endif
}
- colo_info_init();
-
if (net_init_clients(&err) < 0) {
error_report_err(err);
exit(1);
--
2.17.GIT
^ permalink raw reply related [flat|nested] 35+ messages in thread* [Qemu-devel] [PATCH V11 07/20] COLO: Load dirty pages into SVM's RAM cache firstly
2018-08-11 20:59 [Qemu-devel] [PATCH V11 00/20] COLO: integrate colo frame with block replication and COLO proxy Zhang Chen
` (5 preceding siblings ...)
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 06/20] COLO: Remove colo_state migration struct Zhang Chen
@ 2018-08-11 20:59 ` Zhang Chen
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 08/20] ram/COLO: Record the dirty pages that SVM received Zhang Chen
` (13 subsequent siblings)
20 siblings, 0 replies; 35+ messages in thread
From: Zhang Chen @ 2018-08-11 20:59 UTC (permalink / raw)
To: qemu-devel, Paolo Bonzini, Juan Quintela, Dr . David Alan Gilbert,
Jason Wang, Eric Blake, Markus Armbruster
Cc: Zhang Chen, zhanghailiang, Li Zhijian, Zhang Chen
We should not load PVM's state directly into SVM, because there maybe some
errors happen when SVM is receving data, which will break SVM.
We need to ensure receving all data before load the state into SVM. We use
an extra memory to cache these data (PVM's ram). The ram cache in secondary side
is initially the same as SVM/PVM's memory. And in the process of checkpoint,
we cache the dirty pages of PVM into this ram cache firstly, so this ram cache
always the same as PVM's memory at every checkpoint, then we flush this cached ram
to SVM after we receive all PVM's state.
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
Signed-off-by: Zhang Chen <zhangckid@gmail.com>
Signed-off-by: Zhang Chen <chen.zhang@intel.com>
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
---
include/exec/ram_addr.h | 1 +
migration/migration.c | 6 +++
migration/ram.c | 83 ++++++++++++++++++++++++++++++++++++++++-
migration/ram.h | 4 ++
migration/savevm.c | 2 +-
5 files changed, 93 insertions(+), 3 deletions(-)
diff --git a/include/exec/ram_addr.h b/include/exec/ram_addr.h
index cf4ce06248..a78c1c99a7 100644
--- a/include/exec/ram_addr.h
+++ b/include/exec/ram_addr.h
@@ -27,6 +27,7 @@ struct RAMBlock {
struct rcu_head rcu;
struct MemoryRegion *mr;
uint8_t *host;
+ uint8_t *colo_cache; /* For colo, VM's ram cache */
ram_addr_t offset;
ram_addr_t used_length;
ram_addr_t max_length;
diff --git a/migration/migration.c b/migration/migration.c
index e5330dbede..d702a847f2 100644
--- a/migration/migration.c
+++ b/migration/migration.c
@@ -441,6 +441,10 @@ static void process_incoming_migration_co(void *opaque)
error_report_err(local_err);
exit(EXIT_FAILURE);
}
+ if (colo_init_ram_cache() < 0) {
+ error_report("Init ram cache failed");
+ exit(EXIT_FAILURE);
+ }
mis->migration_incoming_co = qemu_coroutine_self();
qemu_thread_create(&mis->colo_incoming_thread, "COLO incoming",
colo_process_incoming_thread, mis, QEMU_THREAD_JOINABLE);
@@ -449,6 +453,8 @@ static void process_incoming_migration_co(void *opaque)
/* Wait checkpoint incoming thread exit before free resource */
qemu_thread_join(&mis->colo_incoming_thread);
+ /* We hold the global iothread lock, so it is safe here */
+ colo_release_ram_cache();
}
if (ret < 0) {
diff --git a/migration/ram.c b/migration/ram.c
index 24dea2730c..ce2cf99bf5 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -3321,6 +3321,20 @@ static inline void *host_from_ram_block_offset(RAMBlock *block,
return block->host + offset;
}
+static inline void *colo_cache_from_block_offset(RAMBlock *block,
+ ram_addr_t offset)
+{
+ if (!offset_in_ramblock(block, offset)) {
+ return NULL;
+ }
+ if (!block->colo_cache) {
+ error_report("%s: colo_cache is NULL in block :%s",
+ __func__, block->idstr);
+ return NULL;
+ }
+ return block->colo_cache + offset;
+}
+
/**
* ram_handle_compressed: handle the zero page case
*
@@ -3525,6 +3539,58 @@ static void decompress_data_with_multi_threads(QEMUFile *f,
qemu_mutex_unlock(&decomp_done_lock);
}
+/*
+ * colo cache: this is for secondary VM, we cache the whole
+ * memory of the secondary VM, it is need to hold the global lock
+ * to call this helper.
+ */
+int colo_init_ram_cache(void)
+{
+ RAMBlock *block;
+
+ rcu_read_lock();
+ QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+ block->colo_cache = qemu_anon_ram_alloc(block->used_length,
+ NULL,
+ false);
+ if (!block->colo_cache) {
+ error_report("%s: Can't alloc memory for COLO cache of block %s,"
+ "size 0x" RAM_ADDR_FMT, __func__, block->idstr,
+ block->used_length);
+ goto out_locked;
+ }
+ memcpy(block->colo_cache, block->host, block->used_length);
+ }
+ rcu_read_unlock();
+ return 0;
+
+out_locked:
+ QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+ if (block->colo_cache) {
+ qemu_anon_ram_free(block->colo_cache, block->used_length);
+ block->colo_cache = NULL;
+ }
+ }
+
+ rcu_read_unlock();
+ return -errno;
+}
+
+/* It is need to hold the global lock to call this helper */
+void colo_release_ram_cache(void)
+{
+ RAMBlock *block;
+
+ rcu_read_lock();
+ QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+ if (block->colo_cache) {
+ qemu_anon_ram_free(block->colo_cache, block->used_length);
+ block->colo_cache = NULL;
+ }
+ }
+ rcu_read_unlock();
+}
+
/**
* ram_load_setup: Setup RAM for migration incoming side
*
@@ -3541,6 +3607,7 @@ static int ram_load_setup(QEMUFile *f, void *opaque)
xbzrle_load_setup();
ramblock_recv_map_init();
+
return 0;
}
@@ -3554,6 +3621,7 @@ static int ram_load_cleanup(void *opaque)
g_free(rb->receivedmap);
rb->receivedmap = NULL;
}
+
return 0;
}
@@ -3791,13 +3859,24 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
RAM_SAVE_FLAG_COMPRESS_PAGE | RAM_SAVE_FLAG_XBZRLE)) {
RAMBlock *block = ram_block_from_stream(f, flags);
- host = host_from_ram_block_offset(block, addr);
+ /*
+ * After going into COLO, we should load the Page into colo_cache.
+ */
+ if (migration_incoming_in_colo_state()) {
+ host = colo_cache_from_block_offset(block, addr);
+ } else {
+ host = host_from_ram_block_offset(block, addr);
+ }
if (!host) {
error_report("Illegal RAM offset " RAM_ADDR_FMT, addr);
ret = -EINVAL;
break;
}
- ramblock_recv_bitmap_set(block, host);
+
+ if (!migration_incoming_in_colo_state()) {
+ ramblock_recv_bitmap_set(block, host);
+ }
+
trace_ram_load_loop(block->idstr, (uint64_t)addr, flags, host);
}
diff --git a/migration/ram.h b/migration/ram.h
index 457bf54b8c..d009480494 100644
--- a/migration/ram.h
+++ b/migration/ram.h
@@ -70,4 +70,8 @@ int64_t ramblock_recv_bitmap_send(QEMUFile *file,
const char *block_name);
int ram_dirty_bitmap_reload(MigrationState *s, RAMBlock *rb);
+/* ram cache */
+int colo_init_ram_cache(void);
+void colo_release_ram_cache(void);
+
#endif
diff --git a/migration/savevm.c b/migration/savevm.c
index 437308877a..33e9e7cda0 100644
--- a/migration/savevm.c
+++ b/migration/savevm.c
@@ -1929,7 +1929,7 @@ static int loadvm_handle_recv_bitmap(MigrationIncomingState *mis,
static int loadvm_process_enable_colo(MigrationIncomingState *mis)
{
migration_incoming_enable_colo();
- return 0;
+ return colo_init_ram_cache();
}
/*
--
2.17.GIT
^ permalink raw reply related [flat|nested] 35+ messages in thread* [Qemu-devel] [PATCH V11 08/20] ram/COLO: Record the dirty pages that SVM received
2018-08-11 20:59 [Qemu-devel] [PATCH V11 00/20] COLO: integrate colo frame with block replication and COLO proxy Zhang Chen
` (6 preceding siblings ...)
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 07/20] COLO: Load dirty pages into SVM's RAM cache firstly Zhang Chen
@ 2018-08-11 20:59 ` Zhang Chen
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 09/20] COLO: Flush memory data from ram cache Zhang Chen
` (12 subsequent siblings)
20 siblings, 0 replies; 35+ messages in thread
From: Zhang Chen @ 2018-08-11 20:59 UTC (permalink / raw)
To: qemu-devel, Paolo Bonzini, Juan Quintela, Dr . David Alan Gilbert,
Jason Wang, Eric Blake, Markus Armbruster
Cc: Zhang Chen, zhanghailiang, Li Zhijian, Zhang Chen
We record the address of the dirty pages that received,
it will help flushing pages that cached into SVM.
Here, it is a trick, we record dirty pages by re-using migration
dirty bitmap. In the later patch, we will start the dirty log
for SVM, just like migration, in this way, we can record both
the dirty pages caused by PVM and SVM, we only flush those dirty
pages from RAM cache while do checkpoint.
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Zhang Chen <zhangckid@gmail.com>
Signed-off-by: Zhang Chen <chen.zhang@intel.com>
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
---
migration/ram.c | 43 ++++++++++++++++++++++++++++++++++++++++---
1 file changed, 40 insertions(+), 3 deletions(-)
diff --git a/migration/ram.c b/migration/ram.c
index ce2cf99bf5..3dc403adb5 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -3332,6 +3332,15 @@ static inline void *colo_cache_from_block_offset(RAMBlock *block,
__func__, block->idstr);
return NULL;
}
+
+ /*
+ * During colo checkpoint, we need bitmap of these migrated pages.
+ * It help us to decide which pages in ram cache should be flushed
+ * into VM's RAM later.
+ */
+ if (!test_and_set_bit(offset >> TARGET_PAGE_BITS, block->bmap)) {
+ ram_state->migration_dirty_pages++;
+ }
return block->colo_cache + offset;
}
@@ -3549,7 +3558,7 @@ int colo_init_ram_cache(void)
RAMBlock *block;
rcu_read_lock();
- QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+ RAMBLOCK_FOREACH_MIGRATABLE(block) {
block->colo_cache = qemu_anon_ram_alloc(block->used_length,
NULL,
false);
@@ -3562,10 +3571,29 @@ int colo_init_ram_cache(void)
memcpy(block->colo_cache, block->host, block->used_length);
}
rcu_read_unlock();
+ /*
+ * Record the dirty pages that sent by PVM, we use this dirty bitmap together
+ * with to decide which page in cache should be flushed into SVM's RAM. Here
+ * we use the same name 'ram_bitmap' as for migration.
+ */
+ if (ram_bytes_total()) {
+ RAMBlock *block;
+
+ RAMBLOCK_FOREACH_MIGRATABLE(block) {
+ unsigned long pages = block->max_length >> TARGET_PAGE_BITS;
+
+ block->bmap = bitmap_new(pages);
+ bitmap_set(block->bmap, 0, pages);
+ }
+ }
+ ram_state = g_new0(RAMState, 1);
+ ram_state->migration_dirty_pages = 0;
+
return 0;
out_locked:
- QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+
+ RAMBLOCK_FOREACH_MIGRATABLE(block) {
if (block->colo_cache) {
qemu_anon_ram_free(block->colo_cache, block->used_length);
block->colo_cache = NULL;
@@ -3581,14 +3609,23 @@ void colo_release_ram_cache(void)
{
RAMBlock *block;
+ RAMBLOCK_FOREACH_MIGRATABLE(block) {
+ g_free(block->bmap);
+ block->bmap = NULL;
+ }
+
rcu_read_lock();
- QLIST_FOREACH_RCU(block, &ram_list.blocks, next) {
+
+ RAMBLOCK_FOREACH_MIGRATABLE(block) {
if (block->colo_cache) {
qemu_anon_ram_free(block->colo_cache, block->used_length);
block->colo_cache = NULL;
}
}
+
rcu_read_unlock();
+ g_free(ram_state);
+ ram_state = NULL;
}
/**
--
2.17.GIT
^ permalink raw reply related [flat|nested] 35+ messages in thread* [Qemu-devel] [PATCH V11 09/20] COLO: Flush memory data from ram cache
2018-08-11 20:59 [Qemu-devel] [PATCH V11 00/20] COLO: integrate colo frame with block replication and COLO proxy Zhang Chen
` (7 preceding siblings ...)
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 08/20] ram/COLO: Record the dirty pages that SVM received Zhang Chen
@ 2018-08-11 20:59 ` Zhang Chen
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 10/20] qmp event: Add COLO_EXIT event to notify users while exited COLO Zhang Chen
` (11 subsequent siblings)
20 siblings, 0 replies; 35+ messages in thread
From: Zhang Chen @ 2018-08-11 20:59 UTC (permalink / raw)
To: qemu-devel, Paolo Bonzini, Juan Quintela, Dr . David Alan Gilbert,
Jason Wang, Eric Blake, Markus Armbruster
Cc: Zhang Chen, zhanghailiang, Li Zhijian
During the time of VM's running, PVM may dirty some pages, we will transfer
PVM's dirty pages to SVM and store them into SVM's RAM cache at next checkpoint
time. So, the content of SVM's RAM cache will always be same with PVM's memory
after checkpoint.
Instead of flushing all content of PVM's RAM cache into SVM's MEMORY,
we do this in a more efficient way:
Only flush any page that dirtied by PVM since last checkpoint.
In this way, we can ensure SVM's memory same with PVM's.
Besides, we must ensure flush RAM cache before load device state.
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
---
migration/ram.c | 37 +++++++++++++++++++++++++++++++++++++
migration/trace-events | 2 ++
2 files changed, 39 insertions(+)
diff --git a/migration/ram.c b/migration/ram.c
index 3dc403adb5..b4cbcf783f 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -3841,6 +3841,39 @@ static bool postcopy_is_running(void)
return ps >= POSTCOPY_INCOMING_LISTENING && ps < POSTCOPY_INCOMING_END;
}
+/*
+ * Flush content of RAM cache into SVM's memory.
+ * Only flush the pages that be dirtied by PVM or SVM or both.
+ */
+static void colo_flush_ram_cache(void)
+{
+ RAMBlock *block = NULL;
+ void *dst_host;
+ void *src_host;
+ unsigned long offset = 0;
+
+ trace_colo_flush_ram_cache_begin(ram_state->migration_dirty_pages);
+ rcu_read_lock();
+ block = QLIST_FIRST_RCU(&ram_list.blocks);
+
+ while (block) {
+ offset = migration_bitmap_find_dirty(ram_state, block, offset);
+
+ if (offset << TARGET_PAGE_BITS >= block->used_length) {
+ offset = 0;
+ block = QLIST_NEXT_RCU(block, next);
+ } else {
+ migration_bitmap_clear_dirty(ram_state, block, offset);
+ dst_host = block->host + (offset << TARGET_PAGE_BITS);
+ src_host = block->colo_cache + (offset << TARGET_PAGE_BITS);
+ memcpy(dst_host, src_host, TARGET_PAGE_SIZE);
+ }
+ }
+
+ rcu_read_unlock();
+ trace_colo_flush_ram_cache_end();
+}
+
static int ram_load(QEMUFile *f, void *opaque, int version_id)
{
int flags = 0, ret = 0, invalid_flags = 0;
@@ -4017,6 +4050,10 @@ static int ram_load(QEMUFile *f, void *opaque, int version_id)
ret |= wait_for_decompress_done();
rcu_read_unlock();
trace_ram_load_complete(ret, seq_iter);
+
+ if (!ret && migration_incoming_in_colo_state()) {
+ colo_flush_ram_cache();
+ }
return ret;
}
diff --git a/migration/trace-events b/migration/trace-events
index fa0ff3f3bf..bd2d0cd25a 100644
--- a/migration/trace-events
+++ b/migration/trace-events
@@ -102,6 +102,8 @@ ram_dirty_bitmap_sync_start(void) ""
ram_dirty_bitmap_sync_wait(void) ""
ram_dirty_bitmap_sync_complete(void) ""
ram_state_resume_prepare(uint64_t v) "%" PRId64
+colo_flush_ram_cache_begin(uint64_t dirty_pages) "dirty_pages %" PRIu64
+colo_flush_ram_cache_end(void) ""
# migration/migration.c
await_return_path_close_on_source_close(void) ""
--
2.17.GIT
^ permalink raw reply related [flat|nested] 35+ messages in thread* [Qemu-devel] [PATCH V11 10/20] qmp event: Add COLO_EXIT event to notify users while exited COLO
2018-08-11 20:59 [Qemu-devel] [PATCH V11 00/20] COLO: integrate colo frame with block replication and COLO proxy Zhang Chen
` (8 preceding siblings ...)
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 09/20] COLO: Flush memory data from ram cache Zhang Chen
@ 2018-08-11 20:59 ` Zhang Chen
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 11/20] qapi/migration.json: Rename COLO unknown mode to none mode Zhang Chen
` (10 subsequent siblings)
20 siblings, 0 replies; 35+ messages in thread
From: Zhang Chen @ 2018-08-11 20:59 UTC (permalink / raw)
To: qemu-devel, Paolo Bonzini, Juan Quintela, Dr . David Alan Gilbert,
Jason Wang, Eric Blake, Markus Armbruster
Cc: Zhang Chen, zhanghailiang, Li Zhijian, Zhang Chen
From: zhanghailiang <zhang.zhanghailiang@huawei.com>
If some errors happen during VM's COLO FT stage, it's important to
notify the users of this event. Together with 'x-colo-lost-heartbeat',
Users can intervene in COLO's failover work immediately.
If users don't want to get involved in COLO's failover verdict,
it is still necessary to notify users that we exited COLO mode.
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
Signed-off-by: Zhang Chen <zhangckid@gmail.com>
Signed-off-by: Zhang Chen <chen.zhang@intel.com>
Reviewed-by: Eric Blake <eblake@redhat.com>
Reviewed-by: Markus Armbruster <armbru@redhat.com>
---
migration/colo.c | 31 +++++++++++++++++++++++++++++++
qapi/migration.json | 38 ++++++++++++++++++++++++++++++++++++++
2 files changed, 69 insertions(+)
diff --git a/migration/colo.c b/migration/colo.c
index c083d3696f..ab484ad754 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -28,6 +28,7 @@
#include "net/colo-compare.h"
#include "net/colo.h"
#include "block/block.h"
+#include "qapi/qapi-events-migration.h"
static bool vmstate_loading;
static Notifier packets_compare_notifier;
@@ -514,6 +515,23 @@ out:
qemu_fclose(fb);
}
+ /*
+ * There are only two reasons we can get here, some error happened
+ * or the user triggered failover.
+ */
+ switch (failover_get_state()) {
+ case FAILOVER_STATUS_NONE:
+ qapi_event_send_colo_exit(COLO_MODE_PRIMARY,
+ COLO_EXIT_REASON_ERROR, NULL);
+ break;
+ case FAILOVER_STATUS_REQUIRE:
+ qapi_event_send_colo_exit(COLO_MODE_PRIMARY,
+ COLO_EXIT_REASON_REQUEST, NULL);
+ break;
+ default:
+ abort();
+ }
+
/* Hope this not to be too long to wait here */
qemu_sem_wait(&s->colo_exit_sem);
qemu_sem_destroy(&s->colo_exit_sem);
@@ -745,6 +763,19 @@ out:
error_report_err(local_err);
}
+ switch (failover_get_state()) {
+ case FAILOVER_STATUS_NONE:
+ qapi_event_send_colo_exit(COLO_MODE_SECONDARY,
+ COLO_EXIT_REASON_ERROR, NULL);
+ break;
+ case FAILOVER_STATUS_REQUIRE:
+ qapi_event_send_colo_exit(COLO_MODE_SECONDARY,
+ COLO_EXIT_REASON_REQUEST, NULL);
+ break;
+ default:
+ abort();
+ }
+
if (fb) {
qemu_fclose(fb);
}
diff --git a/qapi/migration.json b/qapi/migration.json
index 186e8a7303..36faea59c1 100644
--- a/qapi/migration.json
+++ b/qapi/migration.json
@@ -900,6 +900,44 @@
{ 'enum': 'FailoverStatus',
'data': [ 'none', 'require', 'active', 'completed', 'relaunch' ] }
+##
+# @COLO_EXIT:
+#
+# Emitted when VM finishes COLO mode due to some errors happening or
+# at the request of users.
+#
+# @mode: report COLO mode when COLO exited.
+#
+# @reason: describes the reason for the COLO exit.
+#
+# Since: 3.1
+#
+# Example:
+#
+# <- { "timestamp": {"seconds": 2032141960, "microseconds": 417172},
+# "event": "COLO_EXIT", "data": {"mode": "primary", "reason": "request" } }
+#
+##
+{ 'event': 'COLO_EXIT',
+ 'data': {'mode': 'COLOMode', 'reason': 'COLOExitReason' } }
+
+##
+# @COLOExitReason:
+#
+# The reason for a COLO exit
+#
+# @none: no failover has ever happened. This can't occur in the
+# COLO_EXIT event, only in the result of query-colo-status.
+#
+# @request: COLO exit is due to an external request
+#
+# @error: COLO exit is due to an internal error
+#
+# Since: 3.1
+##
+{ 'enum': 'COLOExitReason',
+ 'data': [ 'none', 'request', 'error' ] }
+
##
# @x-colo-lost-heartbeat:
#
--
2.17.GIT
^ permalink raw reply related [flat|nested] 35+ messages in thread* [Qemu-devel] [PATCH V11 11/20] qapi/migration.json: Rename COLO unknown mode to none mode.
2018-08-11 20:59 [Qemu-devel] [PATCH V11 00/20] COLO: integrate colo frame with block replication and COLO proxy Zhang Chen
` (9 preceding siblings ...)
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 10/20] qmp event: Add COLO_EXIT event to notify users while exited COLO Zhang Chen
@ 2018-08-11 20:59 ` Zhang Chen
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 12/20] qapi: Add new command to query colo status Zhang Chen
` (9 subsequent siblings)
20 siblings, 0 replies; 35+ messages in thread
From: Zhang Chen @ 2018-08-11 20:59 UTC (permalink / raw)
To: qemu-devel, Paolo Bonzini, Juan Quintela, Dr . David Alan Gilbert,
Jason Wang, Eric Blake, Markus Armbruster
Cc: Zhang Chen, zhanghailiang, Li Zhijian, Zhang Chen
From: Zhang Chen <chen.zhang@intel.com>
Suggested by Markus Armbruster rename COLO unknown mode to none mode.
Signed-off-by: Zhang Chen <zhangckid@gmail.com>
Signed-off-by: Zhang Chen <chen.zhang@intel.com>
Reviewed-by: Eric Blake <eblake@redhat.com>
Reviewed-by: Markus Armbruster <armbru@redhat.com>
---
migration/colo-failover.c | 2 +-
migration/colo.c | 2 +-
qapi/migration.json | 10 +++++-----
3 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/migration/colo-failover.c b/migration/colo-failover.c
index 0ae0c41221..4854a96c92 100644
--- a/migration/colo-failover.c
+++ b/migration/colo-failover.c
@@ -77,7 +77,7 @@ FailoverStatus failover_get_state(void)
void qmp_x_colo_lost_heartbeat(Error **errp)
{
- if (get_colo_mode() == COLO_MODE_UNKNOWN) {
+ if (get_colo_mode() == COLO_MODE_NONE) {
error_setg(errp, QERR_FEATURE_DISABLED, "colo");
return;
}
diff --git a/migration/colo.c b/migration/colo.c
index ab484ad754..8fdb79ac73 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -160,7 +160,7 @@ COLOMode get_colo_mode(void)
} else if (migration_incoming_in_colo_state()) {
return COLO_MODE_SECONDARY;
} else {
- return COLO_MODE_UNKNOWN;
+ return COLO_MODE_NONE;
}
}
diff --git a/qapi/migration.json b/qapi/migration.json
index 36faea59c1..78932206dd 100644
--- a/qapi/migration.json
+++ b/qapi/migration.json
@@ -867,18 +867,18 @@
##
# @COLOMode:
#
-# The colo mode
+# The COLO current mode.
#
-# @unknown: unknown mode
+# @none: COLO is disabled.
#
-# @primary: master side
+# @primary: COLO node in primary side.
#
-# @secondary: slave side
+# @secondary: COLO node in slave side.
#
# Since: 2.8
##
{ 'enum': 'COLOMode',
- 'data': [ 'unknown', 'primary', 'secondary'] }
+ 'data': [ 'none', 'primary', 'secondary'] }
##
# @FailoverStatus:
--
2.17.GIT
^ permalink raw reply related [flat|nested] 35+ messages in thread* [Qemu-devel] [PATCH V11 12/20] qapi: Add new command to query colo status
2018-08-11 20:59 [Qemu-devel] [PATCH V11 00/20] COLO: integrate colo frame with block replication and COLO proxy Zhang Chen
` (10 preceding siblings ...)
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 11/20] qapi/migration.json: Rename COLO unknown mode to none mode Zhang Chen
@ 2018-08-11 20:59 ` Zhang Chen
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 13/20] savevm: split the process of different stages for loadvm/savevm Zhang Chen
` (8 subsequent siblings)
20 siblings, 0 replies; 35+ messages in thread
From: Zhang Chen @ 2018-08-11 20:59 UTC (permalink / raw)
To: qemu-devel, Paolo Bonzini, Juan Quintela, Dr . David Alan Gilbert,
Jason Wang, Eric Blake, Markus Armbruster
Cc: Zhang Chen, zhanghailiang, Li Zhijian, Zhang Chen
Libvirt or other high level software can use this command query colo status.
You can test this command like that:
{'execute':'query-colo-status'}
Signed-off-by: Zhang Chen <zhangckid@gmail.com>
Signed-off-by: Zhang Chen <chen.zhang@intel.com>
---
migration/colo.c | 21 +++++++++++++++++++++
qapi/migration.json | 32 ++++++++++++++++++++++++++++++++
2 files changed, 53 insertions(+)
diff --git a/migration/colo.c b/migration/colo.c
index 8fdb79ac73..b355de3f01 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -29,6 +29,7 @@
#include "net/colo.h"
#include "block/block.h"
#include "qapi/qapi-events-migration.h"
+#include "qapi/qmp/qerror.h"
static bool vmstate_loading;
static Notifier packets_compare_notifier;
@@ -237,6 +238,26 @@ void qmp_xen_colo_do_checkpoint(Error **errp)
#endif
}
+COLOStatus *qmp_query_colo_status(Error **errp)
+{
+ COLOStatus *s = g_new0(COLOStatus, 1);
+
+ s->mode = get_colo_mode();
+
+ switch (failover_get_state()) {
+ case FAILOVER_STATUS_NONE:
+ s->reason = COLO_EXIT_REASON_NONE;
+ break;
+ case FAILOVER_STATUS_REQUIRE:
+ s->reason = COLO_EXIT_REASON_REQUEST;
+ break;
+ default:
+ s->reason = COLO_EXIT_REASON_ERROR;
+ }
+
+ return s;
+}
+
static void colo_send_message(QEMUFile *f, COLOMessage msg,
Error **errp)
{
diff --git a/qapi/migration.json b/qapi/migration.json
index 78932206dd..87312dd093 100644
--- a/qapi/migration.json
+++ b/qapi/migration.json
@@ -1251,6 +1251,38 @@
##
{ 'command': 'xen-colo-do-checkpoint' }
+##
+# @COLOStatus:
+#
+# The result format for 'query-colo-status'.
+#
+# @mode: COLO running mode. If COLO is running, this field will return
+# 'primary' or 'secondary'.
+#
+# @reason: describes the reason for the COLO exit.
+#
+# Since: 3.0
+##
+{ 'struct': 'COLOStatus',
+ 'data': { 'mode': 'COLOMode', 'reason': 'COLOExitReason' } }
+
+##
+# @query-colo-status:
+#
+# Query COLO status while the vm is running.
+#
+# Returns: A @COLOStatus object showing the status.
+#
+# Example:
+#
+# -> { "execute": "query-colo-status" }
+# <- { "return": { "mode": "primary", "active": true, "reason": "request" } }
+#
+# Since: 3.0
+##
+{ 'command': 'query-colo-status',
+ 'returns': 'COLOStatus' }
+
##
# @migrate-recover:
#
--
2.17.GIT
^ permalink raw reply related [flat|nested] 35+ messages in thread* [Qemu-devel] [PATCH V11 13/20] savevm: split the process of different stages for loadvm/savevm
2018-08-11 20:59 [Qemu-devel] [PATCH V11 00/20] COLO: integrate colo frame with block replication and COLO proxy Zhang Chen
` (11 preceding siblings ...)
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 12/20] qapi: Add new command to query colo status Zhang Chen
@ 2018-08-11 20:59 ` Zhang Chen
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 14/20] COLO: flush host dirty ram from cache Zhang Chen
` (7 subsequent siblings)
20 siblings, 0 replies; 35+ messages in thread
From: Zhang Chen @ 2018-08-11 20:59 UTC (permalink / raw)
To: qemu-devel, Paolo Bonzini, Juan Quintela, Dr . David Alan Gilbert,
Jason Wang, Eric Blake, Markus Armbruster
Cc: Zhang Chen, zhanghailiang, Li Zhijian, Zhang Chen
There are several stages during loadvm/savevm process. In different stage,
migration incoming processes different types of sections.
We want to control these stages more accuracy, it will benefit COLO
performance, we don't have to save type of QEMU_VM_SECTION_START
sections everytime while do checkpoint, besides, we want to separate
the process of saving/loading memory and devices state.
So we add three new helper functions: qemu_load_device_state() and
qemu_savevm_live_state() to achieve different process during migration.
Besides, we make qemu_loadvm_state_main() and qemu_save_device_state()
public, and simplify the codes of qemu_save_device_state() by calling the
wrapper qemu_savevm_state_header().
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
Signed-off-by: Zhang Chen <zhangckid@gmail.com>
Signed-off-by: Zhang Chen <chen.zhang@intel.com>
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
---
migration/colo.c | 41 ++++++++++++++++++++++++++++++++---------
migration/savevm.c | 36 +++++++++++++++++++++++++++++-------
migration/savevm.h | 4 ++++
3 files changed, 65 insertions(+), 16 deletions(-)
diff --git a/migration/colo.c b/migration/colo.c
index b355de3f01..688d6f40b2 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -30,6 +30,7 @@
#include "block/block.h"
#include "qapi/qapi-events-migration.h"
#include "qapi/qmp/qerror.h"
+#include "sysemu/cpus.h"
static bool vmstate_loading;
static Notifier packets_compare_notifier;
@@ -401,23 +402,34 @@ static int colo_do_checkpoint_transaction(MigrationState *s,
/* Disable block migration */
migrate_set_block_enabled(false, &local_err);
- qemu_savevm_state_header(fb);
- qemu_savevm_state_setup(fb);
qemu_mutex_lock_iothread();
replication_do_checkpoint_all(&local_err);
if (local_err) {
qemu_mutex_unlock_iothread();
goto out;
}
- qemu_savevm_state_complete_precopy(fb, false, false);
- qemu_mutex_unlock_iothread();
-
- qemu_fflush(fb);
colo_send_message(s->to_dst_file, COLO_MESSAGE_VMSTATE_SEND, &local_err);
if (local_err) {
+ qemu_mutex_unlock_iothread();
+ goto out;
+ }
+ /* Note: device state is saved into buffer */
+ ret = qemu_save_device_state(fb);
+
+ qemu_mutex_unlock_iothread();
+ if (ret < 0) {
goto out;
}
+ /*
+ * Only save VM's live state, which not including device state.
+ * TODO: We may need a timeout mechanism to prevent COLO process
+ * to be blocked here.
+ */
+ qemu_savevm_live_state(s->to_dst_file);
+
+ qemu_fflush(fb);
+
/*
* We need the size of the VMstate data in Secondary side,
* With which we can decide how much data should be read.
@@ -635,6 +647,7 @@ void *colo_process_incoming_thread(void *opaque)
uint64_t total_size;
uint64_t value;
Error *local_err = NULL;
+ int ret;
qemu_sem_init(&mis->colo_incoming_sem, 0);
@@ -707,6 +720,16 @@ void *colo_process_incoming_thread(void *opaque)
goto out;
}
+ qemu_mutex_lock_iothread();
+ cpu_synchronize_all_pre_loadvm();
+ ret = qemu_loadvm_state_main(mis->from_src_file, mis);
+ qemu_mutex_unlock_iothread();
+
+ if (ret < 0) {
+ error_report("Load VM's live state (ram) error");
+ goto out;
+ }
+
value = colo_receive_message_value(mis->from_src_file,
COLO_MESSAGE_VMSTATE_SIZE, &local_err);
if (local_err) {
@@ -738,10 +761,10 @@ void *colo_process_incoming_thread(void *opaque)
}
qemu_mutex_lock_iothread();
- qemu_system_reset(SHUTDOWN_CAUSE_NONE);
vmstate_loading = true;
- if (qemu_loadvm_state(fb) < 0) {
- error_report("COLO: loadvm failed");
+ ret = qemu_load_device_state(fb);
+ if (ret < 0) {
+ error_report("COLO: load device state failed");
qemu_mutex_unlock_iothread();
goto out;
}
diff --git a/migration/savevm.c b/migration/savevm.c
index 33e9e7cda0..3a6aa747f9 100644
--- a/migration/savevm.c
+++ b/migration/savevm.c
@@ -1378,13 +1378,21 @@ done:
return ret;
}
-static int qemu_save_device_state(QEMUFile *f)
+void qemu_savevm_live_state(QEMUFile *f)
{
- SaveStateEntry *se;
+ /* save QEMU_VM_SECTION_END section */
+ qemu_savevm_state_complete_precopy(f, true, false);
+ qemu_put_byte(f, QEMU_VM_EOF);
+}
- qemu_put_be32(f, QEMU_VM_FILE_MAGIC);
- qemu_put_be32(f, QEMU_VM_FILE_VERSION);
+int qemu_save_device_state(QEMUFile *f)
+{
+ SaveStateEntry *se;
+ if (!migration_in_colo_state()) {
+ qemu_put_be32(f, QEMU_VM_FILE_MAGIC);
+ qemu_put_be32(f, QEMU_VM_FILE_VERSION);
+ }
cpu_synchronize_all_states();
QTAILQ_FOREACH(se, &savevm_state.handlers, entry) {
@@ -1440,8 +1448,6 @@ enum LoadVMExitCodes {
LOADVM_QUIT = 1,
};
-static int qemu_loadvm_state_main(QEMUFile *f, MigrationIncomingState *mis);
-
/* ------ incoming postcopy messages ------ */
/* 'advise' arrives before any transfers just to tell us that a postcopy
* *might* happen - it might be skipped if precopy transferred everything
@@ -2241,7 +2247,7 @@ static bool postcopy_pause_incoming(MigrationIncomingState *mis)
return true;
}
-static int qemu_loadvm_state_main(QEMUFile *f, MigrationIncomingState *mis)
+int qemu_loadvm_state_main(QEMUFile *f, MigrationIncomingState *mis)
{
uint8_t section_type;
int ret = 0;
@@ -2410,6 +2416,22 @@ int qemu_loadvm_state(QEMUFile *f)
return ret;
}
+int qemu_load_device_state(QEMUFile *f)
+{
+ MigrationIncomingState *mis = migration_incoming_get_current();
+ int ret;
+
+ /* Load QEMU_VM_SECTION_FULL section */
+ ret = qemu_loadvm_state_main(f, mis);
+ if (ret < 0) {
+ error_report("Failed to load device state: %d", ret);
+ return ret;
+ }
+
+ cpu_synchronize_all_post_init();
+ return 0;
+}
+
int save_snapshot(const char *name, Error **errp)
{
BlockDriverState *bs, *bs1;
diff --git a/migration/savevm.h b/migration/savevm.h
index 8373c2f6bd..51a4b9caa8 100644
--- a/migration/savevm.h
+++ b/migration/savevm.h
@@ -56,8 +56,12 @@ void qemu_savevm_send_postcopy_ram_discard(QEMUFile *f, const char *name,
uint64_t *start_list,
uint64_t *length_list);
void qemu_savevm_send_colo_enable(QEMUFile *f);
+void qemu_savevm_live_state(QEMUFile *f);
+int qemu_save_device_state(QEMUFile *f);
int qemu_loadvm_state(QEMUFile *f);
void qemu_loadvm_state_cleanup(void);
+int qemu_loadvm_state_main(QEMUFile *f, MigrationIncomingState *mis);
+int qemu_load_device_state(QEMUFile *f);
#endif
--
2.17.GIT
^ permalink raw reply related [flat|nested] 35+ messages in thread* [Qemu-devel] [PATCH V11 14/20] COLO: flush host dirty ram from cache
2018-08-11 20:59 [Qemu-devel] [PATCH V11 00/20] COLO: integrate colo frame with block replication and COLO proxy Zhang Chen
` (12 preceding siblings ...)
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 13/20] savevm: split the process of different stages for loadvm/savevm Zhang Chen
@ 2018-08-11 20:59 ` Zhang Chen
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 15/20] net/net.c: Add net client type check function for COLO Zhang Chen
` (6 subsequent siblings)
20 siblings, 0 replies; 35+ messages in thread
From: Zhang Chen @ 2018-08-11 20:59 UTC (permalink / raw)
To: qemu-devel, Paolo Bonzini, Juan Quintela, Dr . David Alan Gilbert,
Jason Wang, Eric Blake, Markus Armbruster
Cc: Zhang Chen, zhanghailiang, Li Zhijian, Zhang Chen
From: zhanghailiang <zhang.zhanghailiang@huawei.com>
Don't need to flush all VM's ram from cache, only
flush the dirty pages since last checkpoint
Signed-off-by: Li Zhijian <lizhijian@cn.fujitsu.com>
Signed-off-by: Zhang Chen <zhangckid@gmail.com>
Signed-off-by: Zhang Chen <chen.zhang@intel.com>
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
---
migration/ram.c | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/migration/ram.c b/migration/ram.c
index b4cbcf783f..734e944a21 100644
--- a/migration/ram.c
+++ b/migration/ram.c
@@ -3588,6 +3588,7 @@ int colo_init_ram_cache(void)
}
ram_state = g_new0(RAMState, 1);
ram_state->migration_dirty_pages = 0;
+ memory_global_dirty_log_start();
return 0;
@@ -3609,6 +3610,7 @@ void colo_release_ram_cache(void)
{
RAMBlock *block;
+ memory_global_dirty_log_stop();
RAMBLOCK_FOREACH_MIGRATABLE(block) {
g_free(block->bmap);
block->bmap = NULL;
@@ -3852,6 +3854,13 @@ static void colo_flush_ram_cache(void)
void *src_host;
unsigned long offset = 0;
+ memory_global_dirty_log_sync();
+ rcu_read_lock();
+ RAMBLOCK_FOREACH_MIGRATABLE(block) {
+ migration_bitmap_sync_range(ram_state, block, 0, block->used_length);
+ }
+ rcu_read_unlock();
+
trace_colo_flush_ram_cache_begin(ram_state->migration_dirty_pages);
rcu_read_lock();
block = QLIST_FIRST_RCU(&ram_list.blocks);
--
2.17.GIT
^ permalink raw reply related [flat|nested] 35+ messages in thread* [Qemu-devel] [PATCH V11 15/20] net/net.c: Add net client type check function for COLO
2018-08-11 20:59 [Qemu-devel] [PATCH V11 00/20] COLO: integrate colo frame with block replication and COLO proxy Zhang Chen
` (13 preceding siblings ...)
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 14/20] COLO: flush host dirty ram from cache Zhang Chen
@ 2018-08-11 20:59 ` Zhang Chen
2018-08-21 3:26 ` Jason Wang
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 16/20] filter: Add handle_event method for NetFilterClass Zhang Chen
` (5 subsequent siblings)
20 siblings, 1 reply; 35+ messages in thread
From: Zhang Chen @ 2018-08-11 20:59 UTC (permalink / raw)
To: qemu-devel, Paolo Bonzini, Juan Quintela, Dr . David Alan Gilbert,
Jason Wang, Eric Blake, Markus Armbruster
Cc: Zhang Chen, zhanghailiang, Li Zhijian, Zhang Chen
From: Zhang Chen <chen.zhang@intel.com>
We add is_colo_support_client_type() to check the net client type for
COLO-compare. Currently we just support TAP.
Suggested by Jason.
Signed-off-by: Zhang Chen <zhangckid@gmail.com>
Signed-off-by: Zhang Chen <chen.zhang@intel.com>
---
include/net/net.h | 1 +
net/colo-compare.c | 5 +++++
net/net.c | 14 ++++++++++++++
3 files changed, 20 insertions(+)
diff --git a/include/net/net.h b/include/net/net.h
index 1425960f76..dcbc7ba9c0 100644
--- a/include/net/net.h
+++ b/include/net/net.h
@@ -212,6 +212,7 @@ void hmp_host_net_add(Monitor *mon, const QDict *qdict);
void hmp_host_net_remove(Monitor *mon, const QDict *qdict);
void netdev_add(QemuOpts *opts, Error **errp);
void qmp_netdev_add(QDict *qdict, QObject **ret, Error **errp);
+bool is_colo_support_client_type(void);
int net_hub_id_for_client(NetClientState *nc, int *id);
NetClientState *net_hub_port_find(int hub_id);
diff --git a/net/colo-compare.c b/net/colo-compare.c
index 426eab5973..b8c0240725 100644
--- a/net/colo-compare.c
+++ b/net/colo-compare.c
@@ -996,6 +996,11 @@ static void colo_compare_complete(UserCreatable *uc, Error **errp)
return;
}
+ if (!is_colo_support_client_type()) {
+ error_setg(errp, "COLO-compare: Net client type is not supported");
+ return;
+ }
+
net_socket_rs_init(&s->pri_rs, compare_pri_rs_finalize, s->vnet_hdr);
net_socket_rs_init(&s->sec_rs, compare_sec_rs_finalize, s->vnet_hdr);
diff --git a/net/net.c b/net/net.c
index 2a3133990c..a77ea88fff 100644
--- a/net/net.c
+++ b/net/net.c
@@ -1733,3 +1733,17 @@ int net_fill_rstate(SocketReadState *rs, const uint8_t *buf, int size)
assert(size == 0);
return 0;
}
+
+/* Currently, COLO just support TAP */
+bool is_colo_support_client_type(void)
+{
+ NetClientState *nc;
+
+ QTAILQ_FOREACH(nc, &net_clients, next) {
+ if (nc->info->type != NET_CLIENT_DRIVER_TAP) {
+ return false;
+ }
+ }
+
+ return true;
+}
--
2.17.GIT
^ permalink raw reply related [flat|nested] 35+ messages in thread* Re: [Qemu-devel] [PATCH V11 15/20] net/net.c: Add net client type check function for COLO
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 15/20] net/net.c: Add net client type check function for COLO Zhang Chen
@ 2018-08-21 3:26 ` Jason Wang
2018-08-21 9:26 ` Zhang Chen
0 siblings, 1 reply; 35+ messages in thread
From: Jason Wang @ 2018-08-21 3:26 UTC (permalink / raw)
To: Zhang Chen, qemu-devel, Paolo Bonzini, Juan Quintela,
Dr . David Alan Gilbert, Eric Blake, Markus Armbruster
Cc: zhanghailiang, Li Zhijian, Zhang Chen
On 2018年08月12日 04:59, Zhang Chen wrote:
> From: Zhang Chen <chen.zhang@intel.com>
>
> We add is_colo_support_client_type() to check the net client type for
> COLO-compare. Currently we just support TAP.
> Suggested by Jason.
>
> Signed-off-by: Zhang Chen <zhangckid@gmail.com>
> Signed-off-by: Zhang Chen <chen.zhang@intel.com>
> ---
This needs more works:
- Forbid vhost
- Deal with hotplug
Looks not a must for this series, so to speed up the process, you may do
it on top.
Thanks
> include/net/net.h | 1 +
> net/colo-compare.c | 5 +++++
> net/net.c | 14 ++++++++++++++
> 3 files changed, 20 insertions(+)
>
> diff --git a/include/net/net.h b/include/net/net.h
> index 1425960f76..dcbc7ba9c0 100644
> --- a/include/net/net.h
> +++ b/include/net/net.h
> @@ -212,6 +212,7 @@ void hmp_host_net_add(Monitor *mon, const QDict *qdict);
> void hmp_host_net_remove(Monitor *mon, const QDict *qdict);
> void netdev_add(QemuOpts *opts, Error **errp);
> void qmp_netdev_add(QDict *qdict, QObject **ret, Error **errp);
> +bool is_colo_support_client_type(void);
>
> int net_hub_id_for_client(NetClientState *nc, int *id);
> NetClientState *net_hub_port_find(int hub_id);
> diff --git a/net/colo-compare.c b/net/colo-compare.c
> index 426eab5973..b8c0240725 100644
> --- a/net/colo-compare.c
> +++ b/net/colo-compare.c
> @@ -996,6 +996,11 @@ static void colo_compare_complete(UserCreatable *uc, Error **errp)
> return;
> }
>
> + if (!is_colo_support_client_type()) {
> + error_setg(errp, "COLO-compare: Net client type is not supported");
> + return;
> + }
> +
> net_socket_rs_init(&s->pri_rs, compare_pri_rs_finalize, s->vnet_hdr);
> net_socket_rs_init(&s->sec_rs, compare_sec_rs_finalize, s->vnet_hdr);
>
> diff --git a/net/net.c b/net/net.c
> index 2a3133990c..a77ea88fff 100644
> --- a/net/net.c
> +++ b/net/net.c
> @@ -1733,3 +1733,17 @@ int net_fill_rstate(SocketReadState *rs, const uint8_t *buf, int size)
> assert(size == 0);
> return 0;
> }
> +
> +/* Currently, COLO just support TAP */
> +bool is_colo_support_client_type(void)
> +{
> + NetClientState *nc;
> +
> + QTAILQ_FOREACH(nc, &net_clients, next) {
> + if (nc->info->type != NET_CLIENT_DRIVER_TAP) {
> + return false;
> + }
> + }
> +
> + return true;
> +}
^ permalink raw reply [flat|nested] 35+ messages in thread* Re: [Qemu-devel] [PATCH V11 15/20] net/net.c: Add net client type check function for COLO
2018-08-21 3:26 ` Jason Wang
@ 2018-08-21 9:26 ` Zhang Chen
2018-08-22 8:22 ` Jason Wang
0 siblings, 1 reply; 35+ messages in thread
From: Zhang Chen @ 2018-08-21 9:26 UTC (permalink / raw)
To: jasowang
Cc: qemu-devel, Paolo Bonzini, Juan Quintela, Dr . David Alan Gilbert,
Eric Blake, Markus Armbruster, zhang.zhanghailiang, lizhijian,
Zhang Chen
On Tue, Aug 21, 2018 at 11:27 AM Jason Wang <jasowang@redhat.com> wrote:
>
>
> On 2018年08月12日 04:59, Zhang Chen wrote:
> > From: Zhang Chen <chen.zhang@intel.com>
> >
> > We add is_colo_support_client_type() to check the net client type for
> > COLO-compare. Currently we just support TAP.
> > Suggested by Jason.
> >
> > Signed-off-by: Zhang Chen <zhangckid@gmail.com>
> > Signed-off-by: Zhang Chen <chen.zhang@intel.com>
> > ---
>
> This needs more works:
>
> - Forbid vhost
> - Deal with hotplug
>
> Looks not a must for this series, so to speed up the process, you may do
> it on top.
>
>
Do you mean this patch we can stay this status currently?
Thanks
Zhang Chen
> Thanks
>
> > include/net/net.h | 1 +
> > net/colo-compare.c | 5 +++++
> > net/net.c | 14 ++++++++++++++
> > 3 files changed, 20 insertions(+)
> >
> > diff --git a/include/net/net.h b/include/net/net.h
> > index 1425960f76..dcbc7ba9c0 100644
> > --- a/include/net/net.h
> > +++ b/include/net/net.h
> > @@ -212,6 +212,7 @@ void hmp_host_net_add(Monitor *mon, const QDict
> *qdict);
> > void hmp_host_net_remove(Monitor *mon, const QDict *qdict);
> > void netdev_add(QemuOpts *opts, Error **errp);
> > void qmp_netdev_add(QDict *qdict, QObject **ret, Error **errp);
> > +bool is_colo_support_client_type(void);
> >
> > int net_hub_id_for_client(NetClientState *nc, int *id);
> > NetClientState *net_hub_port_find(int hub_id);
> > diff --git a/net/colo-compare.c b/net/colo-compare.c
> > index 426eab5973..b8c0240725 100644
> > --- a/net/colo-compare.c
> > +++ b/net/colo-compare.c
> > @@ -996,6 +996,11 @@ static void colo_compare_complete(UserCreatable
> *uc, Error **errp)
> > return;
> > }
> >
> > + if (!is_colo_support_client_type()) {
> > + error_setg(errp, "COLO-compare: Net client type is not
> supported");
> > + return;
> > + }
> > +
> > net_socket_rs_init(&s->pri_rs, compare_pri_rs_finalize,
> s->vnet_hdr);
> > net_socket_rs_init(&s->sec_rs, compare_sec_rs_finalize,
> s->vnet_hdr);
> >
> > diff --git a/net/net.c b/net/net.c
> > index 2a3133990c..a77ea88fff 100644
> > --- a/net/net.c
> > +++ b/net/net.c
> > @@ -1733,3 +1733,17 @@ int net_fill_rstate(SocketReadState *rs, const
> uint8_t *buf, int size)
> > assert(size == 0);
> > return 0;
> > }
> > +
> > +/* Currently, COLO just support TAP */
> > +bool is_colo_support_client_type(void)
> > +{
> > + NetClientState *nc;
> > +
> > + QTAILQ_FOREACH(nc, &net_clients, next) {
> > + if (nc->info->type != NET_CLIENT_DRIVER_TAP) {
> > + return false;
> > + }
> > + }
> > +
> > + return true;
> > +}
>
>
^ permalink raw reply [flat|nested] 35+ messages in thread* Re: [Qemu-devel] [PATCH V11 15/20] net/net.c: Add net client type check function for COLO
2018-08-21 9:26 ` Zhang Chen
@ 2018-08-22 8:22 ` Jason Wang
2018-08-23 9:37 ` Zhang Chen
0 siblings, 1 reply; 35+ messages in thread
From: Jason Wang @ 2018-08-22 8:22 UTC (permalink / raw)
To: Zhang Chen
Cc: qemu-devel, Paolo Bonzini, Juan Quintela, Dr . David Alan Gilbert,
Eric Blake, Markus Armbruster, zhang.zhanghailiang, lizhijian,
Zhang Chen
On 2018年08月21日 17:26, Zhang Chen wrote:
>
>
> On Tue, Aug 21, 2018 at 11:27 AM Jason Wang <jasowang@redhat.com
> <mailto:jasowang@redhat.com>> wrote:
>
>
>
> On 2018年08月12日 04:59, Zhang Chen wrote:
> > From: Zhang Chen <chen.zhang@intel.com
> <mailto:chen.zhang@intel.com>>
> >
> > We add is_colo_support_client_type() to check the net client
> type for
> > COLO-compare. Currently we just support TAP.
> > Suggested by Jason.
> >
> > Signed-off-by: Zhang Chen <zhangckid@gmail.com
> <mailto:zhangckid@gmail.com>>
> > Signed-off-by: Zhang Chen <chen.zhang@intel.com
> <mailto:chen.zhang@intel.com>>
> > ---
>
> This needs more works:
>
> - Forbid vhost
> - Deal with hotplug
>
> Looks not a must for this series, so to speed up the process, you
> may do
> it on top.
>
>
> Do you mean this patch we can stay this status currently?
>
> Thanks
> Zhang Chen
Sorry for being unclear, I mean we need a complete solution rather than
partial one.
Thanks
^ permalink raw reply [flat|nested] 35+ messages in thread
* Re: [Qemu-devel] [PATCH V11 15/20] net/net.c: Add net client type check function for COLO
2018-08-22 8:22 ` Jason Wang
@ 2018-08-23 9:37 ` Zhang Chen
0 siblings, 0 replies; 35+ messages in thread
From: Zhang Chen @ 2018-08-23 9:37 UTC (permalink / raw)
To: jasowang
Cc: qemu-devel, Paolo Bonzini, Juan Quintela, Dr . David Alan Gilbert,
Eric Blake, Markus Armbruster, zhang.zhanghailiang, lizhijian,
Zhang Chen
On Wed, Aug 22, 2018 at 4:22 PM Jason Wang <jasowang@redhat.com> wrote:
>
>
> On 2018年08月21日 17:26, Zhang Chen wrote:
> >
> >
> > On Tue, Aug 21, 2018 at 11:27 AM Jason Wang <jasowang@redhat.com
> > <mailto:jasowang@redhat.com>> wrote:
> >
> >
> >
> > On 2018年08月12日 04:59, Zhang Chen wrote:
> > > From: Zhang Chen <chen.zhang@intel.com
> > <mailto:chen.zhang@intel.com>>
> > >
> > > We add is_colo_support_client_type() to check the net client
> > type for
> > > COLO-compare. Currently we just support TAP.
> > > Suggested by Jason.
> > >
> > > Signed-off-by: Zhang Chen <zhangckid@gmail.com
> > <mailto:zhangckid@gmail.com>>
> > > Signed-off-by: Zhang Chen <chen.zhang@intel.com
> > <mailto:chen.zhang@intel.com>>
> > > ---
> >
> > This needs more works:
> >
> > - Forbid vhost
> > - Deal with hotplug
> >
> > Looks not a must for this series, so to speed up the process, you
> > may do
> > it on top.
> >
> >
> > Do you mean this patch we can stay this status currently?
> >
> > Thanks
> > Zhang Chen
>
> Sorry for being unclear, I mean we need a complete solution rather than
> partial one.
>
OK, I will remove this patch from this series, and send a complete solution
in the future.
Thanks
Zhang Chen
>
> Thanks
>
>
^ permalink raw reply [flat|nested] 35+ messages in thread
* [Qemu-devel] [PATCH V11 16/20] filter: Add handle_event method for NetFilterClass
2018-08-11 20:59 [Qemu-devel] [PATCH V11 00/20] COLO: integrate colo frame with block replication and COLO proxy Zhang Chen
` (14 preceding siblings ...)
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 15/20] net/net.c: Add net client type check function for COLO Zhang Chen
@ 2018-08-11 20:59 ` Zhang Chen
2018-08-21 3:30 ` Jason Wang
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 17/20] filter-rewriter: handle checkpoint and failover event Zhang Chen
` (4 subsequent siblings)
20 siblings, 1 reply; 35+ messages in thread
From: Zhang Chen @ 2018-08-11 20:59 UTC (permalink / raw)
To: qemu-devel, Paolo Bonzini, Juan Quintela, Dr . David Alan Gilbert,
Jason Wang, Eric Blake, Markus Armbruster
Cc: Zhang Chen, zhanghailiang, Li Zhijian
Filter needs to process the event of checkpoint/failover or
other event passed by COLO frame.
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
---
include/net/filter.h | 5 +++++
net/filter.c | 17 +++++++++++++++++
net/net.c | 28 ++++++++++++++++++++++++++++
3 files changed, 50 insertions(+)
diff --git a/include/net/filter.h b/include/net/filter.h
index 435acd6f82..49da666ac0 100644
--- a/include/net/filter.h
+++ b/include/net/filter.h
@@ -38,6 +38,8 @@ typedef ssize_t (FilterReceiveIOV)(NetFilterState *nc,
typedef void (FilterStatusChanged) (NetFilterState *nf, Error **errp);
+typedef void (FilterHandleEvent) (NetFilterState *nf, int event, Error **errp);
+
typedef struct NetFilterClass {
ObjectClass parent_class;
@@ -45,6 +47,7 @@ typedef struct NetFilterClass {
FilterSetup *setup;
FilterCleanup *cleanup;
FilterStatusChanged *status_changed;
+ FilterHandleEvent *handle_event;
/* mandatory */
FilterReceiveIOV *receive_iov;
} NetFilterClass;
@@ -77,4 +80,6 @@ ssize_t qemu_netfilter_pass_to_next(NetClientState *sender,
int iovcnt,
void *opaque);
+void colo_notify_filters_event(int event, Error **errp);
+
#endif /* QEMU_NET_FILTER_H */
diff --git a/net/filter.c b/net/filter.c
index 2fd7d7d663..0f17eba143 100644
--- a/net/filter.c
+++ b/net/filter.c
@@ -17,6 +17,8 @@
#include "net/vhost_net.h"
#include "qom/object_interfaces.h"
#include "qemu/iov.h"
+#include "net/colo.h"
+#include "migration/colo.h"
static inline bool qemu_can_skip_netfilter(NetFilterState *nf)
{
@@ -245,11 +247,26 @@ static void netfilter_finalize(Object *obj)
g_free(nf->netdev_id);
}
+static void dummy_handle_event(NetFilterState *nf, int event, Error **errp)
+{
+ switch (event) {
+ case COLO_EVENT_CHECKPOINT:
+ break;
+ case COLO_EVENT_FAILOVER:
+ object_property_set_str(OBJECT(nf), "off", "status", errp);
+ break;
+ default:
+ break;
+ }
+}
+
static void netfilter_class_init(ObjectClass *oc, void *data)
{
UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc);
+ NetFilterClass *nfc = NETFILTER_CLASS(oc);
ucc->complete = netfilter_complete;
+ nfc->handle_event = dummy_handle_event;
}
static const TypeInfo netfilter_info = {
diff --git a/net/net.c b/net/net.c
index a77ea88fff..b4f6a2efb2 100644
--- a/net/net.c
+++ b/net/net.c
@@ -1331,6 +1331,34 @@ void hmp_info_network(Monitor *mon, const QDict *qdict)
}
}
+void colo_notify_filters_event(int event, Error **errp)
+{
+ NetClientState *nc, *peer;
+ NetClientDriver type;
+ NetFilterState *nf;
+ NetFilterClass *nfc = NULL;
+ Error *local_err = NULL;
+
+ QTAILQ_FOREACH(nc, &net_clients, next) {
+ peer = nc->peer;
+ type = nc->info->type;
+ if (!peer || type != NET_CLIENT_DRIVER_TAP) {
+ continue;
+ }
+ QTAILQ_FOREACH(nf, &nc->filters, next) {
+ nfc = NETFILTER_GET_CLASS(OBJECT(nf));
+ if (!nfc->handle_event) {
+ continue;
+ }
+ nfc->handle_event(nf, event, &local_err);
+ if (local_err) {
+ error_propagate(errp, local_err);
+ return;
+ }
+ }
+ }
+}
+
void qmp_set_link(const char *name, bool up, Error **errp)
{
NetClientState *ncs[MAX_QUEUE_NUM];
--
2.17.GIT
^ permalink raw reply related [flat|nested] 35+ messages in thread* Re: [Qemu-devel] [PATCH V11 16/20] filter: Add handle_event method for NetFilterClass
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 16/20] filter: Add handle_event method for NetFilterClass Zhang Chen
@ 2018-08-21 3:30 ` Jason Wang
2018-08-21 9:25 ` Zhang Chen
0 siblings, 1 reply; 35+ messages in thread
From: Jason Wang @ 2018-08-21 3:30 UTC (permalink / raw)
To: Zhang Chen, qemu-devel, Paolo Bonzini, Juan Quintela,
Dr . David Alan Gilbert, Eric Blake, Markus Armbruster
Cc: zhanghailiang, Li Zhijian
On 2018年08月12日 04:59, Zhang Chen wrote:
> Filter needs to process the event of checkpoint/failover or
> other event passed by COLO frame.
>
> Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
> ---
> include/net/filter.h | 5 +++++
> net/filter.c | 17 +++++++++++++++++
> net/net.c | 28 ++++++++++++++++++++++++++++
> 3 files changed, 50 insertions(+)
>
> diff --git a/include/net/filter.h b/include/net/filter.h
> index 435acd6f82..49da666ac0 100644
> --- a/include/net/filter.h
> +++ b/include/net/filter.h
> @@ -38,6 +38,8 @@ typedef ssize_t (FilterReceiveIOV)(NetFilterState *nc,
>
> typedef void (FilterStatusChanged) (NetFilterState *nf, Error **errp);
>
> +typedef void (FilterHandleEvent) (NetFilterState *nf, int event, Error **errp);
> +
> typedef struct NetFilterClass {
> ObjectClass parent_class;
>
> @@ -45,6 +47,7 @@ typedef struct NetFilterClass {
> FilterSetup *setup;
> FilterCleanup *cleanup;
> FilterStatusChanged *status_changed;
> + FilterHandleEvent *handle_event;
> /* mandatory */
> FilterReceiveIOV *receive_iov;
> } NetFilterClass;
> @@ -77,4 +80,6 @@ ssize_t qemu_netfilter_pass_to_next(NetClientState *sender,
> int iovcnt,
> void *opaque);
>
> +void colo_notify_filters_event(int event, Error **errp);
> +
> #endif /* QEMU_NET_FILTER_H */
> diff --git a/net/filter.c b/net/filter.c
> index 2fd7d7d663..0f17eba143 100644
> --- a/net/filter.c
> +++ b/net/filter.c
> @@ -17,6 +17,8 @@
> #include "net/vhost_net.h"
> #include "qom/object_interfaces.h"
> #include "qemu/iov.h"
> +#include "net/colo.h"
> +#include "migration/colo.h"
>
> static inline bool qemu_can_skip_netfilter(NetFilterState *nf)
> {
> @@ -245,11 +247,26 @@ static void netfilter_finalize(Object *obj)
> g_free(nf->netdev_id);
> }
>
> +static void dummy_handle_event(NetFilterState *nf, int event, Error **errp)
> +{
It's in fact not a "dummy" handler, Maybe it's better to rename it as
"default".
> + switch (event) {
> + case COLO_EVENT_CHECKPOINT:
> + break;
> + case COLO_EVENT_FAILOVER:
> + object_property_set_str(OBJECT(nf), "off", "status", errp);
I think filter is a generic infrastructure, so it's better not have COLO
specific things like this. You can either add a generic name or have a
dedicated helper to just disable all net filters.
> + break;
> + default:
> + break;
> + }
> +}
> +
> static void netfilter_class_init(ObjectClass *oc, void *data)
> {
> UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc);
> + NetFilterClass *nfc = NETFILTER_CLASS(oc);
>
> ucc->complete = netfilter_complete;
> + nfc->handle_event = dummy_handle_event;
> }
>
> static const TypeInfo netfilter_info = {
> diff --git a/net/net.c b/net/net.c
> index a77ea88fff..b4f6a2efb2 100644
> --- a/net/net.c
> +++ b/net/net.c
> @@ -1331,6 +1331,34 @@ void hmp_info_network(Monitor *mon, const QDict *qdict)
> }
> }
>
> +void colo_notify_filters_event(int event, Error **errp)
> +{
> + NetClientState *nc, *peer;
> + NetClientDriver type;
> + NetFilterState *nf;
> + NetFilterClass *nfc = NULL;
> + Error *local_err = NULL;
> +
> + QTAILQ_FOREACH(nc, &net_clients, next) {
> + peer = nc->peer;
> + type = nc->info->type;
> + if (!peer || type != NET_CLIENT_DRIVER_TAP) {
> + continue;
> + }
The check the TAP is redundant with previous patch.
> + QTAILQ_FOREACH(nf, &nc->filters, next) {
> + nfc = NETFILTER_GET_CLASS(OBJECT(nf));
> + if (!nfc->handle_event) {
Looks like this won't happen.
Thanks
> + continue;
> + }
> + nfc->handle_event(nf, event, &local_err);
> + if (local_err) {
> + error_propagate(errp, local_err);
> + return;
> + }
> + }
> + }
> +}
> +
> void qmp_set_link(const char *name, bool up, Error **errp)
> {
> NetClientState *ncs[MAX_QUEUE_NUM];
^ permalink raw reply [flat|nested] 35+ messages in thread* Re: [Qemu-devel] [PATCH V11 16/20] filter: Add handle_event method for NetFilterClass
2018-08-21 3:30 ` Jason Wang
@ 2018-08-21 9:25 ` Zhang Chen
2018-08-22 8:21 ` Jason Wang
0 siblings, 1 reply; 35+ messages in thread
From: Zhang Chen @ 2018-08-21 9:25 UTC (permalink / raw)
To: jasowang
Cc: qemu-devel, Paolo Bonzini, Juan Quintela, Dr . David Alan Gilbert,
Eric Blake, Markus Armbruster, zhang.zhanghailiang, lizhijian
On Tue, Aug 21, 2018 at 11:30 AM Jason Wang <jasowang@redhat.com> wrote:
>
>
> On 2018年08月12日 04:59, Zhang Chen wrote:
> > Filter needs to process the event of checkpoint/failover or
> > other event passed by COLO frame.
> >
> > Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
> > ---
> > include/net/filter.h | 5 +++++
> > net/filter.c | 17 +++++++++++++++++
> > net/net.c | 28 ++++++++++++++++++++++++++++
> > 3 files changed, 50 insertions(+)
> >
> > diff --git a/include/net/filter.h b/include/net/filter.h
> > index 435acd6f82..49da666ac0 100644
> > --- a/include/net/filter.h
> > +++ b/include/net/filter.h
> > @@ -38,6 +38,8 @@ typedef ssize_t (FilterReceiveIOV)(NetFilterState *nc,
> >
> > typedef void (FilterStatusChanged) (NetFilterState *nf, Error **errp);
> >
> > +typedef void (FilterHandleEvent) (NetFilterState *nf, int event, Error
> **errp);
> > +
> > typedef struct NetFilterClass {
> > ObjectClass parent_class;
> >
> > @@ -45,6 +47,7 @@ typedef struct NetFilterClass {
> > FilterSetup *setup;
> > FilterCleanup *cleanup;
> > FilterStatusChanged *status_changed;
> > + FilterHandleEvent *handle_event;
> > /* mandatory */
> > FilterReceiveIOV *receive_iov;
> > } NetFilterClass;
> > @@ -77,4 +80,6 @@ ssize_t qemu_netfilter_pass_to_next(NetClientState
> *sender,
> > int iovcnt,
> > void *opaque);
> >
> > +void colo_notify_filters_event(int event, Error **errp);
> > +
> > #endif /* QEMU_NET_FILTER_H */
> > diff --git a/net/filter.c b/net/filter.c
> > index 2fd7d7d663..0f17eba143 100644
> > --- a/net/filter.c
> > +++ b/net/filter.c
> > @@ -17,6 +17,8 @@
> > #include "net/vhost_net.h"
> > #include "qom/object_interfaces.h"
> > #include "qemu/iov.h"
> > +#include "net/colo.h"
> > +#include "migration/colo.h"
> >
> > static inline bool qemu_can_skip_netfilter(NetFilterState *nf)
> > {
> > @@ -245,11 +247,26 @@ static void netfilter_finalize(Object *obj)
> > g_free(nf->netdev_id);
> > }
> >
> > +static void dummy_handle_event(NetFilterState *nf, int event, Error
> **errp)
> > +{
>
> It's in fact not a "dummy" handler, Maybe it's better to rename it as
> "default".
>
OK, I will rename it in next version.
>
> > + switch (event) {
> > + case COLO_EVENT_CHECKPOINT:
> > + break;
> > + case COLO_EVENT_FAILOVER:
> > + object_property_set_str(OBJECT(nf), "off", "status", errp);
>
> I think filter is a generic infrastructure, so it's better not have COLO
> specific things like this. You can either add a generic name or have a
> dedicated helper to just disable all net filters.
>
Maybe we can rename it to "EVENT_CHECKPOINT" and "EVENT_FAILOVER" looks
better?
>
> > + break;
> > + default:
> > + break;
> > + }
> > +}
> > +
> > static void netfilter_class_init(ObjectClass *oc, void *data)
> > {
> > UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc);
> > + NetFilterClass *nfc = NETFILTER_CLASS(oc);
> >
> > ucc->complete = netfilter_complete;
> > + nfc->handle_event = dummy_handle_event;
> > }
> >
> > static const TypeInfo netfilter_info = {
> > diff --git a/net/net.c b/net/net.c
> > index a77ea88fff..b4f6a2efb2 100644
> > --- a/net/net.c
> > +++ b/net/net.c
> > @@ -1331,6 +1331,34 @@ void hmp_info_network(Monitor *mon, const QDict
> *qdict)
> > }
> > }
> >
> > +void colo_notify_filters_event(int event, Error **errp)
> > +{
> > + NetClientState *nc, *peer;
> > + NetClientDriver type;
> > + NetFilterState *nf;
> > + NetFilterClass *nfc = NULL;
> > + Error *local_err = NULL;
> > +
> > + QTAILQ_FOREACH(nc, &net_clients, next) {
> > + peer = nc->peer;
> > + type = nc->info->type;
> > + if (!peer || type != NET_CLIENT_DRIVER_TAP) {
> > + continue;
> > + }
>
> The check the TAP is redundant with previous patch.
>
OK, I will remove it.
>
> > + QTAILQ_FOREACH(nf, &nc->filters, next) {
> > + nfc = NETFILTER_GET_CLASS(OBJECT(nf));
> > + if (!nfc->handle_event) {
>
> Looks like this won't happen.
>
It will happen. like filter-mirror and filter-redirector haven't this event.
Thanks
Zhang Chen
>
> Thanks
>
> > + continue;
> > + }
> > + nfc->handle_event(nf, event, &local_err);
> > + if (local_err) {
> > + error_propagate(errp, local_err);
> > + return;
> > + }
> > + }
> > + }
> > +}
> > +
> > void qmp_set_link(const char *name, bool up, Error **errp)
> > {
> > NetClientState *ncs[MAX_QUEUE_NUM];
>
>
^ permalink raw reply [flat|nested] 35+ messages in thread* Re: [Qemu-devel] [PATCH V11 16/20] filter: Add handle_event method for NetFilterClass
2018-08-21 9:25 ` Zhang Chen
@ 2018-08-22 8:21 ` Jason Wang
2018-08-24 5:57 ` Zhang Chen
0 siblings, 1 reply; 35+ messages in thread
From: Jason Wang @ 2018-08-22 8:21 UTC (permalink / raw)
To: Zhang Chen
Cc: zhang.zhanghailiang, lizhijian, Juan Quintela, Markus Armbruster,
qemu-devel, Paolo Bonzini, Dr . David Alan Gilbert
On 2018年08月21日 17:25, Zhang Chen wrote:
> On Tue, Aug 21, 2018 at 11:30 AM Jason Wang <jasowang@redhat.com> wrote:
>
>>
>> On 2018年08月12日 04:59, Zhang Chen wrote:
>>> Filter needs to process the event of checkpoint/failover or
>>> other event passed by COLO frame.
>>>
>>> Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
>>> ---
>>> include/net/filter.h | 5 +++++
>>> net/filter.c | 17 +++++++++++++++++
>>> net/net.c | 28 ++++++++++++++++++++++++++++
>>> 3 files changed, 50 insertions(+)
>>>
>>> diff --git a/include/net/filter.h b/include/net/filter.h
>>> index 435acd6f82..49da666ac0 100644
>>> --- a/include/net/filter.h
>>> +++ b/include/net/filter.h
>>> @@ -38,6 +38,8 @@ typedef ssize_t (FilterReceiveIOV)(NetFilterState *nc,
>>>
>>> typedef void (FilterStatusChanged) (NetFilterState *nf, Error **errp);
>>>
>>> +typedef void (FilterHandleEvent) (NetFilterState *nf, int event, Error
>> **errp);
>>> +
>>> typedef struct NetFilterClass {
>>> ObjectClass parent_class;
>>>
>>> @@ -45,6 +47,7 @@ typedef struct NetFilterClass {
>>> FilterSetup *setup;
>>> FilterCleanup *cleanup;
>>> FilterStatusChanged *status_changed;
>>> + FilterHandleEvent *handle_event;
>>> /* mandatory */
>>> FilterReceiveIOV *receive_iov;
>>> } NetFilterClass;
>>> @@ -77,4 +80,6 @@ ssize_t qemu_netfilter_pass_to_next(NetClientState
>> *sender,
>>> int iovcnt,
>>> void *opaque);
>>>
>>> +void colo_notify_filters_event(int event, Error **errp);
>>> +
>>> #endif /* QEMU_NET_FILTER_H */
>>> diff --git a/net/filter.c b/net/filter.c
>>> index 2fd7d7d663..0f17eba143 100644
>>> --- a/net/filter.c
>>> +++ b/net/filter.c
>>> @@ -17,6 +17,8 @@
>>> #include "net/vhost_net.h"
>>> #include "qom/object_interfaces.h"
>>> #include "qemu/iov.h"
>>> +#include "net/colo.h"
>>> +#include "migration/colo.h"
>>>
>>> static inline bool qemu_can_skip_netfilter(NetFilterState *nf)
>>> {
>>> @@ -245,11 +247,26 @@ static void netfilter_finalize(Object *obj)
>>> g_free(nf->netdev_id);
>>> }
>>>
>>> +static void dummy_handle_event(NetFilterState *nf, int event, Error
>> **errp)
>>> +{
>> It's in fact not a "dummy" handler, Maybe it's better to rename it as
>> "default".
>>
> OK, I will rename it in next version.
>
>
>>> + switch (event) {
>>> + case COLO_EVENT_CHECKPOINT:
>>> + break;
>>> + case COLO_EVENT_FAILOVER:
>>> + object_property_set_str(OBJECT(nf), "off", "status", errp);
>> I think filter is a generic infrastructure, so it's better not have COLO
>> specific things like this. You can either add a generic name or have a
>> dedicated helper to just disable all net filters.
>>
> Maybe we can rename it to "EVENT_CHECKPOINT" and "EVENT_FAILOVER" looks
> better?
>
I think registering notifier to COLO is better. For disabling filter, we
can have a generic dedicated helpers to do this.
Btw, I remember we allow disabling filter through qmp, if it's true, we
probably need some notification to management, but this can be done in
the future.
>>> + break;
>>> + default:
>>> + break;
>>> + }
>>> +}
>>> +
>>> static void netfilter_class_init(ObjectClass *oc, void *data)
>>> {
>>> UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc);
>>> + NetFilterClass *nfc = NETFILTER_CLASS(oc);
>>>
>>> ucc->complete = netfilter_complete;
>>> + nfc->handle_event = dummy_handle_event;
>>> }
>>>
>>> static const TypeInfo netfilter_info = {
>>> diff --git a/net/net.c b/net/net.c
>>> index a77ea88fff..b4f6a2efb2 100644
>>> --- a/net/net.c
>>> +++ b/net/net.c
>>> @@ -1331,6 +1331,34 @@ void hmp_info_network(Monitor *mon, const QDict
>> *qdict)
>>> }
>>> }
>>>
>>> +void colo_notify_filters_event(int event, Error **errp)
>>> +{
>>> + NetClientState *nc, *peer;
>>> + NetClientDriver type;
>>> + NetFilterState *nf;
>>> + NetFilterClass *nfc = NULL;
>>> + Error *local_err = NULL;
>>> +
>>> + QTAILQ_FOREACH(nc, &net_clients, next) {
>>> + peer = nc->peer;
>>> + type = nc->info->type;
>>> + if (!peer || type != NET_CLIENT_DRIVER_TAP) {
>>> + continue;
>>> + }
>> The check the TAP is redundant with previous patch.
>>
> OK, I will remove it.
>
>
>>> + QTAILQ_FOREACH(nf, &nc->filters, next) {
>>> + nfc = NETFILTER_GET_CLASS(OBJECT(nf));
>>> + if (!nfc->handle_event) {
>> Looks like this won't happen.
>>
> It will happen. like filter-mirror and filter-redirector haven't this event.
But you do:
static void netfilter_class_init(ObjectClass *oc, void *data)
{
UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc);
+ NetFilterClass *nfc = NETFILTER_CLASS(oc);
ucc->complete = netfilter_complete;
+ nfc->handle_event = dummy_handle_event;
}
?
Thanks
>
> Thanks
> Zhang Chen
>
>
>> Thanks
>>
>>> + continue;
>>> + }
>>> + nfc->handle_event(nf, event, &local_err);
>>> + if (local_err) {
>>> + error_propagate(errp, local_err);
>>> + return;
>>> + }
>>> + }
>>> + }
>>> +}
>>> +
>>> void qmp_set_link(const char *name, bool up, Error **errp)
>>> {
>>> NetClientState *ncs[MAX_QUEUE_NUM];
>>
^ permalink raw reply [flat|nested] 35+ messages in thread* Re: [Qemu-devel] [PATCH V11 16/20] filter: Add handle_event method for NetFilterClass
2018-08-22 8:21 ` Jason Wang
@ 2018-08-24 5:57 ` Zhang Chen
2018-08-28 7:19 ` Jason Wang
0 siblings, 1 reply; 35+ messages in thread
From: Zhang Chen @ 2018-08-24 5:57 UTC (permalink / raw)
To: jasowang
Cc: zhang.zhanghailiang, lizhijian, Juan Quintela, Markus Armbruster,
qemu-devel, Paolo Bonzini, Dr . David Alan Gilbert
On Wed, Aug 22, 2018 at 4:22 PM Jason Wang <jasowang@redhat.com> wrote:
>
>
> On 2018年08月21日 17:25, Zhang Chen wrote:
> > On Tue, Aug 21, 2018 at 11:30 AM Jason Wang <jasowang@redhat.com> wrote:
> >
> >>
> >> On 2018年08月12日 04:59, Zhang Chen wrote:
> >>> Filter needs to process the event of checkpoint/failover or
> >>> other event passed by COLO frame.
> >>>
> >>> Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
> >>> ---
> >>> include/net/filter.h | 5 +++++
> >>> net/filter.c | 17 +++++++++++++++++
> >>> net/net.c | 28 ++++++++++++++++++++++++++++
> >>> 3 files changed, 50 insertions(+)
> >>>
> >>> diff --git a/include/net/filter.h b/include/net/filter.h
> >>> index 435acd6f82..49da666ac0 100644
> >>> --- a/include/net/filter.h
> >>> +++ b/include/net/filter.h
> >>> @@ -38,6 +38,8 @@ typedef ssize_t (FilterReceiveIOV)(NetFilterState
> *nc,
> >>>
> >>> typedef void (FilterStatusChanged) (NetFilterState *nf, Error
> **errp);
> >>>
> >>> +typedef void (FilterHandleEvent) (NetFilterState *nf, int event, Error
> >> **errp);
> >>> +
> >>> typedef struct NetFilterClass {
> >>> ObjectClass parent_class;
> >>>
> >>> @@ -45,6 +47,7 @@ typedef struct NetFilterClass {
> >>> FilterSetup *setup;
> >>> FilterCleanup *cleanup;
> >>> FilterStatusChanged *status_changed;
> >>> + FilterHandleEvent *handle_event;
> >>> /* mandatory */
> >>> FilterReceiveIOV *receive_iov;
> >>> } NetFilterClass;
> >>> @@ -77,4 +80,6 @@ ssize_t qemu_netfilter_pass_to_next(NetClientState
> >> *sender,
> >>> int iovcnt,
> >>> void *opaque);
> >>>
> >>> +void colo_notify_filters_event(int event, Error **errp);
> >>> +
> >>> #endif /* QEMU_NET_FILTER_H */
> >>> diff --git a/net/filter.c b/net/filter.c
> >>> index 2fd7d7d663..0f17eba143 100644
> >>> --- a/net/filter.c
> >>> +++ b/net/filter.c
> >>> @@ -17,6 +17,8 @@
> >>> #include "net/vhost_net.h"
> >>> #include "qom/object_interfaces.h"
> >>> #include "qemu/iov.h"
> >>> +#include "net/colo.h"
> >>> +#include "migration/colo.h"
> >>>
> >>> static inline bool qemu_can_skip_netfilter(NetFilterState *nf)
> >>> {
> >>> @@ -245,11 +247,26 @@ static void netfilter_finalize(Object *obj)
> >>> g_free(nf->netdev_id);
> >>> }
> >>>
> >>> +static void dummy_handle_event(NetFilterState *nf, int event, Error
> >> **errp)
> >>> +{
> >> It's in fact not a "dummy" handler, Maybe it's better to rename it as
> >> "default".
> >>
> > OK, I will rename it in next version.
> >
> >
> >>> + switch (event) {
> >>> + case COLO_EVENT_CHECKPOINT:
> >>> + break;
> >>> + case COLO_EVENT_FAILOVER:
> >>> + object_property_set_str(OBJECT(nf), "off", "status", errp);
> >> I think filter is a generic infrastructure, so it's better not have COLO
> >> specific things like this. You can either add a generic name or have a
> >> dedicated helper to just disable all net filters.
> >>
> > Maybe we can rename it to "EVENT_CHECKPOINT" and "EVENT_FAILOVER" looks
> > better?
> >
>
> I think registering notifier to COLO is better. For disabling filter, we
> can have a generic dedicated helpers to do this.
>
If we registering notifier to COLO (in migration/colo.c), filter-rewriter
can not get the notify immediately when checkout occur.
filter-reweiter didn't know when checkpoint happened, and I think loop to
check the COLO event in filter-rewriter is a bad choice.
Any thing I misunderstand?
>
> Btw, I remember we allow disabling filter through qmp, if it's true, we
> probably need some notification to management, but this can be done in
> the future.
>
> >>> + break;
> >>> + default:
> >>> + break;
> >>> + }
> >>> +}
> >>> +
> >>> static void netfilter_class_init(ObjectClass *oc, void *data)
> >>> {
> >>> UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc);
> >>> + NetFilterClass *nfc = NETFILTER_CLASS(oc);
> >>>
> >>> ucc->complete = netfilter_complete;
> >>> + nfc->handle_event = dummy_handle_event;
> >>> }
> >>>
> >>> static const TypeInfo netfilter_info = {
> >>> diff --git a/net/net.c b/net/net.c
> >>> index a77ea88fff..b4f6a2efb2 100644
> >>> --- a/net/net.c
> >>> +++ b/net/net.c
> >>> @@ -1331,6 +1331,34 @@ void hmp_info_network(Monitor *mon, const QDict
> >> *qdict)
> >>> }
> >>> }
> >>>
> >>> +void colo_notify_filters_event(int event, Error **errp)
> >>> +{
> >>> + NetClientState *nc, *peer;
> >>> + NetClientDriver type;
> >>> + NetFilterState *nf;
> >>> + NetFilterClass *nfc = NULL;
> >>> + Error *local_err = NULL;
> >>> +
> >>> + QTAILQ_FOREACH(nc, &net_clients, next) {
> >>> + peer = nc->peer;
> >>> + type = nc->info->type;
> >>> + if (!peer || type != NET_CLIENT_DRIVER_TAP) {
> >>> + continue;
> >>> + }
> >> The check the TAP is redundant with previous patch.
> >>
> > OK, I will remove it.
> >
> >
> >>> + QTAILQ_FOREACH(nf, &nc->filters, next) {
> >>> + nfc = NETFILTER_GET_CLASS(OBJECT(nf));
> >>> + if (!nfc->handle_event) {
> >> Looks like this won't happen.
> >>
> > It will happen. like filter-mirror and filter-redirector haven't this
> event.
>
> But you do:
>
> static void netfilter_class_init(ObjectClass *oc, void *data)
> {
> UserCreatableClass *ucc = USER_CREATABLE_CLASS(oc);
> + NetFilterClass *nfc = NETFILTER_CLASS(oc);
>
> ucc->complete = netfilter_complete;
> + nfc->handle_event = dummy_handle_event;
> }
>
> ?
>
Oh, I forgot thiat. you are right, I will remove this in next version.
Thanks
Zhang Chen
>
> Thanks
>
> >
> > Thanks
> > Zhang Chen
> >
> >
> >> Thanks
> >>
> >>> + continue;
> >>> + }
> >>> + nfc->handle_event(nf, event, &local_err);
> >>> + if (local_err) {
> >>> + error_propagate(errp, local_err);
> >>> + return;
> >>> + }
> >>> + }
> >>> + }
> >>> +}
> >>> +
> >>> void qmp_set_link(const char *name, bool up, Error **errp)
> >>> {
> >>> NetClientState *ncs[MAX_QUEUE_NUM];
> >>
>
>
^ permalink raw reply [flat|nested] 35+ messages in thread* Re: [Qemu-devel] [PATCH V11 16/20] filter: Add handle_event method for NetFilterClass
2018-08-24 5:57 ` Zhang Chen
@ 2018-08-28 7:19 ` Jason Wang
2018-08-30 5:56 ` Zhang Chen
0 siblings, 1 reply; 35+ messages in thread
From: Jason Wang @ 2018-08-28 7:19 UTC (permalink / raw)
To: Zhang Chen
Cc: zhang.zhanghailiang, lizhijian, Juan Quintela, qemu-devel,
Markus Armbruster, Paolo Bonzini, Dr . David Alan Gilbert
On 2018年08月24日 13:57, Zhang Chen wrote:
> On Wed, Aug 22, 2018 at 4:22 PM Jason Wang<jasowang@redhat.com> wrote:
>
>> On 2018年08月21日 17:25, Zhang Chen wrote:
>>> On Tue, Aug 21, 2018 at 11:30 AM Jason Wang<jasowang@redhat.com> wrote:
>>>
>>>> On 2018年08月12日 04:59, Zhang Chen wrote:
>>>>> Filter needs to process the event of checkpoint/failover or
>>>>> other event passed by COLO frame.
>>>>>
>>>>> Signed-off-by: zhanghailiang<zhang.zhanghailiang@huawei.com>
>>>>> ---
>>>>> include/net/filter.h | 5 +++++
>>>>> net/filter.c | 17 +++++++++++++++++
>>>>> net/net.c | 28 ++++++++++++++++++++++++++++
>>>>> 3 files changed, 50 insertions(+)
>>>>>
>>>>> diff --git a/include/net/filter.h b/include/net/filter.h
>>>>> index 435acd6f82..49da666ac0 100644
>>>>> --- a/include/net/filter.h
>>>>> +++ b/include/net/filter.h
>>>>> @@ -38,6 +38,8 @@ typedef ssize_t (FilterReceiveIOV)(NetFilterState
>> *nc,
>>>>> typedef void (FilterStatusChanged) (NetFilterState *nf, Error
>> **errp);
>>>>> +typedef void (FilterHandleEvent) (NetFilterState *nf, int event, Error
>>>> **errp);
>>>>> +
>>>>> typedef struct NetFilterClass {
>>>>> ObjectClass parent_class;
>>>>>
>>>>> @@ -45,6 +47,7 @@ typedef struct NetFilterClass {
>>>>> FilterSetup *setup;
>>>>> FilterCleanup *cleanup;
>>>>> FilterStatusChanged *status_changed;
>>>>> + FilterHandleEvent *handle_event;
>>>>> /* mandatory */
>>>>> FilterReceiveIOV *receive_iov;
>>>>> } NetFilterClass;
>>>>> @@ -77,4 +80,6 @@ ssize_t qemu_netfilter_pass_to_next(NetClientState
>>>> *sender,
>>>>> int iovcnt,
>>>>> void *opaque);
>>>>>
>>>>> +void colo_notify_filters_event(int event, Error **errp);
>>>>> +
>>>>> #endif /* QEMU_NET_FILTER_H */
>>>>> diff --git a/net/filter.c b/net/filter.c
>>>>> index 2fd7d7d663..0f17eba143 100644
>>>>> --- a/net/filter.c
>>>>> +++ b/net/filter.c
>>>>> @@ -17,6 +17,8 @@
>>>>> #include "net/vhost_net.h"
>>>>> #include "qom/object_interfaces.h"
>>>>> #include "qemu/iov.h"
>>>>> +#include "net/colo.h"
>>>>> +#include "migration/colo.h"
>>>>>
>>>>> static inline bool qemu_can_skip_netfilter(NetFilterState *nf)
>>>>> {
>>>>> @@ -245,11 +247,26 @@ static void netfilter_finalize(Object *obj)
>>>>> g_free(nf->netdev_id);
>>>>> }
>>>>>
>>>>> +static void dummy_handle_event(NetFilterState *nf, int event, Error
>>>> **errp)
>>>>> +{
>>>> It's in fact not a "dummy" handler, Maybe it's better to rename it as
>>>> "default".
>>>>
>>> OK, I will rename it in next version.
>>>
>>>
>>>>> + switch (event) {
>>>>> + case COLO_EVENT_CHECKPOINT:
>>>>> + break;
>>>>> + case COLO_EVENT_FAILOVER:
>>>>> + object_property_set_str(OBJECT(nf), "off", "status", errp);
>>>> I think filter is a generic infrastructure, so it's better not have COLO
>>>> specific things like this. You can either add a generic name or have a
>>>> dedicated helper to just disable all net filters.
>>>>
>>> Maybe we can rename it to "EVENT_CHECKPOINT" and "EVENT_FAILOVER" looks
>>> better?
>>>
>> I think registering notifier to COLO is better. For disabling filter, we
>> can have a generic dedicated helpers to do this.
>>
> If we registering notifier to COLO (in migration/colo.c), filter-rewriter
> can not get the notify immediately when checkout occur.
> filter-reweiter didn't know when checkpoint happened,
I may miss something, isn't COLO which triggers the checkpoint even?
> and I think loop to
> check the COLO event in filter-rewriter is a bad choice.
Well, anyway you need a loop somewhere to iterate all notifiers?
Thanks
> Any thing I misunderstand?
>
>
>
^ permalink raw reply [flat|nested] 35+ messages in thread* Re: [Qemu-devel] [PATCH V11 16/20] filter: Add handle_event method for NetFilterClass
2018-08-28 7:19 ` Jason Wang
@ 2018-08-30 5:56 ` Zhang Chen
0 siblings, 0 replies; 35+ messages in thread
From: Zhang Chen @ 2018-08-30 5:56 UTC (permalink / raw)
To: jasowang
Cc: zhang.zhanghailiang, lizhijian, Juan Quintela, qemu-devel,
Markus Armbruster, Paolo Bonzini, Dr . David Alan Gilbert
On Tue, Aug 28, 2018 at 3:19 PM Jason Wang <jasowang@redhat.com> wrote:
>
>
> On 2018年08月24日 13:57, Zhang Chen wrote:
> > On Wed, Aug 22, 2018 at 4:22 PM Jason Wang<jasowang@redhat.com> wrote:
> >
> >> On 2018年08月21日 17:25, Zhang Chen wrote:
> >>> On Tue, Aug 21, 2018 at 11:30 AM Jason Wang<jasowang@redhat.com>
> wrote:
> >>>
> >>>> On 2018年08月12日 04:59, Zhang Chen wrote:
> >>>>> Filter needs to process the event of checkpoint/failover or
> >>>>> other event passed by COLO frame.
> >>>>>
> >>>>> Signed-off-by: zhanghailiang<zhang.zhanghailiang@huawei.com>
> >>>>> ---
> >>>>> include/net/filter.h | 5 +++++
> >>>>> net/filter.c | 17 +++++++++++++++++
> >>>>> net/net.c | 28 ++++++++++++++++++++++++++++
> >>>>> 3 files changed, 50 insertions(+)
> >>>>>
> >>>>> diff --git a/include/net/filter.h b/include/net/filter.h
> >>>>> index 435acd6f82..49da666ac0 100644
> >>>>> --- a/include/net/filter.h
> >>>>> +++ b/include/net/filter.h
> >>>>> @@ -38,6 +38,8 @@ typedef ssize_t (FilterReceiveIOV)(NetFilterState
> >> *nc,
> >>>>> typedef void (FilterStatusChanged) (NetFilterState *nf, Error
> >> **errp);
> >>>>> +typedef void (FilterHandleEvent) (NetFilterState *nf, int event,
> Error
> >>>> **errp);
> >>>>> +
> >>>>> typedef struct NetFilterClass {
> >>>>> ObjectClass parent_class;
> >>>>>
> >>>>> @@ -45,6 +47,7 @@ typedef struct NetFilterClass {
> >>>>> FilterSetup *setup;
> >>>>> FilterCleanup *cleanup;
> >>>>> FilterStatusChanged *status_changed;
> >>>>> + FilterHandleEvent *handle_event;
> >>>>> /* mandatory */
> >>>>> FilterReceiveIOV *receive_iov;
> >>>>> } NetFilterClass;
> >>>>> @@ -77,4 +80,6 @@ ssize_t qemu_netfilter_pass_to_next(NetClientState
> >>>> *sender,
> >>>>> int iovcnt,
> >>>>> void *opaque);
> >>>>>
> >>>>> +void colo_notify_filters_event(int event, Error **errp);
> >>>>> +
> >>>>> #endif /* QEMU_NET_FILTER_H */
> >>>>> diff --git a/net/filter.c b/net/filter.c
> >>>>> index 2fd7d7d663..0f17eba143 100644
> >>>>> --- a/net/filter.c
> >>>>> +++ b/net/filter.c
> >>>>> @@ -17,6 +17,8 @@
> >>>>> #include "net/vhost_net.h"
> >>>>> #include "qom/object_interfaces.h"
> >>>>> #include "qemu/iov.h"
> >>>>> +#include "net/colo.h"
> >>>>> +#include "migration/colo.h"
> >>>>>
> >>>>> static inline bool qemu_can_skip_netfilter(NetFilterState *nf)
> >>>>> {
> >>>>> @@ -245,11 +247,26 @@ static void netfilter_finalize(Object *obj)
> >>>>> g_free(nf->netdev_id);
> >>>>> }
> >>>>>
> >>>>> +static void dummy_handle_event(NetFilterState *nf, int event, Error
> >>>> **errp)
> >>>>> +{
> >>>> It's in fact not a "dummy" handler, Maybe it's better to rename it as
> >>>> "default".
> >>>>
> >>> OK, I will rename it in next version.
> >>>
> >>>
> >>>>> + switch (event) {
> >>>>> + case COLO_EVENT_CHECKPOINT:
> >>>>> + break;
> >>>>> + case COLO_EVENT_FAILOVER:
> >>>>> + object_property_set_str(OBJECT(nf), "off", "status", errp);
> >>>> I think filter is a generic infrastructure, so it's better not have
> COLO
> >>>> specific things like this. You can either add a generic name or have a
> >>>> dedicated helper to just disable all net filters.
> >>>>
> >>> Maybe we can rename it to "EVENT_CHECKPOINT" and "EVENT_FAILOVER" looks
> >>> better?
> >>>
> >> I think registering notifier to COLO is better. For disabling filter, we
> >> can have a generic dedicated helpers to do this.
> >>
> > If we registering notifier to COLO (in migration/colo.c), filter-rewriter
> > can not get the notify immediately when checkout occur.
> > filter-reweiter didn't know when checkpoint happened,
>
> I may miss something, isn't COLO which triggers the checkpoint even?
>
Yes, in detail, we have two way to trigger the checkpoint:
case 1. COLO compare trigger.
case 2. periodic checkpoint mechanism trigger.
But we both solve the same problem that how to notify filter-rewriter do
checkpoint in secondary node.
Currently we do this job in that way:
case 1. primary node COLO compare(net/colo-compare.c) ----> primary node
COLO frame(migration/colo.c) ------> secondary node COLO frame
-----------------> secondary node filter-rewriter.
case 2. primary node COLO frame(migration/colo.c) ------> secondary node
COLO frame -----------------> secondary node filter-rewriter.
So, this patch just focus on "secondary node COLO frame ----------------->
secondary node filter-rewriter".
In filter-rewriter hard to get the checkpoint message from COLO frame.
Thanks
Zhang Chen
>
> > and I think loop to
> > check the COLO event in filter-rewriter is a bad choice.
>
> Well, anyway you need a loop somewhere to iterate all notifiers?
>
> Thanks
>
> > Any thing I misunderstand?
> >
> >
> >
>
>
^ permalink raw reply [flat|nested] 35+ messages in thread
* [Qemu-devel] [PATCH V11 17/20] filter-rewriter: handle checkpoint and failover event
2018-08-11 20:59 [Qemu-devel] [PATCH V11 00/20] COLO: integrate colo frame with block replication and COLO proxy Zhang Chen
` (15 preceding siblings ...)
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 16/20] filter: Add handle_event method for NetFilterClass Zhang Chen
@ 2018-08-11 20:59 ` Zhang Chen
2018-08-21 3:40 ` Jason Wang
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 18/20] COLO: notify net filters about checkpoint/failover event Zhang Chen
` (3 subsequent siblings)
20 siblings, 1 reply; 35+ messages in thread
From: Zhang Chen @ 2018-08-11 20:59 UTC (permalink / raw)
To: qemu-devel, Paolo Bonzini, Juan Quintela, Dr . David Alan Gilbert,
Jason Wang, Eric Blake, Markus Armbruster
Cc: Zhang Chen, zhanghailiang, Li Zhijian, Zhang Chen
After one round of checkpoint, the states between PVM and SVM
become consistent, so it is unnecessary to adjust the sequence
of net packets for old connections, besides, while failover
happens, filter-rewriter will into failover mode that needn't
handle the new TCP connection.
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Signed-off-by: Zhang Chen <zhangckid@gmail.com>
Signed-off-by: Zhang Chen <chen.zhang@intel.com>
---
net/colo-compare.c | 12 ++++-----
net/colo.c | 8 ++++++
net/colo.h | 2 ++
net/filter-rewriter.c | 57 +++++++++++++++++++++++++++++++++++++++++++
4 files changed, 73 insertions(+), 6 deletions(-)
diff --git a/net/colo-compare.c b/net/colo-compare.c
index b8c0240725..462e822be6 100644
--- a/net/colo-compare.c
+++ b/net/colo-compare.c
@@ -116,6 +116,12 @@ enum {
SECONDARY_IN,
};
+static void colo_compare_inconsistency_notify(void)
+{
+ notifier_list_notify(&colo_compare_notifiers,
+ migrate_get_current());
+}
+
static int compare_chr_send(CompareState *s,
const uint8_t *buf,
uint32_t size,
@@ -562,12 +568,6 @@ static int colo_old_packet_check_one(Packet *pkt, int64_t *check_time)
}
}
-static void colo_compare_inconsistency_notify(void)
-{
- notifier_list_notify(&colo_compare_notifiers,
- migrate_get_current());
-}
-
void colo_compare_register_notifier(Notifier *notify)
{
notifier_list_add(&colo_compare_notifiers, notify);
diff --git a/net/colo.c b/net/colo.c
index 97c8fc928f..49176bf07b 100644
--- a/net/colo.c
+++ b/net/colo.c
@@ -221,3 +221,11 @@ Connection *connection_get(GHashTable *connection_track_table,
return conn;
}
+
+bool connection_has_tracked(GHashTable *connection_track_table,
+ ConnectionKey *key)
+{
+ Connection *conn = g_hash_table_lookup(connection_track_table, key);
+
+ return conn ? true : false;
+}
diff --git a/net/colo.h b/net/colo.h
index 0277e0e9ba..11c5226488 100644
--- a/net/colo.h
+++ b/net/colo.h
@@ -98,6 +98,8 @@ void connection_destroy(void *opaque);
Connection *connection_get(GHashTable *connection_track_table,
ConnectionKey *key,
GQueue *conn_list);
+bool connection_has_tracked(GHashTable *connection_track_table,
+ ConnectionKey *key);
void connection_hashtable_reset(GHashTable *connection_track_table);
Packet *packet_new(const void *data, int size, int vnet_hdr_len);
void packet_destroy(void *opaque, void *user_data);
diff --git a/net/filter-rewriter.c b/net/filter-rewriter.c
index f18a71bf2e..c463a0c1d0 100644
--- a/net/filter-rewriter.c
+++ b/net/filter-rewriter.c
@@ -20,11 +20,15 @@
#include "qemu/main-loop.h"
#include "qemu/iov.h"
#include "net/checksum.h"
+#include "net/colo.h"
+#include "migration/colo.h"
#define FILTER_COLO_REWRITER(obj) \
OBJECT_CHECK(RewriterState, (obj), TYPE_FILTER_REWRITER)
#define TYPE_FILTER_REWRITER "filter-rewriter"
+#define FAILOVER_MODE_ON true
+#define FAILOVER_MODE_OFF false
typedef struct RewriterState {
NetFilterState parent_obj;
@@ -32,8 +36,14 @@ typedef struct RewriterState {
/* hashtable to save connection */
GHashTable *connection_track_table;
bool vnet_hdr;
+ bool failover_mode;
} RewriterState;
+static void filter_rewriter_failover_mode(RewriterState *s)
+{
+ s->failover_mode = FAILOVER_MODE_ON;
+}
+
static void filter_rewriter_flush(NetFilterState *nf)
{
RewriterState *s = FILTER_COLO_REWRITER(nf);
@@ -269,6 +279,13 @@ static ssize_t colo_rewriter_receive_iov(NetFilterState *nf,
*/
reverse_connection_key(&key);
}
+
+ /* After failover we needn't change new TCP packet */
+ if (s->failover_mode &&
+ connection_has_tracked(s->connection_track_table, &key)) {
+ goto out;
+ }
+
conn = connection_get(s->connection_track_table,
&key,
NULL);
@@ -302,11 +319,49 @@ static ssize_t colo_rewriter_receive_iov(NetFilterState *nf,
}
}
+out:
packet_destroy(pkt, NULL);
pkt = NULL;
return 0;
}
+static void reset_seq_offset(gpointer key, gpointer value, gpointer user_data)
+{
+ Connection *conn = (Connection *)value;
+
+ conn->offset = 0;
+}
+
+static gboolean offset_is_nonzero(gpointer key,
+ gpointer value,
+ gpointer user_data)
+{
+ Connection *conn = (Connection *)value;
+
+ return conn->offset ? true : false;
+}
+
+static void colo_rewriter_handle_event(NetFilterState *nf, int event,
+ Error **errp)
+{
+ RewriterState *rs = FILTER_COLO_REWRITER(nf);
+
+ switch (event) {
+ case COLO_EVENT_CHECKPOINT:
+ g_hash_table_foreach(rs->connection_track_table,
+ reset_seq_offset, NULL);
+ break;
+ case COLO_EVENT_FAILOVER:
+ if (!g_hash_table_find(rs->connection_track_table,
+ offset_is_nonzero, NULL)) {
+ filter_rewriter_failover_mode(rs);
+ }
+ break;
+ default:
+ break;
+ }
+}
+
static void colo_rewriter_cleanup(NetFilterState *nf)
{
RewriterState *s = FILTER_COLO_REWRITER(nf);
@@ -350,6 +405,7 @@ static void filter_rewriter_init(Object *obj)
RewriterState *s = FILTER_COLO_REWRITER(obj);
s->vnet_hdr = false;
+ s->failover_mode = FAILOVER_MODE_OFF;
object_property_add_bool(obj, "vnet_hdr_support",
filter_rewriter_get_vnet_hdr,
filter_rewriter_set_vnet_hdr, NULL);
@@ -362,6 +418,7 @@ static void colo_rewriter_class_init(ObjectClass *oc, void *data)
nfc->setup = colo_rewriter_setup;
nfc->cleanup = colo_rewriter_cleanup;
nfc->receive_iov = colo_rewriter_receive_iov;
+ nfc->handle_event = colo_rewriter_handle_event;
}
static const TypeInfo colo_rewriter_info = {
--
2.17.GIT
^ permalink raw reply related [flat|nested] 35+ messages in thread* Re: [Qemu-devel] [PATCH V11 17/20] filter-rewriter: handle checkpoint and failover event
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 17/20] filter-rewriter: handle checkpoint and failover event Zhang Chen
@ 2018-08-21 3:40 ` Jason Wang
2018-08-21 9:51 ` Zhang Chen
0 siblings, 1 reply; 35+ messages in thread
From: Jason Wang @ 2018-08-21 3:40 UTC (permalink / raw)
To: Zhang Chen, qemu-devel, Paolo Bonzini, Juan Quintela,
Dr . David Alan Gilbert, Eric Blake, Markus Armbruster
Cc: zhanghailiang, Li Zhijian, Zhang Chen
On 2018年08月12日 04:59, Zhang Chen wrote:
> After one round of checkpoint, the states between PVM and SVM
> become consistent, so it is unnecessary to adjust the sequence
> of net packets for old connections, besides, while failover
> happens, filter-rewriter will into failover mode that needn't
> handle the new TCP connection.
>
> Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
> Signed-off-by: Zhang Chen <zhangckid@gmail.com>
> Signed-off-by: Zhang Chen <chen.zhang@intel.com>
> ---
> net/colo-compare.c | 12 ++++-----
> net/colo.c | 8 ++++++
> net/colo.h | 2 ++
> net/filter-rewriter.c | 57 +++++++++++++++++++++++++++++++++++++++++++
> 4 files changed, 73 insertions(+), 6 deletions(-)
>
> diff --git a/net/colo-compare.c b/net/colo-compare.c
> index b8c0240725..462e822be6 100644
> --- a/net/colo-compare.c
> +++ b/net/colo-compare.c
> @@ -116,6 +116,12 @@ enum {
> SECONDARY_IN,
> };
>
> +static void colo_compare_inconsistency_notify(void)
> +{
> + notifier_list_notify(&colo_compare_notifiers,
> + migrate_get_current());
> +}
> +
> static int compare_chr_send(CompareState *s,
> const uint8_t *buf,
> uint32_t size,
> @@ -562,12 +568,6 @@ static int colo_old_packet_check_one(Packet *pkt, int64_t *check_time)
> }
> }
>
> -static void colo_compare_inconsistency_notify(void)
> -{
> - notifier_list_notify(&colo_compare_notifiers,
> - migrate_get_current());
> -}
> -
This part of changes seems unnecessary?
> void colo_compare_register_notifier(Notifier *notify)
> {
> notifier_list_add(&colo_compare_notifiers, notify);
> diff --git a/net/colo.c b/net/colo.c
> index 97c8fc928f..49176bf07b 100644
> --- a/net/colo.c
> +++ b/net/colo.c
> @@ -221,3 +221,11 @@ Connection *connection_get(GHashTable *connection_track_table,
>
> return conn;
> }
> +
> +bool connection_has_tracked(GHashTable *connection_track_table,
> + ConnectionKey *key)
> +{
> + Connection *conn = g_hash_table_lookup(connection_track_table, key);
> +
> + return conn ? true : false;
> +}
> diff --git a/net/colo.h b/net/colo.h
> index 0277e0e9ba..11c5226488 100644
> --- a/net/colo.h
> +++ b/net/colo.h
> @@ -98,6 +98,8 @@ void connection_destroy(void *opaque);
> Connection *connection_get(GHashTable *connection_track_table,
> ConnectionKey *key,
> GQueue *conn_list);
> +bool connection_has_tracked(GHashTable *connection_track_table,
> + ConnectionKey *key);
> void connection_hashtable_reset(GHashTable *connection_track_table);
> Packet *packet_new(const void *data, int size, int vnet_hdr_len);
> void packet_destroy(void *opaque, void *user_data);
> diff --git a/net/filter-rewriter.c b/net/filter-rewriter.c
> index f18a71bf2e..c463a0c1d0 100644
> --- a/net/filter-rewriter.c
> +++ b/net/filter-rewriter.c
> @@ -20,11 +20,15 @@
> #include "qemu/main-loop.h"
> #include "qemu/iov.h"
> #include "net/checksum.h"
> +#include "net/colo.h"
> +#include "migration/colo.h"
>
> #define FILTER_COLO_REWRITER(obj) \
> OBJECT_CHECK(RewriterState, (obj), TYPE_FILTER_REWRITER)
>
> #define TYPE_FILTER_REWRITER "filter-rewriter"
> +#define FAILOVER_MODE_ON true
> +#define FAILOVER_MODE_OFF false
>
> typedef struct RewriterState {
> NetFilterState parent_obj;
> @@ -32,8 +36,14 @@ typedef struct RewriterState {
> /* hashtable to save connection */
> GHashTable *connection_track_table;
> bool vnet_hdr;
> + bool failover_mode;
> } RewriterState;
>
> +static void filter_rewriter_failover_mode(RewriterState *s)
> +{
> + s->failover_mode = FAILOVER_MODE_ON;
> +}
> +
> static void filter_rewriter_flush(NetFilterState *nf)
> {
> RewriterState *s = FILTER_COLO_REWRITER(nf);
> @@ -269,6 +279,13 @@ static ssize_t colo_rewriter_receive_iov(NetFilterState *nf,
> */
> reverse_connection_key(&key);
> }
> +
> + /* After failover we needn't change new TCP packet */
> + if (s->failover_mode &&
> + connection_has_tracked(s->connection_track_table, &key)) {
I think you mean !connection_has_tracked() here?
> + goto out;
> + }
> +
> conn = connection_get(s->connection_track_table,
> &key,
> NULL);
> @@ -302,11 +319,49 @@ static ssize_t colo_rewriter_receive_iov(NetFilterState *nf,
> }
> }
>
> +out:
> packet_destroy(pkt, NULL);
> pkt = NULL;
> return 0;
> }
>
> +static void reset_seq_offset(gpointer key, gpointer value, gpointer user_data)
> +{
> + Connection *conn = (Connection *)value;
> +
> + conn->offset = 0;
> +}
> +
> +static gboolean offset_is_nonzero(gpointer key,
> + gpointer value,
> + gpointer user_data)
> +{
> + Connection *conn = (Connection *)value;
> +
> + return conn->offset ? true : false;
> +}
> +
> +static void colo_rewriter_handle_event(NetFilterState *nf, int event,
> + Error **errp)
> +{
> + RewriterState *rs = FILTER_COLO_REWRITER(nf);
> +
> + switch (event) {
> + case COLO_EVENT_CHECKPOINT:
> + g_hash_table_foreach(rs->connection_track_table,
> + reset_seq_offset, NULL);
> + break;
> + case COLO_EVENT_FAILOVER:
> + if (!g_hash_table_find(rs->connection_track_table,
> + offset_is_nonzero, NULL)) {
> + filter_rewriter_failover_mode(rs);
> + }
> + break;
> + default:
> + break;
> + }
> +}
So I think instead of having a generic filter notifier chain, rewriter
can register notifier to COLO instead. This avoid touching filter core
codes.
Thanks
> +
> static void colo_rewriter_cleanup(NetFilterState *nf)
> {
> RewriterState *s = FILTER_COLO_REWRITER(nf);
> @@ -350,6 +405,7 @@ static void filter_rewriter_init(Object *obj)
> RewriterState *s = FILTER_COLO_REWRITER(obj);
>
> s->vnet_hdr = false;
> + s->failover_mode = FAILOVER_MODE_OFF;
> object_property_add_bool(obj, "vnet_hdr_support",
> filter_rewriter_get_vnet_hdr,
> filter_rewriter_set_vnet_hdr, NULL);
> @@ -362,6 +418,7 @@ static void colo_rewriter_class_init(ObjectClass *oc, void *data)
> nfc->setup = colo_rewriter_setup;
> nfc->cleanup = colo_rewriter_cleanup;
> nfc->receive_iov = colo_rewriter_receive_iov;
> + nfc->handle_event = colo_rewriter_handle_event;
> }
>
> static const TypeInfo colo_rewriter_info = {
^ permalink raw reply [flat|nested] 35+ messages in thread* Re: [Qemu-devel] [PATCH V11 17/20] filter-rewriter: handle checkpoint and failover event
2018-08-21 3:40 ` Jason Wang
@ 2018-08-21 9:51 ` Zhang Chen
0 siblings, 0 replies; 35+ messages in thread
From: Zhang Chen @ 2018-08-21 9:51 UTC (permalink / raw)
To: jasowang
Cc: qemu-devel, Paolo Bonzini, Juan Quintela, Dr . David Alan Gilbert,
Eric Blake, Markus Armbruster, zhang.zhanghailiang, lizhijian,
Zhang Chen
On Tue, Aug 21, 2018 at 11:40 AM Jason Wang <jasowang@redhat.com> wrote:
>
>
> On 2018年08月12日 04:59, Zhang Chen wrote:
> > After one round of checkpoint, the states between PVM and SVM
> > become consistent, so it is unnecessary to adjust the sequence
> > of net packets for old connections, besides, while failover
> > happens, filter-rewriter will into failover mode that needn't
> > handle the new TCP connection.
> >
> > Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
> > Signed-off-by: Zhang Chen <zhangckid@gmail.com>
> > Signed-off-by: Zhang Chen <chen.zhang@intel.com>
> > ---
> > net/colo-compare.c | 12 ++++-----
> > net/colo.c | 8 ++++++
> > net/colo.h | 2 ++
> > net/filter-rewriter.c | 57 +++++++++++++++++++++++++++++++++++++++++++
> > 4 files changed, 73 insertions(+), 6 deletions(-)
> >
> > diff --git a/net/colo-compare.c b/net/colo-compare.c
> > index b8c0240725..462e822be6 100644
> > --- a/net/colo-compare.c
> > +++ b/net/colo-compare.c
> > @@ -116,6 +116,12 @@ enum {
> > SECONDARY_IN,
> > };
> >
> > +static void colo_compare_inconsistency_notify(void)
> > +{
> > + notifier_list_notify(&colo_compare_notifiers,
> > + migrate_get_current());
> > +}
> > +
> > static int compare_chr_send(CompareState *s,
> > const uint8_t *buf,
> > uint32_t size,
> > @@ -562,12 +568,6 @@ static int colo_old_packet_check_one(Packet *pkt,
> int64_t *check_time)
> > }
> > }
> >
> > -static void colo_compare_inconsistency_notify(void)
> > -{
> > - notifier_list_notify(&colo_compare_notifiers,
> > - migrate_get_current());
> > -}
> > -
>
> This part of changes seems unnecessary?
>
colo_compare_tcp want to use this function, so we just move forward this
function.
>
> > void colo_compare_register_notifier(Notifier *notify)
> > {
> > notifier_list_add(&colo_compare_notifiers, notify);
> > diff --git a/net/colo.c b/net/colo.c
> > index 97c8fc928f..49176bf07b 100644
> > --- a/net/colo.c
> > +++ b/net/colo.c
> > @@ -221,3 +221,11 @@ Connection *connection_get(GHashTable
> *connection_track_table,
> >
> > return conn;
> > }
> > +
> > +bool connection_has_tracked(GHashTable *connection_track_table,
> > + ConnectionKey *key)
> > +{
> > + Connection *conn = g_hash_table_lookup(connection_track_table, key);
> > +
> > + return conn ? true : false;
> > +}
> > diff --git a/net/colo.h b/net/colo.h
> > index 0277e0e9ba..11c5226488 100644
> > --- a/net/colo.h
> > +++ b/net/colo.h
> > @@ -98,6 +98,8 @@ void connection_destroy(void *opaque);
> > Connection *connection_get(GHashTable *connection_track_table,
> > ConnectionKey *key,
> > GQueue *conn_list);
> > +bool connection_has_tracked(GHashTable *connection_track_table,
> > + ConnectionKey *key);
> > void connection_hashtable_reset(GHashTable *connection_track_table);
> > Packet *packet_new(const void *data, int size, int vnet_hdr_len);
> > void packet_destroy(void *opaque, void *user_data);
> > diff --git a/net/filter-rewriter.c b/net/filter-rewriter.c
> > index f18a71bf2e..c463a0c1d0 100644
> > --- a/net/filter-rewriter.c
> > +++ b/net/filter-rewriter.c
> > @@ -20,11 +20,15 @@
> > #include "qemu/main-loop.h"
> > #include "qemu/iov.h"
> > #include "net/checksum.h"
> > +#include "net/colo.h"
> > +#include "migration/colo.h"
> >
> > #define FILTER_COLO_REWRITER(obj) \
> > OBJECT_CHECK(RewriterState, (obj), TYPE_FILTER_REWRITER)
> >
> > #define TYPE_FILTER_REWRITER "filter-rewriter"
> > +#define FAILOVER_MODE_ON true
> > +#define FAILOVER_MODE_OFF false
> >
> > typedef struct RewriterState {
> > NetFilterState parent_obj;
> > @@ -32,8 +36,14 @@ typedef struct RewriterState {
> > /* hashtable to save connection */
> > GHashTable *connection_track_table;
> > bool vnet_hdr;
> > + bool failover_mode;
> > } RewriterState;
> >
> > +static void filter_rewriter_failover_mode(RewriterState *s)
> > +{
> > + s->failover_mode = FAILOVER_MODE_ON;
> > +}
> > +
> > static void filter_rewriter_flush(NetFilterState *nf)
> > {
> > RewriterState *s = FILTER_COLO_REWRITER(nf);
> > @@ -269,6 +279,13 @@ static ssize_t
> colo_rewriter_receive_iov(NetFilterState *nf,
> > */
> > reverse_connection_key(&key);
> > }
> > +
> > + /* After failover we needn't change new TCP packet */
> > + if (s->failover_mode &&
> > + connection_has_tracked(s->connection_track_table, &key)) {
>
> I think you mean !connection_has_tracked() here?
>
Yes, you are right.
I will fix it in next version.
Thanks
Zhang Chen
>
> > + goto out;
> > + }
> > +
> > conn = connection_get(s->connection_track_table,
> > &key,
> > NULL);
> > @@ -302,11 +319,49 @@ static ssize_t
> colo_rewriter_receive_iov(NetFilterState *nf,
> > }
> > }
> >
> > +out:
> > packet_destroy(pkt, NULL);
> > pkt = NULL;
> > return 0;
> > }
> >
> > +static void reset_seq_offset(gpointer key, gpointer value, gpointer
> user_data)
> > +{
> > + Connection *conn = (Connection *)value;
> > +
> > + conn->offset = 0;
> > +}
> > +
> > +static gboolean offset_is_nonzero(gpointer key,
> > + gpointer value,
> > + gpointer user_data)
> > +{
> > + Connection *conn = (Connection *)value;
> > +
> > + return conn->offset ? true : false;
> > +}
> > +
> > +static void colo_rewriter_handle_event(NetFilterState *nf, int event,
> > + Error **errp)
> > +{
> > + RewriterState *rs = FILTER_COLO_REWRITER(nf);
> > +
> > + switch (event) {
> > + case COLO_EVENT_CHECKPOINT:
> > + g_hash_table_foreach(rs->connection_track_table,
> > + reset_seq_offset, NULL);
> > + break;
> > + case COLO_EVENT_FAILOVER:
> > + if (!g_hash_table_find(rs->connection_track_table,
> > + offset_is_nonzero, NULL)) {
> > + filter_rewriter_failover_mode(rs);
> > + }
> > + break;
> > + default:
> > + break;
> > + }
> > +}
>
> So I think instead of having a generic filter notifier chain, rewriter
> can register notifier to COLO instead. This avoid touching filter core
> codes.
>
> Thanks
>
> > +
> > static void colo_rewriter_cleanup(NetFilterState *nf)
> > {
> > RewriterState *s = FILTER_COLO_REWRITER(nf);
> > @@ -350,6 +405,7 @@ static void filter_rewriter_init(Object *obj)
> > RewriterState *s = FILTER_COLO_REWRITER(obj);
> >
> > s->vnet_hdr = false;
> > + s->failover_mode = FAILOVER_MODE_OFF;
> > object_property_add_bool(obj, "vnet_hdr_support",
> > filter_rewriter_get_vnet_hdr,
> > filter_rewriter_set_vnet_hdr, NULL);
> > @@ -362,6 +418,7 @@ static void colo_rewriter_class_init(ObjectClass
> *oc, void *data)
> > nfc->setup = colo_rewriter_setup;
> > nfc->cleanup = colo_rewriter_cleanup;
> > nfc->receive_iov = colo_rewriter_receive_iov;
> > + nfc->handle_event = colo_rewriter_handle_event;
> > }
> >
> > static const TypeInfo colo_rewriter_info = {
>
>
^ permalink raw reply [flat|nested] 35+ messages in thread
* [Qemu-devel] [PATCH V11 18/20] COLO: notify net filters about checkpoint/failover event
2018-08-11 20:59 [Qemu-devel] [PATCH V11 00/20] COLO: integrate colo frame with block replication and COLO proxy Zhang Chen
` (16 preceding siblings ...)
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 17/20] filter-rewriter: handle checkpoint and failover event Zhang Chen
@ 2018-08-11 20:59 ` Zhang Chen
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 19/20] COLO: quick failover process by kick COLO thread Zhang Chen
` (2 subsequent siblings)
20 siblings, 0 replies; 35+ messages in thread
From: Zhang Chen @ 2018-08-11 20:59 UTC (permalink / raw)
To: qemu-devel, Paolo Bonzini, Juan Quintela, Dr . David Alan Gilbert,
Jason Wang, Eric Blake, Markus Armbruster
Cc: Zhang Chen, zhanghailiang, Li Zhijian
From: zhanghailiang <zhang.zhanghailiang@huawei.com>
Notify all net filters about the checkpoint and failover event.
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
---
migration/colo.c | 15 +++++++++++++++
1 file changed, 15 insertions(+)
diff --git a/migration/colo.c b/migration/colo.c
index 688d6f40b2..8221a18add 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -31,6 +31,7 @@
#include "qapi/qapi-events-migration.h"
#include "qapi/qmp/qerror.h"
#include "sysemu/cpus.h"
+#include "net/filter.h"
static bool vmstate_loading;
static Notifier packets_compare_notifier;
@@ -83,6 +84,12 @@ static void secondary_vm_do_failover(void)
error_report_err(local_err);
}
+ /* Notify all filters of all NIC to do checkpoint */
+ colo_notify_filters_event(COLO_EVENT_FAILOVER, &local_err);
+ if (local_err) {
+ error_report_err(local_err);
+ }
+
if (!autostart) {
error_report("\"-S\" qemu option will be ignored in secondary side");
/* recover runstate to normal migration finish state */
@@ -781,6 +788,14 @@ void *colo_process_incoming_thread(void *opaque)
goto out;
}
+ /* Notify all filters of all NIC to do checkpoint */
+ colo_notify_filters_event(COLO_EVENT_CHECKPOINT, &local_err);
+
+ if (local_err) {
+ qemu_mutex_unlock_iothread();
+ goto out;
+ }
+
vmstate_loading = false;
vm_start();
trace_colo_vm_state_change("stop", "run");
--
2.17.GIT
^ permalink raw reply related [flat|nested] 35+ messages in thread* [Qemu-devel] [PATCH V11 19/20] COLO: quick failover process by kick COLO thread
2018-08-11 20:59 [Qemu-devel] [PATCH V11 00/20] COLO: integrate colo frame with block replication and COLO proxy Zhang Chen
` (17 preceding siblings ...)
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 18/20] COLO: notify net filters about checkpoint/failover event Zhang Chen
@ 2018-08-11 20:59 ` Zhang Chen
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 20/20] docs: Add COLO status diagram to COLO-FT.txt Zhang Chen
2018-08-21 3:06 ` [Qemu-devel] [PATCH V11 00/20] COLO: integrate colo frame with block replication and COLO proxy Zhang Chen
20 siblings, 0 replies; 35+ messages in thread
From: Zhang Chen @ 2018-08-11 20:59 UTC (permalink / raw)
To: qemu-devel, Paolo Bonzini, Juan Quintela, Dr . David Alan Gilbert,
Jason Wang, Eric Blake, Markus Armbruster
Cc: Zhang Chen, zhanghailiang, Li Zhijian
From: zhanghailiang <zhang.zhanghailiang@huawei.com>
COLO thread may sleep at qemu_sem_wait(&s->colo_checkpoint_sem),
while failover works begin, It's better to wakeup it to quick
the process.
Signed-off-by: zhanghailiang <zhang.zhanghailiang@huawei.com>
Reviewed-by: Dr. David Alan Gilbert <dgilbert@redhat.com>
---
migration/colo.c | 8 ++++++++
1 file changed, 8 insertions(+)
diff --git a/migration/colo.c b/migration/colo.c
index 8221a18add..e951159071 100644
--- a/migration/colo.c
+++ b/migration/colo.c
@@ -131,6 +131,11 @@ static void primary_vm_do_failover(void)
migrate_set_state(&s->state, MIGRATION_STATUS_COLO,
MIGRATION_STATUS_COMPLETED);
+ /*
+ * kick COLO thread which might wait at
+ * qemu_sem_wait(&s->colo_checkpoint_sem).
+ */
+ colo_checkpoint_notify(migrate_get_current());
/*
* Wake up COLO thread which may blocked in recv() or send(),
@@ -539,6 +544,9 @@ static void colo_process_checkpoint(MigrationState *s)
qemu_sem_wait(&s->colo_checkpoint_sem);
+ if (s->state != MIGRATION_STATUS_COLO) {
+ goto out;
+ }
ret = colo_do_checkpoint_transaction(s, bioc, fb);
if (ret < 0) {
goto out;
--
2.17.GIT
^ permalink raw reply related [flat|nested] 35+ messages in thread* [Qemu-devel] [PATCH V11 20/20] docs: Add COLO status diagram to COLO-FT.txt
2018-08-11 20:59 [Qemu-devel] [PATCH V11 00/20] COLO: integrate colo frame with block replication and COLO proxy Zhang Chen
` (18 preceding siblings ...)
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 19/20] COLO: quick failover process by kick COLO thread Zhang Chen
@ 2018-08-11 20:59 ` Zhang Chen
2018-08-21 3:06 ` [Qemu-devel] [PATCH V11 00/20] COLO: integrate colo frame with block replication and COLO proxy Zhang Chen
20 siblings, 0 replies; 35+ messages in thread
From: Zhang Chen @ 2018-08-11 20:59 UTC (permalink / raw)
To: qemu-devel, Paolo Bonzini, Juan Quintela, Dr . David Alan Gilbert,
Jason Wang, Eric Blake, Markus Armbruster
Cc: Zhang Chen, zhanghailiang, Li Zhijian, Zhang Chen
From: Zhang Chen <chen.zhang@intel.com>
This diagram make user better understand COLO.
Suggested by Markus Armbruster.
Signed-off-by: Zhang Chen <zhangckid@gmail.com>
Signed-off-by: Zhang Chen <chen.zhang@intel.com>
---
docs/COLO-FT.txt | 34 ++++++++++++++++++++++++++++++++++
1 file changed, 34 insertions(+)
diff --git a/docs/COLO-FT.txt b/docs/COLO-FT.txt
index d7c7dcda8f..d5007895d1 100644
--- a/docs/COLO-FT.txt
+++ b/docs/COLO-FT.txt
@@ -110,6 +110,40 @@ Note:
HeartBeat has not been implemented yet, so you need to trigger failover process
by using 'x-colo-lost-heartbeat' command.
+== COLO operation status ==
+
++-----------------+
+| |
+| Start COLO |
+| |
++--------+--------+
+ |
+ | Main qmp command:
+ | migrate-set-capabilities with x-colo
+ | migrate
+ |
+ v
++--------+--------+
+| |
+| COLO running |
+| |
++--------+--------+
+ |
+ | Main qmp command:
+ | x-colo-lost-heartbeat
+ | or
+ | some error happened
+ v
++--------+--------+
+| | send qmp event:
+| COLO failover | COLO_EXIT
+| |
++-----------------+
+
+COLO use the qmp command switching and report operation status.
+The diagram just write the main qmp command, you can get the detail
+in test procedure.
+
== Test procedure ==
1. Startup qemu
Primary:
--
2.17.GIT
^ permalink raw reply related [flat|nested] 35+ messages in thread* Re: [Qemu-devel] [PATCH V11 00/20] COLO: integrate colo frame with block replication and COLO proxy
2018-08-11 20:59 [Qemu-devel] [PATCH V11 00/20] COLO: integrate colo frame with block replication and COLO proxy Zhang Chen
` (19 preceding siblings ...)
2018-08-11 20:59 ` [Qemu-devel] [PATCH V11 20/20] docs: Add COLO status diagram to COLO-FT.txt Zhang Chen
@ 2018-08-21 3:06 ` Zhang Chen
20 siblings, 0 replies; 35+ messages in thread
From: Zhang Chen @ 2018-08-21 3:06 UTC (permalink / raw)
To: qemu-devel, Paolo Bonzini, Juan Quintela, Dr . David Alan Gilbert,
jasowang, Eric Blake, Markus Armbruster
Cc: zhang.zhanghailiang, lizhijian
Hi Jason and Markus,
Ping...
Do you have time to review this series?
Thanks
Zhang Chen
On Sun, Aug 12, 2018 at 4:59 AM Zhang Chen <zhangckid@gmail.com> wrote:
> Hi~ All~
>
> COLO Frame, block replication and COLO proxy(colo-compare,filter-mirror,
> filter-redirector,filter-rewriter) have been exist in qemu
> for long time, it's time to integrate these three parts to make COLO
> really works.
>
> In this series, we have some optimizations for COLO frame, including
> separating the
> process of saving ram and device state, using an COLO_EXIT event to notify
> users that
> VM exits COLO, for these parts, most of them have been reviewed long time
> ago in old version,
> but since this series have just rebased on upstream which had merged a new
> series of migration,
> parts of pathes in this series deserve review again.
>
> We use notifier/callback method for COLO compare to notify COLO frame about
> net packets inconsistent event, and add a handle_event method for
> NetFilterClass to
> help COLO frame to notify filters and colo-compare about
> checkpoint/failover event,
> it is flexible.
>
> For the neweset version, please refer to:
> https://github.com/zhangckid/qemu/tree/qemu-colo-18aug12
>
> Please review, thanks.
>
> V11:
> - Rebased on upstream.
> - Used "RAMBLOCK_FOREACH_MIGRATABLE()" to replace "QLIST_FOREACH_RCU()"
> in patch 08/20.
> - Fixed COLO related qapi command's since version in patch 10/20.
>
> V10:
> - Rebased on upstream.
> - Removed the "active" in COLOState.
> - Fixed some comments.
>
> V9:
> - Rebased on upstream codes.
> - Addressed Jason's comments add TCP state machine track in
> filter-rewriter.
> - Fix some bug in colo-compare.
> - Fix typo.
> - Add filter-rewriter failover handle.
> - Add net client type check in colo-compare.
> - Add COLO state diagram.
> - Addressed Markus and Daive's comments.
>
>
> V8:
> - Rebased on upstream codes.
> - Addressed Markus's comments in patch 10/17.
> - Addressed Markus's comments in patch 11/17.
> - Removed some comments in patch 4/17.
> - Moved the "migration_bitmap_clear_dirty()" to suitable position in
> patch 9/17.
> - Rewrote the patch 07/17 to address Davie's comments.
> - Moved the "qemu_savevm_live_state" out of the
> qemu_mutex_lock_iothread.
> - Fixed the bug that in some status COLO vm crash with segmentation fault.
>
> V7:
> - Addressed Markus's comments in 11/17.
> - Rebased on upstream.
>
> V6:
> - Addressed Eric Blake's comments, use the enum to feedback in patch
> 11/17.
> - Fixed QAPI command separator problem in patch 11/17.
>
>
>
> Zhang Chen (16):
> filter-rewriter: Add TCP state machine and fix memory leak in
> connection_track_table
> colo-compare: implement the process of checkpoint
> colo-compare: use notifier to notify packets comparing result
> COLO: integrate colo compare with colo frame
> COLO: Add block replication into colo process
> COLO: Remove colo_state migration struct
> COLO: Load dirty pages into SVM's RAM cache firstly
> ram/COLO: Record the dirty pages that SVM received
> COLO: Flush memory data from ram cache
> qapi/migration.json: Rename COLO unknown mode to none mode.
> qapi: Add new command to query colo status
> savevm: split the process of different stages for loadvm/savevm
> net/net.c: Add net client type check function for COLO
> filter: Add handle_event method for NetFilterClass
> filter-rewriter: handle checkpoint and failover event
> docs: Add COLO status diagram to COLO-FT.txt
>
> zhanghailiang (4):
> qmp event: Add COLO_EXIT event to notify users while exited COLO
> COLO: flush host dirty ram from cache
> COLO: notify net filters about checkpoint/failover event
> COLO: quick failover process by kick COLO thread
>
> docs/COLO-FT.txt | 34 ++++++
> include/exec/ram_addr.h | 1 +
> include/migration/colo.h | 11 +-
> include/net/filter.h | 5 +
> include/net/net.h | 1 +
> migration/Makefile.objs | 2 +-
> migration/colo-comm.c | 76 --------------
> migration/colo-failover.c | 2 +-
> migration/colo.c | 212 ++++++++++++++++++++++++++++++++++++--
> migration/migration.c | 44 +++++++-
> migration/ram.c | 166 ++++++++++++++++++++++++++++-
> migration/ram.h | 4 +
> migration/savevm.c | 53 ++++++++--
> migration/savevm.h | 5 +
> migration/trace-events | 3 +
> net/colo-compare.c | 120 +++++++++++++++++++--
> net/colo-compare.h | 24 +++++
> net/colo.c | 10 +-
> net/colo.h | 11 +-
> net/filter-rewriter.c | 162 +++++++++++++++++++++++++++--
> net/filter.c | 17 +++
> net/net.c | 42 ++++++++
> qapi/migration.json | 80 +++++++++++++-
> vl.c | 2 -
> 24 files changed, 948 insertions(+), 139 deletions(-)
> delete mode 100644 migration/colo-comm.c
> create mode 100644 net/colo-compare.h
>
> --
> 2.17.GIT
>
>
^ permalink raw reply [flat|nested] 35+ messages in thread