From mboxrd@z Thu Jan 1 00:00:00 1970 From: jbrassow@sourceware.org Date: 23 Jan 2008 21:21:11 -0000 Subject: [Cluster-devel] cluster/cmirror/src clogd.c cluster.c cluster. ... Message-ID: <20080123212111.22310.qmail@sourceware.org> List-Id: To: cluster-devel.redhat.com MIME-Version: 1.0 Content-Type: text/plain; charset="us-ascii" Content-Transfer-Encoding: 7bit CVSROOT: /cvs/cluster Module name: cluster Branch: RHEL5 Changes by: jbrassow at sourceware.org 2008-01-23 21:21:06 Modified files: cmirror/src : clogd.c cluster.c cluster.h functions.c local.c local.h queues.c queues.h Log message: - better cleanup on exit - pull rbtrees in favor of simple list until later Patches: http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/cmirror/src/clogd.c.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.1.2.4&r2=1.1.2.5 http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/cmirror/src/cluster.c.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.1.2.10&r2=1.1.2.11 http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/cmirror/src/cluster.h.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.1.2.1&r2=1.1.2.2 http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/cmirror/src/functions.c.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.1.2.9&r2=1.1.2.10 http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/cmirror/src/local.c.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.1.2.8&r2=1.1.2.9 http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/cmirror/src/local.h.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.1.2.1&r2=1.1.2.2 http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/cmirror/src/queues.c.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.1.2.3&r2=1.1.2.4 http://sourceware.org/cgi-bin/cvsweb.cgi/cluster/cmirror/src/queues.h.diff?cvsroot=cluster&only_with_tag=RHEL5&r1=1.1.2.2&r2=1.1.2.3 --- cluster/cmirror/src/Attic/clogd.c 2008/01/21 20:18:44 1.1.2.4 +++ cluster/cmirror/src/Attic/clogd.c 2008/01/23 21:21:06 1.1.2.5 @@ -30,6 +30,7 @@ static void process_signals(void); static void daemonize(void); static void init_all(void); +static void cleanup_all(void); int main(int argc, char *argv[]) { @@ -129,6 +130,7 @@ if (!r) { LOG_DBG("No current cluster logs... safe to exit."); + cleanup_all(); exit(EXIT_SUCCESS); } @@ -247,3 +249,14 @@ } } +/* + * cleanup_all + * + * Clean up before exiting + */ +static void cleanup_all(void) +{ + cleanup_local(); + cleanup_cluster(); + cleanup_queues(); +} --- cluster/cmirror/src/Attic/cluster.c 2008/01/18 21:00:15 1.1.2.10 +++ cluster/cmirror/src/Attic/cluster.c 2008/01/23 21:21:06 1.1.2.11 @@ -68,6 +68,9 @@ cpg_flow_control_state_t flow_control_state; cpg_error_t error; + /* FIXME: no flow control for now (cmirror should self regulate) */ + return 0; + error = cpg_flow_control_state_get(handle, &flow_control_state); if (error != CPG_OK) { LOG_ERROR("Failed to get flow control state. Reason: %d", error); @@ -981,3 +984,12 @@ EXIT(); return 0; } + +void cleanup_cluster(void) +{ + SaAisErrorT err; + + err = saCkptFinalize(ckpt_handle); + if (err != SA_AIS_OK) + LOG_ERROR("Failed to finalize checkpoint handle"); +} --- cluster/cmirror/src/Attic/cluster.h 2007/08/23 19:57:31 1.1.2.1 +++ cluster/cmirror/src/Attic/cluster.h 2008/01/23 21:21:06 1.1.2.2 @@ -2,6 +2,7 @@ #define __CLUSTER_LOG_CLUSTER_DOT_H__ int init_cluster(void); +void cleanup_cluster(void); int create_cluster_cpg(char *str); int destroy_cluster_cpg(char *str); --- cluster/cmirror/src/Attic/functions.c 2008/01/18 17:11:07 1.1.2.9 +++ cluster/cmirror/src/Attic/functions.c 2008/01/23 21:21:06 1.1.2.10 @@ -15,7 +15,6 @@ #include "common.h" #include "cluster.h" #include "logging.h" -#include "rbtree.h" #define BYTE_SHIFT 3 @@ -67,7 +66,7 @@ uint32_t state; /* current operational state of the log */ - struct rb_tree mark_tree; /* Tree that tracks all mark requests */ + struct list_head mark_list; uint32_t recovery_halted; struct recovery_request *recovery_request_list; @@ -80,6 +79,7 @@ }; struct mark_entry { + struct list_head list; uint32_t nodeid; uint64_t region; }; @@ -288,47 +288,6 @@ return 0; } -static void *get_mark_entry_region(void *data) -{ - struct mark_entry *m = data; - - return (void *)&m->region; -} - -static int cmp_mark_entry_regions(void *a, void *b) -{ - uint64_t _a = *((uint64_t *)a); - uint64_t _b = *((uint64_t *)b); - - return (_a == _b) ? 0 : (_a < _b) ? -1 : 1; -} - -/* - * srsm_count - Same Region, Same Machine count - * @data - data held in the tree node (a mark_entry ptr) - * @adata - additional data passed in (nodeid) - * - * This function always returns 1 - allowing the RBT search - * to continuing finding additional matches. It's useful - * feature is that it counts all the tree nodes that match - * the given machine (not just the region). Results of the - * count is placed in 'srsm_count_var'. - * - * Returns: 1 - */ -static int srsm_count_var = 0; -static int srsm_count(void *data, void *adata) -{ - uint32_t nodeid = *((uint32_t *)adata); - struct mark_entry *m = data; - - if (nodeid == m->nodeid) - srsm_count_var++; - - return 1; -} - - static int find_disk_path(char *major_minor_str, char *path_rtn, int *unlink_path) { int r; @@ -467,8 +426,7 @@ lc->disk_fd = -1; lc->log_dev_failed = 0; - rbt_init(&lc->mark_tree, sizeof(struct mark_entry), - get_mark_entry_region, cmp_mark_entry_regions); + INIT_LIST_HEAD(&lc->mark_list); lc->bitset_uint32_count = region_count / (sizeof(*lc->clean_bits) << BYTE_SHIFT); @@ -523,8 +481,6 @@ list_add(&lc->list, &log_pending_list); - LOG_DBG("Log UUID = %s, mark_tree = 0x%p", - SHORT_UUID(lc->uuid), &lc->mark_tree); EXIT(); return 0; fail: @@ -965,39 +921,37 @@ */ static int mark_region(struct log_c *lc, uint64_t region, uint32_t who) { - struct rb_node *new; + int found = 0; struct mark_entry *m; + struct list_head *p, *n; - /* - * The search will find every node in the tree that has - * the same region marked. The additional function - * passed to 'rbt_search_plus' detects if the matching - * nodes also are from the machine who is performing this - * request. - */ - srsm_count_var = 0; - if (!rbt_search_plus(&lc->mark_tree, ®ion, srsm_count, &who)) - log_clear_bit(lc, lc->clean_bits, region); + list_for_each_safe(p, n, &lc->mark_list) { + /* FIXME: Use proper macros */ + m = (struct mark_entry *)p; + if (m->region == region) { + found = 1; + if (m->nodeid == who) + return 0; + } + } - /* Requesting region/nodeid is already in the tree */ - if (srsm_count_var) - return 0; + if (!found) + log_clear_bit(lc, lc->clean_bits, region); /* * Save allocation until here - if there is a failure, * at least we have cleared the bit. */ - new = rbt_alloc_node(&lc->mark_tree); - if (!new) { + m = malloc(sizeof(*m)); + if (!m) { LOG_ERROR("Unable to allocate space for mark_entry: %llu/%u", (unsigned long long)region, who); return -ENOMEM; } - m = new->rb_data; m->nodeid = who; m->region = region; - rbt_insert(&lc->mark_tree, new); + list_add_tail(&m->list, &lc->mark_list); return 0; } @@ -1042,40 +996,28 @@ static int clear_region(struct log_c *lc, uint64_t region, uint32_t who) { - int set_bit = 1; - struct rb_node *mark_list; + int other_matches = 0; struct mark_entry *m; + struct list_head *p, *n; - srsm_count_var = 0; - mark_list = rbt_search_plus(&lc->mark_tree, ®ion, srsm_count, &who); - if (!mark_list || !srsm_count_var) { - LOG_DBG("[%s] Clear issued by %u on region not marked: %llu", - SHORT_UUID(lc->uuid), who, (unsigned long long)region); - goto set_bit; - } - - /* If rb_next is set, it means more than one node has this marked */ - if (mark_list->rb_next) - set_bit = 0; - - /* Must find this machine's entry to remove it */ - for (; mark_list; mark_list = mark_list->rb_next) { - m = mark_list->rb_data; - if (m->nodeid == who) - break; - } - - if (!mark_list) { - LOG_ERROR("Bad programming: searches disagree on results"); - goto set_bit; - } - - rbt_remove(&lc->mark_tree, mark_list); - rbt_free_node(&lc->mark_tree, mark_list); + list_for_each_safe(p, n, &lc->mark_list) { + /* FIXME: Use proper macros */ + m = (struct mark_entry *)p; + if (m->region == region) { + if (m->nodeid == who) { + list_del_init(&m->list); + free(m); + } else + other_matches = 1; + } + } -set_bit: - /* Only clear the region if it is also in-sync */ - if (set_bit && log_test_bit(lc->sync_bits, region)) + /* + * Clear region if: + * 1) It is in-sync + * 2) There are no other machines that have it marked + */ + if (!other_matches && log_test_bit(lc->sync_bits, region)) log_set_bit(lc, lc->clean_bits, region); return 0; --- cluster/cmirror/src/Attic/local.c 2008/01/18 17:11:07 1.1.2.8 +++ cluster/cmirror/src/Attic/local.c 2008/01/23 21:21:06 1.1.2.9 @@ -16,6 +16,7 @@ #include "local.h" static int cn_fd; /* Connector (netlink) socket fd */ +static int request_array[20]; /* for request counting (debugging) */ static int kernel_recv_helper(void *data, int in_size) { @@ -143,7 +144,8 @@ */ static int do_local_work(void *data) { - int r; + static int request_count = 0; + int r, i; struct clog_tfr *tfr = NULL; ENTER(); @@ -151,6 +153,12 @@ if (r) return r; + request_array[tfr->request_type]++; + request_count++; + if (!(request_count % 10000)) + for (i = 0; i < 20; i++) + LOG_PRINT(" %s: %d", RQ_TYPE(i), request_array[i]); + LOG_DBG("Request from kernel recieved [%s/%s/%llu]", RQ_TYPE(tfr->request_type), SHORT_UUID(tfr->uuid), (unsigned long long)tfr->seq); @@ -287,6 +295,8 @@ ENTER(); + memset(request_array, 0, sizeof(int)*20); + cn_fd = socket(PF_NETLINK, SOCK_DGRAM, NETLINK_CONNECTOR); if (cn_fd < 0) { EXIT(); @@ -321,3 +331,14 @@ EXIT(); return 0; } + +/* + * cleanup_local + * + * Clean up before exiting + */ +void cleanup_local(void) +{ + links_unregister(cn_fd); + close(cn_fd); +} --- cluster/cmirror/src/Attic/local.h 2007/08/23 19:57:31 1.1.2.1 +++ cluster/cmirror/src/Attic/local.h 2008/01/23 21:21:06 1.1.2.2 @@ -2,6 +2,7 @@ #define __CLUSTER_LOG_LOCAL_DOT_H__ int init_local(void); +void cleanup_local(void); int kernel_send(struct clog_tfr *tfr); --- cluster/cmirror/src/Attic/queues.c 2008/01/21 20:18:44 1.1.2.3 +++ cluster/cmirror/src/Attic/queues.c 2008/01/23 21:21:06 1.1.2.4 @@ -123,6 +123,17 @@ } /* + * cleanup_queues + * + * Clean up before exiting + */ +void cleanup_queues(void) +{ + queue_dtr(cluster_queue); + queue_dtr(free_queue); +} + +/* * queue_add_tail * @tfr: item to add * @q: receiving queue --- cluster/cmirror/src/Attic/queues.h 2008/01/14 22:52:17 1.1.2.2 +++ cluster/cmirror/src/Attic/queues.h 2008/01/23 21:21:06 1.1.2.3 @@ -16,6 +16,7 @@ extern struct queue *free_queue; int init_queues(void); +void cleanup_queues(void); int queue_status(void); void queue_add_tail(struct clog_tfr *tfr, struct queue *q); void queue_add(struct clog_tfr *tfr, struct queue *q);