[PATCH v3 01/10] station: always add BSS to network blacklist on failure

[PATCH v3 01/10] station: always add BSS to network blacklist on failure

[James Prestwood]

Allowing the timeout blacklist to be disabled has introduced a bug
where a failed connection will not result in the BSS list to be
traversed. This causes IWD to retry the same BSS over and over which
be either a) have some issue preventing a connection or b) may simply
be unreachable/out of range.

This is because IWD was inherently relying on the timeout blacklist
to flag BSS's on failures. With it disabled there was nothing to tell
network_bss_select that we should skip the BSS and it would return
the same BSS indefinitely.

To fix this some of the blacklisting logic was re-worked in station.
Now, a BSS will always get network blacklisted upon a failure. This
allows network.c to traverse to the next BSS upon failure.

For auth/assoc failures we will then only timeout blacklist under
certain conditions, i.e. the status code was not in the temporary
list.

Fixes: 77639d2d452e ("blacklist: allow configuration to disable the blacklist")
---
 src/station.c | 25 +++++++++++++++++++++----
 1 file changed, 21 insertions(+), 4 deletions(-)

diff --git a/src/station.c b/src/station.c
index 5403c332..0b20e785 100644
--- a/src/station.c
+++ b/src/station.c
@@ -3402,6 +3402,13 @@ static bool station_retry_with_reason(struct station *station,
 
 	blacklist_add_bss(station->connected_bss->addr);
 
+	/*
+	 * Network blacklist the BSS as well, since the timeout blacklist could
+	 * be disabled
+	 */
+	network_blacklist_add(station->connected_network,
+				station->connected_bss);
+
 try_next:
 	return station_try_next_bss(station);
 }
@@ -3449,6 +3456,10 @@ static bool station_pmksa_fallback(struct station *station, uint16_t status)
 static bool station_retry_with_status(struct station *station,
 					uint16_t status_code)
 {
+	/* If PMKSA failed don't blacklist so we can retry this BSS */
+	if (station_pmksa_fallback(station, status_code))
+		goto try_next;
+
 	/*
 	 * Certain Auth/Assoc failures should not cause a timeout blacklist.
 	 * In these cases we want to only temporarily blacklist the BSS until
@@ -3459,12 +3470,18 @@ static bool station_retry_with_status(struct station *station,
 	 *       specific BSS on our next attempt. There is currently no way to
 	 *       obtain that IE, but this should be done in the future.
 	 */
-	if (IS_TEMPORARY_STATUS(status_code))
-		network_blacklist_add(station->connected_network,
-						station->connected_bss);
-	else if (!station_pmksa_fallback(station, status_code))
+	if (!IS_TEMPORARY_STATUS(status_code))
 		blacklist_add_bss(station->connected_bss->addr);
 
+	/*
+	 * Unconditionally network blacklist the BSS if we are retrying. This
+	 * will allow network_bss_select to traverse the BSS list and ignore
+	 * BSS's which have previously failed
+	 */
+	network_blacklist_add(station->connected_network,
+				station->connected_bss);
+
+try_next:
 	iwd_notice(IWD_NOTICE_CONNECT_FAILED, "status: %u", status_code);
 
 	return station_try_next_bss(station);
-- 
2.34.1

[PATCH v3 02/10] auto-t: add test for disabling the timeout blacklist

[James Prestwood]

---
 autotests/testBSSBlacklist/TestBlacklist.psk  |  2 +
 autotests/testBSSBlacklist/connection_test.py | 57 +++++++++++++++++++
 .../{main.conf => main.conf.default}          |  0
 autotests/testBSSBlacklist/main.conf.disabled |  2 +
 4 files changed, 61 insertions(+)
 create mode 100644 autotests/testBSSBlacklist/TestBlacklist.psk
 rename autotests/testBSSBlacklist/{main.conf => main.conf.default} (100%)
 create mode 100644 autotests/testBSSBlacklist/main.conf.disabled

diff --git a/autotests/testBSSBlacklist/TestBlacklist.psk b/autotests/testBSSBlacklist/TestBlacklist.psk
new file mode 100644
index 00000000..abafdb66
--- /dev/null
+++ b/autotests/testBSSBlacklist/TestBlacklist.psk
@@ -0,0 +1,2 @@
+[Security]
+Passphrase=secret123
diff --git a/autotests/testBSSBlacklist/connection_test.py b/autotests/testBSSBlacklist/connection_test.py
index 9631a322..ac1cb86d 100644
--- a/autotests/testBSSBlacklist/connection_test.py
+++ b/autotests/testBSSBlacklist/connection_test.py
@@ -260,12 +260,69 @@ class Test(unittest.TestCase):
 
         self.wd.unregister_psk_agent(psk_agent)
 
+    def test_blacklist_disabled(self):
+        wd = self.wd
+        bss_hostapd = self.bss_hostapd
+
+        rule0 = self.rule0
+        rule1 = self.rule1
+        rule2 = self.rule2
+
+        psk_agent = PSKAgent(["secret123", 'secret123'])
+        wd.register_psk_agent(psk_agent)
+
+        devices = wd.list_devices(1)
+        device = devices[0]
+
+        rule0.drop = True
+        rule0.enabled = True
+
+        device.autoconnect = True
+
+        condition = 'obj.state == DeviceState.connected'
+        wd.wait_for_object_condition(device, condition)
+
+        ordered_network = device.get_ordered_network("TestBlacklist", full_scan=True)
+
+        self.assertEqual(ordered_network.type, NetworkType.psk)
+
+        # The first BSS should fail, and we should connect to the second. This
+        # should not result in a connection blacklist though since its disabled.
+        bss_hostapd[1].wait_for_event('AP-STA-CONNECTED %s' % device.address)
+
+        device.disconnect()
+
+        rule0.drop = False
+        device.autoconnect = True
+
+        # Verify the first BSS wasn't blacklisted.
+        bss_hostapd[0].wait_for_event('AP-STA-CONNECTED %s' % device.address)
+
     def setUp(self):
+        _, _, name = self.id().split(".")
+
+        # TODO: If we have this pattern elsewhere it might be nice to turn this
+        # into a decorator e.g.
+        #
+        # @config("main.conf.disabled")
+        # @profile("TestBlacklist.psk")
+        # def test_blacklist_disabled(self)
+        #    ...
+        #
+        if name == "test_blacklist_disabled":
+            IWD.copy_to_storage("main.conf.disabled", IWD_CONFIG_DIR, "main.conf")
+            IWD.copy_to_storage("TestBlacklist.psk")
+        else:
+            IWD.copy_to_storage("main.conf.default", IWD_CONFIG_DIR, "main.conf")
+
         self.wd = IWD(True)
 
     def tearDown(self):
         IWD.clear_storage()
         self.wd = None
+        self.rule0.drop = False
+        self.rule1.drop = False
+        self.rule2.drop = False
 
     @classmethod
     def setUpClass(cls):
diff --git a/autotests/testBSSBlacklist/main.conf b/autotests/testBSSBlacklist/main.conf.default
similarity index 100%
rename from autotests/testBSSBlacklist/main.conf
rename to autotests/testBSSBlacklist/main.conf.default
diff --git a/autotests/testBSSBlacklist/main.conf.disabled b/autotests/testBSSBlacklist/main.conf.disabled
new file mode 100644
index 00000000..aae6bc12
--- /dev/null
+++ b/autotests/testBSSBlacklist/main.conf.disabled
@@ -0,0 +1,2 @@
+[Blacklist]
+InitialTimeout=0
-- 
2.34.1

[PATCH v3 03/10] blacklist: include a blacklist reason when adding/finding

[James Prestwood]

To both prepare for some new blacklisting behavior and allow for
easier consolidation of the network-specific blacklist include a
reason enum for each entry. This allows IWD to differentiate
between multiple blacklist types. For now only the existing
"permanent" type is being added which prevents connections to that
BSS via autoconnect until it expires.

By including a type into each entry we now have additional search
criteria and can have multiple entires of the same BSS with
different reasons. This was done versus a bitmask because each
blacklist reason may have a different expiration time. We want to
maintain individual expirations to have the best "memory" of past
events rather than overwriting them.

Future patches will lump in the temporary network blacklist as well
as a new roaming blacklist type.
---
 src/blacklist.c | 61 +++++++++++++++++++++++++++++++++++++++++--------
 src/blacklist.h | 14 +++++++++---
 src/network.c   |  3 ++-
 src/station.c   | 12 ++++++----
 4 files changed, 72 insertions(+), 18 deletions(-)

diff --git a/src/blacklist.c b/src/blacklist.c
index 21f85a75..8ae474b3 100644
--- a/src/blacklist.c
+++ b/src/blacklist.c
@@ -51,10 +51,27 @@ struct blacklist_entry {
 	uint8_t addr[6];
 	uint64_t added_time;
 	uint64_t expire_time;
+	enum blacklist_reason reason;
+};
+
+struct blacklist_search {
+	const uint8_t *addr;
+	enum blacklist_reason reason;
 };
 
 static struct l_queue *blacklist;
 
+static uint64_t get_reason_timeout(enum blacklist_reason reason)
+{
+	switch (reason) {
+	case BLACKLIST_REASON_CONNECT_FAILED:
+		return blacklist_initial_timeout;
+	default:
+		l_warn("Unhandled blacklist reason: %u", reason);
+		return 0;
+	}
+}
+
 static bool check_if_expired(void *data, void *user_data)
 {
 	struct blacklist_entry *entry = data;
@@ -87,15 +104,31 @@ static bool match_addr(const void *a, const void *b)
 	return false;
 }
 
-void blacklist_add_bss(const uint8_t *addr)
+static bool match_addr_and_reason(const void *a, const void *b)
 {
-	struct blacklist_entry *entry;
+	const struct blacklist_entry *entry = a;
+	const struct blacklist_search *search = b;
 
-	if (!blacklist_initial_timeout)
-		return;
+	if (entry->reason != search->reason)
+		return false;
+
+	if (!memcmp(entry->addr, search->addr, 6))
+		return true;
+
+	return false;
+}
+
+void blacklist_add_bss(const uint8_t *addr, enum blacklist_reason reason)
+{
+	struct blacklist_entry *entry;
+	uint64_t timeout;
 
 	blacklist_prune();
 
+	timeout = get_reason_timeout(reason);
+	if (!timeout)
+		return;
+
 	entry = l_queue_find(blacklist, match_addr, addr);
 
 	if (entry) {
@@ -115,22 +148,26 @@ void blacklist_add_bss(const uint8_t *addr)
 	entry = l_new(struct blacklist_entry, 1);
 
 	entry->added_time = l_time_now();
-	entry->expire_time = l_time_offset(entry->added_time,
-						blacklist_initial_timeout);
+	entry->expire_time = l_time_offset(entry->added_time, timeout);
+	entry->reason = reason;
 	memcpy(entry->addr, addr, 6);
 
 	l_queue_push_tail(blacklist, entry);
 }
 
-bool blacklist_contains_bss(const uint8_t *addr)
+bool blacklist_contains_bss(const uint8_t *addr, enum blacklist_reason reason)
 {
 	bool ret;
 	uint64_t time_now;
 	struct blacklist_entry *entry;
+	struct blacklist_search search = {
+		.addr = addr,
+		.reason = reason
+	};
 
 	blacklist_prune();
 
-	entry = l_queue_find(blacklist, match_addr, addr);
+	entry = l_queue_find(blacklist, match_addr_and_reason, &search);
 
 	if (!entry)
 		return false;
@@ -142,13 +179,17 @@ bool blacklist_contains_bss(const uint8_t *addr)
 	return ret;
 }
 
-void blacklist_remove_bss(const uint8_t *addr)
+void blacklist_remove_bss(const uint8_t *addr, enum blacklist_reason reason)
 {
 	struct blacklist_entry *entry;
+	struct blacklist_search search = {
+		.addr = addr,
+		.reason = reason
+	};
 
 	blacklist_prune();
 
-	entry = l_queue_remove_if(blacklist, match_addr, addr);
+	entry = l_queue_remove_if(blacklist, match_addr_and_reason, &search);
 
 	if (!entry)
 		return;
diff --git a/src/blacklist.h b/src/blacklist.h
index 56260e20..a87e5eca 100644
--- a/src/blacklist.h
+++ b/src/blacklist.h
@@ -20,6 +20,14 @@
  *
  */
 
-void blacklist_add_bss(const uint8_t *addr);
-bool blacklist_contains_bss(const uint8_t *addr);
-void blacklist_remove_bss(const uint8_t *addr);
+enum blacklist_reason {
+	/*
+	 * When a BSS is blacklisted using this reason IWD will refuse to
+	 * connect to it via autoconnect
+	 */
+	BLACKLIST_REASON_CONNECT_FAILED,
+};
+
+void blacklist_add_bss(const uint8_t *addr, enum blacklist_reason reason);
+bool blacklist_contains_bss(const uint8_t *addr, enum blacklist_reason reason);
+void blacklist_remove_bss(const uint8_t *addr, enum blacklist_reason reason);
diff --git a/src/network.c b/src/network.c
index 0a40a6c5..4602a110 100644
--- a/src/network.c
+++ b/src/network.c
@@ -1280,7 +1280,8 @@ struct scan_bss *network_bss_select(struct network *network,
 		if (l_queue_find(network->blacklist, match_bss, bss))
 			continue;
 
-		if (blacklist_contains_bss(bss->addr))
+		if (blacklist_contains_bss(bss->addr,
+					BLACKLIST_REASON_CONNECT_FAILED))
 			continue;
 
 		/* OWE Transition BSS */
diff --git a/src/station.c b/src/station.c
index 0b20e785..e2ed78f3 100644
--- a/src/station.c
+++ b/src/station.c
@@ -2880,7 +2880,8 @@ static bool station_roam_scan_notify(int err, struct l_queue *bss_list,
 		if (network_can_connect_bss(network, bss) < 0)
 			goto next;
 
-		if (blacklist_contains_bss(bss->addr))
+		if (blacklist_contains_bss(bss->addr,
+					BLACKLIST_REASON_CONNECT_FAILED))
 			goto next;
 
 		rank = bss->rank;
@@ -3400,7 +3401,8 @@ static bool station_retry_with_reason(struct station *station,
 		break;
 	}
 
-	blacklist_add_bss(station->connected_bss->addr);
+	blacklist_add_bss(station->connected_bss->addr,
+				BLACKLIST_REASON_CONNECT_FAILED);
 
 	/*
 	 * Network blacklist the BSS as well, since the timeout blacklist could
@@ -3471,7 +3473,8 @@ static bool station_retry_with_status(struct station *station,
 	 *       obtain that IE, but this should be done in the future.
 	 */
 	if (!IS_TEMPORARY_STATUS(status_code))
-		blacklist_add_bss(station->connected_bss->addr);
+		blacklist_add_bss(station->connected_bss->addr,
+					BLACKLIST_REASON_CONNECT_FAILED);
 
 	/*
 	 * Unconditionally network blacklist the BSS if we are retrying. This
@@ -3566,7 +3569,8 @@ static void station_connect_cb(struct netdev *netdev, enum netdev_result result,
 
 	switch (result) {
 	case NETDEV_RESULT_OK:
-		blacklist_remove_bss(station->connected_bss->addr);
+		blacklist_remove_bss(station->connected_bss->addr,
+					BLACKLIST_REASON_CONNECT_FAILED);
 		station_connect_ok(station);
 		return;
 	case NETDEV_RESULT_DISCONNECTED:
-- 
2.34.1

[PATCH v3 04/10] blacklist: fix pruning to remove the entry if its expired

[James Prestwood]

When pruning the list check_if_expired was comparing to the maximum
amount of time a BSS can be blacklisted, not if the current time had
exceeded the expirationt time. This results in blacklist entries
hanging around longer than they should, which would result in them
poentially being blacklisted even longer if there was another reason
to blacklist in the future.

Instead on prune check the actual expiration and remove the entry if
its expired. Doing this removes the need to check any of the times
in blacklist_contains_bss since prune will remove any expired entries
correctly.
---
 src/blacklist.c | 16 ++--------------
 1 file changed, 2 insertions(+), 14 deletions(-)

diff --git a/src/blacklist.c b/src/blacklist.c
index 8ae474b3..2539c5e0 100644
--- a/src/blacklist.c
+++ b/src/blacklist.c
@@ -77,7 +77,7 @@ static bool check_if_expired(void *data, void *user_data)
 	struct blacklist_entry *entry = data;
 	uint64_t now = l_get_u64(user_data);
 
-	if (l_time_diff(now, entry->added_time) > blacklist_max_timeout) {
+	if (l_time_after(now, entry->expire_time)) {
 		l_debug("Removing entry "MAC" on prune", MAC_STR(entry->addr));
 		l_free(entry);
 		return true;
@@ -157,9 +157,6 @@ void blacklist_add_bss(const uint8_t *addr, enum blacklist_reason reason)
 
 bool blacklist_contains_bss(const uint8_t *addr, enum blacklist_reason reason)
 {
-	bool ret;
-	uint64_t time_now;
-	struct blacklist_entry *entry;
 	struct blacklist_search search = {
 		.addr = addr,
 		.reason = reason
@@ -167,16 +164,7 @@ bool blacklist_contains_bss(const uint8_t *addr, enum blacklist_reason reason)
 
 	blacklist_prune();
 
-	entry = l_queue_find(blacklist, match_addr_and_reason, &search);
-
-	if (!entry)
-		return false;
-
-	time_now = l_time_now();
-
-	ret = l_time_after(time_now, entry->expire_time) ? false : true;
-
-	return ret;
+	return l_queue_find(blacklist, match_addr_and_reason, &search) != NULL;
 }
 
 void blacklist_remove_bss(const uint8_t *addr, enum blacklist_reason reason)
-- 
2.34.1

[PATCH v3 05/10] blacklist: add new blacklist reason, ROAM_REQUESTED

[James Prestwood]

This adds a new (less severe) blacklist reason as well as an option
to configure the timeout. This blacklist reason will be used in cases
where a BSS has requested IWD roam elsewhere. At that time a new
blacklist entry will be added which will be used along with some
other criteria to determine if IWD should connect/roam to that BSS
again.

Now that we have multiple blacklist reasons there may be situations
where a blacklist entry already exists but with a different reason.
This is going to be handled by the reason severity. Since we have
just two reasons we will treat a connection failure as most severe
and a roam requested as less severe. This leaves us with two
possible situations:

1. BSS is roam blacklisted, then gets connection blacklisted:
   The reason will be "promoted" to connection blacklisted.

2. BSS is connection blacklisted, then gets roam blacklisted:
   The blacklist request will be ignored
---
 src/blacklist.c | 27 +++++++++++++++++++++++++--
 src/blacklist.h |  7 +++++++
 2 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/src/blacklist.c b/src/blacklist.c
index 2539c5e0..98713095 100644
--- a/src/blacklist.c
+++ b/src/blacklist.c
@@ -45,6 +45,7 @@
 
 static uint64_t blacklist_multiplier;
 static uint64_t blacklist_initial_timeout;
+static uint64_t blacklist_roam_initial_timeout;
 static uint64_t blacklist_max_timeout;
 
 struct blacklist_entry {
@@ -66,6 +67,8 @@ static uint64_t get_reason_timeout(enum blacklist_reason reason)
 	switch (reason) {
 	case BLACKLIST_REASON_CONNECT_FAILED:
 		return blacklist_initial_timeout;
+	case BLACKLIST_REASON_ROAM_REQUESTED:
+		return blacklist_roam_initial_timeout;
 	default:
 		l_warn("Unhandled blacklist reason: %u", reason);
 		return 0;
@@ -132,8 +135,20 @@ void blacklist_add_bss(const uint8_t *addr, enum blacklist_reason reason)
 	entry = l_queue_find(blacklist, match_addr, addr);
 
 	if (entry) {
-		uint64_t offset = l_time_diff(entry->added_time,
-							entry->expire_time);
+		uint64_t offset;
+
+		if (reason < entry->reason) {
+			l_debug("Promoting "MAC" blacklist to reason %u",
+					MAC_STR(addr), reason);
+			entry->reason = reason;
+		} else if (reason > entry->reason) {
+			l_debug("Ignoring blacklist extension of "MAC", "
+				"current blacklist status is more severe!",
+				MAC_STR(addr));
+			return;
+		}
+
+		offset = l_time_diff(entry->added_time, entry->expire_time);
 
 		offset *= blacklist_multiplier;
 
@@ -196,6 +211,14 @@ static int blacklist_init(void)
 	/* For easier user configuration the timeout values are in seconds */
 	blacklist_initial_timeout *= L_USEC_PER_SEC;
 
+	if (!l_settings_get_uint64(config, "Blacklist",
+					"InitialRoamRequestedTimeout",
+					&blacklist_roam_initial_timeout))
+		blacklist_roam_initial_timeout = BLACKLIST_DEFAULT_TIMEOUT;
+
+	/* For easier user configuration the timeout values are in seconds */
+	blacklist_roam_initial_timeout *= L_USEC_PER_SEC;
+
 	if (!l_settings_get_uint64(config, "Blacklist",
 					"Multiplier",
 					&blacklist_multiplier))
diff --git a/src/blacklist.h b/src/blacklist.h
index a87e5eca..f5c899e0 100644
--- a/src/blacklist.h
+++ b/src/blacklist.h
@@ -26,6 +26,13 @@ enum blacklist_reason {
 	 * connect to it via autoconnect
 	 */
 	BLACKLIST_REASON_CONNECT_FAILED,
+	/*
+	 * This type of blacklist is added when a BSS requests IWD roams
+	 * elsewhere. This is to aid in preventing IWD from roaming/connecting
+	 * back to that BSS in the future unless there are no other "good"
+	 * candidates to connect to.
+	 */
+	BLACKLIST_REASON_ROAM_REQUESTED,
 };
 
 void blacklist_add_bss(const uint8_t *addr, enum blacklist_reason reason);
-- 
2.34.1

[PATCH v3 06/10] netdev: add netdev_get_low_signal_thresholds

[James Prestwood]

---
 src/netdev.c | 9 +++++++++
 src/netdev.h | 1 +
 2 files changed, 10 insertions(+)

diff --git a/src/netdev.c b/src/netdev.c
index 2a6d94fc..31ccf9ab 100644
--- a/src/netdev.c
+++ b/src/netdev.c
@@ -463,6 +463,15 @@ uint8_t netdev_get_rssi_level_idx(struct netdev *netdev)
 	return netdev->cur_rssi_level_idx;
 }
 
+void netdev_get_low_signal_thresholds(int *threshold, int *threshold_5g)
+{
+	if (threshold)
+		*threshold = LOW_SIGNAL_THRESHOLD;
+
+	if (threshold_5g)
+		*threshold_5g = LOW_SIGNAL_THRESHOLD_5GHZ;
+}
+
 static void netdev_set_powered_result(int error, uint16_t type,
 					const void *data,
 					uint32_t len, void *user_data)
diff --git a/src/netdev.h b/src/netdev.h
index 6299934e..3962dc93 100644
--- a/src/netdev.h
+++ b/src/netdev.h
@@ -158,6 +158,7 @@ const char *netdev_get_name(struct netdev *netdev);
 bool netdev_get_is_up(struct netdev *netdev);
 const char *netdev_get_path(struct netdev *netdev);
 uint8_t netdev_get_rssi_level_idx(struct netdev *netdev);
+void netdev_get_low_signal_thresholds(int *threshold, int *threshold_5g);
 
 struct handshake_state *netdev_handshake_state_new(struct netdev *netdev);
 struct handshake_state *netdev_get_handshake(struct netdev *netdev);
-- 
2.34.1

[PATCH v3 07/10] station: roam blacklist BSS's, and consider when roaming

[James Prestwood]

If the BSS is requesting IWD roam elsewhere add this BSS to the
blacklist using BLACKLIST_REASON_ROAM_REQUESTED. This will lower
the chances of IWD roaming/connecting back to this BSS in the
future.

This then allows IWD to consider this blacklist state when picking
a roam candidate. Its undesireable to fully ban a roam blacklisted
BSS, so some additional sorting logic has been added. Prior to
comparing based on rank, BSS's will be sorted into 3 groups:

Optimal - Not roam blacklisted, and above the roaming threshold
Above Threshold - May be blacklisted, and above the roaming threshold
Below Threshold - BSS is below the roaming threshold
---
 src/station.c | 88 ++++++++++++++++++++++++++++++++++++++++++++++++---
 1 file changed, 84 insertions(+), 4 deletions(-)

diff --git a/src/station.c b/src/station.c
index e2ed78f3..e3c7c189 100644
--- a/src/station.c
+++ b/src/station.c
@@ -155,6 +155,57 @@ struct anqp_entry {
 	uint32_t pending;
 };
 
+/*
+ * Rather than sorting BSS's purely based on ranking a higher level grouping
+ * is used. The purpose of this higher order grouping is the consider the BSS's
+ * roam blacklist status. The roam blacklist is a "soft" blacklist in that we
+ * still should connect to these BSS's if they are the only "good" option.
+ * The question here is: what makes a BSS "good" vs "bad".
+ *
+ * For an initial (probably nieve) approach here we can use the RoamThreshod[5G]
+ * and sort BSS's into "above" or "below" this threshold. Within each of these
+ * groups a BSS may be blacklisted, meaning it should get sorted lower on the
+ * list compared to others within the same group.
+ *
+ * Since we have several call sites needing to check/compare these groupings
+ * a bitmask can be used to describe the grouping:
+ *
+ * Each group will have 2 bits. The lower order bit signifies the BSS is in the
+ * group, but also blacklisted. The higher order bit signifies the BSS is in
+ * the group, but not blacklisted. The bitmask between two BSS's can then be
+ * compared directly similar to rank.
+ */
+
+#define UNDER_THRESHOLD_BIT 1
+#define ABOVE_THRESHOLD_BIT 3
+
+static uint32_t evaluate_bss_group(const uint8_t *addr, uint32_t freq,
+					int16_t signal_strength)
+{
+	int threshold;
+	int signal = signal_strength / 100;
+	bool roam_blacklist;
+	uint32_t mask = 0;
+
+	if (blacklist_contains_bss(addr, BLACKLIST_REASON_CONNECT_FAILED))
+		return mask;
+
+	roam_blacklist = blacklist_contains_bss(addr,
+					BLACKLIST_REASON_ROAM_REQUESTED);
+
+	if (freq > 4000)
+		netdev_get_low_signal_thresholds(NULL, &threshold);
+	else
+		netdev_get_low_signal_thresholds(&threshold, NULL);
+
+	if (signal >= threshold)
+		set_bit(&mask, ABOVE_THRESHOLD_BIT - roam_blacklist);
+	else
+		set_bit(&mask, UNDER_THRESHOLD_BIT - roam_blacklist);
+
+	return mask;
+}
+
 /*
  * Used as entries for the roam list since holding scan_bss pointers directly
  * from station->bss_list is not 100% safe due to the possibility of the
@@ -164,11 +215,13 @@ struct roam_bss {
 	uint8_t addr[6];
 	uint16_t rank;
 	int32_t signal_strength;
+	uint32_t group;
 	bool ft_failed: 1;
 };
 
 static struct roam_bss *roam_bss_from_scan_bss(const struct scan_bss *bss,
-						uint16_t rank)
+						uint16_t rank,
+						uint32_t group)
 {
 	struct roam_bss *rbss = l_new(struct roam_bss, 1);
 
@@ -176,6 +229,7 @@ static struct roam_bss *roam_bss_from_scan_bss(const struct scan_bss *bss,
 	rbss->rank = rank;
 	rbss->signal_strength = bss->signal_strength;
 	rbss->ft_failed = false;
+	rbss->group = group;
 
 	return rbss;
 }
@@ -184,6 +238,12 @@ static int roam_bss_rank_compare(const void *a, const void *b, void *user_data)
 {
 	const struct roam_bss *new_bss = a, *bss = b;
 
+	if (bss->group > new_bss->group)
+		return 1;
+	else if (bss->group < new_bss->group)
+		return -1;
+
+	/* BSS's both have the same group, sort by rank */
 	if (bss->rank == new_bss->rank)
 		return (bss->signal_strength >
 					new_bss->signal_strength) ? 1 : -1;
@@ -2805,6 +2865,7 @@ static bool station_roam_scan_notify(int err, struct l_queue *bss_list,
 	struct handshake_state *hs = netdev_get_handshake(station->netdev);
 	struct scan_bss *current_bss = station->connected_bss;
 	struct scan_bss *bss;
+	uint32_t cur_bss_group = 0;
 	double cur_bss_rank = 0.0;
 	static const double RANK_FT_FACTOR = 1.3;
 	uint16_t mdid;
@@ -2835,6 +2896,9 @@ static bool station_roam_scan_notify(int err, struct l_queue *bss_list,
 	bss = l_queue_find(bss_list, bss_match_bssid, current_bss->addr);
 	if (bss && !station->ap_directed_roaming) {
 		cur_bss_rank = bss->rank;
+		cur_bss_group = evaluate_bss_group(current_bss->addr,
+						current_bss->frequency,
+						current_bss->signal_strength);
 
 		if (hs->mde && bss->mde_present && l_get_le16(bss->mde) == mdid)
 			cur_bss_rank *= RANK_FT_FACTOR;
@@ -2859,6 +2923,9 @@ static bool station_roam_scan_notify(int err, struct l_queue *bss_list,
 	while ((bss = l_queue_pop_head(bss_list))) {
 		double rank;
 		struct roam_bss *rbss;
+		uint32_t group = evaluate_bss_group(bss->addr,
+							bss->frequency,
+							bss->signal_strength);
 
 		station_print_scan_bss(bss);
 
@@ -2889,7 +2956,15 @@ static bool station_roam_scan_notify(int err, struct l_queue *bss_list,
 		if (hs->mde && bss->mde_present && l_get_le16(bss->mde) == mdid)
 			rank *= RANK_FT_FACTOR;
 
-		if (rank <= cur_bss_rank)
+		/*
+		 * First check the group:
+		 *   - If worse, disregard BSS candidate
+		 *   - If better, keep BSS candidate
+		 *   - If equal, compare based on rank
+		 */
+		if (group < cur_bss_group)
+			goto next;
+		else if (group == cur_bss_group && rank <= cur_bss_rank)
 			goto next;
 
 		/*
@@ -2898,7 +2973,7 @@ static bool station_roam_scan_notify(int err, struct l_queue *bss_list,
 		 */
 		station_update_roam_bss(station, bss);
 
-		rbss = roam_bss_from_scan_bss(bss, rank);
+		rbss = roam_bss_from_scan_bss(bss, rank, group);
 
 		l_queue_insert(station->roam_bss_list, rbss,
 				roam_bss_rank_compare, NULL);
@@ -3268,6 +3343,10 @@ static void station_ap_directed_roam(struct station *station,
 	l_timeout_remove(station->roam_trigger_timeout);
 	station->roam_trigger_timeout = NULL;
 
+	blacklist_add_bss(station->connected_bss->addr,
+				BLACKLIST_REASON_ROAM_REQUESTED);
+	station_debug_event(station, "ap-roam-blacklist-added");
+
 	if (req_mode & WNM_REQUEST_MODE_PREFERRED_CANDIDATE_LIST) {
 		l_debug("roam: AP sent a preferred candidate list");
 		station_neighbor_report_cb(station->netdev, 0, body + pos,
@@ -5344,7 +5423,8 @@ static bool station_force_roam_scan_notify(int err, struct l_queue *bss_list,
 	/* The various roam routines expect this to be set from scanning */
 	station->preparing_roam = true;
 	l_queue_push_tail(station->roam_bss_list,
-				roam_bss_from_scan_bss(target, target->rank));
+				roam_bss_from_scan_bss(target, target->rank,
+				0xffff));
 
 	station_update_roam_bss(station, target);
 
-- 
2.34.1

[PATCH v3 08/10] station: roam blacklist AP even mid-roam

[James Prestwood]

If an AP directed roam frame comes in while IWD is roaming its
still valuable to parse that frame and blacklist the BSS that
sent it.

This can happen most frequently during a roam scan while connected
to an overloaded BSS that is requesting IWD roams elsewhere.
---
 src/station.c | 28 ++++++++++++++++++++--------
 1 file changed, 20 insertions(+), 8 deletions(-)

diff --git a/src/station.c b/src/station.c
index e3c7c189..c84b777a 100644
--- a/src/station.c
+++ b/src/station.c
@@ -3241,12 +3241,10 @@ static void station_ap_directed_roam(struct station *station,
 	uint8_t req_mode;
 	uint16_t dtimer;
 	uint8_t valid_interval;
+	bool can_roam = !station_cannot_roam(station);
 
 	l_debug("ifindex: %u", netdev_get_ifindex(station->netdev));
 
-	if (station_cannot_roam(station))
-		return;
-
 	if (station->state != STATION_STATE_CONNECTED) {
 		l_debug("roam: unexpected AP directed roam -- ignore");
 		return;
@@ -3310,8 +3308,13 @@ static void station_ap_directed_roam(struct station *station,
 	 * disassociating us. If either of these bits are set, set the
 	 * ap_directed_roaming flag. Otherwise still try roaming but don't
 	 * treat it any different than a normal roam.
+	 *
+	 * The only exception here is if we are in the middle of roaming
+	 * (can_roam == false) since we cannot reliably know if the roam scan
+	 * included frequencies from potential candidates in this request,
+	 * forcing a roam in this case might result in unintended behavior.
 	 */
-	if (req_mode & (WNM_REQUEST_MODE_DISASSOCIATION_IMMINENT |
+	if (can_roam && req_mode & (WNM_REQUEST_MODE_DISASSOCIATION_IMMINENT |
 			WNM_REQUEST_MODE_TERMINATION_IMMINENT |
 			WNM_REQUEST_MODE_ESS_DISASSOCIATION_IMMINENT))
 		station->ap_directed_roaming = true;
@@ -3338,15 +3341,24 @@ static void station_ap_directed_roam(struct station *station,
 		pos += url_len;
 	}
 
+	blacklist_add_bss(station->connected_bss->addr,
+				BLACKLIST_REASON_ROAM_REQUESTED);
+	station_debug_event(station, "ap-roam-blacklist-added");
+
+	/*
+	 * Validating the frame and blacklisting should still be done even if
+	 * we are mid-roam. Its important to track the BSS requesting the
+	 * transition so when the current roam completes IWD will be less likely
+	 * to roam back to the current BSS.
+	 */
+	if (!can_roam)
+		return;
+
 	station->preparing_roam = true;
 
 	l_timeout_remove(station->roam_trigger_timeout);
 	station->roam_trigger_timeout = NULL;
 
-	blacklist_add_bss(station->connected_bss->addr,
-				BLACKLIST_REASON_ROAM_REQUESTED);
-	station_debug_event(station, "ap-roam-blacklist-added");
-
 	if (req_mode & WNM_REQUEST_MODE_PREFERRED_CANDIDATE_LIST) {
 		l_debug("roam: AP sent a preferred candidate list");
 		station_neighbor_report_cb(station->netdev, 0, body + pos,
-- 
2.34.1

[PATCH v3 09/10] auto-t: add tests for AP roam blacklisting

[James Prestwood]

---
 autotests/testAPRoam/connection_test.py     |  56 +++---
 autotests/testAPRoam/hw.conf                |   2 +
 autotests/testAPRoam/main.conf.roaming      |   5 +
 autotests/testAPRoam/roam_blacklist_test.py | 183 ++++++++++++++++++++
 4 files changed, 223 insertions(+), 23 deletions(-)
 create mode 100644 autotests/testAPRoam/main.conf.roaming
 create mode 100644 autotests/testAPRoam/roam_blacklist_test.py

diff --git a/autotests/testAPRoam/connection_test.py b/autotests/testAPRoam/connection_test.py
index a419f4aa..1f95fb96 100644
--- a/autotests/testAPRoam/connection_test.py
+++ b/autotests/testAPRoam/connection_test.py
@@ -11,52 +11,58 @@ from iwd import NetworkType
 from hostapd import HostapdCLI
 
 class Test(unittest.TestCase):
-
-    def validate(self, expect_roam=True):
-        wd = IWD()
-
-        devices = wd.list_devices(1)
-        device = devices[0]
-
-        ordered_network = device.get_ordered_network('TestAPRoam')
+    def initial_connection(self):
+        ordered_network = self.device.get_ordered_network('TestAPRoam')
 
         self.assertEqual(ordered_network.type, NetworkType.psk)
 
         condition = 'not obj.connected'
-        wd.wait_for_object_condition(ordered_network.network_object, condition)
+        self.wd.wait_for_object_condition(ordered_network.network_object, condition)
 
-        device.connect_bssid(self.bss_hostapd[0].bssid)
+        self.device.connect_bssid(self.bss_hostapd[0].bssid)
 
         condition = 'obj.state == DeviceState.connected'
-        wd.wait_for_object_condition(device, condition)
+        self.wd.wait_for_object_condition(self.device, condition)
 
         self.bss_hostapd[0].wait_for_event('AP-STA-CONNECTED')
 
         self.assertFalse(self.bss_hostapd[1].list_sta())
 
-        self.bss_hostapd[0].send_bss_transition(device.address,
-                [(self.bss_hostapd[1].bssid, '8f0000005102060603000000')],
+    def validate_roam(self, from_bss, to_bss, expect_roam=True):
+        from_bss.send_bss_transition(self.device.address,
+                self.neighbor_list,
                 disassoc_imminent=expect_roam)
 
         if expect_roam:
             from_condition = 'obj.state == DeviceState.roaming'
             to_condition = 'obj.state == DeviceState.connected'
-            wd.wait_for_object_change(device, from_condition, to_condition)
+            self.wd.wait_for_object_change(self.device, from_condition, to_condition)
 
-            self.bss_hostapd[1].wait_for_event('AP-STA-CONNECTED %s' % device.address)
+            to_bss.wait_for_event('AP-STA-CONNECTED %s' % self.device.address)
         else:
-            device.wait_for_event("no-roam-candidates")
-
-        device.disconnect()
-
-        condition = 'not obj.connected'
-        wd.wait_for_object_condition(ordered_network.network_object, condition)
+            self.device.wait_for_event("no-roam-candidates")
 
     def test_disassoc_imminent(self):
-        self.validate(expect_roam=True)
+        self.initial_connection()
+        self.validate_roam(self.bss_hostapd[0], self.bss_hostapd[1])
 
     def test_no_candidates(self):
-        self.validate(expect_roam=False)
+        self.initial_connection()
+        # We now have BSS0 roam blacklisted
+        self.validate_roam(self.bss_hostapd[0], self.bss_hostapd[1])
+        # Try and trigger another roam back, which shouldn't happen since now
+        # both BSS's are roam blacklisted
+        self.validate_roam(self.bss_hostapd[1], self.bss_hostapd[0], expect_roam=False)
+
+    def setUp(self):
+        self.wd = IWD(True)
+
+        devices = self.wd.list_devices(1)
+        self.device = devices[0]
+
+    def tearDown(self):
+        self.wd = None
+        self.device = None
 
     @classmethod
     def setUpClass(cls):
@@ -65,6 +71,10 @@ class Test(unittest.TestCase):
         cls.bss_hostapd = [ HostapdCLI(config='ssid1.conf'),
                             HostapdCLI(config='ssid2.conf'),
                             HostapdCLI(config='ssid3.conf') ]
+        cls.neighbor_list = [
+            (cls.bss_hostapd[0].bssid, "8f0000005101060603000000"),
+            (cls.bss_hostapd[1].bssid, "8f0000005102060603000000"),
+        ]
 
     @classmethod
     def tearDownClass(cls):
diff --git a/autotests/testAPRoam/hw.conf b/autotests/testAPRoam/hw.conf
index 00a31063..46b1d4a8 100644
--- a/autotests/testAPRoam/hw.conf
+++ b/autotests/testAPRoam/hw.conf
@@ -1,5 +1,7 @@
 [SETUP]
 num_radios=4
+hwsim_medium=true
+start_iwd=false
 
 [HOSTAPD]
 rad0=ssid1.conf
diff --git a/autotests/testAPRoam/main.conf.roaming b/autotests/testAPRoam/main.conf.roaming
new file mode 100644
index 00000000..e19adbf3
--- /dev/null
+++ b/autotests/testAPRoam/main.conf.roaming
@@ -0,0 +1,5 @@
+[General]
+RoamThreshold=-72
+
+[Blacklist]
+InitialRoamRequestedTimeout=20
diff --git a/autotests/testAPRoam/roam_blacklist_test.py b/autotests/testAPRoam/roam_blacklist_test.py
new file mode 100644
index 00000000..883deea5
--- /dev/null
+++ b/autotests/testAPRoam/roam_blacklist_test.py
@@ -0,0 +1,183 @@
+#!/usr/bin/python3
+
+import unittest
+import sys
+
+sys.path.append('../util')
+import iwd
+from iwd import IWD, IWD_CONFIG_DIR
+from iwd import NetworkType
+
+from hostapd import HostapdCLI
+from hwsim import Hwsim
+
+class Test(unittest.TestCase):
+    def validate_connected(self, hostapd):
+        ordered_network = self.device.get_ordered_network('TestAPRoam')
+
+        self.assertEqual(ordered_network.type, NetworkType.psk)
+
+        condition = 'not obj.connected'
+        self.wd.wait_for_object_condition(ordered_network.network_object, condition)
+
+        self.device.connect_bssid(hostapd.bssid)
+
+        condition = 'obj.state == DeviceState.connected'
+        self.wd.wait_for_object_condition(self.device, condition)
+
+        hostapd.wait_for_event('AP-STA-CONNECTED')
+
+    def validate_ap_roamed(self, from_hostapd, to_hostapd):
+        from_hostapd.send_bss_transition(
+            self.device.address, self.neighbor_list, disassoc_imminent=True
+        )
+
+        from_condition = 'obj.state == DeviceState.roaming'
+        to_condition = 'obj.state == DeviceState.connected'
+        self.wd.wait_for_object_change(self.device, from_condition, to_condition)
+
+        to_hostapd.wait_for_event('AP-STA-CONNECTED %s' % self.device.address)
+
+        self.device.wait_for_event("ap-roam-blacklist-added")
+
+    def test_roam_to_optimal_candidates(self):
+        # In this test IWD will naturally transition down the list after each
+        # BSS gets roam blacklisted. All BSS's are above the RSSI thresholds.
+        self.rule_ssid1.signal = -5000
+        self.rule_ssid2.signal = -6500
+        self.rule_ssid3.signal = -6900
+
+        # Connect to BSS0
+        self.validate_connected(self.bss_hostapd[0])
+
+        # AP directed roam to BSS1
+        self.validate_ap_roamed(self.bss_hostapd[0], self.bss_hostapd[1])
+
+        # AP directed roam to BSS2
+        self.validate_ap_roamed(self.bss_hostapd[1], self.bss_hostapd[2])
+
+    def test_avoiding_under_threshold_bss(self):
+        # In this test IWD will blacklist BSS0, then roam the BSS1. BSS1 will
+        # then tell IWD to roam, but it should go back to BSS0 since the only
+        # non-blacklisted BSS is under the roam threshold.
+        self.rule_ssid1.signal = -5000
+        self.rule_ssid2.signal = -6500
+        self.rule_ssid3.signal = -7300
+
+        # Connect to BSS0
+        self.validate_connected(self.bss_hostapd[0])
+
+        # AP directed roam to BSS1
+        self.validate_ap_roamed(self.bss_hostapd[0], self.bss_hostapd[1])
+
+        # AP directed roam, but IWD should choose BSS0 since BSS2 is -73dB
+        self.validate_ap_roamed(self.bss_hostapd[1], self.bss_hostapd[0])
+
+    def test_connect_to_roam_blacklisted_bss(self):
+        # In this test a BSS will be roam blacklisted, but all other options are
+        # below the RSSI threshold so IWD should roam back to the blacklisted
+        # BSS.
+        self.rule_ssid1.signal = -5000
+        self.rule_ssid2.signal = -8000
+        self.rule_ssid3.signal = -8500
+
+        # Connect to BSS0
+        self.validate_connected(self.bss_hostapd[0])
+
+        # AP directed roam, should connect to BSS1 as its the next best
+        self.validate_ap_roamed(self.bss_hostapd[0], self.bss_hostapd[1])
+
+        # Connected to BSS1, but the signal is bad, so IWD should try to roam
+        # again. BSS0 is still blacklisted, but its the only reasonable option
+        # since both BSS1 and BSS2 are below the set RSSI threshold (-72dB)
+
+        from_condition = 'obj.state == DeviceState.roaming'
+        to_condition = 'obj.state == DeviceState.connected'
+        self.wd.wait_for_object_change(self.device, from_condition, to_condition)
+
+        # IWD should have connected to BSS0, even though its roam blacklisted
+        self.bss_hostapd[0].wait_for_event('AP-STA-CONNECTED %s' % self.device.address)
+
+    def test_blacklist_during_roam_scan(self):
+        # Tests that an AP roam request mid-roam results in the AP still being
+        # blacklisted even though the request itself doesn't directly trigger
+        # a roam.
+        self.rule_ssid1.signal = -7300
+        self.rule_ssid2.signal = -7500
+        self.rule_ssid3.signal = -8500
+
+        # Connect to BSS0 under the roam threshold so IWD will immediately try
+        # roaming elsewhere
+        self.validate_connected(self.bss_hostapd[0])
+
+        self.device.wait_for_event("roam-scan-triggered")
+
+        self.bss_hostapd[0].send_bss_transition(
+            self.device.address, self.neighbor_list, disassoc_imminent=True
+        )
+        self.device.wait_for_event("ap-roam-blacklist-added")
+
+        # BSS0 should have gotten blacklisted even though IWD was mid-roam,
+        # causing IWD to choose BSS1 when it gets is results.
+
+        from_condition = 'obj.state == DeviceState.roaming'
+        to_condition = 'obj.state == DeviceState.connected'
+        self.wd.wait_for_object_change(self.device, from_condition, to_condition)
+
+        self.bss_hostapd[1].wait_for_event('AP-STA-CONNECTED %s' % self.device.address)
+
+    def setUp(self):
+        self.wd = IWD(True)
+
+        devices = self.wd.list_devices(1)
+        self.device = devices[0]
+
+
+    def tearDown(self):
+        self.wd = None
+        self.device = None
+
+
+    @classmethod
+    def setUpClass(cls):
+        IWD.copy_to_storage("main.conf.roaming", IWD_CONFIG_DIR, "main.conf")
+        IWD.copy_to_storage('TestAPRoam.psk')
+        hwsim = Hwsim()
+
+        cls.bss_hostapd = [ HostapdCLI(config='ssid1.conf'),
+                            HostapdCLI(config='ssid2.conf'),
+                            HostapdCLI(config='ssid3.conf') ]
+        HostapdCLI.group_neighbors(*cls.bss_hostapd)
+
+        rad0 = hwsim.get_radio('rad0')
+        rad1 = hwsim.get_radio('rad1')
+        rad2 = hwsim.get_radio('rad2')
+
+        cls.neighbor_list = [
+            (cls.bss_hostapd[0].bssid, "8f0000005101060603000000"),
+            (cls.bss_hostapd[1].bssid, "8f0000005102060603000000"),
+            (cls.bss_hostapd[2].bssid, "8f0000005103060603000000"),
+        ]
+
+
+        cls.rule_ssid1 = hwsim.rules.create()
+        cls.rule_ssid1.source = rad0.addresses[0]
+        cls.rule_ssid1.bidirectional = True
+        cls.rule_ssid1.enabled = True
+
+        cls.rule_ssid2 = hwsim.rules.create()
+        cls.rule_ssid2.source = rad1.addresses[0]
+        cls.rule_ssid2.bidirectional = True
+        cls.rule_ssid2.enabled = True
+
+        cls.rule_ssid3 = hwsim.rules.create()
+        cls.rule_ssid3.source = rad2.addresses[0]
+        cls.rule_ssid3.bidirectional = True
+        cls.rule_ssid3.enabled = True
+
+    @classmethod
+    def tearDownClass(cls):
+        IWD.clear_storage()
+
+if __name__ == '__main__':
+    unittest.main(exit=True)
-- 
2.34.1

[PATCH v3 10/10] doc: document InitialRoamRequestedTimeout

[James Prestwood]

---
 src/iwd.config.rst | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/src/iwd.config.rst b/src/iwd.config.rst
index 895a1012..9761e8fd 100644
--- a/src/iwd.config.rst
+++ b/src/iwd.config.rst
@@ -292,6 +292,12 @@ control how long a misbehaved BSS spends on the blacklist.
 
        The initial time that a BSS spends on the blacklist. Setting this to zero
        will disable blacklisting functionality in IWD.
+   * - InitialRoamRequestedTimeout
+     - Values: uint64 value in seconds (default: **30**)
+
+       The initial time that a BSS will be marked after a BSS requests a roam.
+       This is to aid in avoiding roaming back to BSS's which are likely
+       overloaded. Setting this to zero will disabled this form of blacklisting.
    * - Multiplier
      - Values: unsigned int value greater than zero, in seconds
        (default: **30**)
-- 
2.34.1

Re: [PATCH v3 07/10] station: roam blacklist BSS's, and consider when roaming

[James Prestwood]

On 3/25/25 11:00 AM, James Prestwood wrote:
> If the BSS is requesting IWD roam elsewhere add this BSS to the
> blacklist using BLACKLIST_REASON_ROAM_REQUESTED. This will lower
> the chances of IWD roaming/connecting back to this BSS in the
> future.
>
> This then allows IWD to consider this blacklist state when picking
> a roam candidate. Its undesireable to fully ban a roam blacklisted
> BSS, so some additional sorting logic has been added. Prior to
> comparing based on rank, BSS's will be sorted into 3 groups:
>
> Optimal - Not roam blacklisted, and above the roaming threshold
> Above Threshold - May be blacklisted, and above the roaming threshold
> Below Threshold - BSS is below the roaming threshold
> ---
>   src/station.c | 88 ++++++++++++++++++++++++++++++++++++++++++++++++---
>   1 file changed, 84 insertions(+), 4 deletions(-)
>
> diff --git a/src/station.c b/src/station.c
> index e2ed78f3..e3c7c189 100644
> --- a/src/station.c
> +++ b/src/station.c
> @@ -155,6 +155,57 @@ struct anqp_entry {
>   	uint32_t pending;
>   };
>   
> +/*
> + * Rather than sorting BSS's purely based on ranking a higher level grouping
> + * is used. The purpose of this higher order grouping is the consider the BSS's
> + * roam blacklist status. The roam blacklist is a "soft" blacklist in that we
> + * still should connect to these BSS's if they are the only "good" option.
> + * The question here is: what makes a BSS "good" vs "bad".
> + *
> + * For an initial (probably nieve) approach here we can use the RoamThreshod[5G]
> + * and sort BSS's into "above" or "below" this threshold. Within each of these
> + * groups a BSS may be blacklisted, meaning it should get sorted lower on the
> + * list compared to others within the same group.
> + *
> + * Since we have several call sites needing to check/compare these groupings
> + * a bitmask can be used to describe the grouping:
> + *
> + * Each group will have 2 bits. The lower order bit signifies the BSS is in the
> + * group, but also blacklisted. The higher order bit signifies the BSS is in
> + * the group, but not blacklisted. The bitmask between two BSS's can then be
> + * compared directly similar to rank.
> + */
> +
> +#define UNDER_THRESHOLD_BIT 1
> +#define ABOVE_THRESHOLD_BIT 3
> +
> +static uint32_t evaluate_bss_group(const uint8_t *addr, uint32_t freq,
> +					int16_t signal_strength)
> +{
> +	int threshold;
> +	int signal = signal_strength / 100;
> +	bool roam_blacklist;
> +	uint32_t mask = 0;
> +
> +	if (blacklist_contains_bss(addr, BLACKLIST_REASON_CONNECT_FAILED))
> +		return mask;
> +
> +	roam_blacklist = blacklist_contains_bss(addr,
> +					BLACKLIST_REASON_ROAM_REQUESTED);
> +
> +	if (freq > 4000)
> +		netdev_get_low_signal_thresholds(NULL, &threshold);
> +	else
> +		netdev_get_low_signal_thresholds(&threshold, NULL);
> +
> +	if (signal >= threshold)
> +		set_bit(&mask, ABOVE_THRESHOLD_BIT - roam_blacklist);
> +	else
> +		set_bit(&mask, UNDER_THRESHOLD_BIT - roam_blacklist);
> +
> +	return mask;
> +}
> +
>   /*
>    * Used as entries for the roam list since holding scan_bss pointers directly
>    * from station->bss_list is not 100% safe due to the possibility of the
> @@ -164,11 +215,13 @@ struct roam_bss {
>   	uint8_t addr[6];
>   	uint16_t rank;
>   	int32_t signal_strength;
> +	uint32_t group;
>   	bool ft_failed: 1;
>   };
>   
>   static struct roam_bss *roam_bss_from_scan_bss(const struct scan_bss *bss,
> -						uint16_t rank)
> +						uint16_t rank,
> +						uint32_t group)
>   {
>   	struct roam_bss *rbss = l_new(struct roam_bss, 1);
>   
> @@ -176,6 +229,7 @@ static struct roam_bss *roam_bss_from_scan_bss(const struct scan_bss *bss,
>   	rbss->rank = rank;
>   	rbss->signal_strength = bss->signal_strength;
>   	rbss->ft_failed = false;
> +	rbss->group = group;
>   
>   	return rbss;
>   }
> @@ -184,6 +238,12 @@ static int roam_bss_rank_compare(const void *a, const void *b, void *user_data)
>   {
>   	const struct roam_bss *new_bss = a, *bss = b;
>   
> +	if (bss->group > new_bss->group)
> +		return 1;
> +	else if (bss->group < new_bss->group)
> +		return -1;
> +
> +	/* BSS's both have the same group, sort by rank */
>   	if (bss->rank == new_bss->rank)
>   		return (bss->signal_strength >
>   					new_bss->signal_strength) ? 1 : -1;
> @@ -2805,6 +2865,7 @@ static bool station_roam_scan_notify(int err, struct l_queue *bss_list,
>   	struct handshake_state *hs = netdev_get_handshake(station->netdev);
>   	struct scan_bss *current_bss = station->connected_bss;
>   	struct scan_bss *bss;
> +	uint32_t cur_bss_group = 0;
>   	double cur_bss_rank = 0.0;
>   	static const double RANK_FT_FACTOR = 1.3;
>   	uint16_t mdid;
> @@ -2835,6 +2896,9 @@ static bool station_roam_scan_notify(int err, struct l_queue *bss_list,
>   	bss = l_queue_find(bss_list, bss_match_bssid, current_bss->addr);
>   	if (bss && !station->ap_directed_roaming) {
>   		cur_bss_rank = bss->rank;
> +		cur_bss_group = evaluate_bss_group(current_bss->addr,
> +						current_bss->frequency,
> +						current_bss->signal_strength);
This is what caused the CI to fail on testPSK-roam. Instead I should 
have done the group evaluation on the "bss" pointer, not "current_bss". 
This will take into account the most recent scan, rather than the stale 
object we have from past scans.
>   
>   		if (hs->mde && bss->mde_present && l_get_le16(bss->mde) == mdid)
>   			cur_bss_rank *= RANK_FT_FACTOR;
> @@ -2859,6 +2923,9 @@ static bool station_roam_scan_notify(int err, struct l_queue *bss_list,
>   	while ((bss = l_queue_pop_head(bss_list))) {
>   		double rank;
>   		struct roam_bss *rbss;
> +		uint32_t group = evaluate_bss_group(bss->addr,
> +							bss->frequency,
> +							bss->signal_strength);
>   
>   		station_print_scan_bss(bss);
>   
> @@ -2889,7 +2956,15 @@ static bool station_roam_scan_notify(int err, struct l_queue *bss_list,
>   		if (hs->mde && bss->mde_present && l_get_le16(bss->mde) == mdid)
>   			rank *= RANK_FT_FACTOR;
>   
> -		if (rank <= cur_bss_rank)
> +		/*
> +		 * First check the group:
> +		 *   - If worse, disregard BSS candidate
> +		 *   - If better, keep BSS candidate
> +		 *   - If equal, compare based on rank
> +		 */
> +		if (group < cur_bss_group)
> +			goto next;
> +		else if (group == cur_bss_group && rank <= cur_bss_rank)
>   			goto next;
>   
>   		/*
> @@ -2898,7 +2973,7 @@ static bool station_roam_scan_notify(int err, struct l_queue *bss_list,
>   		 */
>   		station_update_roam_bss(station, bss);
>   
> -		rbss = roam_bss_from_scan_bss(bss, rank);
> +		rbss = roam_bss_from_scan_bss(bss, rank, group);
>   
>   		l_queue_insert(station->roam_bss_list, rbss,
>   				roam_bss_rank_compare, NULL);
> @@ -3268,6 +3343,10 @@ static void station_ap_directed_roam(struct station *station,
>   	l_timeout_remove(station->roam_trigger_timeout);
>   	station->roam_trigger_timeout = NULL;
>   
> +	blacklist_add_bss(station->connected_bss->addr,
> +				BLACKLIST_REASON_ROAM_REQUESTED);
> +	station_debug_event(station, "ap-roam-blacklist-added");
> +
>   	if (req_mode & WNM_REQUEST_MODE_PREFERRED_CANDIDATE_LIST) {
>   		l_debug("roam: AP sent a preferred candidate list");
>   		station_neighbor_report_cb(station->netdev, 0, body + pos,
> @@ -5344,7 +5423,8 @@ static bool station_force_roam_scan_notify(int err, struct l_queue *bss_list,
>   	/* The various roam routines expect this to be set from scanning */
>   	station->preparing_roam = true;
>   	l_queue_push_tail(station->roam_bss_list,
> -				roam_bss_from_scan_bss(target, target->rank));
> +				roam_bss_from_scan_bss(target, target->rank,
> +				0xffff));
>   
>   	station_update_roam_bss(station, target);
>