diff --git a/src/adapter.c b/src/adapter.c
index ab07ca7..d225b86 100644
--- a/src/adapter.c
+++ b/src/adapter.c
@@ -2472,7 +2472,8 @@ void adapter_remove(struct btd_adapter *adapter)
 		device_remove(l->data, FALSE);
 	g_slist_free(adapter->devices);
 
-	unload_drivers(adapter);
+	if (adapter->initialized)
+		unload_drivers(adapter);
 
 	/* Return adapter to down state if it was not up on init */
 	if (adapter->up && !adapter->already_up)