Unflake bootstrap test (#3636)

lib/tests/bootstrap.rs

@@ -48,15 +48,38 @@ async fn bootstrap_removes_unreachable_nodes() -> Result<(), Error> {
 	// Bootstrap
 	dbs.bootstrap().await.unwrap();
 
-	// Verify the incorrect node is deleted, but self and valid still exist
-	let mut tx = dbs.transaction(Write, Optimistic).await.unwrap();
-	let res = tx.scan_nd(1000).await.unwrap();
-	tx.commit().await.unwrap();
-	for node in &res {
-		assert_ne!(node.name, bad_node.to_string());
+	// Declare a function that will assert
+	async fn try_validate(mut tx: &mut Transaction, bad_node: &uuid::Uuid) -> Result<(), String> {
+		let res = tx.scan_nd(1000).await.map_err(|e| e.to_string())?;
+		tx.commit().await.map_err(|e| e.to_string())?;
+		for node in &res {
+			if node.name == bad_node.to_string() {
+				return Err(format!("The node name was actually the bad node {:?}", bad_node));
+			}
+		}
+		// {Node generated by bootstrap} + {valid node who's uuid we don't know}
+		assert_eq!(res.len(), 2);
+		if res.len() != 2 {
+			return Err("Expected 2 nodes".to_string());
+		}
+		Ok(())
 	}
-	// {Node generated by bootstrap} + {valid node who's uuid we don't know}
-	assert_eq!(res.len(), 2);
+
+	// Verify the incorrect node is deleted, but self and valid still exist
+	let res = {
+		let mut err = None;
+		for _ in 0..5 {
+			let mut tx = dbs.transaction(Write, Optimistic).await.unwrap();
+			let res = try_validate(&mut tx, &bad_node).await;
+			if res.is_ok() {
+				return Ok(());
+			}
+			err = Some(res);
+			tokio::time::sleep(std::time::Duration::from_millis(100)).await;
+		}
+		err.unwrap()
+	};
+	res.unwrap();
 	Ok(())
 }