diff --git a/provider/provider_test.go b/provider/provider_test.go
index b455f8c5..de23f611 100644
--- a/provider/provider_test.go
+++ b/provider/provider_test.go
@@ -52,19 +52,20 @@ func TestMain(m *testing.M) {
 
 	code := m.Run()
 
+	// Clean up of tests is not fatal if it fails
 	err := keycloakClient.DeleteRealm(testCtx, testAccRealm.Realm)
 	if err != nil {
-		log.Fatalf("Unable to delete realm %s: %s", testAccRealmUserFederation.Realm, err)
+		log.Printf("Unable to delete realm %s: %s", testAccRealmUserFederation.Realm, err)
 	}
 
 	err = keycloakClient.DeleteRealm(testCtx, testAccRealmTwo.Realm)
 	if err != nil {
-		log.Fatalf("Unable to delete realm %s: %s", testAccRealmUserFederation.Realm, err)
+		log.Printf("Unable to delete realm %s: %s", testAccRealmUserFederation.Realm, err)
 	}
 
 	err = keycloakClient.DeleteRealm(testCtx, testAccRealmUserFederation.Realm)
 	if err != nil {
-		log.Fatalf("Unable to delete realm %s: %s", testAccRealmUserFederation.Realm, err)
+		log.Printf("Unable to delete realm %s: %s", testAccRealmUserFederation.Realm, err)
 	}
 
 	os.Exit(code)
@@ -78,7 +79,15 @@ func createTestRealm(testCtx context.Context) *keycloak.Realm {
 		Enabled: true,
 	}
 
-	err := keycloakClient.NewRealm(testCtx, r)
+	var err error
+	for i := 0; i < 3; i++ { // on CI this sometimes fails and keycloak can't be reached
+		err = keycloakClient.NewRealm(testCtx, r)
+		if err != nil {
+			log.Printf("Unable to create new realm: %s - retrying", err)
+		} else {
+			break
+		}
+	}
 	if err != nil {
 		log.Fatalf("Unable to create new realm: %s", err)
 	}
diff --git a/provider/resource_keycloak_realm_user_profile_test.go b/provider/resource_keycloak_realm_user_profile_test.go
index a07d0470..8c8b4dfa 100644
--- a/provider/resource_keycloak_realm_user_profile_test.go
+++ b/provider/resource_keycloak_realm_user_profile_test.go
@@ -41,6 +41,10 @@ func TestAccKeycloakRealmUserProfile_basicEmpty(t *testing.T) {
 	realmName := acctest.RandomWithPrefix("tf-acc")
 
 	realmUserProfile := &keycloak.RealmUserProfile{}
+	if ok, _ := keycloakClient.VersionIsLessThanOrEqualTo(testCtx, keycloak.Version_23); ok {
+		// Username and email can't be removed in this version
+		realmUserProfile.Attributes = []*keycloak.RealmUserProfileAttribute{{Name: "username"}, {Name: "email"}}
+	}
 
 	resource.Test(t, resource.TestCase{
 		ProviderFactories: testAccProviderFactories,
@@ -62,6 +66,7 @@ func TestAccKeycloakRealmUserProfile_basicFull(t *testing.T) {
 
 	realmUserProfile := &keycloak.RealmUserProfile{
 		Attributes: []*keycloak.RealmUserProfileAttribute{
+			{Name: "username"}, {Name: "email"}, // Version 23 needs these
 			{Name: "attribute1"},
 			{
 				Name:        "attribute2",