diff --git a/wincred/wincred_windows_test.go b/wincred/wincred_windows_test.go
index 27a12a9..9b56c33 100644
--- a/wincred/wincred_windows_test.go
+++ b/wincred/wincred_windows_test.go
@@ -113,15 +113,19 @@ func TestWinCredHelperRetrieveAliases(t *testing.T) {
 	}
 
 	helper := Wincred{}
-	defer func() {
+	t.Cleanup(func() {
 		for _, tc := range tests {
-			helper.Delete(tc.storeURL)
+			if err := helper.Delete(tc.storeURL); err != nil && !credentials.IsErrCredentialsNotFound(err) {
+				t.Errorf("cleanup: failed to delete '%s': %v", tc.storeURL, err)
+			}
 		}
-	}()
+	})
 
 	// Clean store before testing.
 	for _, tc := range tests {
-		helper.Delete(tc.storeURL)
+		if err := helper.Delete(tc.storeURL); err != nil && !credentials.IsErrCredentialsNotFound(err) {
+			t.Errorf("prepare: failed to delete '%s': %v", tc.storeURL, err)
+		}
 	}
 
 	for _, tc := range tests {
@@ -133,7 +137,9 @@ func TestWinCredHelperRetrieveAliases(t *testing.T) {
 		if _, _, err := helper.Get(tc.readURL); err != nil {
 			t.Errorf("Error: failed to read secret for URL %q using %q", tc.storeURL, tc.readURL)
 		}
-		helper.Delete(tc.storeURL)
+		if err := helper.Delete(tc.storeURL); err != nil {
+			t.Error(err)
+		}
 	}
 }
 
@@ -180,15 +186,19 @@ func TestWinCredHelperRetrieveStrict(t *testing.T) {
 	}
 
 	helper := Wincred{}
-	defer func() {
+	t.Cleanup(func() {
 		for _, tc := range tests {
-			helper.Delete(tc.storeURL)
+			if err := helper.Delete(tc.storeURL); err != nil && !credentials.IsErrCredentialsNotFound(err) {
+				t.Errorf("cleanup: failed to delete '%s': %v", tc.storeURL, err)
+			}
 		}
-	}()
+	})
 
 	// Clean store before testing.
 	for _, tc := range tests {
-		helper.Delete(tc.storeURL)
+		if err := helper.Delete(tc.storeURL); err != nil && !credentials.IsErrCredentialsNotFound(err) {
+			t.Errorf("prepare: failed to delete '%s': %v", tc.storeURL, err)
+		}
 	}
 
 	for _, tc := range tests {
@@ -200,7 +210,9 @@ func TestWinCredHelperRetrieveStrict(t *testing.T) {
 		if _, _, err := helper.Get(tc.readURL); err == nil {
 			t.Errorf("Error: managed to read secret for URL %q using %q, but should not be able to", tc.storeURL, tc.readURL)
 		}
-		helper.Delete(tc.storeURL)
+		if err := helper.Delete(tc.storeURL); err != nil {
+			t.Error(err)
+		}
 	}
 }
 
@@ -221,15 +233,19 @@ func TestWinCredHelperStoreRetrieve(t *testing.T) {
 	}
 
 	helper := Wincred{}
-	defer func() {
+	t.Cleanup(func() {
 		for _, tc := range tests {
-			helper.Delete(tc.url)
+			if err := helper.Delete(tc.url); err != nil && !credentials.IsErrCredentialsNotFound(err) {
+				t.Errorf("cleanup: failed to delete '%s': %v", tc.url, err)
+			}
 		}
-	}()
+	})
 
 	// Clean store before testing.
 	for _, tc := range tests {
-		helper.Delete(tc.url)
+		if err := helper.Delete(tc.url); err != nil && !credentials.IsErrCredentialsNotFound(err) {
+			t.Errorf("prepare: failed to delete '%s': %v", tc.url, err)
+		}
 	}
 
 	// Note that we don't delete between individual tests here, to verify that