diff --git a/src/System.Management.Automation/security/wldpNativeMethods.cs b/src/System.Management.Automation/security/wldpNativeMethods.cs
index 35d37d88eed..b1f3060c880 100644
--- a/src/System.Management.Automation/security/wldpNativeMethods.cs
+++ b/src/System.Management.Automation/security/wldpNativeMethods.cs
@@ -277,9 +277,10 @@ private static SystemEnforcementMode GetAppLockerPolicy(string path, SafeHandle
                     }
                     finally
                     {
-                        if (IO.File.Exists(testPathScript)) { IO.File.Delete(testPathScript); }
-
-                        if (IO.File.Exists(testPathModule)) { IO.File.Delete(testPathModule); }
+                        // Ok to leave the test scripts in the temp folder if they happen to be in use
+                        // so that PowerShell will still startup.
+                        PathUtils.TryDeleteFile(testPathScript);
+                        PathUtils.TryDeleteFile(testPathModule);
                     }
 
                     s_cachedSaferSystemPolicy = result;
diff --git a/src/System.Management.Automation/utils/PathUtils.cs b/src/System.Management.Automation/utils/PathUtils.cs
index 71d84109b0f..6c398af7494 100644
--- a/src/System.Management.Automation/utils/PathUtils.cs
+++ b/src/System.Management.Automation/utils/PathUtils.cs
@@ -429,5 +429,23 @@ internal static DirectoryInfo CreateTemporaryDirectory()
             Directory.CreateDirectory(moduleDirectory.FullName);
             return new DirectoryInfo(moduleDirectory.FullName);
         }
+
+        internal static bool TryDeleteFile(string filepath)
+        {
+            if (IO.File.Exists(filepath))
+            {
+                try
+                {
+                    IO.File.Delete(filepath);
+                    return true;
+                }
+                catch (IOException)
+                {
+                    // file is in use on Windows
+                }
+            }
+
+            return false;
+        }
     }
 }