Tests: Do not read system user's data

The tests would initialize Base, PreferencesData with default settings and/or run the arduino executable, without specifying any settings to use. In practice, this would read settings from e.g. `~/.arduino15`, load libraries from the user's sketchbook, etc. This is not a good idea, since this can be influence the test. For example, the presence of invalid libraries would cause extra output to be generated, which breaks the `--version` test. Or having the "Use external editor" setting set would break tests that try to edit a sketch. This commit fixes this. The core of this commit is in `AbstractWithPreferencesTest`, which sets up a clean settings dir (to replace `~/.arduino15`) with a sketchbook and `preferences.txt` file inside before every test (and removes it again after the test. For some tests, this is enough, but some tests create an instance of `Base`, which again initializes everything, including preferences, from the default location. To prevent that, `--preferences-file` is passed to the `Base` constructor, loading the previously set up preferences. This is handled by the new `AbstractWithPreferencesTest.createBase()` method. Furthermore, CommandLineTest calls the actual Arduino executable, which has the same problem. This is fixed by passing the same `--preferences-file` option on the commandline (generated by `AbstractWithPreferencesTest.getBaseArgs()`). This should prevent all tests from reading the the default settings files, fixing some tests on my system and even speeding up the tests somewhat (due less libraries and cores to load, probably).
arduino · cmaglie · May 11, 2020 · May 5, 2020 · May 5, 2020 · May 6, 2020
commit 75e5c9eb43b9d2d8c8fe7a1aa110ed4c308dfd83
diff --git a/app/test/processing/app/AbstractGUITest.java b/app/test/processing/app/AbstractGUITest.java
@@ -60,7 +60,7 @@ public void startUpTheIDE() throws Exception {
    window = GuiActionRunner.execute(new GuiQuery<ArduinoFrameFixture>() {
      @Override
      protected ArduinoFrameFixture executeInEDT() throws Throwable {
-        return new ArduinoFrameFixture(new Base(new String[0]).editors.get(0));
+        return new ArduinoFrameFixture(createBase().editors.get(0));
      }
    });
  }

diff --git a/app/test/processing/app/AbstractWithPreferencesTest.java b/app/test/processing/app/AbstractWithPreferencesTest.java
@@ -29,6 +29,7 @@

 package processing.app;

+import static org.junit.Assert.assertEquals;
 import org.junit.Before;
 import org.junit.After;

@@ -47,12 +48,30 @@ public abstract class AbstractWithPreferencesTest {
   * Subclasses can add files here in @Test or @Before functions.
   */
  protected List<File> deleteAfter = new LinkedList<File>();
+  protected File preferencesFile;

  @Before
  public void init() throws Exception {
+    File settingsDir = Files.createTempDirectory("arduino_test_settings").toFile();
+    deleteAfter.add(settingsDir);
+
+    preferencesFile = new File(settingsDir, "preferences.txt");
+    File sketchbookDir = new File(settingsDir, "sketchbook");
+    sketchbookDir.mkdir();
+
    BaseNoGui.initPlatform();
    BaseNoGui.getPlatform().init();
-    PreferencesData.init(null);
+
+    PreferencesData.init(preferencesFile);
+    // Do not read anything from e.g. ~/.arduino15
+    PreferencesData.set("settings.path", settingsDir.toString());
+    // Do not read or write the default ~/Arduino sketchbook
+    PreferencesData.set("sketchbook.path", sketchbookDir.toString());
+    // Write the defaults, with these changes to file. This allows them
+    // to be reloaded when creating a Base instance (see getBaseArgs()
+    // below).
+    PreferencesData.save();
+
    Theme.init();

    BaseNoGui.initPackages();
@@ -61,6 +80,31 @@ public void init() throws Exception {
    deleteAfter.add(Base.untitledFolder);
  }

+  /**
+   * Returns arguments to be passed to the Base constructor or on the
+   * commandline to set up the created dummy environment.
+   */
+  protected String[] getBaseArgs() {
+    return new String[] {
+      // Preferences are loaded (using --preferences-file) before
+      // processing any other commandline options (e.g. --pref), so only
+      // use --preferences-file here. Also, this does not affect the
+      // "action" mode, for tests that require the GUI to be loaded.
+      "--preferences-file", preferencesFile.toString(),
+    };
+  }
+
+  /**
+   * Creates a new instance of Base. Always use this rather than calling
+   * it directly, to ensure the right settings are used.
+   */
+  protected Base createBase() throws Exception {
+    Base base = new Base(getBaseArgs());
+    // Doublecheck that the right preferencesFile was loaded
+    assertEquals(preferencesFile, PreferencesData.preferencesFile);
+    return base;
+  }
+
  @After
  public void cleanup() throws IOException {
    for (File f : deleteAfter)

diff --git a/app/test/processing/app/CommandLineTest.java b/app/test/processing/app/CommandLineTest.java
@@ -45,7 +45,13 @@
 import processing.app.helpers.OSUtils;
 import processing.app.helpers.PreferencesMap;

-public class CommandLineTest {
+/**
+ * This extends AbstractWithPreferencesTest which initializes part of
+ * the internal Arduino structures. Most of that is not required, but it
+ * also conveniently sets up a settings directory and preferences file,
+ * which we can use here (through getBaseArgs()).
+ */
+public class CommandLineTest extends AbstractWithPreferencesTest {

  private static File buildPath;
  private static File arduinoPath;
@@ -81,6 +87,7 @@ public Process runArduino(boolean output, boolean success, File wd, String[] ext

    List<String> args = new ArrayList<String>();
    args.add(arduinoPath.getAbsolutePath());
+    args.addAll(Arrays.asList(getBaseArgs()));
    args.addAll(Arrays.asList(extraArgs));

    System.out.println("Running: " + String.join(" ", args));

diff --git a/app/test/processing/app/DefaultTargetTest.java b/app/test/processing/app/DefaultTargetTest.java
@@ -57,7 +57,7 @@ public void testDefaultTarget() throws Exception {
    PreferencesData.set("board", "unreal_board");

    // should not raise an exception
-    new Base(new String[0]);
+    createBase();

    // skip test if no target platforms are available
    Assume.assumeNotNull(BaseNoGui.getTargetPlatform());