Overhauled the end to end tests and added the test code from progpedia (

#1279)

* Overhauled the end to end tests and added the test code from progpedia.

* Added missing result files for end to end tests

* Fixed test results

* Reduced epsilon value for end to end tests.

* Generate expected test values for the progpedia dataset.

* Added token output for end to end tests.

* Improved end to end test output.

* Added strict order for submissions in comparison.

* Fixed sonarcloud issues.

* Implemented timurs suggestions.

* Moved progpedia data into a zip file.

* Fixed spotless issues concerning zips.

* spotless.

* Fixed sonarcloud issues.

* Improved language in test descriptions.

* Improved handling of io streams in end to end tests.

* Added logging when permissions cannot be set on temp directory.

* Spotless

* Fixed sonarcloud issues.

---------

Co-authored-by: Dominik Fuchß <dominik.fuchss@kit.edu>
Co-authored-by: Timur Saglam <timur.saglam@kit.edu>

endtoend-testing/pom.xml

@@ -18,7 +18,6 @@
             <groupId>de.jplag</groupId>
             <artifactId>cli</artifactId>
             <version>${revision}</version>
-            <scope>test</scope>
         </dependency>
         <dependency>
             <groupId>de.jplag</groupId>

endtoend-testing/src/main/java/de/jplag/endtoend/constants/TestDirectoryConstants.java

@@ -12,18 +12,17 @@ private TestDirectoryConstants() {
     }
 
     /**
-     * Base path to the saved results
+     * Base path to the resources directory
      */
-    public static final Path BASE_PATH_TO_RESULT_JSON = Path.of("src", "test", "resources", "results");
+    public static final Path BASE_PATH_TO_RESOURCES = Path.of("src", "test", "resources");
 
     /**
-     * Base path to the endToEnd testing resources
+     * Base path to the saved results
      */
-    public static final Path BASE_PATH_TO_LANGUAGE_RESOURCES = Path.of("src", "test", "resources", "languageTestFiles");
+    public static final Path BASE_PATH_TO_RESULT_JSON = BASE_PATH_TO_RESOURCES.resolve(Path.of("results"));
 
     /**
-     * Create the complete path to the submission files. Here the temporary system path is extended with the
-     * "SUBMISSION_DIRECTORY_NAME", which is predefined in this class.
+     * Base path to the data set descriptors
      */
-    public static final Path TEMPORARY_SUBMISSION_DIRECTORY_NAME = Path.of("target", "testing-directory-submission");
+    public static final Path BASE_PATH_TO_DATA_SET_DESCRIPTORS = BASE_PATH_TO_RESOURCES.resolve(Path.of("dataSets"));
 }

endtoend-testing/src/main/java/de/jplag/endtoend/helper/FileHelper.java

@@ -1,12 +1,22 @@
 package de.jplag.endtoend.helper;
 
 import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Enumeration;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
 
 /**
  * Helper class to perform all necessary operations or functions on files or folders.
  */
 public class FileHelper {
+    private static final int ZIP_THRESHOLD_ENTRIES = 100000;
+    private static final int ZIP_THRESHOLD_SIZE = 1000000000;
+    private static final double ZIP_THRESHOLD_RATIO = 10;
+    private static final String ZIP_BOMB_ERROR_MESSAGE = "Refusing to unzip file (%s), because it seems to be a fork bomb";
 
     private FileHelper() {
         // private constructor to prevent instantiation
@@ -52,4 +62,65 @@ public static void createFileIfItDoesNotExist(File file) throws IOException {
     private static String createNewIOExceptionStringForFileOrFOlderCreation(File file) {
         return "The file/folder at the location [" + file.toString() + "] could not be created!";
     }
-}
+
+    /**
+     * Unzips a given zip file into a given directory.
+     * @param zip The zip file to extract
+     * @param targetDirectory The target directory
+     * @throws IOException If io operations go wrong
+     */
+    public static void unzip(File zip, File targetDirectory) throws IOException {
+        try (ZipFile zipFile = new ZipFile(zip)) {
+            Enumeration<? extends ZipEntry> entries = zipFile.entries();
+
+            long totalSizeArchive = 0;
+            long totalEntriesArchive = 0;
+
+            while (entries.hasMoreElements()) {
+                totalEntriesArchive++;
+
+                ZipEntry entry = entries.nextElement();
+                File unzippedFile = new File(targetDirectory, entry.getName()).getCanonicalFile();
+
+                if (unzippedFile.getAbsolutePath().startsWith(targetDirectory.getAbsolutePath())) {
+                    if (entry.isDirectory()) {
+                        unzippedFile.mkdirs();
+                    } else {
+                        unzippedFile.getParentFile().mkdirs();
+                        totalSizeArchive += extractZipElement(entry, zipFile, zip, unzippedFile);
+                    }
+                }
+
+                if (totalSizeArchive > ZIP_THRESHOLD_SIZE) {
+                    throw new IllegalStateException(String.format(ZIP_BOMB_ERROR_MESSAGE, zip.getAbsolutePath()));
+                }
+                if (totalEntriesArchive > ZIP_THRESHOLD_ENTRIES) {
+                    throw new IllegalStateException(String.format(ZIP_BOMB_ERROR_MESSAGE, zip.getAbsolutePath()));
+                }
+            }
+        }
+    }
+
+    private static long extractZipElement(ZipEntry entry, ZipFile zipFile, File zip, File target) throws IOException {
+        long totalSizeEntry = 0;
+
+        try (InputStream inputStream = zipFile.getInputStream(entry)) {
+            try (OutputStream outputStream = new FileOutputStream(target)) {
+                byte[] buffer = new byte[2048];
+                int count;
+                while ((count = inputStream.read(buffer)) > 0) {
+                    outputStream.write(buffer, 0, count);
+
+                    totalSizeEntry += count;
+
+                    double compressionRate = (double) totalSizeEntry / entry.getCompressedSize();
+                    if (compressionRate > ZIP_THRESHOLD_RATIO) {
+                        throw new IllegalStateException(String.format(ZIP_BOMB_ERROR_MESSAGE, zip.getAbsolutePath()));
+                    }
+                }
+            }
+        }
+
+        return totalSizeEntry;
+    }
+}

endtoend-testing/src/main/java/de/jplag/endtoend/helper/LanguageDeserializer.java

@@ -0,0 +1,21 @@
+package de.jplag.endtoend.helper;
+
+import java.io.IOException;
+
+import de.jplag.Language;
+import de.jplag.cli.LanguageLoader;
+
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.databind.DeserializationContext;
+import com.fasterxml.jackson.databind.JsonDeserializer;
+
+/**
+ * Deserialized a language from a json file
+ */
+public class LanguageDeserializer extends JsonDeserializer<Language> {
+    @Override
+    public Language deserialize(JsonParser jsonParser, DeserializationContext deserializationContext) throws IOException {
+        String name = jsonParser.getText();
+        return LanguageLoader.getLanguage(name).orElseThrow(() -> new IllegalStateException(String.format("Language %s not found.", name)));
+    }
+}

endtoend-testing/src/main/java/de/jplag/endtoend/helper/TestSuiteHelper.java

@@ -1,13 +1,10 @@
 package de.jplag.endtoend.helper;
 
-import java.io.File;
-import java.util.List;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import de.jplag.JPlagComparison;
-import de.jplag.Language;
 import de.jplag.Submission;
-import de.jplag.endtoend.constants.TestDirectoryConstants;
 
 /**
  * Helper class to perform all necessary additional functions for the endToEnd tests.
@@ -28,30 +25,8 @@ private TestSuiteHelper() {
      * @return unique identifier for test case recognition
      */
     public static String getTestIdentifier(JPlagComparison jPlagComparison) {
-        return List.of(jPlagComparison.firstSubmission(), jPlagComparison.secondSubmission()).stream().map(Submission::getRoot)
+        return Stream.of(jPlagComparison.firstSubmission(), jPlagComparison.secondSubmission()).map(Submission::getRoot)
                 .map(FileHelper::getFileNameWithoutFileExtension).sorted().collect(Collectors.joining("-"));
 
     }
-
-    /**
-     * Returns the file pointing to the directory of the submissions for the given language and result json. The result
-     * json's name is expected to be equal to the test suite identifier.
-     * @param language is the language for the tests
-     * @param resultJSON is the json containing the expected values
-     * @return returns the directory of the submissions
-     */
-    public static File getSubmissionDirectory(Language language, File resultJSON) {
-        return getSubmissionDirectory(language, FileHelper.getFileNameWithoutFileExtension(resultJSON));
-    }
-
-    /**
-     * Returns the file pointing to the directory of the submissions for the given language and test suite identifier as
-     * described in the Readme.md.
-     * @param language is the langauge for the tests
-     * @param testSuiteIdentifier is the test suite identifier of the tests
-     * @return returns the directory of the submissions
-     */
-    public static File getSubmissionDirectory(Language language, String testSuiteIdentifier) {
-        return TestDirectoryConstants.BASE_PATH_TO_LANGUAGE_RESOURCES.resolve(language.getIdentifier()).resolve(testSuiteIdentifier).toFile();
-    }
 }

endtoend-testing/src/main/java/de/jplag/endtoend/helper/UnzipManager.java

@@ -0,0 +1,59 @@
+package de.jplag.endtoend.helper;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.attribute.FileAttribute;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.logging.Logger;
+
+import org.apache.commons.lang3.SystemUtils;
+
+import de.jplag.endtoend.model.DataSet;
+
+public class UnzipManager {
+    private final Map<DataSet, File> unzippedFiles;
+    private static UnzipManager instance;
+    private final Logger logger = Logger.getLogger("Unzip Manager");
+
+    private static synchronized UnzipManager getInstance() {
+        if (instance == null) {
+            instance = new UnzipManager();
+        }
+
+        return instance;
+    }
+
+    public static File unzipOrCache(DataSet dataSet, File zip) throws IOException {
+        return getInstance().unzipOrCacheInternal(dataSet, zip);
+    }
+
+    private UnzipManager() {
+        this.unzippedFiles = new HashMap<>();
+    }
+
+    private File unzipOrCacheInternal(DataSet dataSet, File zip) throws IOException {
+        if (!unzippedFiles.containsKey(dataSet)) {
+            File target;
+
+            if (SystemUtils.IS_OS_UNIX) {
+                FileAttribute<Set<PosixFilePermission>> attr = PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwx------"));
+                target = Files.createTempDirectory(zip.getName(), attr).toFile();
+            } else {
+                target = Files.createTempDirectory(zip.getName()).toFile();
+                if (!(target.setReadable(true, true) && target.setWritable(true, true) && target.setExecutable(true, true))) {
+                    logger.warning("Could not set permissions for temp directory (" + target.getAbsolutePath() + ").");
+                }
+            }
+
+            FileHelper.unzip(zip, target);
+            this.unzippedFiles.put(dataSet, target);
+        }
+
+        return this.unzippedFiles.get(dataSet);
+    }
+}

endtoend-testing/src/main/java/de/jplag/endtoend/model/ComparisonIdentifier.java

@@ -0,0 +1,57 @@
+package de.jplag.endtoend.model;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.util.HashSet;
+import java.util.Scanner;
+import java.util.Set;
+
+/**
+ * Identifier for a comparison. The order of the names does not matter
+ * @param firstName The first name
+ * @param secondName The second name
+ */
+public record ComparisonIdentifier(String firstName, String secondName) {
+    private static final String INVALID_LINE_ERROR_MESSAGE = "Comparison identifier file (%s) has an invalid line: %s";
+
+    @Override
+    public boolean equals(Object o) {
+        if (!(o instanceof ComparisonIdentifier other)) {
+            return false;
+        }
+
+        return (firstName.equals(other.firstName) && secondName.equals(other.secondName))
+                || (secondName.equals(other.firstName) && firstName.equals(other.secondName));
+    }
+
+    @Override
+    public int hashCode() {
+        return firstName.hashCode() + secondName.hashCode();
+    }
+
+    /**
+     * Loads the identifiers stored in a csv (semicolon separated) file.
+     * @param file The file to load
+     * @return The comparisons in the file
+     */
+    public static Set<ComparisonIdentifier> loadIdentifiersFromFile(File file, String delimiter) {
+        try (Scanner scanner = new Scanner(file)) {
+            Set<ComparisonIdentifier> identifiers = new HashSet<>();
+            while (scanner.hasNextLine()) {
+                String[] parts = scanner.nextLine().split(delimiter);
+                if (parts.length != 2) {
+                    throw new IllegalStateException(String.format(INVALID_LINE_ERROR_MESSAGE, file.getAbsolutePath(), String.join(delimiter, parts)));
+                }
+                identifiers.add(new ComparisonIdentifier(parts[0], parts[1]));
+            }
+            return identifiers;
+        } catch (FileNotFoundException e) {
+            throw new IllegalStateException(String.format("Comparisons could not be loaded for %s.", file.getName()), e);
+        }
+    }
+
+    @Override
+    public String toString() {
+        return firstName + " - " + secondName;
+    }
+}