add command line option --CRLF; use LF as default line separator (#177)

com.sap.adt.abapcleaner.gui/src/com/sap/adt/abapcleaner/gui/FrmMain.java

@@ -190,18 +190,18 @@ public static void cleanAutomatically(CommandLineArgs commandLineArgs, PrintStre
 		if (commandLineArgs.isInSingleSourceMode()) {
 			cleanSingleSourceAutomatically(commandLineArgs, commandLineArgs.sourceCode, out, profile);
 		} else {
-			cleanMultiAutomatically(commandLineArgs, out, profile);
+			cleanMultiSourceAutomatically(commandLineArgs, out, profile);
 		}
 	}
 
-	private static void cleanMultiAutomatically(CommandLineArgs commandLineArgs, PrintStream out, Profile profile) {
+	private static void cleanMultiSourceAutomatically(CommandLineArgs commandLineArgs, PrintStream out, Profile profile) {
 		Persistency persistency = Persistency.get();
 		int baseSourcePathLength = commandLineArgs.sourceDir.length();
 
 		for (String sourcePath : commandLineArgs.sourcePaths) {
 			String sourceCode = persistency.readAllTextFromFile(sourcePath);
 
-			CleanupResult result = cleanAutomatically(sourceCode, commandLineArgs.abapRelease, commandLineArgs.cleanupRange, profile, commandLineArgs.showStats);
+			CleanupResult result = cleanAutomatically(sourceCode, commandLineArgs.abapRelease, commandLineArgs.cleanupRange, profile, commandLineArgs.showStats, commandLineArgs.lineSeparator);
 			if (result == null) {
 				out.println("Cleanup for file " + sourcePath + " cancelled.");
 				continue;
@@ -217,7 +217,7 @@ private static void cleanMultiAutomatically(CommandLineArgs commandLineArgs, Pri
 	}
 
 	private static void cleanSingleSourceAutomatically(CommandLineArgs commandLineArgs, String sourceCode, PrintStream out, Profile profile) {
-		CleanupResult result = cleanAutomatically(commandLineArgs.sourceCode, commandLineArgs.abapRelease, commandLineArgs.cleanupRange, profile, commandLineArgs.showStats);
+		CleanupResult result = cleanAutomatically(commandLineArgs.sourceCode, commandLineArgs.abapRelease, commandLineArgs.cleanupRange, profile, commandLineArgs.showStats, commandLineArgs.lineSeparator);
 		if (result == null) {
 			out.println("Cleanup cancelled.");
 			return;
@@ -264,14 +264,14 @@ private static void writeCleanUpResult(CommandLineArgs commandLineArgs, PrintStr
 			if (commandLineArgs.overwrite || !persistency.fileExists(targetPath)) {
 				persistency.ensureDirectoryExistsForPath(targetPath);
 				// for abapGit, ensure a final line separator
-				if (output.length() > 0 && ABAP.LINE_SEPARATOR.indexOf(output.charAt(output.length() - 1)) < 0)
-					output += ABAP.LINE_SEPARATOR;
+				if (output.length() > 0 && commandLineArgs.lineSeparator.indexOf(output.charAt(output.length() - 1)) < 0)
+					output += commandLineArgs.lineSeparator;
 				persistency.writeAllTextToFile(targetPath, output);
 			}
 		}
 	}
 
-	public static CleanupResult cleanAutomatically(String sourceCode, String abapRelease, CleanupRange cleanupRange, Profile profile, boolean provideRuleStats) {
+	public static CleanupResult cleanAutomatically(String sourceCode, String abapRelease, CleanupRange cleanupRange, Profile profile, boolean provideRuleStats, String lineSeparator) {
 		initialize();
 
 		MainSettings settings = new MainSettings();
@@ -285,7 +285,7 @@ public static CleanupResult cleanAutomatically(String sourceCode, String abapRel
 		Task result = job.getResult();
 
 		if (result.getSuccess()) {
-			CleanupResult cleanupResult = result.getResultingCode().toCleanupResult();
+			CleanupResult cleanupResult = result.getResultingCode().toCleanupResult(lineSeparator);
 
 			if (provideRuleStats) {
 				StringBuilder stats = new StringBuilder();
@@ -319,7 +319,7 @@ public static CleanupResult cleanInteractively(String sourceCode, String abapRel
 		window.open(isPlugin, sourcePageTitle, sourceCode, abapRelease, cleanupRange, isReadOnly);
 
 		if (window.resultCode != null) {
-			return window.resultCode.toCleanupResult();
+			return window.resultCode.toCleanupResult(ABAP.LINE_SEPARATOR);
 		} else if (!StringUtil.isNullOrEmpty(window.resultErrorMessage)) {
 			return CleanupResult.createError(window.resultErrorMessage);
 		} else {

com.sap.adt.abapcleaner.gui/src/com/sap/adt/abapcleaner/gui/eclipse/AbapCleanerHandlerBase.java

@@ -27,6 +27,7 @@
 import org.eclipse.ui.statushandlers.StatusManager;
 import org.osgi.framework.FrameworkUtil;
 
+import com.sap.adt.abapcleaner.base.ABAP;
 import com.sap.adt.abapcleaner.gui.CodeDisplayColors;
 import com.sap.adt.abapcleaner.gui.FrmMain;
 import com.sap.adt.abapcleaner.parser.CleanupRange;
@@ -97,7 +98,7 @@ public Object execute(ExecutionEvent event) throws ExecutionException {
 				CodeDisplayColors codeDisplayColors = createCodeDisplayColors(); 
 				result = FrmMain.cleanInteractively(oldSource, abapRelease, cleanupRange, true, adtSourcePage.getTitle(), codeDisplayColors, readOnly);
 			} else {
-				result = FrmMain.cleanAutomatically(oldSource, abapRelease, cleanupRange, null, false);
+				result = FrmMain.cleanAutomatically(oldSource, abapRelease, cleanupRange, null, false, ABAP.LINE_SEPARATOR);
 			}
 
 			if (result == null) 

com.sap.adt.abapcleaner/src/com/sap/adt/abapcleaner/base/ABAP.java

@@ -59,6 +59,7 @@ public static enum SyField {
    }
 
    public static final String LINE_SEPARATOR = "\r\n";
+   public static final String LINE_SEP_FOR_COMMAND_LINE = "\n";
    public static final char COMPONENT_SELECTOR = '-';
    public static final String COMPONENT_SELECTOR_STRING = "-";
    public static final char COMMENT_SIGN = '\"';

com.sap.adt.abapcleaner/src/com/sap/adt/abapcleaner/parser/Code.java

@@ -126,10 +126,14 @@ final void finishBuild() {
 
 	@Override
 	public String toString() {
+		return toString(ABAP.LINE_SEPARATOR);
+	}
+
+	public String toString(String lineSeparator) {
 		StringBuilder result = new StringBuilder();
 		Command command = firstCommand;
 		while (command != null) {
-			result.append(command.toString());
+			result.append(command.toString(lineSeparator));
 			command = command.getNext();
 		}
 		return result.toString();
@@ -478,9 +482,9 @@ final Command findLastCommandInCleanupRange() {
 		return null;
 	}
 
-	public CleanupResult toCleanupResult() {
+	public CleanupResult toCleanupResult(String lineSeparator) {
 		if (!hasCleanupRange())
-			return CleanupResult.createWithoutRange(this.toString());
+			return CleanupResult.createWithoutRange(this.toString(lineSeparator));
 
 		int startLine = -1;
 		int endLine = -1;
@@ -493,7 +497,7 @@ public CleanupResult toCleanupResult() {
 
 		// determine the char and line range of the selection that was cleaned up 
 		while (command != null) {
-			int commandLength = command.toString().length();
+			int commandLength = command.toString(lineSeparator).length();
 
 			// since Rules like IfBlockAtLineEndRule / IfBlockAtMethodEndRule may change the sequence of Commands, 
 			// it is not guaranteed that the Commands that are in the cleanup range are still consecutive; 
@@ -503,7 +507,7 @@ public CleanupResult toCleanupResult() {
 					// do not include the initial line breaks in the selection 
 					int firstTokenLineBreaks = command.getFirstTokenLineBreaks();
 					startLine = line + firstTokenLineBreaks;
-					startPos = pos + 2 * firstTokenLineBreaks;
+					startPos = pos + lineSeparator.length() * firstTokenLineBreaks;
 				}
 				endLine = line;
 				endPos = pos + commandLength;
@@ -514,7 +518,7 @@ public CleanupResult toCleanupResult() {
 			command = command.getNext();
 		}
 
-		return CleanupResult.createForRange(this.toString(), startLine, endLine, startPos, endPos - startPos);
+		return CleanupResult.createForRange(toString(lineSeparator), startLine, endLine, startPos, endPos - startPos);
 	}
 
 	public void expandCleanupRange(CleanupRangeExpandMode mode) {

com.sap.adt.abapcleaner/src/com/sap/adt/abapcleaner/parser/Command.java

@@ -1378,10 +1378,14 @@ public final ArrayList<Command> splitOutCommentLinesAfter(Token token) throws Un
 
 	@Override
 	public String toString() {
+		return toString(ABAP.LINE_SEPARATOR);
+	}
+
+	public String toString(String lineSeparator) {
 		StringBuilder result = new StringBuilder();
 		Token token = firstToken;
 		while (token != null) {
-			result.append(token.toString());
+			result.append(token.toString(lineSeparator));
 			token = token.getNext();
 		}
 		return result.toString();

com.sap.adt.abapcleaner/src/com/sap/adt/abapcleaner/parser/Token.java

@@ -447,9 +447,13 @@ private void addChild(Token newToken) {
 
 	@Override
 	public String toString() {
+		return toString(ABAP.LINE_SEPARATOR);
+	}
+
+	public String toString(String lineSeparator) {
 		StringBuilder result = new StringBuilder();
 		for (int i = 0; i < lineBreaks; ++i)
-			result.append(ABAP.LINE_SEPARATOR);
+			result.append(lineSeparator);
 		if (spacesLeft > 0)
 			result.append(StringUtil.repeatChar(' ', spacesLeft));
 		result.append(text);

com.sap.adt.abapcleaner/src/com/sap/adt/abapcleaner/parser/Tokenizer.java

@@ -17,6 +17,8 @@
  * {@link #changeToNonAbapLanguage(Token, Language, String)}.</p>
  */
 public class Tokenizer {
+	/** regardless of ABAP.LINE_SEPARATOR, contains all possible chars with which lines could be separated */ 
+	private final static String lineSeparatorChars = "\r\n";
 	private final static String spaceChars = " ";
 	private final static char[] lineFeedChars = new char[] { '\r', '\n' };
 
@@ -73,7 +75,7 @@ Token getNext() throws UnexpectedSyntaxException {
 		int spaceCount;
 		do {
 			// count line separators (tolerating both \r\n and \n)
-			while (readPos < text.length() && ABAP.LINE_SEPARATOR.indexOf(text.charAt(readPos)) >= 0) {
+			while (readPos < text.length() && lineSeparatorChars.indexOf(text.charAt(readPos)) >= 0) {
 				if (text.charAt(readPos) == '\n') {
 					++lineFeedCount;
 					++lineNum;
@@ -89,7 +91,7 @@ Token getNext() throws UnexpectedSyntaxException {
 			}
 
 			// in sample code, lines never end with spaces or tabs; if such a line is encountered, ignore such trailing spaces
-		} while (readPos < text.length() && ABAP.LINE_SEPARATOR.indexOf(text.charAt(readPos)) >= 0);
+		} while (readPos < text.length() && lineSeparatorChars.indexOf(text.charAt(readPos)) >= 0);
 
 		// ignore final line feed
 		if (readPos == text.length() && lineFeedCount == 1 && spaceCount == 0)

com.sap.adt.abapcleaner/src/com/sap/adt/abapcleaner/programbase/CommandLineArgs.java

@@ -1,5 +1,6 @@
 package com.sap.adt.abapcleaner.programbase;
 
+import com.sap.adt.abapcleaner.base.ABAP;
 import com.sap.adt.abapcleaner.base.StringUtil;
 import com.sap.adt.abapcleaner.parser.CleanupRange;
 
@@ -19,12 +20,14 @@ public class CommandLineArgs {
 	private static final String OPT_PROFILE = "--profile";
 	private static final String OPT_PROFILE_DATA = "--profiledata";
 	private static final String OPT_RELEASE = "--release";
+
+	private static final String OPT_CRLF = "--crlf";
 	private static final String OPT_OVERWRITE = "--overwrite";
 	private static final String OPT_PARTIAL_RESULT = "--partialresult";
 	private static final String OPT_STATS = "--stats";
 	private static final String OPT_USED_RULES = "--usedrules";
 
-	private static final String[] allOptions = new String[] { OPT_SOURCE_FILE, OPT_SOURCE_CODE, OPT_LINE_RANGE, OPT_TARGET_FILE, OPT_SOURCE_DIR, OPT_RECURSIVE, OPT_TARGET_DIR, OPT_FILE_FILTER, OPT_PROFILE, OPT_PROFILE_DATA, OPT_RELEASE, OPT_OVERWRITE, OPT_PARTIAL_RESULT, OPT_STATS, OPT_USED_RULES };
+	private static final String[] allOptions = new String[] { OPT_SOURCE_FILE, OPT_SOURCE_CODE, OPT_LINE_RANGE, OPT_TARGET_FILE, OPT_SOURCE_DIR, OPT_RECURSIVE, OPT_TARGET_DIR, OPT_FILE_FILTER, OPT_PROFILE, OPT_PROFILE_DATA, OPT_RELEASE, OPT_CRLF, OPT_OVERWRITE, OPT_PARTIAL_RESULT, OPT_STATS, OPT_USED_RULES };
 
 	private static final String EXECUTABLE_NAME = ".\\abap-cleanerc.exe"; 
 	private static final String OPT_HELP_WINDOWS = "/?";
@@ -60,6 +63,7 @@ public static CommandLineArgs create(Persistency persistency, String[] args) {
 
 		String profileData = null;
 		String abapRelease = null;
+		String lineSeparator = ABAP.LINE_SEP_FOR_COMMAND_LINE;
 		boolean overwrite = false;
 		boolean partialResult = false;
 		boolean showStats = false;
@@ -130,6 +134,9 @@ public static CommandLineArgs create(Persistency persistency, String[] args) {
 			} else if (arg.equals(OPT_RELEASE)) {
 				abapRelease = nextArg;
 
+			} else if (arg.equals(OPT_CRLF)) {
+				lineSeparator = "\r\n";
+
 			} else if (arg.equals(OPT_TARGET_FILE)) {
 				targetPath = nextArg;
 
@@ -196,9 +203,9 @@ public static CommandLineArgs create(Persistency persistency, String[] args) {
 		}
 
 		if (sourceCode != null) {
-			return new CommandLineArgs(sourceCode, targetPath, cleanupRange, profileData, abapRelease, overwrite, partialResult, showStats, showUsedRules, errors.toString(), showHelp);
+			return new CommandLineArgs(sourceCode, targetPath, cleanupRange, profileData, abapRelease, lineSeparator, overwrite, partialResult, showStats, showUsedRules, errors.toString(), showHelp);
 		} else {
-			return new CommandLineArgs(sourceDir, sourcePaths, targetDir, profileData, abapRelease, overwrite, showStats, showUsedRules, errors.toString(), showHelp);
+			return new CommandLineArgs(sourceDir, sourcePaths, targetDir, profileData, abapRelease, lineSeparator, overwrite, showStats, showUsedRules, errors.toString(), showHelp);
 		}
 	}
 
@@ -223,6 +230,7 @@ public static String getHelp(Persistency persistency) {
 		sb.append(" [" + OPT_RELEASE + " release]");
 		sb.append(LINE_SEP);
 		sb.append(spacePrefix);
+		sb.append(" [" + OPT_CRLF + "]");
 		sb.append(" [" + OPT_TARGET_FILE + " targetfile");
 		sb.append(" [" + OPT_OVERWRITE + "]]");
 		sb.append(" [" + OPT_PARTIAL_RESULT + "]");
@@ -297,6 +305,7 @@ public static String getHelp(Persistency persistency) {
 		sb.append(getOptionHelp(OPT_RELEASE, "ABAP release to restrict syntax of cleanup changes, e.g. \"758\""));
 		sb.append(getOptionHelp(null, "Without this option, the latest ABAP syntax will be allowed."));
 		sb.append(LINE_SEP);
+		sb.append(getOptionHelp(OPT_CRLF, "Use CRLF = \"\\r\\n\" as line separator (default: LF = \"\\n\")."));
 		sb.append(getOptionHelp(OPT_TARGET_FILE, "Target file name to which the cleanup result will be saved."));
 		sb.append(getOptionHelp(null, "Without this option, the cleanup result will be written to the standard output."));
 		sb.append(getOptionHelp(OPT_TARGET_DIR, "Target directory name to which the cleanup files will be saved"));
@@ -336,20 +345,22 @@ private static String getOptionsLinePrefix(String option) {
 
 	public final String profileData;
 	public final String abapRelease;
+
+	public final String lineSeparator;
 	public final boolean overwrite;
 	public final boolean partialResult;
 	public final boolean showStats;
 	public final boolean showUsedRules;
 	public final String errors;
 	public final boolean showHelp;
 
-	public boolean writesResultCodeToOutput() { return isInSingleSourceMode() && StringUtil.isNullOrEmpty(targetPath); }
+	public boolean hasErrors() { return !StringUtil.isNullOrEmpty(errors); }
 
 	public boolean isInSingleSourceMode() { return sourceDir == null; }
 
-	public boolean hasErrors() { return !StringUtil.isNullOrEmpty(errors); }
+	public boolean writesResultCodeToOutput() { return isInSingleSourceMode() && StringUtil.isNullOrEmpty(targetPath); }
 
-	private CommandLineArgs(String sourceCode, String targetPath, CleanupRange cleanupRange, String profileData, String abapRelease, boolean overwrite, boolean partialResult, boolean showStats, boolean showUsedRules, String errors, boolean showHelp) {
+	private CommandLineArgs(String sourceCode, String targetPath, CleanupRange cleanupRange, String profileData, String abapRelease, String lineSeparator, boolean overwrite, boolean partialResult, boolean showStats, boolean showUsedRules, String errors, boolean showHelp) {
 		this.sourceCode = sourceCode;
 		this.targetPath = targetPath;
 		this.cleanupRange = cleanupRange;
@@ -360,6 +371,8 @@ private CommandLineArgs(String sourceCode, String targetPath, CleanupRange clean
 
 		this.profileData = profileData;
 		this.abapRelease = abapRelease;
+
+		this.lineSeparator = lineSeparator;
 		this.overwrite = overwrite;
 		this.partialResult = partialResult;
 		this.showStats = showStats;
@@ -369,7 +382,7 @@ private CommandLineArgs(String sourceCode, String targetPath, CleanupRange clean
 
 	}
 
-	private CommandLineArgs(String sourceDir, String[] sourcePaths, String targetDir, String profileData, String abapRelease, boolean overwrite, boolean showStats, boolean showUsedRules, String errors, boolean showHelp) {
+	private CommandLineArgs(String sourceDir, String[] sourcePaths, String targetDir, String profileData, String abapRelease, String lineSeparator, boolean overwrite, boolean showStats, boolean showUsedRules, String errors, boolean showHelp) {
 		this.sourceCode = null;
 		this.cleanupRange = null;
 		this.targetPath = null;
@@ -380,6 +393,8 @@ private CommandLineArgs(String sourceDir, String[] sourcePaths, String targetDir
 
 		this.profileData = profileData;
 		this.abapRelease = abapRelease;
+
+		this.lineSeparator = lineSeparator;
 		this.overwrite = overwrite;
 		this.partialResult = false;
 		this.showStats = showStats;

test/com.sap.adt.abapcleaner.test/src/com/sap/adt/abapcleaner/parser/CodeTest.java

@@ -632,21 +632,25 @@ void testToCleanupResult() {
 
 		try {
 			Code code = Code.parse(null, ParseParams.createForWholeCode("test", codeText, ABAP.NEWEST_RELEASE));
-			CleanupResult cleanupResult = code.toCleanupResult();
+			CleanupResult cleanupResult = code.toCleanupResult(LINE_SEP);
 			assertTrue(cleanupResult.hasCleanedCode());
 			assertEquals(codeText, cleanupResult.getCleanedCode());
 			assertFalse(cleanupResult.hasLineSelection());
 			assertFalse(cleanupResult.hasErrorMessage());
 			assertEquals("", cleanupResult.getSelectedText());
 
 			code = Code.parse(null, ParseParams.createForCleanupRange("test", codeText, ABAP.NEWEST_RELEASE, CleanupRange.create(1, 4, false), CleanupRangeExpandMode.FULL_METHOD));
-			cleanupResult = code.toCleanupResult();
+			cleanupResult = code.toCleanupResult(LINE_SEP);
 			assertTrue(cleanupResult.hasCleanedCode());
 			assertEquals(codeText, cleanupResult.getCleanedCode());
 			assertTrue(cleanupResult.hasLineSelection());
 			assertFalse(cleanupResult.hasErrorMessage());
 			assertEquals("  IF a = 1." + LINE_SEP + "    RETURN." + LINE_SEP + "  ENDIF.", cleanupResult.getSelectedText());
-
+
+			final String LF = ABAP.LINE_SEP_FOR_COMMAND_LINE;
+			cleanupResult = code.toCleanupResult(LF);
+			assertEquals("  IF a = 1." + LF + "    RETURN." + LF + "  ENDIF.", cleanupResult.getSelectedText());
+
 		} catch (ParseException e) {
 			fail();
 		}

test/com.sap.adt.abapcleaner.test/src/com/sap/adt/abapcleaner/parser/CommandTest.java

@@ -1095,15 +1095,14 @@ void testRemoveAllFurtherChainColons() throws UnexpectedSyntaxAfterChanges {
 		command.removeAllChainColons();
 		assertEquals("DATA a TYPE i.", command.toString());
 
-		// except whitespace before last chain colon to be transferred to the period .
+		// expect whitespace before last chain colon to be transferred to the period .
 		command = buildCommand("DATA a : TYPE : i \" comment" + SEP + "  : .");
 		command.removeAllChainColons();
 		assertEquals("DATA a TYPE i \" comment" + SEP + "  .", command.toString());
 
-		// except whitespace before period . to be kept
-		command = buildCommand("DATA a : TYPE : i \" comment" + SEP + "  :" + SEP + ".");
+		command = buildCommand("DATA a : TYPE : i \" comment" + SEP + "  : .");
 		command.removeAllChainColons();
-		assertEquals("DATA a TYPE i \" comment" + SEP + ".", command.toString());
+		assertEquals("DATA a TYPE i \" comment" + SEP + "  .", command.toString());
 	}
 
 	@Test
@@ -1545,4 +1544,24 @@ void testInsertStressTestToken() throws IntegrityBrokenException {
 		buildCommand("EXEC SQL." + SEP + " SQL." + SEP + "ENDEXEC.");
 		assertFalse(commands[1].insertStressTestTokenAt(0, StressTestType.COLON));
 	}
+
+	@Test
+	void testToString() throws UnexpectedSyntaxAfterChanges {
+		// ensure that the Command can be parsed with any line separator, 
+		// and that .toString(String) returns a result with the supplied line separator
+
+		final String CRLF = "\r\n";
+		final String LF = "\n";
+
+		Command command = buildCommand("DATA: a TYPE i," + CRLF + "      b TYPE string.");
+		assertEquals("DATA: a TYPE i," + ABAP.LINE_SEPARATOR + "      b TYPE string.", command.toString());
+		assertEquals("DATA: a TYPE i," + CRLF + "      b TYPE string.", command.toString(CRLF));
+		assertEquals("DATA: a TYPE i," + LF + "      b TYPE string.", command.toString(LF));
+
+		command = buildCommand("DATA: a TYPE i," + LF + "      b TYPE string.");
+		assertEquals("DATA: a TYPE i," + ABAP.LINE_SEPARATOR + "      b TYPE string.", command.toString());
+		assertEquals("DATA: a TYPE i," + CRLF + "      b TYPE string.", command.toString(CRLF));
+		assertEquals("DATA: a TYPE i," + LF + "      b TYPE string.", command.toString(LF));
+	}
+
 }