outputencoding and fixlastline attributes for <concat>

PR: 12511
Submitted by:	Peter Reilly <peter dot reilly at corvil dot com>


git-svn-id: https://svn.apache.org/repos/asf/ant/core/trunk@274557 13f79535-47bb-0310-9956-ffa450edef68

WHATSNEW

@@ -308,6 +308,10 @@ Other changes:
 * There is a new data type <propertyset> that can be used to collect
   properties.
 
+* <concat> can now control the encoding of the output as well and optionally
+  add new-line characters at the end files that get concatenated but
+  don't end in newlines.  Bugzilla Report 12511.
+
 Changes from Ant 1.5.2 to Ant 1.5.3
 ===================================
 

docs/manual/CoreTasks/concat.html

@@ -76,6 +76,53 @@
           </td>
           <td valign="top" align="center">No</td>
         </tr>
+        <tr>
+          <td valign="top">outputencoding</td>
+          <td valign="top">
+            The encoding to use when writing the output file
+            <em>since Ant 1.6</em>.
+            Defaults to the value of the encoding attribute
+            if given or the default JVM encoding otherwise.
+          </td>
+          <td valign="top" align="center">No</td>
+        </tr>
+        <tr>
+          <td valign="top">fixlastline</td>
+          <td valign="top">
+ 	        Specifies whether or not to check if
+            each file concatenated is terminated by
+            a new line. If this attribute is &quot;yes&quot;
+            a new line will be appended to the stream if
+            the file did not end in a new line.
+            <em>since Ant 1.6</em>.
+            Defaults to &quot;no&quot;.
+            This attribute does not apply to embedded text.
+          </td>
+          <td valign="top" align="center">No</td>
+        </tr>
+        <tr>
+          <td valign="top">eol</td>
+          <td valign="top">
+            Specifies what the end of line character are
+            for use by the fixlastline attribute.
+            <em>since Ant 1.6</em>
+            Valid values for this property are:
+            <ul>
+	      <li>cr: a single CR</li>
+	      <li>lf: a single LF</li>
+	      <li>crlf: the pair CRLF</li>
+	      <li>mac: a single CR</li>
+	      <li>unix: a single LF</li>
+	      <li>dos: the pair CRLF</li>
+	    </ul>
+            The default is &quot;platform&quot;.
+            For Unix platforms, the default is &quot;lf&quot;.
+	    For DOS based systems (including Windows),
+            the default is &quot;crlf&quot;. 
+            For Mac OS, the default is &quot;cr&quot;.
+          </td>
+          <td valign="top" align="center">No</td>
+        </tr>
 
       </table>
 
@@ -86,7 +133,7 @@
       <p>
         This is a <a href="../CoreTypes/path.html">Path</a>. This is
         used to select file files to be concatenated. Note that
-        if a file can only appear once in a path. If this is
+        a file can only appear once in a path. If this is
         an issue consider using multiple paths.
       </p>
 

src/etc/testcases/taskdefs/concat.xml

@@ -11,7 +11,11 @@
     <delete file="TESTDEST"/>
     <delete file="${tmp.file}"/>
     <delete file="${tmp.file.2}"/>
-  </target>    
+    <delete file="concat.line4"/>
+    <delete file="concat.noeol"/>
+    <delete file="concat.linecr"/>
+    <delete file="concat.utf8"/>
+  </target>
 
   <target name="test1">
     <concat>
@@ -153,4 +157,32 @@
       </filterchain>
     </concat>
   </target>
+  
+  <target name="create-noel">
+    <concat destfile="concat.noeol">This has no end of line</concat>
+  </target>
+
+  <target name="testfixlastline" depends="create-noel">
+    <concat destfile="concat.line4" fixlastline="yes">
+      <path path="concat.noeol"/>
+      <path path="concat.noeol"/>
+      <path path="concat.noeol"/>
+      <path path="concat.noeol"/>
+    </concat>
+  </target>
+
+  <target name="testfixlastlineeol" depends="create-noel">
+    <concat destfile="concat.linecr" fixlastline="yes" eol="mac">
+      <path path="concat.noeol"/>
+      <path path="concat.noeol"/>
+    </concat>
+  </target>
+
+  <target name="testTranscoding">
+    <concat destfile="concat.utf8"
+            encoding="ISO8859_1" outputencoding="UTF8">
+      <path path="copy/input/iso8859-1"/>
+    </concat>
+  </target>
+
 </project>

src/main/org/apache/tools/ant/taskdefs/Concat.java

@@ -122,6 +122,9 @@ public class Concat extends Task {
      */
     private String encoding = null;
 
+    /** Stores the output file encoding. */
+    private String outputEncoding = null;
+
     // Child elements.
 
     /**
@@ -143,6 +146,14 @@ public class Concat extends Task {
     private TextElement   footer;
     /** String to place at the end of the concatented stream */
     private TextElement   header;
+    /** add missing line.separator to files **/
+    private boolean       fixLastLine = false;
+    /** endofline for fixlast line */
+    private String       eolString = System.getProperty("line.separator");
+    /** outputwriter */
+    private Writer       outputWriter = null;
+
+    /** internal variable - used to collect the source files from sources */
     private Vector        sourceFiles = new Vector();
 
     /** 1.1 utilities and copy utilities */
@@ -179,6 +190,17 @@ public class Concat extends Task {
      */
     public void setEncoding(String encoding) {
         this.encoding = encoding;
+        if (outputEncoding == null) {
+            outputEncoding = encoding;
+        }
+    }
+
+    /**
+     * Sets the character encoding for outputting
+     * @since Ant 1.6
+     */
+    public void setOutputEncoding(String outputEncoding) {
+        this.outputEncoding = outputEncoding;
     }
 
     /**
@@ -220,8 +242,9 @@ public class Concat extends Task {
      * @since Ant 1.6
      */
     public void addFilterChain(FilterChain filterChain) {
-        if (filterChains == null)
+        if (filterChains == null) {
             filterChains = new Vector();
+        }
         filterChains.addElement(filterChain);
     }
 
@@ -244,18 +267,51 @@ public class Concat extends Task {
      * Add a header to the concatenated output
      * @since Ant 1.6
      */
-    public void addHeader(TextElement el) {
-        this.header = el;
+    public void addHeader(TextElement header) {
+        this.header = header;
     }
 
     /**
      * Add a footer to the concatenated output
      * @since Ant 1.6
      */
-    public void addFooter(TextElement el) {
-        this.footer = el;
+    public void addFooter(TextElement footer) {
+        this.footer = footer;
     }
 
+    /**
+     * Append line.separator to files that do not end
+     * with a line.separator, default false.
+     * @since Ant 1.6
+     */
+    public void setFixLastLine(boolean fixLastLine) {
+        this.fixLastLine = fixLastLine;
+    }
+
+    /**
+     * Specify the end of line to find and to add if
+     * not present at end of each input file.
+     */
+    public void setEol(FixCRLF.CrLf enum) {
+        String s = enum.getValue();
+        if (s.equals("cr") || s.equals("mac")) {
+            eolString = "\r";
+        } else if (s.equals("lf") || s.equals("unix")) {
+            eolString = "\n";
+        } else if (s.equals("crlf") || s.equals("dos")) {
+            eolString = "\r\n";
+        }
+    }
+
+    /**
+     * set the output writer, this is to allow
+     * concat to be used as a nested element
+     * @since Ant 1.6
+     */
+    public void setWriter(Writer outputWriter) {
+        this.outputWriter = outputWriter;
+    }
+    
     /**
      * This method performs the concatenation.
      */
@@ -265,6 +321,11 @@ public class Concat extends Task {
         // treat empty nested text as no text
         sanitizeText();
 
+        if (destinationFile != null && outputWriter != null) {
+            throw new BuildException(
+                "Cannot specify both a destination file and an output writer");
+        }
+        
         // Sanity check our inputs.
         if (sources.size() == 0 && textBuffer == null) {
             // Nothing to concatenate!
@@ -339,6 +400,8 @@ public class Concat extends Task {
         forceOverwrite = true;
         destinationFile = null;
         encoding = null;
+        outputEncoding = null;
+        fixLastLine = false;
         sources.removeAllElements();
         sourceFiles.removeAllElements();
         filterChains = null;
@@ -371,32 +434,36 @@ public class Concat extends Task {
 
         try {
 
-            if (destinationFile == null) {
-                // Log using WARN so it displays in 'quiet' mode.
-                os = new LogOutputStream(this, Project.MSG_WARN);
+            PrintWriter writer = null;
+
+            if (outputWriter != null) {
+                writer = new PrintWriter(outputWriter);
             } else {
-                // ensure that the parent dir of dest file exists
-                File parent = fileUtils.getParentFile(destinationFile);
-                if (!parent.exists()) {
-                    parent.mkdirs();
-                }
+                if (destinationFile == null) {
+                    // Log using WARN so it displays in 'quiet' mode.
+                    os = new LogOutputStream(this, Project.MSG_WARN);
+                } else {
+                    // ensure that the parent dir of dest file exists
+                    File parent = fileUtils.getParentFile(destinationFile);
+                    if (!parent.exists()) {
+                        parent.mkdirs();
+                    }
 
-                os = new FileOutputStream(destinationFile.getAbsolutePath(),
-                                          append);
-            }
+                    os = new FileOutputStream(destinationFile.getAbsolutePath(),
+                                              append);
+                }
 
-            PrintWriter writer = null;
-            if (encoding == null) {
-                writer = new PrintWriter(
-                    new BufferedWriter(
-                        new OutputStreamWriter(os)));
-            } else {
-                writer = new PrintWriter(
-                    new BufferedWriter(
-                        new OutputStreamWriter(os, encoding)));
+                if (outputEncoding == null) {
+                    writer = new PrintWriter(
+                        new BufferedWriter(
+                            new OutputStreamWriter(os)));
+                } else {
+                    writer = new PrintWriter(
+                        new BufferedWriter(
+                            new OutputStreamWriter(os, outputEncoding)));
+                }
             }
 
-
             if (header != null) {
                 if (header.getFiltering()) {
                     concatenate(
@@ -425,7 +492,9 @@ public class Concat extends Task {
             }
 
             writer.flush();
-            os.flush();
+            if (os != null) {
+                os.flush();
+            }
 
         } catch (IOException ioex) {
             throw new BuildException("Error while concatenating: "
@@ -486,6 +555,7 @@ public class Concat extends Task {
         private boolean  trimLeading = false;
         private boolean  trim = false;
         private boolean  filtering = true;
+        private String   encoding = null;
 
         /**
          * whether to filter the text in this element
@@ -502,6 +572,10 @@ public class Concat extends Task {
         private boolean getFiltering() {
             return filtering;
         }
+
+        public void setEncoding(String encoding) {
+            this.encoding = encoding;
+        }
         
         /**
          * set the text using a file
@@ -517,7 +591,13 @@ public class Concat extends Task {
 
             BufferedReader reader = null;
             try {
-                reader = new BufferedReader(new FileReader(file));
+                if (this.encoding == null) {
+                    reader = new BufferedReader(new FileReader(file));
+                } else {
+                    reader = new BufferedReader(
+                        new InputStreamReader(new FileInputStream(file),
+                                              this.encoding));
+                }
                 value = fileUtils.readFully(reader);
             } catch (IOException ex) {
                 throw new BuildException(ex);
@@ -596,9 +676,12 @@ public class Concat extends Task {
      * a single stream.
      */
     private class MultiReader extends Reader {
-        private int pos = 0;
+        private int    pos = 0;
         private Reader reader = null;
-
+        private int    lastPos = 0;
+        private char[] lastChars = new char[eolString.length()];
+        private boolean needAddSeparator = false;
+        
         private Reader getReader() throws IOException {
             if (reader == null) {
                 if (encoding == null) {
@@ -611,7 +694,10 @@ public class Concat extends Task {
                             new FileInputStream(
                                 (File) sourceFiles.elementAt(pos)),
                             encoding));
-                }                
+                }
+                for (int i = 0; i < lastChars.length; ++i) {
+                    lastChars[i] = 0;
+                }
             }
             return reader;
         }
@@ -624,12 +710,26 @@ public class Concat extends Task {
          *            object.
          */
         public int read() throws IOException {
+            if (needAddSeparator) {
+                int ret = eolString.charAt(lastPos++);
+                if (lastPos >= eolString.length()) {
+                    lastPos = 0;
+                    needAddSeparator = false;
+                }
+                return ret;
+            }
+            
             while (pos < sourceFiles.size()) {
                 int ch = getReader().read();
                 if (ch == -1) {
                     reader.close();
                     reader = null;
+                    if (fixLastLine && isMissingEndOfLine()) {
+                        needAddSeparator = true;
+                        lastPos = 0;
+                    }
                 } else {
+                    addLastChar((char) ch);
                     return ch;
                 }
                 pos++; 
@@ -647,15 +747,45 @@ public class Concat extends Task {
          */
         public int read(char cbuf[], int off, int len)
             throws IOException {
+
             int amountRead = 0;
             int iOff = off;
-            while (pos < sourceFiles.size()) {
+            while (pos < sourceFiles.size() || (needAddSeparator)) {
+                if (needAddSeparator) {
+                    cbuf[off] = eolString.charAt(lastPos++);
+                    if (lastPos >= eolString.length()) {
+                        lastPos = 0;
+                        needAddSeparator = false;
+                        pos++;
+                    }
+                    len--;
+                    off++;
+                    amountRead++;
+                    if (len == 0)
+                        return amountRead;
+                    continue;
+                }
+
                 int nRead = getReader().read(cbuf, off, len);
                 if (nRead == -1 || nRead == 0) {
                     reader.close();
                     reader = null;
-                    pos++;
+                    if (fixLastLine && isMissingEndOfLine()) {
+                        needAddSeparator = true;
+                        lastPos = 0;
+                    } else {
+                        pos++;
+                    }
                 } else {
+                    if (fixLastLine) {
+                        for (int i = nRead; i > (nRead-lastChars.length);
+                             --i) {
+                            if (i < 0) {
+                                break;
+                            }
+                            addLastChar(cbuf[off+i]);
+                        }
+                    }
                     len -= nRead;
                     off += nRead;
                     amountRead += nRead;
@@ -671,11 +801,37 @@ public class Concat extends Task {
             }
         }
 
+        /**
+         * Close the current reader
+         */
         public void close() throws IOException {
             if (reader != null) {
                 reader.close();
             }
         }
+        /**
+         * if checking for end of line at end of file
+         * add a character to the lastchars buffer
+         */
+        private void addLastChar(char ch) {
+            for (int i = lastChars.length-2; i >= 0; --i) {
+                lastChars[i] = lastChars[i+1];
+            }
+            lastChars[lastChars.length-1] = ch;
+        }
+
+        /**
+         * return true if the lastchars buffer does
+         * not contain the lineseparator
+         */
+        private boolean isMissingEndOfLine() {
+            for (int i = 0; i < lastChars.length; ++i) {
+                if (lastChars[i] != eolString.charAt(i))
+                    return true;
+            }
+            return false;
+        }
     }
- }
+
+}
 

src/testcases/org/apache/tools/ant/taskdefs/ConcatTest.java

@@ -55,8 +55,12 @@
 package org.apache.tools.ant.taskdefs;
 
 import org.apache.tools.ant.BuildFileTest;
+import org.apache.tools.ant.util.FileUtils;
 
 import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.Reader;
 
 /**
  * A test class for the 'concat' task, used to concatenate a series of
@@ -241,5 +245,70 @@ public class ConcatTest
         assertTrue(getLog().indexOf("Bye") > -1);
         assertTrue(getLog().indexOf("Hello") == -1);
     }        
+    /**
+     * Check if fixlastline works
+     */
+    public void testfixlastline()
+        throws IOException
+    {
+        expectFileContains(
+            "testfixlastline", "concat.line4",
+            "end of line" + System.getProperty("line.separator")
+            + "This has");
+    }        
+
+    /**
+     * Check if fixlastline works with eol
+     */
+    public void testfixlastlineeol()
+        throws IOException
+    {
+        expectFileContains(
+            "testfixlastlineeol", "concat.linecr",
+            "end of line\rThis has");
+    }        
+
+    // ------------------------------------------------------
+    //   Helper methods - should be in BuildFileTest
+    // -----------------------------------------------------
+
+    private String getFileString(String filename)
+        throws IOException
+    {
+        Reader r = null;
+        try {
+            r = new FileReader(getProject().resolveFile(filename));
+            return  FileUtils.newFileUtils().readFully(r);
+        }
+        finally {
+            try {r.close();} catch (Throwable ignore) {}
+        }
+        
+    }
         
+    private String getFileString(String target, String filename)
+        throws IOException
+    {
+        executeTarget(target);
+        return getFileString(filename);
+    }
+    
+    private void expectFileContains(
+        String target, String filename, String contains)
+        throws IOException
+    {
+        String content = getFileString(target, filename);
+        assertTrue(
+            "expecting file " + filename + " to contain " +
+            contains +
+            " but got " + content, content.indexOf(contains) > -1);
+    }
+
+    public void testTranscoding() throws IOException {
+        executeTarget("testTranscoding");
+        FileUtils fileUtils = FileUtils.newFileUtils();
+        File f1 = getProject().resolveFile("copy/expected/utf-8");
+        File f2 = getProject().resolveFile("concat.utf8");
+        assertTrue(fileUtils.contentEquals(f1, f2));
+    }
 }