Add encoding attributes to <tar>, <untar> and <tarfileset>

This is a workaround for Bugzilla Issue 57822
https://bz.apache.org/bugzilla/show_bug.cgi?id=57822

WHATSNEW

@@ -84,6 +84,16 @@ Fixed bugs:
  * TarEntry's constructor with a File and a String arg didn't
    normalize the name.
 
+ * Between 1.8.4 and 1.9.5 TarArchiveInputStream started to parse file
+   names using the platform's default encoding rather than as ASCII.
+   This has been a breaking change that has never been marked as such
+   (in fact it went unnoticed).  In order to allow <untar> and
+   <tarfileset> to work on platforms who's encoding doesn't match the
+   encoding of file names inside the archive, the both now support
+   encoding attributes.
+   The attribute has also been added to <tar> for symmetry.
+   Bugzilla Report 57822
+
 Other changes:
 --------------
 

manual/Tasks/tar.html

@@ -146,6 +146,15 @@ or "bzip2".</p>
        &quot;none&quot;.</td>
     <td valign="top" align="center">No</td>
   </tr>
+  <tr>
+    <td valign="top">encoding</td>
+    <td valign="top">The character encoding to use for filenames
+    inside the tar file.  For a list of possible values see the <a
+                href="http://docs.oracle.com/javase/7/docs/technotes/guides/intl/encoding.doc.html">Supported Encodings</a>.<br/>
+    Defaults to the platform's default character encoding.
+      <em>Since Ant 1.9.4</em>
+    <td align="center" valign="top">No</td>
+  </tr>
 </table>
 
 <h3>Nested Elements</h3>

manual/Tasks/unzip.html

@@ -100,12 +100,13 @@ archive.</p>
   </tr>
   <tr>
     <td valign="top">encoding</td>
-    <td valign="top"><b>Note:</b> This attribute is not available for
-    the <code>untar</code> task.<br>
+    <td valign="top">
     The character encoding that has been used for filenames
     inside the zip file.  For a list of possible values see the <a
     href="http://docs.oracle.com/javase/7/docs/technotes/guides/intl/encoding.doc.html">Supported Encodings</a>.<br/>
-    Defaults to &quot;UTF8&quot;, use the magic value
+    Defaults to &quot;UTF8&quot; for the <code>unzip</code> and the
+    platform's default encoding for the <code>untar</code> task.  Use
+    the magic value
     <code>native-encoding</code> for the platform's default character
     encoding.
       <br/>See also the <a href="zip.html#encoding">discussion in the

manual/Types/tarfileset.html

@@ -130,6 +130,15 @@ directories.  Default is 755.</td>
       </td>
       <td valign="top" align="center">No</td>
     </tr>
+    <tr>
+      <td valign="top">encoding</td>
+      <td valign="top">The character encoding to use for filenames
+      inside the zip file.  For a list of possible values see the <a
+      href="http://docs.oracle.com/javase/7/docs/technotes/guides/intl/encoding.doc.html">Supported Encodings</a>.
+      Defaults to the platform's default character encoding.
+        <em>Since Ant 1.9.5</em>
+      <td align="center" valign="top">No</td>
+    </tr>
   </tbody>
 </table>
 <p>The <i>fullpath</i> attribute can only be set for filesets that

manual/Types/zipfileset.html

@@ -97,7 +97,6 @@ directories.  Default is 755. <em>since Ant 1.5.2</em>.</td>
       inside the zip file.  For a list of possible values see the <a
       href="http://docs.oracle.com/javase/7/docs/technotes/guides/intl/encoding.doc.html">Supported Encodings</a>.
       Defaults to the platform's default character encoding.
-      <b>Only supported by zipfileset.</b></td>
       <td align="center" valign="top">No</td>
     </tr>
     <tr>

src/etc/testcases/taskdefs/untar.xml

@@ -57,8 +57,12 @@
     <untar src="." dest="${output}/untar" />
   </target>
 
-  <target name="encoding">
-    <untar src="expected/asf-logo.gif.tar" dest="${output}/untar" encoding="foo"/>
+  <target name="encodingTest">
+    <mkdir dir="${output}/untartestin"/>
+    <touch file="${output}/untartestin/foo"/>
+    <tar tarfile="${output}/untartest.tar" basedir="${output}/untartestin" encoding="UnicodeBig"/>
+    <mkdir dir="${output}/untartestout"/>
+    <untar src="${output}/untartest.tar" dest="${output}/untartestout" encoding="UnicodeBig"/>
   </target>
 
   <target name="resourceCollection">

src/main/org/apache/tools/ant/taskdefs/Expand.java

@@ -72,12 +72,28 @@ public class Expand extends Task {
 
     public static final String NATIVE_ENCODING = "native-encoding";
 
-    private String encoding = "UTF8";
+    private String encoding;
     /** Error message when more that one mapper is defined */
     public static final String ERROR_MULTIPLE_MAPPERS = "Cannot define more than one mapper";
 
     private static final FileUtils FILE_UTILS = FileUtils.getFileUtils();
 
+    /**
+     * Creates an Expand instance and sets encoding to UTF-8.
+     */
+    public Expand() {
+        this("UTF8");
+    }
+
+    /**
+     * Creates an Expand instance and sets the given encoding.
+     *
+     * @since Ant 1.9.5
+     */
+    protected Expand(String encoding) {
+        this.encoding = encoding;
+    }
+
     /**
      * Whether try ing to expand an empty archive would be an error.
      *

src/main/org/apache/tools/ant/taskdefs/Tar.java

@@ -123,6 +123,12 @@ public class Tar extends MatchingTask {
 
     private TarCompressionMethod compression = new TarCompressionMethod();
 
+    /**
+     * Encoding to use for filenames, defaults to the platform's
+     * default encoding.
+     */
+    private String encoding;
+
     /**
      * Add a new fileset with the option to specify permissions
      * @return the tar fileset to be used as the nested element.
@@ -231,6 +237,20 @@ public class Tar extends MatchingTask {
         this.compression = mode;
     }
 
+    /**
+     * Encoding to use for filenames, defaults to the platform's
+     * default encoding.
+     *
+     * <p>For a list of possible values see <a
+     * href="http://java.sun.com/j2se/1.5.0/docs/guide/intl/encoding.doc.html">http://java.sun.com/j2se/1.5.0/docs/guide/intl/encoding.doc.html</a>.</p>
+     * @param encoding the encoding name
+     *
+     * @since Ant 1.9.5
+     */
+    public void setEncoding(final String encoding) {
+        this.encoding = encoding;
+    }
+
     /**
      * do the business
      * @throws BuildException on error
@@ -304,7 +324,8 @@ public class Tar extends MatchingTask {
                 tOut = new TarOutputStream(
                     compression.compress(
                         new BufferedOutputStream(
-                            new FileOutputStream(tarFile))));
+                            new FileOutputStream(tarFile))),
+                    encoding);
                 tOut.setDebug(true);
                 if (longFileMode.isTruncateMode()) {
                     tOut.setLongFileMode(TarOutputStream.LONGFILE_TRUNCATE);

src/main/org/apache/tools/ant/taskdefs/Untar.java

@@ -58,6 +58,10 @@ public class Untar extends Expand {
      */
     private UntarCompressionMethod compression = new UntarCompressionMethod();
 
+    public Untar() {
+        super(null);
+    }
+
     /**
      * Set decompression algorithm to use; default=none.
      *
@@ -74,18 +78,6 @@ public class Untar extends Expand {
         compression = method;
     }
 
-    /**
-     * No encoding support in Untar.
-     * @param encoding not used
-     * @throws BuildException always
-     * @since Ant 1.6
-     */
-    public void setEncoding(String encoding) {
-        throw new BuildException("The " + getTaskName()
-                                 + " task doesn't support the encoding"
-                                 + " attribute", getLocation());
-    }
-
     /**
      * No unicode extra fields in tar.
      *
@@ -157,7 +149,8 @@ public class Untar extends Expand {
         try {
             tis =
                 new TarInputStream(compression.decompress(name,
-                                                          new BufferedInputStream(stream)));
+                                                          new BufferedInputStream(stream)),
+                                   getEncoding());
             log("Expanding: " + name + " into " + dir, Project.MSG_INFO);
             TarEntry te = null;
             boolean empty = true;

src/main/org/apache/tools/ant/types/ArchiveFileSet.java

@@ -72,6 +72,8 @@ public abstract class ArchiveFileSet extends FileSet {
 
     private boolean errorOnMissingArchive = true;
 
+    private String encoding = null;
+
     /** Constructor for ArchiveFileSet */
     public ArchiveFileSet() {
         super();
@@ -100,6 +102,7 @@ public abstract class ArchiveFileSet extends FileSet {
         fileModeHasBeenSet = fileset.fileModeHasBeenSet;
         dirModeHasBeenSet = fileset.dirModeHasBeenSet;
         errorOnMissingArchive = fileset.errorOnMissingArchive;
+        encoding = fileset.encoding;
     }
 
     /**
@@ -267,6 +270,33 @@ public abstract class ArchiveFileSet extends FileSet {
         return fullpath;
     }
 
+    /**
+     * Set the encoding used for this ZipFileSet.
+     * @param enc encoding as String.
+     * @since Ant 1.9.5
+     */
+    public void setEncoding(String enc) {
+        checkAttributesAllowed();
+        this.encoding = enc;
+    }
+
+    /**
+     * Get the encoding used for this ZipFileSet.
+     * @return String encoding.
+     * @since Ant 1.9.5
+     */
+    public String getEncoding() {
+        if (isReference()) {
+            AbstractFileSet ref = getRef(getProject());
+            if (ref instanceof ArchiveFileSet) {
+                return ((ArchiveFileSet) ref).getEncoding();
+            } else {
+                return null;
+            }
+        }
+        return encoding;
+    }
+
     /**
      * Creates a scanner for this type of archive.
      * @return the scanner.

src/main/org/apache/tools/ant/types/TarFileSet.java

@@ -180,6 +180,7 @@ public class TarFileSet extends ArchiveFileSet {
      */
     protected ArchiveScanner newArchiveScanner() {
         TarScanner zs = new TarScanner();
+        zs.setEncoding(getEncoding());
         return zs;
     }
 

src/main/org/apache/tools/ant/types/TarScanner.java

@@ -58,7 +58,7 @@ public class TarScanner extends ArchiveScanner {
 
         try {
             try {
-                ti = new TarInputStream(src.getInputStream());
+                ti = new TarInputStream(src.getInputStream(), encoding);
             } catch (IOException ex) {
                 throw new BuildException("problem opening " + srcFile, ex);
             }
@@ -84,4 +84,4 @@ public class TarScanner extends ArchiveScanner {
             FileUtils.close(ti);
         }
     }
-}
+}

src/main/org/apache/tools/ant/types/ZipFileSet.java

@@ -33,8 +33,6 @@ import org.apache.tools.ant.Project;
  */
 public class ZipFileSet extends ArchiveFileSet {
 
-    private String encoding = null;
-
     /** Constructor for ZipFileSet */
     public ZipFileSet() {
         super();
@@ -54,34 +52,6 @@ public class ZipFileSet extends ArchiveFileSet {
      */
     protected ZipFileSet(ZipFileSet fileset) {
         super(fileset);
-        encoding = fileset.encoding;
-    }
-
-    /**
-     * Set the encoding used for this ZipFileSet.
-     * @param enc encoding as String.
-     * @since Ant 1.7
-     */
-    public void setEncoding(String enc) {
-        checkZipFileSetAttributesAllowed();
-        this.encoding = enc;
-    }
-
-    /**
-     * Get the encoding used for this ZipFileSet.
-     * @return String encoding.
-     * @since Ant 1.7
-     */
-    public String getEncoding() {
-        if (isReference()) {
-            AbstractFileSet ref = getRef(getProject());
-            if (ref instanceof ZipFileSet) {
-                return ((ZipFileSet) ref).getEncoding();
-            } else {
-                return null;
-            }
-        }
-        return encoding;
     }
 
     /**
@@ -90,7 +60,7 @@ public class ZipFileSet extends ArchiveFileSet {
      */
     protected ArchiveScanner newArchiveScanner() {
         ZipScanner zs = new ZipScanner();
-        zs.setEncoding(encoding);
+        zs.setEncoding(getEncoding());
         return zs;
     }
 

src/tests/antunit/types/tarfileset-test.xml

@@ -34,4 +34,15 @@
     </copy>
   </target>
 
+  <target name="test-refid-check-encoding">
+    <tarfileset id="test-refid2"
+                encoding="utf-8"
+                dir="${basedir}"/>
+    <au:expectfailure>
+      <tarfileset id="ref4"
+                  encoding="utf-8"
+                  refid="test-refid2"/>
+    </au:expectfailure>
+  </target>
+
 </project>

src/tests/junit/org/apache/tools/ant/taskdefs/UntarTest.java

@@ -84,12 +84,10 @@ public class UntarTest {
 
     @Test
     public void testEncoding() {
-        try {
-            buildRule.executeTarget("encoding");
-            fail("<untar> overrides setEncoding.");
-        } catch (BuildException ex) {
-            assertEquals("The untar task doesn't support the encoding attribute", ex.getMessage());
-        }
+        buildRule.executeTarget("encodingTest");
+        String filename = buildRule.getProject().getProperty("output") + "/untartestout/foo";
+        assertTrue("foo has been properly named",
+                   buildRule.getProject().resolveFile(filename).exists());
     }
 
     @Test