From 34fdc2f62a8e282875c134c78aaf14105af92cf9 Mon Sep 17 00:00:00 2001 From: Stefan Bodewig Date: Sat, 21 Jun 2014 07:05:20 +0200 Subject: [PATCH 1/6] PR 56641 cannot read entries with empty gid/uid anymore manually merging http://svn.apache.org/viewvc?view=revision&revision=1588618 from Commons Compress --- WHATSNEW | 4 ++++ src/main/org/apache/tools/tar/TarUtils.java | 22 ++++++++++++--------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/WHATSNEW b/WHATSNEW index 74e79b4c7..c8fbe501d 100644 --- a/WHATSNEW +++ b/WHATSNEW @@ -7,6 +7,10 @@ Changes that could break older environments: Fixed bugs: ----------- +* TarArchiveInputStream failed to read archives with empty gid/uid + fields. + Bugzilla Report 56641 + Other changes: -------------- diff --git a/src/main/org/apache/tools/tar/TarUtils.java b/src/main/org/apache/tools/tar/TarUtils.java index cc4106349..264ff9921 100644 --- a/src/main/org/apache/tools/tar/TarUtils.java +++ b/src/main/org/apache/tools/tar/TarUtils.java @@ -60,7 +60,7 @@ public class TarUtils { public String decode(byte[] buffer) { final int length = buffer.length; - StringBuffer result = new StringBuffer(length); + StringBuilder result = new StringBuilder(length); for (int i = 0; i < length; ++i) { byte b = buffer[i]; @@ -130,10 +130,6 @@ public class TarUtils { end--; trailer = buffer[end - 1]; } - if (start == end) { - throw new IllegalArgumentException( - exceptionMessage(buffer, offset, length, start, trailer)); - } for ( ;start < end; start++) { final byte currentByte = buffer[start]; @@ -194,7 +190,7 @@ public class TarUtils { if (negative) { // 2's complement val--; - val ^= ((long) Math.pow(2, (length - 1) * 8) - 1); + val ^= (long) Math.pow(2, (length - 1) * 8) - 1; } return negative ? -val : val; } @@ -236,7 +232,15 @@ public class TarUtils { // Helper method to generate the exception message private static String exceptionMessage(byte[] buffer, final int offset, final int length, int current, final byte currentByte) { - String string = new String(buffer, offset, length); // TODO default charset? + // default charset is good enough for an exception message, + // + // the alternative was to modify parseOctal and + // parseOctalOrBinary to receive the ZipEncoding of the + // archive (deprecating the existing public methods, of + // course) and dealing with the fact that ZipEncoding#decode + // can throw an IOException which parseOctal* doesn't declare + String string = new String(buffer, offset, length); + string=string.replaceAll("\0", "{NUL}"); // Replace NULs to allow string to be printed final String s = "Invalid byte "+currentByte+" at offset "+(current-offset)+" in '"+string+"' len="+length; return s; @@ -549,8 +553,8 @@ public class TarUtils { public static long computeCheckSum(final byte[] buf) { long sum = 0; - for (int i = 0; i < buf.length; ++i) { - sum += BYTE_MASK & buf[i]; + for (byte element : buf) { + sum += BYTE_MASK & element; } return sum; From 8819ee167bec189fb17e8d25ae3aca5268e0ec23 Mon Sep 17 00:00:00 2001 From: Stefan Bodewig Date: Sat, 21 Jun 2014 07:25:55 +0200 Subject: [PATCH 2/6] Deal with InputStreams that don't return the full PAX header in one read() This used to be https://issues.apache.org/jira/browse/COMPRESS-270 --- WHATSNEW | 3 +++ src/main/org/apache/tools/tar/TarInputStream.java | 14 +++++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/WHATSNEW b/WHATSNEW index c8fbe501d..f4c64ef80 100644 --- a/WHATSNEW +++ b/WHATSNEW @@ -11,6 +11,9 @@ Fixed bugs: fields. Bugzilla Report 56641 +* TarArchiveInputStream could throw IOException when reading PAX + headers from a "slow" InputStream. + Other changes: -------------- diff --git a/src/main/org/apache/tools/tar/TarInputStream.java b/src/main/org/apache/tools/tar/TarInputStream.java index 62bbd625f..154f14e2e 100644 --- a/src/main/org/apache/tools/tar/TarInputStream.java +++ b/src/main/org/apache/tools/tar/TarInputStream.java @@ -431,18 +431,22 @@ public class TarInputStream extends FilterInputStream { if (ch == '='){ // end of keyword String keyword = coll.toString("UTF-8"); // Get rest of entry - byte[] rest = new byte[len - read]; - int got = i.read(rest); - if (got != len - read){ + final int restLen = len - read; + byte[] rest = new byte[restLen]; + int got = 0; + while (got < restLen && (ch = i.read()) != -1) { + rest[got++] = (byte) ch; + } + if (got != restLen) { throw new IOException("Failed to read " + "Paxheader. Expected " - + (len - read) + + restLen + " bytes, read " + got); } // Drop trailing NL String value = new String(rest, 0, - len - read - 1, "UTF-8"); + restLen - 1, "UTF-8"); headers.put(keyword, value); break; } From ca34ebdbd1bedbca42b277d34f19db01697dbbb5 Mon Sep 17 00:00:00 2001 From: Stefan Bodewig Date: Sat, 21 Jun 2014 07:39:47 +0200 Subject: [PATCH 3/6] whitespace --- WHATSNEW | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/WHATSNEW b/WHATSNEW index f4c64ef80..ab0212420 100644 --- a/WHATSNEW +++ b/WHATSNEW @@ -7,12 +7,13 @@ Changes that could break older environments: Fixed bugs: ----------- -* TarArchiveInputStream failed to read archives with empty gid/uid - fields. - Bugzilla Report 56641 + * TarArchiveInputStream failed to read archives with empty gid/uid + fields. + Bugzilla Report 56641 + + * TarArchiveInputStream could throw IOException when reading PAX + headers from a "slow" InputStream. -* TarArchiveInputStream could throw IOException when reading PAX - headers from a "slow" InputStream. Other changes: -------------- From a6b49f948ddad457bee248c1efea1ab08d249ba5 Mon Sep 17 00:00:00 2001 From: Stefan Bodewig Date: Sat, 21 Jun 2014 07:42:44 +0200 Subject: [PATCH 4/6] PR 56593 XMLJUnitResultFormatter may throw NPE when Java cannot determine hostname. Submitted by Joel Tucci. --- CONTRIBUTORS | 1 + WHATSNEW | 3 +++ contributors.xml | 4 ++++ .../taskdefs/optional/junit/XMLJUnitResultFormatter.java | 8 ++++++-- 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTORS b/CONTRIBUTORS index a533fb8b9..4a10c42b4 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -182,6 +182,7 @@ Jesse Glick Jesse Stockall Jim Allers Joerg Wassmer +Joel Tucci Joey Richey Johann Herunter John Elion diff --git a/WHATSNEW b/WHATSNEW index ab0212420..6ebed1eca 100644 --- a/WHATSNEW +++ b/WHATSNEW @@ -14,6 +14,9 @@ Fixed bugs: * TarArchiveInputStream could throw IOException when reading PAX headers from a "slow" InputStream. + * XMLJunitResultFormatter could throw NullPointerException if Java + cannot determine the local hostname. + Bugzilla Report 56593 Other changes: -------------- diff --git a/contributors.xml b/contributors.xml index 488d784e4..a86950aff 100644 --- a/contributors.xml +++ b/contributors.xml @@ -751,6 +751,10 @@ Joerg Wassmer + + Joel + Tucci + Joey Richey diff --git a/src/main/org/apache/tools/ant/taskdefs/optional/junit/XMLJUnitResultFormatter.java b/src/main/org/apache/tools/ant/taskdefs/optional/junit/XMLJUnitResultFormatter.java index 4ff3ab1cd..7f064f963 100644 --- a/src/main/org/apache/tools/ant/taskdefs/optional/junit/XMLJUnitResultFormatter.java +++ b/src/main/org/apache/tools/ant/taskdefs/optional/junit/XMLJUnitResultFormatter.java @@ -161,11 +161,15 @@ public class XMLJUnitResultFormatter implements JUnitResultFormatter, XMLConstan * @return the name of the local host, or "localhost" if we cannot work it out */ private String getHostname() { + String hostname = "localhost"; try { - return InetAddress.getLocalHost().getHostName(); + InetAddress localHost = InetAddress.getLocalHost(); + if (localHost != null) { + hostname = localHost.getHostName(); + } } catch (UnknownHostException e) { - return "localhost"; } + return hostname; } /** From ca4d619112299ab28dd289a2e0e407ddebc6340c Mon Sep 17 00:00:00 2001 From: Stefan Bodewig Date: Sat, 21 Jun 2014 08:28:18 +0200 Subject: [PATCH 5/6] freecode is no more --- ReleaseInstructions | 6 ------ 1 file changed, 6 deletions(-) diff --git a/ReleaseInstructions b/ReleaseInstructions index bb7be3068..165cdf03d 100644 --- a/ReleaseInstructions +++ b/ReleaseInstructions @@ -173,9 +173,6 @@ Note: This document was updated in the context of releasing Ant 1.9.3. days pass and there are no major problems, a wider announcement is made (ant website, announce@apache.org, etc). - Announce beta releases at freecode.com (Stefan Bodewig is the - owner of Ant's project entry - bug him ;-). - 17. As problems in the beta are discovered, there may be a need for one or more subsequent betas. The release manager makes this call. Each time, the versions are updated and the above process is @@ -256,9 +253,6 @@ Note: This document was updated in the context of releasing Ant 1.9.3. Apache mailing lists that should get the announcements: announce@apache.org, dev@ant and user@ant. - Announce release at freecode.com - (Stefan Bodewig is the owner of Ant's project entry - bug him ;-). - 25. Add a new release tag to doap_Ant.rdf in Ant's site. 26. You can now reacquaint yourself with your family and friends. From 6e88f92ead6a5f67935055661a9e8041f5ed8bae Mon Sep 17 00:00:00 2001 From: Stefan Bodewig Date: Tue, 24 Jun 2014 06:31:55 +0200 Subject: [PATCH 6/6] PR 56584 allow ReplaceTokens filter to use multi-character token separators. Submitted by Ralf Hergert. --- CONTRIBUTORS | 1 + WHATSNEW | 9 + contributors.xml | 4 + manual/Types/filterchain.html | 29 ++- src/etc/testcases/filters/build.xml | 33 ++++ .../expected/replacetokens.double.test | 2 + .../filters/input/replacetokens.double.test | 2 + .../filters/input/replacetokens.mustache.test | 2 + .../tools/ant/filters/ReplaceTokens.java | 185 ++++++++---------- .../tools/ant/filters/ReplaceTokensTest.java | 26 ++- 10 files changed, 188 insertions(+), 105 deletions(-) create mode 100644 src/etc/testcases/filters/expected/replacetokens.double.test create mode 100644 src/etc/testcases/filters/input/replacetokens.double.test create mode 100644 src/etc/testcases/filters/input/replacetokens.mustache.test diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 4a10c42b4..4d65e56e8 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -300,6 +300,7 @@ Pierre Delisle Pierre Dittgen riasol R Handerson +Ralf Hergert Rami Ojares Randy Watler Raphael Pierquin diff --git a/WHATSNEW b/WHATSNEW index 6ebed1eca..d80727dcb 100644 --- a/WHATSNEW +++ b/WHATSNEW @@ -4,6 +4,15 @@ Changes from Ant 1.9.4 TO Ant 1.9.5 Changes that could break older environments: ------------------------------------------- + * The ReplaceTokens filter can now use token-separators longer than + one character. This means it can be used to replace mustache-style + {{patterns}} and similar templates. This is going to break code + that invokes the setters on ReplaceTokens via the Java API as their + parameters have been changed from char to String. It may also + break build files that specified multi character tokens and relied + on Ant silently ignoring all but the first character. + Bugzilla Report 56584 + Fixed bugs: ----------- diff --git a/contributors.xml b/contributors.xml index a86950aff..8bdae0f70 100644 --- a/contributors.xml +++ b/contributors.xml @@ -1213,6 +1213,10 @@ R Handerson + + Ralf + Hergert + Rami Ojares diff --git a/manual/Types/filterchain.html b/manual/Types/filterchain.html index b7c95a189..4de10ea3c 100644 --- a/manual/Types/filterchain.html +++ b/manual/Types/filterchain.html @@ -551,14 +551,14 @@ user defined values. tokenchar begintoken - Character marking the + String marking the beginning of a token. Defaults to @ No tokenchar endtoken - Character marking the + String marking the end of a token. Defaults to @ No @@ -626,6 +626,31 @@ Convenience method: </loadfile> +This replaces occurrences of the string {{DATE}} in the data +with today's date and stores it in the property ${src.file.replaced}. +
+<loadfile srcfile="${src.file}" property="${src.file.replaced}">
+  <filterchain>
+    <filterreader classname="org.apache.tools.ant.filters.ReplaceTokens">
+      <param type="tokenchar" name="begintoken" value="{{"/>
+      <param type="tokenchar" name="endtoken" value="}}"/>
+    </filterreader>
+  </filterchain>
+</loadfile>
+
+ +Convenience method: +
+<tstamp/>
+<loadfile srcfile="${src.file}" property="${src.file.replaced}">
+  <filterchain>
+    <replacetokens begintoken="{{" endtoken="}}">
+      <token key="DATE" value="${TODAY}"/>
+    </replacetokens>
+  </filterchain>
+</loadfile>
+
+ This will treat each properties file entry in sample.properties as a token/key pair : 
 <loadfile srcfile="${src.file}" property="${src.file.replaced}">
diff --git a/src/etc/testcases/filters/build.xml b/src/etc/testcases/filters/build.xml
index cc3798ac1..b70b7786b 100644
--- a/src/etc/testcases/filters/build.xml
+++ b/src/etc/testcases/filters/build.xml
@@ -100,6 +100,39 @@
     
   
 
+  
+    
+      
+      
+        
+          
+        
+      
+    
+  
+
+  
+    
+      
+      
+        
+          
+        
+      
+    
+  
+
+  
+    
+      
+      
+        
+          
+        
+      
+    
+  
+
   
     This has no new lines
     
diff --git a/src/etc/testcases/filters/expected/replacetokens.double.test b/src/etc/testcases/filters/expected/replacetokens.double.test
new file mode 100644
index 000000000..72eaee7eb
--- /dev/null
+++ b/src/etc/testcases/filters/expected/replacetokens.double.test
@@ -0,0 +1,2 @@
+1@@2
+3
diff --git a/src/etc/testcases/filters/input/replacetokens.double.test b/src/etc/testcases/filters/input/replacetokens.double.test
new file mode 100644
index 000000000..163417d35
--- /dev/null
+++ b/src/etc/testcases/filters/input/replacetokens.double.test
@@ -0,0 +1,2 @@
+1@@foo@@2
+3
diff --git a/src/etc/testcases/filters/input/replacetokens.mustache.test b/src/etc/testcases/filters/input/replacetokens.mustache.test
new file mode 100644
index 000000000..62df44555
--- /dev/null
+++ b/src/etc/testcases/filters/input/replacetokens.mustache.test
@@ -0,0 +1,2 @@
+1{{foo}}2
+3
diff --git a/src/main/org/apache/tools/ant/filters/ReplaceTokens.java b/src/main/org/apache/tools/ant/filters/ReplaceTokens.java
index c414dcacb..efef83b72 100644
--- a/src/main/org/apache/tools/ant/filters/ReplaceTokens.java
+++ b/src/main/org/apache/tools/ant/filters/ReplaceTokens.java
@@ -24,7 +24,9 @@ import java.io.Reader;
 import java.util.Enumeration;
 import java.util.Hashtable;
 import java.util.Properties;
-import org.apache.tools.ant.BuildException;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
 import org.apache.tools.ant.types.Parameter;
 import org.apache.tools.ant.types.Resource;
 import org.apache.tools.ant.types.resources.FileResource;
@@ -52,13 +54,19 @@ public final class ReplaceTokens
     extends BaseParamFilterReader
     implements ChainableReader {
     /** Default "begin token" character. */
-    private static final char DEFAULT_BEGIN_TOKEN = '@';
+    private static final String DEFAULT_BEGIN_TOKEN = "@";
 
     /** Default "end token" character. */
-    private static final char DEFAULT_END_TOKEN = '@';
+    private static final String DEFAULT_END_TOKEN = "@";
+
+    /** Hashtable to holds the original replacee-replacer pairs (String to String). */
+    private Hashtable hash = new Hashtable();
 
-    /** Data to be used before reading from stream again */
-    private String queuedData = null;
+    /** This map holds the "resolved" tokens (begin- and end-tokens are added to make searching simpler) */
+    private final TreeMap resolvedTokens = new TreeMap();
+    private boolean resolvedTokensBuilt = false;
+    /** Used for comparisons and lookup into the resolvedTokens map. */
+    private String readBuffer = "";
 
     /** replacement test from a token */
     private String replaceData = null;
@@ -66,26 +74,18 @@ public final class ReplaceTokens
     /** Index into replacement data */
     private int replaceIndex = -1;
 
-    /** Index into queue data */
-    private int queueIndex = -1;
-
-    /** Hashtable to hold the replacee-replacer pairs (String to String). */
-    private Hashtable hash = new Hashtable();
-
     /** Character marking the beginning of a token. */
-    private char beginToken = DEFAULT_BEGIN_TOKEN;
+    private String beginToken = DEFAULT_BEGIN_TOKEN;
 
     /** Character marking the end of a token. */
-    private char endToken = DEFAULT_END_TOKEN;
+    private String endToken = DEFAULT_END_TOKEN;
 
     /**
      * Constructor for "dummy" instances.
      *
      * @see BaseFilterReader#BaseFilterReader()
      */
-    public ReplaceTokens() {
-        super();
-    }
+    public ReplaceTokens() {}
 
     /**
      * Creates a new filtered reader.
@@ -97,18 +97,6 @@ public final class ReplaceTokens
         super(in);
     }
 
-    private int getNextChar() throws IOException {
-        if (queueIndex != -1) {
-            final int ch = queuedData.charAt(queueIndex++);
-            if (queueIndex >= queuedData.length()) {
-                queueIndex = -1;
-            }
-            return ch;
-        }
-
-        return in.read();
-    }
-
     /**
      * Returns the next character in the filtered stream, replacing tokens
      * from the original stream.
@@ -125,63 +113,66 @@ public final class ReplaceTokens
             setInitialized(true);
         }
 
-        if (replaceIndex != -1) {
-            final int ch = replaceData.charAt(replaceIndex++);
-            if (replaceIndex >= replaceData.length()) {
-                replaceIndex = -1;
+        if (!resolvedTokensBuilt) {
+            // build the resolved tokens tree map.
+            for (String key : hash.keySet()) {
+                resolvedTokens.put(beginToken + key + endToken, hash.get(key));
             }
-            return ch;
+            resolvedTokensBuilt = true;
         }
 
-        int ch = getNextChar();
-
-        if (ch == beginToken) {
-            final StringBuffer key = new StringBuffer("");
-            do  {
-                ch = getNextChar();
-                if (ch != -1) {
-                    key.append((char) ch);
-                } else {
-                    break;
-                }
-            } while (ch != endToken);
-
-            if (ch == -1) {
-                if (queuedData == null || queueIndex == -1) {
-                    queuedData = key.toString();
-                } else {
-                    queuedData
-                        = key.toString() + queuedData.substring(queueIndex);
-                }
-                if (queuedData.length() > 0) {
-                    queueIndex = 0;
-                } else {
-                    queueIndex = -1;
-                }
-                return beginToken;
+        // are we currently serving replace data?
+        if (replaceData != null) {
+            if (replaceIndex < replaceData.length()) {
+                return replaceData.charAt(replaceIndex++);
             } else {
-                key.setLength(key.length() - 1);
+                replaceData = null;
+            }
+        }
 
-                final String replaceWith = (String) hash.get(key.toString());
-                if (replaceWith != null) {
-                    if (replaceWith.length() > 0) {
-                        replaceData = replaceWith;
-                        replaceIndex = 0;
-                    }
-                    return read();
+        // is the read buffer empty?
+        if (readBuffer.length() == 0) {
+            int next = in.read();
+            if (next == -1) {
+                return next; // end of stream. all buffers empty.
+            }
+            readBuffer += (char)next;
+        }
+
+        for (;;) {
+            // get the closest tokens
+            SortedMap possibleTokens = resolvedTokens.tailMap(readBuffer);
+            if (possibleTokens.isEmpty() || !possibleTokens.firstKey().startsWith(readBuffer)) { // if there is none, then deliver the first char from the buffer.
+                return getFirstCharacterFromReadBuffer();
+            } else if (readBuffer.equals(possibleTokens.firstKey())) { // there exists a nearest token - is it an exact match?
+                // we have found a token. prepare the replaceData buffer.
+                replaceData = resolvedTokens.get(readBuffer);
+                replaceIndex = 0;
+                readBuffer = ""; // destroy the readBuffer - it's contents are being replaced entirely.
+                // get the first character via recursive call.
+                return read();
+            } else { // nearest token is not matching exactly - read one character more.
+                int next = in.read();
+                if (next != -1) {
+                    readBuffer += (char)next;
                 } else {
-                    String newData = key.toString() + endToken;
-                    if (queuedData == null || queueIndex == -1) {
-                        queuedData = newData;
-                    } else {
-                        queuedData = newData + queuedData.substring(queueIndex);
-                    }
-                    queueIndex = 0;
-                    return beginToken;
+                    return getFirstCharacterFromReadBuffer(); // end of stream. deliver remaining characters from buffer.
                 }
             }
         }
-        return ch;
+    }
+
+    /**
+     * @return the first character from the read buffer or -1 if read buffer is empty.
+     */
+    private int getFirstCharacterFromReadBuffer() {
+        if (readBuffer.length() > 0) {
+            int chr = readBuffer.charAt(0);
+            readBuffer = readBuffer.substring(1);
+            return chr;
+        } else {
+            return -1;
+        }
     }
 
     /**
@@ -189,7 +180,7 @@ public final class ReplaceTokens
      *
      * @param beginToken the character used to denote the beginning of a token
      */
-    public void setBeginToken(final char beginToken) {
+    public void setBeginToken(final String beginToken) {
         this.beginToken = beginToken;
     }
 
@@ -198,7 +189,7 @@ public final class ReplaceTokens
      *
      * @return the character used to denote the beginning of a token
      */
-    private char getBeginToken() {
+    private String getBeginToken() {
         return beginToken;
     }
 
@@ -207,7 +198,7 @@ public final class ReplaceTokens
      *
      * @param endToken the character used to denote the end of a token
      */
-    public void setEndToken(final char endToken) {
+    public void setEndToken(final String endToken) {
         this.endToken = endToken;
     }
 
@@ -216,7 +207,7 @@ public final class ReplaceTokens
      *
      * @return the character used to denote the end of a token
      */
-    private char getEndToken() {
+    private String getEndToken() {
         return endToken;
     }
 
@@ -238,18 +229,19 @@ public final class ReplaceTokens
      */
     public void addConfiguredToken(final Token token) {
         hash.put(token.getKey(), token.getValue());
+        resolvedTokensBuilt = false; // invalidate to build them again if they have been built already.
     }
 
     /**
      * Returns properties from a specified properties file.
      *
-     * @param fileName The file to load properties from.
+     * @param resource The resource to load properties from.
      */
-    private Properties getProperties(Resource r) {
+    private Properties getProperties(Resource resource) {
         InputStream in = null;
         Properties props = new Properties();
         try {
-            in = r.getInputStream();
+            in = resource.getInputStream();
             props.load(in);
         } catch (IOException ioe) {
             ioe.printStackTrace();
@@ -305,32 +297,23 @@ public final class ReplaceTokens
     private void initialize() {
         Parameter[] params = getParameters();
         if (params != null) {
-            for (int i = 0; i < params.length; i++) {
-                if (params[i] != null) {
-                    final String type = params[i].getType();
+            for (Parameter param : params) {
+                if (param != null) {
+                    final String type = param.getType();
                     if ("tokenchar".equals(type)) {
-                        final String name = params[i].getName();
-                        String value = params[i].getValue();
+                        final String name = param.getName();
                         if ("begintoken".equals(name)) {
-                            if (value.length() == 0) {
-                                throw new BuildException("Begin token cannot "
-                                    + "be empty");
-                            }
-                            beginToken = params[i].getValue().charAt(0);
+                            beginToken = param.getValue();
                         } else if ("endtoken".equals(name)) {
-                            if (value.length() == 0) {
-                                throw new BuildException("End token cannot "
-                                    + "be empty");
-                            }
-                            endToken = params[i].getValue().charAt(0);
+                            endToken = param.getValue();
                         }
                     } else if ("token".equals(type)) {
-                        final String name = params[i].getName();
-                        final String value = params[i].getValue();
+                        final String name = param.getName();
+                        final String value = param.getValue();
                         hash.put(name, value);
                     } else if ("propertiesfile".equals(type)) {
                         makeTokensFromProperties(
-                            new FileResource(new File(params[i].getValue())));
+                                new FileResource(new File(param.getValue())));
                     }
                 }
             }
diff --git a/src/tests/junit/org/apache/tools/ant/filters/ReplaceTokensTest.java b/src/tests/junit/org/apache/tools/ant/filters/ReplaceTokensTest.java
index c58dcc6d5..4db8d7a01 100644
--- a/src/tests/junit/org/apache/tools/ant/filters/ReplaceTokensTest.java
+++ b/src/tests/junit/org/apache/tools/ant/filters/ReplaceTokensTest.java
@@ -31,7 +31,6 @@ import static org.junit.Assert.assertEquals;
 
 public class ReplaceTokensTest {
 
-
     @Rule
     public BuildFileRule buildRule = new BuildFileRule();
 
@@ -45,7 +44,7 @@ public class ReplaceTokensTest {
         buildRule.executeTarget("testReplaceTokens");
         File expected = buildRule.getProject().resolveFile("expected/replacetokens.test");
         File result = new File(buildRule.getProject().getProperty("output"), "replacetokens.test");
-       assertEquals(FileUtilities.getFileContents(expected), FileUtilities.getFileContents(result));
+        assertEquals(FileUtilities.getFileContents(expected), FileUtilities.getFileContents(result));
     }
 
     @Test
@@ -56,4 +55,27 @@ public class ReplaceTokensTest {
         assertEquals(FileUtilities.getFileContents(expected), FileUtilities.getFileContents(result));
     }
 
+    @Test
+    public void testReplaceTokensDoubleEncoded() throws IOException {
+        buildRule.executeTarget("testReplaceTokensDoubleEncoded");
+        File expected = buildRule.getProject().resolveFile("expected/replacetokens.double.test");
+        File result = new File(buildRule.getProject().getProperty("output"), "replacetokens.double.test");
+        assertEquals(FileUtilities.getFileContents(expected), FileUtilities.getFileContents(result));
+    }
+
+    @Test
+    public void testReplaceTokensDoubleEncodedToSimple() throws IOException {
+        buildRule.executeTarget("testReplaceTokensDoubleEncodedToSimple");
+        File expected = buildRule.getProject().resolveFile("expected/replacetokens.test");
+        File result = new File(buildRule.getProject().getProperty("output"), "replacetokens.double.test");
+        assertEquals(FileUtilities.getFileContents(expected), FileUtilities.getFileContents(result));
+    }
+
+    @Test
+    public void testReplaceTokensMustacheStyle() throws IOException {
+        buildRule.executeTarget("testReplaceTokensMustacheStyle");
+        File expected = buildRule.getProject().resolveFile("expected/replacetokens.test");
+        File result = new File(buildRule.getProject().getProperty("output"), "replacetokens.mustache.test");
+        assertEquals(FileUtilities.getFileContents(expected), FileUtilities.getFileContents(result));
+    }
 }