PR 56584 allow ReplaceTokens filter to use multi-character token separators. Submitted by Ralf Hergert.

CONTRIBUTORS

@@ -300,6 +300,7 @@ Pierre Delisle
 Pierre Dittgen
 riasol
 R Handerson
+Ralf Hergert
 Rami Ojares
 Randy Watler
 Raphael Pierquin

WHATSNEW

@@ -4,6 +4,15 @@ Changes from Ant 1.9.4 TO Ant 1.9.5
 Changes that could break older environments:
 -------------------------------------------
 
+ * The ReplaceTokens filter can now use token-separators longer than
+   one character.  This means it can be used to replace mustache-style
+   {{patterns}} and similar templates.  This is going to break code
+   that invokes the setters on ReplaceTokens via the Java API as their
+   parameters have been changed from char to String.  It may also
+   break build files that specified multi character tokens and relied
+   on Ant silently ignoring all but the first character.
+   Bugzilla Report 56584
+
 Fixed bugs:
 -----------
 

contributors.xml

@@ -1213,6 +1213,10 @@
     <first>R</first>
     <last>Handerson</last>
   </name>
+  <name>
+    <first>Ralf</first>
+    <last>Hergert</last>
+  </name>
   <name>
     <first>Rami</first>
     <last>Ojares</last>

manual/Types/filterchain.html

@@ -551,14 +551,14 @@ user defined values.
   <tr>
     <td vAlign=top>tokenchar</td>
     <td vAlign=top>begintoken</td>
-    <td vAlign=top>Character marking the
+    <td vAlign=top>String marking the
     beginning of a token.  Defaults to @</td>
     <td vAlign=top align="center">No</td>
   </tr>
   <tr>
     <td vAlign=top>tokenchar</td>
     <td vAlign=top>endtoken</td>
-    <td vAlign=top>Character marking the
+    <td vAlign=top>String marking the
     end of a token.  Defaults to @</td>
     <td vAlign=top align="center">No</td>
   </tr>
@@ -626,6 +626,31 @@ Convenience method:
 &lt;/loadfile&gt;
 </pre></blockquote>
 
+This replaces occurrences of the string {{DATE}} in the data
+with today's date and stores it in the property ${src.file.replaced}.
+<blockquote><pre>
+&lt;loadfile srcfile=&quot;${src.file}&quot; property=&quot;${src.file.replaced}&quot;&gt;
+  &lt;filterchain&gt;
+    &lt;filterreader classname=&quot;org.apache.tools.ant.filters.ReplaceTokens&quot;&gt;
+      &lt;param type=&quot;tokenchar&quot; name=&quot;begintoken&quot; value=&quot;{{&quot;/&gt;
+      &lt;param type=&quot;tokenchar&quot; name=&quot;endtoken&quot; value=&quot;}}&quot;/&gt;
+    &lt;/filterreader&gt;
+  &lt;/filterchain&gt;
+&lt;/loadfile&gt;
+</pre></blockquote>
+
+Convenience method:
+<blockquote><pre>
+&lt;tstamp/&gt;
+&lt;loadfile srcfile=&quot;${src.file}&quot; property=&quot;${src.file.replaced}&quot;&gt;
+  &lt;filterchain&gt;
+    &lt;replacetokens begintoken=&quot;{{&quot; endtoken=&quot;}}&quot;&gt;
+      &lt;token key=&quot;DATE&quot; value=&quot;${TODAY}&quot;/&gt;
+    &lt;/replacetokens&gt;
+  &lt;/filterchain&gt;
+&lt;/loadfile&gt;
+</pre></blockquote>
+
 This will treat each properties file entry in sample.properties as a token/key pair :
 <blockquote><pre>
 &lt;loadfile srcfile=&quot;${src.file}&quot; property=&quot;${src.file.replaced}&quot;&gt;

src/etc/testcases/filters/build.xml

@@ -100,6 +100,39 @@
     </copy>
   </target>
 
+  <target name="testReplaceTokensDoubleEncoded" depends="setUp">
+    <copy todir="${output}">
+      <fileset dir="input" includes="replacetokens.double.test" />
+      <filterchain>
+        <replacetokens>
+          <token key="foo" value=""/>
+        </replacetokens>
+      </filterchain>
+    </copy>
+  </target>
+
+  <target name="testReplaceTokensDoubleEncodedToSimple" depends="setUp">
+    <copy todir="${output}">
+      <fileset dir="input" includes="replacetokens.double.test" />
+      <filterchain>
+        <replacetokens begintoken="@@" endtoken="@@">
+          <token key="foo" value=""/>
+        </replacetokens>
+      </filterchain>
+    </copy>
+  </target>
+
+  <target name="testReplaceTokensMustacheStyle" depends="setUp">
+    <copy todir="${output}">
+      <fileset dir="input" includes="replacetokens.mustache.test" />
+      <filterchain>
+        <replacetokens begintoken="{{" endtoken="}}">
+          <token key="foo" value=""/>
+        </replacetokens>
+      </filterchain>
+    </copy>
+  </target>
+
   <target name="testNoAddNewLine" depends="setUp">
     <concat destfile="${output}/nonl">This has no new lines</concat>
     <copy file="${output}/nonl" tofile="${output}/nonl-copyfilter">

src/etc/testcases/filters/expected/replacetokens.double.test

@@ -0,0 +1,2 @@
+1@@2
+3

src/etc/testcases/filters/input/replacetokens.double.test

@@ -0,0 +1,2 @@
+1@@foo@@2
+3

src/etc/testcases/filters/input/replacetokens.mustache.test

@@ -0,0 +1,2 @@
+1{{foo}}2
+3

src/main/org/apache/tools/ant/filters/ReplaceTokens.java

@@ -24,7 +24,9 @@ import java.io.Reader;
 import java.util.Enumeration;
 import java.util.Hashtable;
 import java.util.Properties;
-import org.apache.tools.ant.BuildException;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
 import org.apache.tools.ant.types.Parameter;
 import org.apache.tools.ant.types.Resource;
 import org.apache.tools.ant.types.resources.FileResource;
@@ -52,13 +54,19 @@ public final class ReplaceTokens
     extends BaseParamFilterReader
     implements ChainableReader {
     /** Default "begin token" character. */
-    private static final char DEFAULT_BEGIN_TOKEN = '@';
+    private static final String DEFAULT_BEGIN_TOKEN = "@";
 
     /** Default "end token" character. */
-    private static final char DEFAULT_END_TOKEN = '@';
+    private static final String DEFAULT_END_TOKEN = "@";
+
+    /** Hashtable to holds the original replacee-replacer pairs (String to String). */
+    private Hashtable<String, String> hash = new Hashtable<String, String>();
 
-    /** Data to be used before reading from stream again */
-    private String queuedData = null;
+    /** This map holds the "resolved" tokens (begin- and end-tokens are added to make searching simpler) */
+    private final TreeMap<String, String> resolvedTokens = new TreeMap<String, String>();
+    private boolean resolvedTokensBuilt = false;
+    /** Used for comparisons and lookup into the resolvedTokens map. */
+    private String readBuffer = "";
 
     /** replacement test from a token */
     private String replaceData = null;
@@ -66,26 +74,18 @@ public final class ReplaceTokens
     /** Index into replacement data */
     private int replaceIndex = -1;
 
-    /** Index into queue data */
-    private int queueIndex = -1;
-
-    /** Hashtable to hold the replacee-replacer pairs (String to String). */
-    private Hashtable<String, String> hash = new Hashtable<String, String>();
-
     /** Character marking the beginning of a token. */
-    private char beginToken = DEFAULT_BEGIN_TOKEN;
+    private String beginToken = DEFAULT_BEGIN_TOKEN;
 
     /** Character marking the end of a token. */
-    private char endToken = DEFAULT_END_TOKEN;
+    private String endToken = DEFAULT_END_TOKEN;
 
     /**
      * Constructor for "dummy" instances.
      *
      * @see BaseFilterReader#BaseFilterReader()
      */
-    public ReplaceTokens() {
-        super();
-    }
+    public ReplaceTokens() {}
 
     /**
      * Creates a new filtered reader.
@@ -97,18 +97,6 @@ public final class ReplaceTokens
         super(in);
     }
 
-    private int getNextChar() throws IOException {
-        if (queueIndex != -1) {
-            final int ch = queuedData.charAt(queueIndex++);
-            if (queueIndex >= queuedData.length()) {
-                queueIndex = -1;
-            }
-            return ch;
-        }
-
-        return in.read();
-    }
-
     /**
      * Returns the next character in the filtered stream, replacing tokens
      * from the original stream.
@@ -125,63 +113,66 @@ public final class ReplaceTokens
             setInitialized(true);
         }
 
-        if (replaceIndex != -1) {
-            final int ch = replaceData.charAt(replaceIndex++);
-            if (replaceIndex >= replaceData.length()) {
-                replaceIndex = -1;
+        if (!resolvedTokensBuilt) {
+            // build the resolved tokens tree map.
+            for (String key : hash.keySet()) {
+                resolvedTokens.put(beginToken + key + endToken, hash.get(key));
             }
-            return ch;
+            resolvedTokensBuilt = true;
         }
 
-        int ch = getNextChar();
-
-        if (ch == beginToken) {
-            final StringBuffer key = new StringBuffer("");
-            do  {
-                ch = getNextChar();
-                if (ch != -1) {
-                    key.append((char) ch);
-                } else {
-                    break;
-                }
-            } while (ch != endToken);
-
-            if (ch == -1) {
-                if (queuedData == null || queueIndex == -1) {
-                    queuedData = key.toString();
-                } else {
-                    queuedData
-                        = key.toString() + queuedData.substring(queueIndex);
-                }
-                if (queuedData.length() > 0) {
-                    queueIndex = 0;
-                } else {
-                    queueIndex = -1;
-                }
-                return beginToken;
+        // are we currently serving replace data?
+        if (replaceData != null) {
+            if (replaceIndex < replaceData.length()) {
+                return replaceData.charAt(replaceIndex++);
             } else {
-                key.setLength(key.length() - 1);
+                replaceData = null;
+            }
+        }
 
-                final String replaceWith = (String) hash.get(key.toString());
-                if (replaceWith != null) {
-                    if (replaceWith.length() > 0) {
-                        replaceData = replaceWith;
-                        replaceIndex = 0;
-                    }
-                    return read();
+        // is the read buffer empty?
+        if (readBuffer.length() == 0) {
+            int next = in.read();
+            if (next == -1) {
+                return next; // end of stream. all buffers empty.
+            }
+            readBuffer += (char)next;
+        }
+
+        for (;;) {
+            // get the closest tokens
+            SortedMap<String,String> possibleTokens = resolvedTokens.tailMap(readBuffer);
+            if (possibleTokens.isEmpty() || !possibleTokens.firstKey().startsWith(readBuffer)) { // if there is none, then deliver the first char from the buffer.
+                return getFirstCharacterFromReadBuffer();
+            } else if (readBuffer.equals(possibleTokens.firstKey())) { // there exists a nearest token - is it an exact match?
+                // we have found a token. prepare the replaceData buffer.
+                replaceData = resolvedTokens.get(readBuffer);
+                replaceIndex = 0;
+                readBuffer = ""; // destroy the readBuffer - it's contents are being replaced entirely.
+                // get the first character via recursive call.
+                return read();
+            } else { // nearest token is not matching exactly - read one character more.
+                int next = in.read();
+                if (next != -1) {
+                    readBuffer += (char)next;
                 } else {
-                    String newData = key.toString() + endToken;
-                    if (queuedData == null || queueIndex == -1) {
-                        queuedData = newData;
-                    } else {
-                        queuedData = newData + queuedData.substring(queueIndex);
-                    }
-                    queueIndex = 0;
-                    return beginToken;
+                    return getFirstCharacterFromReadBuffer(); // end of stream. deliver remaining characters from buffer.
                 }
             }
         }
-        return ch;
+    }
+
+    /**
+     * @return the first character from the read buffer or -1 if read buffer is empty.
+     */
+    private int getFirstCharacterFromReadBuffer() {
+        if (readBuffer.length() > 0) {
+            int chr = readBuffer.charAt(0);
+            readBuffer = readBuffer.substring(1);
+            return chr;
+        } else {
+            return -1;
+        }
     }
 
     /**
@@ -189,7 +180,7 @@ public final class ReplaceTokens
      *
      * @param beginToken the character used to denote the beginning of a token
      */
-    public void setBeginToken(final char beginToken) {
+    public void setBeginToken(final String beginToken) {
         this.beginToken = beginToken;
     }
 
@@ -198,7 +189,7 @@ public final class ReplaceTokens
      *
      * @return the character used to denote the beginning of a token
      */
-    private char getBeginToken() {
+    private String getBeginToken() {
         return beginToken;
     }
 
@@ -207,7 +198,7 @@ public final class ReplaceTokens
      *
      * @param endToken the character used to denote the end of a token
      */
-    public void setEndToken(final char endToken) {
+    public void setEndToken(final String endToken) {
         this.endToken = endToken;
     }
 
@@ -216,7 +207,7 @@ public final class ReplaceTokens
      *
      * @return the character used to denote the end of a token
      */
-    private char getEndToken() {
+    private String getEndToken() {
         return endToken;
     }
 
@@ -238,18 +229,19 @@ public final class ReplaceTokens
      */
     public void addConfiguredToken(final Token token) {
         hash.put(token.getKey(), token.getValue());
+        resolvedTokensBuilt = false; // invalidate to build them again if they have been built already.
     }
 
     /**
      * Returns properties from a specified properties file.
      *
-     * @param fileName The file to load properties from.
+     * @param resource The resource to load properties from.
      */
-    private Properties getProperties(Resource r) {
+    private Properties getProperties(Resource resource) {
         InputStream in = null;
         Properties props = new Properties();
         try {
-            in = r.getInputStream();
+            in = resource.getInputStream();
             props.load(in);
         } catch (IOException ioe) {
             ioe.printStackTrace();
@@ -305,32 +297,23 @@ public final class ReplaceTokens
     private void initialize() {
         Parameter[] params = getParameters();
         if (params != null) {
-            for (int i = 0; i < params.length; i++) {
-                if (params[i] != null) {
-                    final String type = params[i].getType();
+            for (Parameter param : params) {
+                if (param != null) {
+                    final String type = param.getType();
                     if ("tokenchar".equals(type)) {
-                        final String name = params[i].getName();
-                        String value = params[i].getValue();
+                        final String name = param.getName();
                         if ("begintoken".equals(name)) {
-                            if (value.length() == 0) {
-                                throw new BuildException("Begin token cannot "
-                                    + "be empty");
-                            }
-                            beginToken = params[i].getValue().charAt(0);
+                            beginToken = param.getValue();
                         } else if ("endtoken".equals(name)) {
-                            if (value.length() == 0) {
-                                throw new BuildException("End token cannot "
-                                    + "be empty");
-                            }
-                            endToken = params[i].getValue().charAt(0);
+                            endToken = param.getValue();
                         }
                     } else if ("token".equals(type)) {
-                        final String name = params[i].getName();
-                        final String value = params[i].getValue();
+                        final String name = param.getName();
+                        final String value = param.getValue();
                         hash.put(name, value);
                     } else if ("propertiesfile".equals(type)) {
                         makeTokensFromProperties(
-                            new FileResource(new File(params[i].getValue())));
+                                new FileResource(new File(param.getValue())));
                     }
                 }
             }

src/tests/junit/org/apache/tools/ant/filters/ReplaceTokensTest.java

@@ -31,7 +31,6 @@ import static org.junit.Assert.assertEquals;
 
 public class ReplaceTokensTest {
 
-
     @Rule
     public BuildFileRule buildRule = new BuildFileRule();
 
@@ -45,7 +44,7 @@ public class ReplaceTokensTest {
         buildRule.executeTarget("testReplaceTokens");
         File expected = buildRule.getProject().resolveFile("expected/replacetokens.test");
         File result = new File(buildRule.getProject().getProperty("output"), "replacetokens.test");
-       assertEquals(FileUtilities.getFileContents(expected), FileUtilities.getFileContents(result));
+        assertEquals(FileUtilities.getFileContents(expected), FileUtilities.getFileContents(result));
     }
 
     @Test
@@ -56,4 +55,27 @@ public class ReplaceTokensTest {
         assertEquals(FileUtilities.getFileContents(expected), FileUtilities.getFileContents(result));
     }
 
+    @Test
+    public void testReplaceTokensDoubleEncoded() throws IOException {
+        buildRule.executeTarget("testReplaceTokensDoubleEncoded");
+        File expected = buildRule.getProject().resolveFile("expected/replacetokens.double.test");
+        File result = new File(buildRule.getProject().getProperty("output"), "replacetokens.double.test");
+        assertEquals(FileUtilities.getFileContents(expected), FileUtilities.getFileContents(result));
+    }
+
+    @Test
+    public void testReplaceTokensDoubleEncodedToSimple() throws IOException {
+        buildRule.executeTarget("testReplaceTokensDoubleEncodedToSimple");
+        File expected = buildRule.getProject().resolveFile("expected/replacetokens.test");
+        File result = new File(buildRule.getProject().getProperty("output"), "replacetokens.double.test");
+        assertEquals(FileUtilities.getFileContents(expected), FileUtilities.getFileContents(result));
+    }
+
+    @Test
+    public void testReplaceTokensMustacheStyle() throws IOException {
+        buildRule.executeTarget("testReplaceTokensMustacheStyle");
+        File expected = buildRule.getProject().resolveFile("expected/replacetokens.test");
+        File result = new File(buildRule.getProject().getProperty("output"), "replacetokens.mustache.test");
+        assertEquals(FileUtilities.getFileContents(expected), FileUtilities.getFileContents(result));
+    }
 }