Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/ant

CONTRIBUTORS

@@ -182,6 +182,7 @@ Jesse Glick
 Jesse Stockall
 Jim Allers
 Joerg Wassmer
+Joel Tucci
 Joey Richey
 Johann Herunter
 John Elion
@@ -299,6 +300,7 @@ Pierre Delisle
 Pierre Dittgen
 riasol
 R Handerson
+Ralf Hergert
 Rami Ojares
 Randy Watler
 Raphael Pierquin

ReleaseInstructions

@@ -173,9 +173,6 @@ Note: This document was updated in the context of releasing Ant 1.9.3.
     days pass and there are no major problems, a wider announcement is
     made (ant website, announce@apache.org, etc).
 
-    Announce beta releases at freecode.com (Stefan Bodewig is the
-    owner of Ant's project entry - bug him ;-).
-
 17. As problems in the beta are discovered, there may be a need for
     one or more subsequent betas. The release manager makes this
     call. Each time, the versions are updated and the above process is
@@ -256,9 +253,6 @@ Note: This document was updated in the context of releasing Ant 1.9.3.
     Apache mailing lists that should get the announcements:
     announce@apache.org, dev@ant and user@ant.
 
-    Announce release at freecode.com
-    (Stefan Bodewig is the owner of Ant's project entry - bug him ;-).
-
 25. Add a new release tag to doap_Ant.rdf in Ant's site.
 
 26. You can now reacquaint yourself with your family and friends.

WHATSNEW

@@ -4,9 +4,29 @@ Changes from Ant 1.9.4 TO Ant 1.9.5
 Changes that could break older environments:
 -------------------------------------------
 
+ * The ReplaceTokens filter can now use token-separators longer than
+   one character.  This means it can be used to replace mustache-style
+   {{patterns}} and similar templates.  This is going to break code
+   that invokes the setters on ReplaceTokens via the Java API as their
+   parameters have been changed from char to String.  It may also
+   break build files that specified multi character tokens and relied
+   on Ant silently ignoring all but the first character.
+   Bugzilla Report 56584
+
 Fixed bugs:
 -----------
 
+ * TarArchiveInputStream failed to read archives with empty gid/uid
+   fields.
+   Bugzilla Report 56641
+
+ * TarArchiveInputStream could throw IOException when reading PAX
+   headers from a "slow" InputStream.
+
+ * XMLJunitResultFormatter could throw NullPointerException if Java
+   cannot determine the local hostname.
+   Bugzilla Report 56593
+
 Other changes:
 --------------
 

contributors.xml

@@ -751,6 +751,10 @@
     <first>Joerg</first>
     <last>Wassmer</last>
   </name>
+  <name>
+    <first>Joel</first>
+    <last>Tucci</last>
+  </name>
   <name>
     <first>Joey</first>
     <last>Richey</last>
@@ -1209,6 +1213,10 @@
     <first>R</first>
     <last>Handerson</last>
   </name>
+  <name>
+    <first>Ralf</first>
+    <last>Hergert</last>
+  </name>
   <name>
     <first>Rami</first>
     <last>Ojares</last>

manual/Types/filterchain.html

@@ -551,14 +551,14 @@ user defined values.
   <tr>
     <td vAlign=top>tokenchar</td>
     <td vAlign=top>begintoken</td>
-    <td vAlign=top>Character marking the
+    <td vAlign=top>String marking the
     beginning of a token.  Defaults to @</td>
     <td vAlign=top align="center">No</td>
   </tr>
   <tr>
     <td vAlign=top>tokenchar</td>
     <td vAlign=top>endtoken</td>
-    <td vAlign=top>Character marking the
+    <td vAlign=top>String marking the
     end of a token.  Defaults to @</td>
     <td vAlign=top align="center">No</td>
   </tr>
@@ -626,6 +626,31 @@ Convenience method:
 &lt;/loadfile&gt;
 </pre></blockquote>
 
+This replaces occurrences of the string {{DATE}} in the data
+with today's date and stores it in the property ${src.file.replaced}.
+<blockquote><pre>
+&lt;loadfile srcfile=&quot;${src.file}&quot; property=&quot;${src.file.replaced}&quot;&gt;
+  &lt;filterchain&gt;
+    &lt;filterreader classname=&quot;org.apache.tools.ant.filters.ReplaceTokens&quot;&gt;
+      &lt;param type=&quot;tokenchar&quot; name=&quot;begintoken&quot; value=&quot;{{&quot;/&gt;
+      &lt;param type=&quot;tokenchar&quot; name=&quot;endtoken&quot; value=&quot;}}&quot;/&gt;
+    &lt;/filterreader&gt;
+  &lt;/filterchain&gt;
+&lt;/loadfile&gt;
+</pre></blockquote>
+
+Convenience method:
+<blockquote><pre>
+&lt;tstamp/&gt;
+&lt;loadfile srcfile=&quot;${src.file}&quot; property=&quot;${src.file.replaced}&quot;&gt;
+  &lt;filterchain&gt;
+    &lt;replacetokens begintoken=&quot;{{&quot; endtoken=&quot;}}&quot;&gt;
+      &lt;token key=&quot;DATE&quot; value=&quot;${TODAY}&quot;/&gt;
+    &lt;/replacetokens&gt;
+  &lt;/filterchain&gt;
+&lt;/loadfile&gt;
+</pre></blockquote>
+
 This will treat each properties file entry in sample.properties as a token/key pair :
 <blockquote><pre>
 &lt;loadfile srcfile=&quot;${src.file}&quot; property=&quot;${src.file.replaced}&quot;&gt;

src/etc/testcases/filters/build.xml

@@ -100,6 +100,39 @@
     </copy>
   </target>
 
+  <target name="testReplaceTokensDoubleEncoded" depends="setUp">
+    <copy todir="${output}">
+      <fileset dir="input" includes="replacetokens.double.test" />
+      <filterchain>
+        <replacetokens>
+          <token key="foo" value=""/>
+        </replacetokens>
+      </filterchain>
+    </copy>
+  </target>
+
+  <target name="testReplaceTokensDoubleEncodedToSimple" depends="setUp">
+    <copy todir="${output}">
+      <fileset dir="input" includes="replacetokens.double.test" />
+      <filterchain>
+        <replacetokens begintoken="@@" endtoken="@@">
+          <token key="foo" value=""/>
+        </replacetokens>
+      </filterchain>
+    </copy>
+  </target>
+
+  <target name="testReplaceTokensMustacheStyle" depends="setUp">
+    <copy todir="${output}">
+      <fileset dir="input" includes="replacetokens.mustache.test" />
+      <filterchain>
+        <replacetokens begintoken="{{" endtoken="}}">
+          <token key="foo" value=""/>
+        </replacetokens>
+      </filterchain>
+    </copy>
+  </target>
+
   <target name="testNoAddNewLine" depends="setUp">
     <concat destfile="${output}/nonl">This has no new lines</concat>
     <copy file="${output}/nonl" tofile="${output}/nonl-copyfilter">

src/etc/testcases/filters/expected/replacetokens.double.test

@@ -0,0 +1,2 @@
+1@@2
+3

src/etc/testcases/filters/input/replacetokens.double.test

@@ -0,0 +1,2 @@
+1@@foo@@2
+3

src/etc/testcases/filters/input/replacetokens.mustache.test

@@ -0,0 +1,2 @@
+1{{foo}}2
+3

src/main/org/apache/tools/ant/filters/ReplaceTokens.java

@@ -24,7 +24,9 @@ import java.io.Reader;
 import java.util.Enumeration;
 import java.util.Hashtable;
 import java.util.Properties;
-import org.apache.tools.ant.BuildException;
+import java.util.SortedMap;
+import java.util.TreeMap;
+
 import org.apache.tools.ant.types.Parameter;
 import org.apache.tools.ant.types.Resource;
 import org.apache.tools.ant.types.resources.FileResource;
@@ -52,13 +54,19 @@ public final class ReplaceTokens
     extends BaseParamFilterReader
     implements ChainableReader {
     /** Default "begin token" character. */
-    private static final char DEFAULT_BEGIN_TOKEN = '@';
+    private static final String DEFAULT_BEGIN_TOKEN = "@";
 
     /** Default "end token" character. */
-    private static final char DEFAULT_END_TOKEN = '@';
+    private static final String DEFAULT_END_TOKEN = "@";
+
+    /** Hashtable to holds the original replacee-replacer pairs (String to String). */
+    private Hashtable<String, String> hash = new Hashtable<String, String>();
 
-    /** Data to be used before reading from stream again */
-    private String queuedData = null;
+    /** This map holds the "resolved" tokens (begin- and end-tokens are added to make searching simpler) */
+    private final TreeMap<String, String> resolvedTokens = new TreeMap<String, String>();
+    private boolean resolvedTokensBuilt = false;
+    /** Used for comparisons and lookup into the resolvedTokens map. */
+    private String readBuffer = "";
 
     /** replacement test from a token */
     private String replaceData = null;
@@ -66,26 +74,18 @@ public final class ReplaceTokens
     /** Index into replacement data */
     private int replaceIndex = -1;
 
-    /** Index into queue data */
-    private int queueIndex = -1;
-
-    /** Hashtable to hold the replacee-replacer pairs (String to String). */
-    private Hashtable<String, String> hash = new Hashtable<String, String>();
-
     /** Character marking the beginning of a token. */
-    private char beginToken = DEFAULT_BEGIN_TOKEN;
+    private String beginToken = DEFAULT_BEGIN_TOKEN;
 
     /** Character marking the end of a token. */
-    private char endToken = DEFAULT_END_TOKEN;
+    private String endToken = DEFAULT_END_TOKEN;
 
     /**
      * Constructor for "dummy" instances.
      *
      * @see BaseFilterReader#BaseFilterReader()
      */
-    public ReplaceTokens() {
-        super();
-    }
+    public ReplaceTokens() {}
 
     /**
      * Creates a new filtered reader.
@@ -97,18 +97,6 @@ public final class ReplaceTokens
         super(in);
     }
 
-    private int getNextChar() throws IOException {
-        if (queueIndex != -1) {
-            final int ch = queuedData.charAt(queueIndex++);
-            if (queueIndex >= queuedData.length()) {
-                queueIndex = -1;
-            }
-            return ch;
-        }
-
-        return in.read();
-    }
-
     /**
      * Returns the next character in the filtered stream, replacing tokens
      * from the original stream.
@@ -125,63 +113,66 @@ public final class ReplaceTokens
             setInitialized(true);
         }
 
-        if (replaceIndex != -1) {
-            final int ch = replaceData.charAt(replaceIndex++);
-            if (replaceIndex >= replaceData.length()) {
-                replaceIndex = -1;
+        if (!resolvedTokensBuilt) {
+            // build the resolved tokens tree map.
+            for (String key : hash.keySet()) {
+                resolvedTokens.put(beginToken + key + endToken, hash.get(key));
             }
-            return ch;
+            resolvedTokensBuilt = true;
         }
 
-        int ch = getNextChar();
-
-        if (ch == beginToken) {
-            final StringBuffer key = new StringBuffer("");
-            do  {
-                ch = getNextChar();
-                if (ch != -1) {
-                    key.append((char) ch);
-                } else {
-                    break;
-                }
-            } while (ch != endToken);
-
-            if (ch == -1) {
-                if (queuedData == null || queueIndex == -1) {
-                    queuedData = key.toString();
-                } else {
-                    queuedData
-                        = key.toString() + queuedData.substring(queueIndex);
-                }
-                if (queuedData.length() > 0) {
-                    queueIndex = 0;
-                } else {
-                    queueIndex = -1;
-                }
-                return beginToken;
+        // are we currently serving replace data?
+        if (replaceData != null) {
+            if (replaceIndex < replaceData.length()) {
+                return replaceData.charAt(replaceIndex++);
             } else {
-                key.setLength(key.length() - 1);
+                replaceData = null;
+            }
+        }
 
-                final String replaceWith = (String) hash.get(key.toString());
-                if (replaceWith != null) {
-                    if (replaceWith.length() > 0) {
-                        replaceData = replaceWith;
-                        replaceIndex = 0;
-                    }
-                    return read();
+        // is the read buffer empty?
+        if (readBuffer.length() == 0) {
+            int next = in.read();
+            if (next == -1) {
+                return next; // end of stream. all buffers empty.
+            }
+            readBuffer += (char)next;
+        }
+
+        for (;;) {
+            // get the closest tokens
+            SortedMap<String,String> possibleTokens = resolvedTokens.tailMap(readBuffer);
+            if (possibleTokens.isEmpty() || !possibleTokens.firstKey().startsWith(readBuffer)) { // if there is none, then deliver the first char from the buffer.
+                return getFirstCharacterFromReadBuffer();
+            } else if (readBuffer.equals(possibleTokens.firstKey())) { // there exists a nearest token - is it an exact match?
+                // we have found a token. prepare the replaceData buffer.
+                replaceData = resolvedTokens.get(readBuffer);
+                replaceIndex = 0;
+                readBuffer = ""; // destroy the readBuffer - it's contents are being replaced entirely.
+                // get the first character via recursive call.
+                return read();
+            } else { // nearest token is not matching exactly - read one character more.
+                int next = in.read();
+                if (next != -1) {
+                    readBuffer += (char)next;
                 } else {
-                    String newData = key.toString() + endToken;
-                    if (queuedData == null || queueIndex == -1) {
-                        queuedData = newData;
-                    } else {
-                        queuedData = newData + queuedData.substring(queueIndex);
-                    }
-                    queueIndex = 0;
-                    return beginToken;
+                    return getFirstCharacterFromReadBuffer(); // end of stream. deliver remaining characters from buffer.
                 }
             }
         }
-        return ch;
+    }
+
+    /**
+     * @return the first character from the read buffer or -1 if read buffer is empty.
+     */
+    private int getFirstCharacterFromReadBuffer() {
+        if (readBuffer.length() > 0) {
+            int chr = readBuffer.charAt(0);
+            readBuffer = readBuffer.substring(1);
+            return chr;
+        } else {
+            return -1;
+        }
     }
 
     /**
@@ -189,7 +180,7 @@ public final class ReplaceTokens
      *
      * @param beginToken the character used to denote the beginning of a token
      */
-    public void setBeginToken(final char beginToken) {
+    public void setBeginToken(final String beginToken) {
         this.beginToken = beginToken;
     }
 
@@ -198,7 +189,7 @@ public final class ReplaceTokens
      *
      * @return the character used to denote the beginning of a token
      */
-    private char getBeginToken() {
+    private String getBeginToken() {
         return beginToken;
     }
 
@@ -207,7 +198,7 @@ public final class ReplaceTokens
      *
      * @param endToken the character used to denote the end of a token
      */
-    public void setEndToken(final char endToken) {
+    public void setEndToken(final String endToken) {
         this.endToken = endToken;
     }
 
@@ -216,7 +207,7 @@ public final class ReplaceTokens
      *
      * @return the character used to denote the end of a token
      */
-    private char getEndToken() {
+    private String getEndToken() {
         return endToken;
     }
 
@@ -238,18 +229,19 @@ public final class ReplaceTokens
      */
     public void addConfiguredToken(final Token token) {
         hash.put(token.getKey(), token.getValue());
+        resolvedTokensBuilt = false; // invalidate to build them again if they have been built already.
     }
 
     /**
      * Returns properties from a specified properties file.
      *
-     * @param fileName The file to load properties from.
+     * @param resource The resource to load properties from.
      */
-    private Properties getProperties(Resource r) {
+    private Properties getProperties(Resource resource) {
         InputStream in = null;
         Properties props = new Properties();
         try {
-            in = r.getInputStream();
+            in = resource.getInputStream();
             props.load(in);
         } catch (IOException ioe) {
             ioe.printStackTrace();
@@ -305,32 +297,23 @@ public final class ReplaceTokens
     private void initialize() {
         Parameter[] params = getParameters();
         if (params != null) {
-            for (int i = 0; i < params.length; i++) {
-                if (params[i] != null) {
-                    final String type = params[i].getType();
+            for (Parameter param : params) {
+                if (param != null) {
+                    final String type = param.getType();
                     if ("tokenchar".equals(type)) {
-                        final String name = params[i].getName();
-                        String value = params[i].getValue();
+                        final String name = param.getName();
                         if ("begintoken".equals(name)) {
-                            if (value.length() == 0) {
-                                throw new BuildException("Begin token cannot "
-                                    + "be empty");
-                            }
-                            beginToken = params[i].getValue().charAt(0);
+                            beginToken = param.getValue();
                         } else if ("endtoken".equals(name)) {
-                            if (value.length() == 0) {
-                                throw new BuildException("End token cannot "
-                                    + "be empty");
-                            }
-                            endToken = params[i].getValue().charAt(0);
+                            endToken = param.getValue();
                         }
                     } else if ("token".equals(type)) {
-                        final String name = params[i].getName();
-                        final String value = params[i].getValue();
+                        final String name = param.getName();
+                        final String value = param.getValue();
                         hash.put(name, value);
                     } else if ("propertiesfile".equals(type)) {
                         makeTokensFromProperties(
-                            new FileResource(new File(params[i].getValue())));
+                                new FileResource(new File(param.getValue())));
                     }
                 }
             }

src/main/org/apache/tools/ant/taskdefs/optional/junit/XMLJUnitResultFormatter.java

@@ -161,11 +161,15 @@ public class XMLJUnitResultFormatter implements JUnitResultFormatter, XMLConstan
      * @return the name of the local host, or "localhost" if we cannot work it out
      */
     private String getHostname()  {
+        String hostname = "localhost";
         try {
-            return InetAddress.getLocalHost().getHostName();
+            InetAddress localHost = InetAddress.getLocalHost();
+            if (localHost != null) {
+                hostname = localHost.getHostName();
+            }
         } catch (UnknownHostException e) {
-            return "localhost";
         }
+        return hostname;
     }
 
     /**

src/main/org/apache/tools/tar/TarInputStream.java

@@ -431,18 +431,22 @@ public class TarInputStream extends FilterInputStream {
                         if (ch == '='){ // end of keyword
                             String keyword = coll.toString("UTF-8");
                             // Get rest of entry
-                            byte[] rest = new byte[len - read];
-                            int got = i.read(rest);
-                            if (got != len - read){
+                            final int restLen = len - read;
+                            byte[] rest = new byte[restLen];
+                            int got = 0;
+                            while (got < restLen && (ch = i.read()) != -1) {
+                                rest[got++] = (byte) ch;
+                            }
+                            if (got != restLen) {
                                 throw new IOException("Failed to read "
                                                       + "Paxheader. Expected "
-                                                      + (len - read)
+                                                      + restLen
                                                       + " bytes, read "
                                                       + got);
                             }
                             // Drop trailing NL
                             String value = new String(rest, 0,
-                                                      len - read - 1, "UTF-8");
+                                                      restLen - 1, "UTF-8");
                             headers.put(keyword, value);
                             break;
                         }

src/main/org/apache/tools/tar/TarUtils.java

@@ -60,7 +60,7 @@ public class TarUtils {
 
             public String decode(byte[] buffer) {
                 final int length = buffer.length;
-                StringBuffer result = new StringBuffer(length);
+                StringBuilder result = new StringBuilder(length);
 
                 for (int i = 0; i < length; ++i) {
                     byte b = buffer[i];
@@ -130,10 +130,6 @@ public class TarUtils {
             end--;
             trailer = buffer[end - 1];
         }
-        if (start == end) {
-            throw new IllegalArgumentException(
-                    exceptionMessage(buffer, offset, length, start, trailer));
-        }
 
         for ( ;start < end; start++) {
             final byte currentByte = buffer[start];
@@ -194,7 +190,7 @@ public class TarUtils {
         if (negative) {
             // 2's complement
             val--;
-            val ^= ((long) Math.pow(2, (length - 1) * 8) - 1);
+            val ^= (long) Math.pow(2, (length - 1) * 8) - 1;
         }
         return negative ? -val : val;
     }
@@ -236,7 +232,15 @@ public class TarUtils {
     // Helper method to generate the exception message
     private static String exceptionMessage(byte[] buffer, final int offset,
             final int length, int current, final byte currentByte) {
-        String string = new String(buffer, offset, length); // TODO default charset?
+        // default charset is good enough for an exception message,
+        //
+        // the alternative was to modify parseOctal and
+        // parseOctalOrBinary to receive the ZipEncoding of the
+        // archive (deprecating the existing public methods, of
+        // course) and dealing with the fact that ZipEncoding#decode
+        // can throw an IOException which parseOctal* doesn't declare
+        String string = new String(buffer, offset, length);
+
         string=string.replaceAll("\0", "{NUL}"); // Replace NULs to allow string to be printed
         final String s = "Invalid byte "+currentByte+" at offset "+(current-offset)+" in '"+string+"' len="+length;
         return s;
@@ -549,8 +553,8 @@ public class TarUtils {
     public static long computeCheckSum(final byte[] buf) {
         long sum = 0;
 
-        for (int i = 0; i < buf.length; ++i) {
-            sum += BYTE_MASK & buf[i];
+        for (byte element : buf) {
+            sum += BYTE_MASK & element;
         }
 
         return sum;

src/tests/junit/org/apache/tools/ant/filters/ReplaceTokensTest.java

@@ -31,7 +31,6 @@ import static org.junit.Assert.assertEquals;
 
 public class ReplaceTokensTest {
 
-
     @Rule
     public BuildFileRule buildRule = new BuildFileRule();
 
@@ -45,7 +44,7 @@ public class ReplaceTokensTest {
         buildRule.executeTarget("testReplaceTokens");
         File expected = buildRule.getProject().resolveFile("expected/replacetokens.test");
         File result = new File(buildRule.getProject().getProperty("output"), "replacetokens.test");
-       assertEquals(FileUtilities.getFileContents(expected), FileUtilities.getFileContents(result));
+        assertEquals(FileUtilities.getFileContents(expected), FileUtilities.getFileContents(result));
     }
 
     @Test
@@ -56,4 +55,27 @@ public class ReplaceTokensTest {
         assertEquals(FileUtilities.getFileContents(expected), FileUtilities.getFileContents(result));
     }
 
+    @Test
+    public void testReplaceTokensDoubleEncoded() throws IOException {
+        buildRule.executeTarget("testReplaceTokensDoubleEncoded");
+        File expected = buildRule.getProject().resolveFile("expected/replacetokens.double.test");
+        File result = new File(buildRule.getProject().getProperty("output"), "replacetokens.double.test");
+        assertEquals(FileUtilities.getFileContents(expected), FileUtilities.getFileContents(result));
+    }
+
+    @Test
+    public void testReplaceTokensDoubleEncodedToSimple() throws IOException {
+        buildRule.executeTarget("testReplaceTokensDoubleEncodedToSimple");
+        File expected = buildRule.getProject().resolveFile("expected/replacetokens.test");
+        File result = new File(buildRule.getProject().getProperty("output"), "replacetokens.double.test");
+        assertEquals(FileUtilities.getFileContents(expected), FileUtilities.getFileContents(result));
+    }
+
+    @Test
+    public void testReplaceTokensMustacheStyle() throws IOException {
+        buildRule.executeTarget("testReplaceTokensMustacheStyle");
+        File expected = buildRule.getProject().resolveFile("expected/replacetokens.test");
+        File result = new File(buildRule.getProject().getProperty("output"), "replacetokens.mustache.test");
+        assertEquals(FileUtilities.getFileContents(expected), FileUtilities.getFileContents(result));
+    }
 }