New filter <escapeunicode> that translates non-ASCII characters to

\u1234 escapes.

addresses PR: 17506


git-svn-id: https://svn.apache.org/repos/asf/ant/core/trunk@274258 13f79535-47bb-0310-9956-ffa450edef68

docs/manual/CoreTypes/filterchain.html

@@ -89,6 +89,7 @@ A FilterChain is formed by defining zero or more of the following
 nested elements.<BR>
 <a href="#filterreader">FilterReader</a><BR>
 <a href="#classconstants">ClassConstants</a><BR>
+<a href="#escapeunicode">EscapeUnicode</a><BR>
 <a href="#expandproperties">ExpandProperties</a><BR>
 <a href="#headfilter">HeadFilter</a><BR>
 <a href="#linecontains">LineContains</a><BR>
@@ -162,6 +163,33 @@ Convenience method:
 &lt;/loadproperties&gt;
 </PRE></BLOCKQUOTE>
 
+<H3><a name="escapeunicode">EscapeUnicode</a></H3>
+<P>
+This filter converts its input by changing all non US-ASCII characters
+into their equivalent unicode escape backslash u plus 4 digits.</p>
+
+<P><em>since Ant 1.6</em></p>
+
+<H4>Example:</H4>
+
+This loads the basic constants defined in a Java class as Ant properties.
+<BLOCKQUOTE><PRE>
+&lt;loadproperties srcfile=&quot;non_ascii_property.properties&quot;&gt;
+  &lt;filterchain&gt;
+    &lt;filterreader classname=&quot;org.apache.tools.ant.filters.EscapeUnicode&quot;/&gt;
+  &lt;/filterchain&gt;
+&lt;/loadproperties&gt;
+</PRE></BLOCKQUOTE>
+
+Convenience method:
+<BLOCKQUOTE><PRE>
+&lt;loadproperties srcfile=&quot;non_ascii_property.properties&quot;&gt;
+  &lt;filterchain&gt;
+    &lt;escapeunicode/&gt;
+  &lt;/filterchain&gt;
+&lt;/loadproperties&gt;
+</PRE></BLOCKQUOTE>
+
 <H3><a name="expandproperties">ExpandProperties</a></H3>
 <P>
 If the data contains data that represents Ant
@@ -629,5 +657,5 @@ Convenience method:
 
 <HR>
 
-<P align=center>Copyright &copy; 2002 Apache Software Foundation. All rights
+<P align=center>Copyright &copy; 2002-2003 Apache Software Foundation. All rights
 Reserved.</P></BODY></HTML>

src/etc/testcases/filters/build.xml

@@ -6,18 +6,37 @@
   </target>
 
   <target name="cleanup">
-    <delete dir="result" />
+    <delete dir="result"/>
   </target>
 
   <target name="testLineContains" depends="init">
     <copy todir="result">
-      <fileset dir="input"/>
+      <fileset dir="input">
+        <include name="linecontains.test"/>
+      </fileset>
       <filterchain>
         <filterreader classname="org.apache.tools.ant.filters.LineContains">
           <param type="contains" value="beta"/>
         </filterreader>
       </filterchain>
     </copy>
+    <!--fixcrlf srcdir="result" eol="lf">
+      <include name="linecontains.test"/>
+    </fixcrlf-->
+  </target>
+
+  <target name="testEscapeUnicode" depends="init">
+    <copy todir="result" encoding="UTF-8">
+      <fileset dir="input">
+        <include name="escapeunicode.test"/>
+      </fileset>
+      <filterchain>
+        <escapeunicode/>
+      </filterchain>
+    </copy>
+    <fixcrlf srcdir="result" eol="crlf">
+      <include name="escapeunicode.test"/>
+    </fixcrlf>
   </target>
 
 </project>

src/etc/testcases/filters/expected/escapeunicode.test

@@ -0,0 +1,9 @@
+#hebrew shalom olam (hello world)
+text.hebrew=\u05e9\u05dc\u05d5\u05dd \u05e2\u05d5\u05dc\u05dd
+#goethe gingko biloba
+text.german.1=Sp\u00fcrst du nicht an meinen Liedern,
+text.german.2=Da\u00df ich eins und doppelt bin ?
+# Francois Villon Ballade des Pendus
+text.french=Fr\u00e8res humains qui apr\u00e8s nous vivez
+# Usual IT example
+text.basic.latin=Hello World 

src/etc/testcases/filters/input/escapeunicode.test

@@ -0,0 +1,9 @@
+#hebrew shalom olam (hello world)
+text.hebrew=שלום עולם
+#goethe gingko biloba
+text.german.1=Spürst du nicht an meinen Liedern,
+text.german.2=Daß ich eins und doppelt bin ?
+# Francois Villon Ballade des Pendus
+text.french=Frères humains qui après nous vivez
+# Usual IT example
+text.basic.latin=Hello World 

src/main/org/apache/tools/ant/filters/EscapeUnicode.java

@@ -0,0 +1,170 @@
+/*
+ * The Apache Software License, Version 1.1
+ *
+ * Copyright (c) 2003 The Apache Software Foundation.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * 3. The end-user documentation included with the redistribution, if
+ *    any, must include the following acknowlegement:
+ *       "This product includes software developed by the
+ *        Apache Software Foundation (http://www.apache.org/)."
+ *    Alternately, this acknowlegement may appear in the software itself,
+ *    if and wherever such third-party acknowlegements normally appear.
+ *
+ * 4. The names "Ant" and "Apache Software
+ *    Foundation" must not be used to endorse or promote products derived
+ *    from this software without prior written permission. For written
+ *    permission, please contact apache@apache.org.
+ *
+ * 5. Products derived from this software may not be called "Apache"
+ *    nor may "Apache" appear in their names without prior written
+ *    permission of the Apache Group.
+ *
+ * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED.  IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
+ * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ * ====================================================================
+ *
+ * This software consists of voluntary contributions made by many
+ * individuals on behalf of the Apache Software Foundation.  For more
+ * information on the Apache Software Foundation, please see
+ * <http://www.apache.org/>.
+ */
+package org.apache.tools.ant.filters;
+
+import java.io.IOException;
+import java.io.Reader;
+import org.apache.tools.ant.types.Parameter;
+
+/**
+ * Converts non latin characters to unicode escapes
+ * Useful to load properties containing non latin
+ * Example:
+ *
+ * <pre>&lt;escapeunicode&gt;</pre>
+ *
+ * Or:
+ *
+ * <pre>&lt;filterreader classname=&quot;org.apache.tools.ant.filters.EscapeUnicode&quot;/&gt;
+*  </pre>
+ *
+ * @author <a href="mailto:levylambert@tiscali-dsl.de">Antoine Levy-Lambert</a>
+ * @since Ant 1.6
+ */
+public class EscapeUnicode
+    extends BaseParamFilterReader
+    implements ChainableReader {
+    //this field will hold unnnn right after reading a non latin character
+    //afterwards it will be truncated of one char every call to read
+    private StringBuffer unicodeBuf;
+
+    /**
+     * Constructor for "dummy" instances.
+     *
+     * @see BaseFilterReader#BaseFilterReader()
+     */
+    public EscapeUnicode() {
+        super();
+        unicodeBuf = new StringBuffer();
+    }
+
+    /**
+     * Creates a new filtered reader.
+     *
+     * @param in A Reader object providing the underlying stream.
+     *           Must not be <code>null</code>.
+     */
+    public EscapeUnicode(final Reader in) {
+        super(in);
+        unicodeBuf = new StringBuffer();
+    }
+
+    /**
+     * Returns the next character in the filtered stream, converting non latin
+     * characters to unicode escapes.
+     *
+     * @return the next character in the resulting stream, or -1
+     * if the end of the resulting stream has been reached
+     *
+     * @exception java.io.IOException if the underlying stream throws
+     * an IOException during reading
+     */
+    public final int read() throws IOException {
+        if (!getInitialized()) {
+            initialize();
+            setInitialized(true);
+        }
+
+        int ch= - 1;
+        if (unicodeBuf.length() == 0) {
+            ch = in.read();
+            if (ch != -1) {
+                char achar = (char) ch;
+                if (achar >= '\u0080') {
+                    unicodeBuf = new StringBuffer("u0000");
+                    String s = Integer.toHexString(ch);
+                    //replace the last 0s by the chars contained in s
+                    for (int i = 0; i < s.length(); i++) {
+                        unicodeBuf.setCharAt(unicodeBuf.length()
+                                             - s.length() + i,
+                                             s.charAt(i));
+                    }
+                    ch='\\';
+                }
+            }
+        } else {
+            ch = (int) unicodeBuf.charAt(0);
+            unicodeBuf.deleteCharAt(0);
+        }
+        return ch;
+    }
+
+    /**
+     * Creates a new EscapeUnicode using the passed in
+     * Reader for instantiation.
+     *
+     * @param rdr A Reader object providing the underlying stream.
+     *            Must not be <code>null</code>.
+     *
+     * @return a new filter based on this configuration, but filtering
+     *         the specified reader
+     */
+    public final Reader chain(final Reader rdr) {
+        EscapeUnicode newFilter = new EscapeUnicode(rdr);
+        newFilter.setInitialized(true);
+        return newFilter;
+    }
+
+    /**
+     * Parses the parameters (currently unused)
+     */
+    private final void initialize() {
+        Parameter[] params = getParameters();
+        if (params != null) {
+            for (int i = 0; i < params.length; i++) {
+            }
+        }
+    }
+}
+

src/main/org/apache/tools/ant/types/FilterChain.java

@@ -1,7 +1,7 @@
 /*
  * The Apache Software License, Version 1.1
  *
- * Copyright (c) 2002 The Apache Software Foundation.  All rights
+ * Copyright (c) 2002-2003 The Apache Software Foundation.  All rights
  * reserved.
  *
  * Redistribution and use in source and binary forms, with or without
@@ -56,6 +56,7 @@ package org.apache.tools.ant.types;
 import java.util.Vector;
 import org.apache.tools.ant.BuildException;
 import org.apache.tools.ant.filters.ClassConstants;
+import org.apache.tools.ant.filters.EscapeUnicode;
 import org.apache.tools.ant.filters.ExpandProperties;
 import org.apache.tools.ant.filters.HeadFilter;
 import org.apache.tools.ant.filters.LineContains;
@@ -137,6 +138,13 @@ public final class FilterChain extends DataType implements Cloneable {
         filterReaders.addElement(tailFilter);
     }
 
+    /**
+     * @since Ant 1.6
+     */
+    public final void addEscapeUnicode(final EscapeUnicode escapeUnicode) {
+        filterReaders.addElement(escapeUnicode);
+    }
+
     /**
      * Makes this instance in effect a reference to another FilterChain 
      * instance.

src/testcases/org/apache/tools/ant/filters/EscapeUnicodeTest.java

@@ -0,0 +1,87 @@
+/*
+ * The Apache Software License, Version 1.1
+ *
+ * Copyright (c) 2003 The Apache Software Foundation.  All rights
+ * reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ *
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in
+ *    the documentation and/or other materials provided with the
+ *    distribution.
+ *
+ * 3. The end-user documentation included with the redistribution, if
+ *    any, must include the following acknowlegement:
+ *       "This product includes software developed by the
+ *        Apache Software Foundation (http://www.apache.org/)."
+ *    Alternately, this acknowlegement may appear in the software itself,
+ *    if and wherever such third-party acknowlegements normally appear.
+ *
+ * 4. The names "Ant" and "Apache Software
+ *    Foundation" must not be used to endorse or promote products derived
+ *    from this software without prior written permission. For written
+ *    permission, please contact apache@apache.org.
+ *
+ * 5. Products derived from this software may not be called "Apache"
+ *    nor may "Apache" appear in their names without prior written
+ *    permission of the Apache Group.
+ *
+ * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED
+ * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED.  IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR
+ * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
+ * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
+ * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ * ====================================================================
+ *
+ * This software consists of voluntary contributions made by many
+ * individuals on behalf of the Apache Software Foundation.  For more
+ * information on the Apache Software Foundation, please see
+ * <http://www.apache.org/>.
+ */
+
+package org.apache.tools.ant.filters;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.tools.ant.BuildFileTest;
+import org.apache.tools.ant.util.FileUtils;
+
+/**
+ */
+public class EscapeUnicodeTest extends BuildFileTest {
+
+    public EscapeUnicodeTest(String name) {
+        super(name);
+    }
+
+    public void setUp() { 
+        configureProject("src/etc/testcases/filters/build.xml");
+    }
+    
+    public void tearDown() { 
+        executeTarget("cleanup");
+    }
+    
+    public void testEscapeUnicode() throws IOException {
+        executeTarget("testEscapeUnicode");
+        File expected = getProject().resolveFile("expected/escapeunicode.test");
+        File result = getProject().resolveFile("result/escapeunicode.test");
+        FileUtils fu = FileUtils.newFileUtils();
+        assertTrue(fu.contentEquals(expected, result));
+    }
+
+}