Merge branch '1.9.x'

WHATSNEW

@@ -53,6 +53,11 @@ Other changes:
  * <junit> now supports JDK9 modules
    Github Pull Request #18
 
+ * a new implementation "builtin" has been added to <native2ascii> and
+   is the default when running on JDK9+ since the tool itself has been
+   removed from the JDK.
+   Bugzilla Report 59855
+
 Changes from Ant 1.9.6 TO Ant 1.9.7
 ===================================
 

manual/Tasks/native2ascii.html

@@ -61,9 +61,14 @@
       with the <code>implementation</code> attribute or a nested element.
       <a name="implementationvalues">Here are the choices of the attribute</a>:</p>
     <ul>
-      <li>default - the default converter (kaffe or sun) for the platform.</li>
-      <li>sun (the standard converter of the JDK)</li>
-      <li>kaffe (the standard converter of <a href="http://www.kaffe.org" target="_top">Kaffe</a>)</li>
+      <li>default - the default converter for the platform - kaffee
+        when run on Kaffee, builtin if JDK9 or newer is detected, sun
+        otherwise.</li>
+      <li>sun (the standard converter of the JDK < 9)</li>
+      <li>kaffe (the standard converter
+        of <a href="http://www.kaffe.org" target="_top">Kaffe</a>)</li>
+      <li>builtin - Ant's internal implementation used for
+        JDK9+. <em>since ant 1.9.8</em></li>
     </ul>
 
     <table border="1" cellpadding="2" cellspacing="0">
@@ -76,7 +81,7 @@
         <td>reverse</td>
         <td>Reverse the sense of the conversion,
           i.e. convert from ASCII to native <b>only supported by the
-            sun converter</b></td>
+            sun and builtin converters</b></td>
         <td align="center">No</td>
       </tr>
       <tr>

src/main/org/apache/tools/ant/taskdefs/optional/native2ascii/BuiltinNative2Ascii.java

@@ -0,0 +1,104 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+package org.apache.tools.ant.taskdefs.optional.native2ascii;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+
+import org.apache.tools.ant.BuildException;
+import org.apache.tools.ant.taskdefs.optional.Native2Ascii;
+import org.apache.tools.ant.util.FileUtils;
+import org.apache.tools.ant.util.Native2AsciiUtils;
+import org.apache.tools.ant.util.StringUtils;
+
+/**
+ * Encapsulates the built-in Native2Ascii implementation.
+ *
+ * @since Ant 1.9.8
+ */
+public class BuiltinNative2Ascii implements Native2AsciiAdapter {
+
+    static final String IMPLEMENTATION_NAME = "builtin";
+
+    public final boolean convert(Native2Ascii args, File srcFile,
+                                 File destFile) throws BuildException {
+        boolean reverse = args.getReverse();
+        String encoding = args.getEncoding();
+        BufferedReader input = null;
+        try {
+            input = getReader(srcFile, encoding, reverse);
+            try {
+                Writer output = getWriter(destFile, encoding, reverse);
+                try {
+                    translate(input, output, reverse);
+                } finally {
+                    FileUtils.close(output);
+                }
+            } finally {
+                FileUtils.close(input);
+            }
+            return true;
+        } catch (IOException ex) {
+            throw new BuildException("Exception trying to translate data", ex);
+        }
+    }
+
+    private BufferedReader getReader(File srcFile, String encoding,
+                                     boolean reverse) throws IOException {
+        if (!reverse && encoding != null) {
+            return new BufferedReader(new InputStreamReader(
+                new FileInputStream(srcFile), encoding));
+        }
+        return new BufferedReader(new FileReader(srcFile));
+    }
+
+    private Writer getWriter(File destFile, String encoding,
+                             boolean reverse) throws IOException {
+        if (!reverse) {
+            encoding = "ASCII";
+        }
+        if (encoding != null) {
+            return new BufferedWriter(
+                new OutputStreamWriter(new FileOutputStream(destFile),
+                                       encoding));
+        }
+        return new BufferedWriter(new FileWriter(destFile));
+    }
+
+    private void translate(BufferedReader input, Writer output,
+                           boolean reverse) throws IOException {
+        String line = null;
+        while ((line = input.readLine()) != null) {
+            if (!reverse) {
+                output.write(Native2AsciiUtils.native2ascii(line));
+            } else {
+                output.write(Native2AsciiUtils.ascii2native(line));
+            }
+            output.write(StringUtils.LINE_SEP);
+        }
+    }
+}

src/main/org/apache/tools/ant/taskdefs/optional/native2ascii/Native2AsciiAdapterFactory.java

@@ -40,10 +40,13 @@ public class Native2AsciiAdapterFactory {
      * vendor
      */
     public static String getDefault() {
-        if (JavaEnvUtils.isKaffe() || JavaEnvUtils.isClasspathBased()) {
+        if (shouldUseKaffee()) {
             return KaffeNative2Ascii.IMPLEMENTATION_NAME;
         }
-        return SunNative2Ascii.IMPLEMENTATION_NAME;
+        if (shouldUseSun()) {
+            return SunNative2Ascii.IMPLEMENTATION_NAME;
+        }
+        return BuiltinNative2Ascii.IMPLEMENTATION_NAME;
     }
 
     /**
@@ -79,11 +82,14 @@ public class Native2AsciiAdapterFactory {
                                                  ProjectComponent log,
                                                  Path classpath)
         throws BuildException {
-        if (((JavaEnvUtils.isKaffe() || JavaEnvUtils.isClasspathBased()) && choice == null)
+        if ((shouldUseKaffee() && choice == null)
             || KaffeNative2Ascii.IMPLEMENTATION_NAME.equals(choice)) {
             return new KaffeNative2Ascii();
-        } else if (SunNative2Ascii.IMPLEMENTATION_NAME.equals(choice)) {
+        } else if ((shouldUseSun() && choice == null)
+                   || SunNative2Ascii.IMPLEMENTATION_NAME.equals(choice)) {
             return new SunNative2Ascii();
+        } else if (BuiltinNative2Ascii.IMPLEMENTATION_NAME.equals(choice)) {
+            return new BuiltinNative2Ascii();
         } else if (choice != null) {
             return resolveClassName(choice,
                                     // Memory leak in line below
@@ -91,9 +97,7 @@ public class Native2AsciiAdapterFactory {
                                     .createClassLoader(classpath));
         }
 
-        // This default has been good enough until Ant 1.6.3, so stick
-        // with it
-        return new SunNative2Ascii();
+        return new BuiltinNative2Ascii();
     }
 
     /**
@@ -113,4 +117,15 @@ public class Native2AsciiAdapterFactory {
             Native2AsciiAdapterFactory.class.getClassLoader(),
             Native2AsciiAdapter.class);
     }
+
+    private static final boolean shouldUseKaffee() {
+        return JavaEnvUtils.isKaffe() || JavaEnvUtils.isClasspathBased();
+    }
+
+    private static final boolean shouldUseSun() {
+        return JavaEnvUtils.isJavaVersion(JavaEnvUtils.JAVA_1_5)
+            || JavaEnvUtils.isJavaVersion(JavaEnvUtils.JAVA_1_6)
+            || JavaEnvUtils.isJavaVersion(JavaEnvUtils.JAVA_1_7)
+            || JavaEnvUtils.isJavaVersion(JavaEnvUtils.JAVA_1_8);
+    }
 }

src/main/org/apache/tools/ant/util/Native2AsciiUtils.java

@@ -0,0 +1,89 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+package org.apache.tools.ant.util;
+
+/**
+ * Contains helper methods for Ant's built-in implementation of native2ascii.
+ *
+ * @since Ant 1.9.8
+ */
+public class Native2AsciiUtils {
+
+    private static final int MAX_ASCII = 127;
+
+    /**
+     * Replaces non-ASCII characters with their Unicode-Escapes.
+     * <p>Expects to be called once per line if applied to a file.</p>
+     * @param line the input line
+     * @return the translated line
+     */
+    public static String native2ascii(String line) {
+        StringBuilder sb = new StringBuilder();
+        for (char c : line.toCharArray()) {
+            if (c <= MAX_ASCII) {
+                sb.append(c);
+            } else {
+                sb.append("\\u");
+                String encoded = Integer.toHexString(c);
+                for (int i = encoded.length(); i < 4; i++) {
+                    sb.append("0");
+                }
+                sb.append(encoded);
+            }
+        }
+        return sb.toString();
+    }
+
+    /**
+     * Replaces Unicode-Escapes.
+     * <p>Expects to be called once per line if applied to a file.</p>
+     * @param line the input line
+     * @return the translated line
+     */
+    public static String ascii2native(String line) {
+        StringBuilder sb = new StringBuilder();
+        int inputLen = line.length();
+        for (int i = 0; i < inputLen; i++) {
+            char c = line.charAt(i);
+            if (c != '\\' || i >= inputLen - 5) {
+                sb.append(c);
+            } else { // backslash with enough remaining characters
+                char u = line.charAt(++i);
+                if (u == 'u') {
+                    int unescaped = tryParse(line, i + 1);
+                    if (unescaped >= 0) {
+                        sb.append((char) unescaped);
+                        i += 4;
+                        continue;
+                    }
+                }
+                // not a unicode escape
+                sb.append(c).append(u);
+            }
+        }
+        return sb.toString();
+    }
+
+    private static int tryParse(String line, int startIdx) {
+        try {
+            return Integer.parseInt(line.substring(startIdx, startIdx + 4), 16);
+        } catch (NumberFormatException ex) {
+            return -1;
+        }
+    }
+}

src/tests/antunit/taskdefs/optional/native2ascii-test.xml

@@ -65,4 +65,71 @@ public class Adapter implements Native2AsciiAdapter {
     </native2ascii>
     <au:assertLogContains text="adapter called"/>
   </target>
+
+  <target name="-real-test-macros">
+    <macrodef name="assertTranslatedOutput">
+      <attribute name="file"/>
+      <attribute name="expected"/>
+      <attribute name="encoding"/>
+      <sequential>
+        <loadfile srcFile="${output}/@{file}" encoding="@{encoding}"
+                  property="@{file}.actual">
+          <filterchain>
+            <striplinebreaks/>
+          </filterchain>
+        </loadfile>
+        <au:assertEquals expected="@{expected}" actual="${@{file}.actual}"/>
+      </sequential>
+    </macrodef>
+    <presetdef name="native2ascii-def">
+      <native2ascii src="${input}" dest="${output}" encoding="UTF-8"
+                    includes="**/*.properties"/>
+    </presetdef>
+  </target>
+
+  <target name="-setup-UTF8-To-ASCII" depends="-real-test-macros">
+    <mkdir dir="${input}"/>
+    <mkdir dir="${output}"/>
+    <echo file="${input}/umlauts.properties" encoding="UTF-8">äöü=ÄÖÜ</echo>
+    <property name="umlauts.expected"
+              value="\u00e4\u00f6\u00fc=\u00c4\u00d6\u00dc"/>
+  </target>
+
+  <target name="testUTF8-To-ASCII-sun" depends="-setup-UTF8-To-ASCII"
+          description="https://bz.apache.org/bugzilla/show_bug.cgi?id=59855"
+          unless="jdk1.9+">
+    <native2ascii-def implementation="sun"/>
+    <assertTranslatedOutput file="umlauts.properties" encoding="ASCII"
+                            expected="${umlauts.expected}"/>
+  </target>
+
+  <target name="testUTF8-To-ASCII-builtin" depends="-setup-UTF8-To-ASCII"
+          description="https://bz.apache.org/bugzilla/show_bug.cgi?id=59855">
+    <native2ascii-def implementation="builtin"/>
+    <assertTranslatedOutput file="umlauts.properties" encoding="ASCII"
+                            expected="${umlauts.expected}"/>
+  </target>
+
+  <target name="-setup-ASCII-To-UTF8" depends="-real-test-macros">
+    <mkdir dir="${input}"/>
+    <mkdir dir="${output}"/>
+    <echo file="${input}/umlauts.properties" encoding="ASCII">\u00e4\u00f6\u00fc=\u00c4\u00d6\u00dc</echo>
+    <property name="umlauts.expected" value="äöü=ÄÖÜ"/>
+  </target>
+
+  <target name="testASCII-To-UTF8-sun" depends="-setup-ASCII-To-UTF8"
+          description="https://bz.apache.org/bugzilla/show_bug.cgi?id=59855"
+          unless="jdk1.9+">
+    <native2ascii-def implementation="sun" reverse="true"/>
+    <assertTranslatedOutput file="umlauts.properties" encoding="UTF-8"
+                            expected="${umlauts.expected}"/>
+  </target>
+
+  <target name="testASCII-To-UTF8-builtin" depends="-setup-ASCII-To-UTF8"
+          description="https://bz.apache.org/bugzilla/show_bug.cgi?id=59855">
+    <native2ascii-def implementation="builtin" reverse="true"/>
+    <assertTranslatedOutput file="umlauts.properties" encoding="UTF-8"
+                            expected="${umlauts.expected}"/>
+  </target>
+
 </project>

src/tests/junit/org/apache/tools/ant/util/Native2AsciiUtilsTest.java

@@ -0,0 +1,76 @@
+/*
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+package org.apache.tools.ant.util;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class Native2AsciiUtilsTest {
+
+    @Test
+    public void doesntTouchAscii() {
+        StringBuilder sb = new StringBuilder();
+        for (char i = 0; i < 128; i++) {
+            sb.append(i);
+        }
+        assertEquals(sb.toString(), Native2AsciiUtils.native2ascii(sb.toString()));
+    }
+
+    @Test
+    public void escapes() {
+        assertEquals("\\u00e4\\u00f6\\u00fc",
+                     Native2AsciiUtils.native2ascii("\u00e4\u00f6\u00fc"));
+    }
+
+    @Test
+    public void pads() {
+        assertEquals("\\u00e4\\u01f6\\u12fc",
+                     Native2AsciiUtils.native2ascii("\u00e4\u01f6\u12fc"));
+    }
+
+    @Test
+    public void doesntTouchNonEscapes() {
+        StringBuilder sb = new StringBuilder();
+        for (char i = 0; i < 128; i++) {
+            sb.append(i);
+        }
+        assertEquals(sb.toString(), Native2AsciiUtils.ascii2native(sb.toString()));
+    }
+
+    @Test
+    public void unescapes() {
+        assertEquals("\u00e4\u00f6\u00fc",
+                     Native2AsciiUtils.ascii2native("\\u00e4\\u00f6\\u00fc"));
+    }
+
+    @Test
+    public void leavesNonUnicodeBackslashesAlone() {
+        assertEquals("\\abcdef", Native2AsciiUtils.ascii2native("\\abcdef"));
+        assertEquals("\\u012j", Native2AsciiUtils.ascii2native("\\u012j"));
+    }
+
+    @Test
+    public void dealsWithUnfinishedEscapes() {
+        assertEquals("\u00e4", Native2AsciiUtils.ascii2native("\\u00e4"));
+        assertEquals("\\u00e", Native2AsciiUtils.ascii2native("\\u00e"));
+        assertEquals("\\u00", Native2AsciiUtils.ascii2native("\\u00"));
+        assertEquals("\\u0", Native2AsciiUtils.ascii2native("\\u0"));
+    }
+
+}