Make <get> support resource collections and mappers

git-svn-id: https://svn.apache.org/repos/asf/ant/core/trunk@818129 13f79535-47bb-0310-9956-ffa450edef68

WHATSNEW

@@ -985,6 +985,10 @@ Other changes:
    makes it ignore differences between / and \ separators.
    Bugzilla Report 47858.
 
+ * <get> now supports resource collections (as long as the resources
+   contained provide URLs) and can get multiple resources in a single
+   task.
+
 Changes from Ant 1.7.0 TO Ant 1.7.1
 =============================================
 

docs/manual/CoreTasks/get.html

@@ -26,31 +26,27 @@
 
 <h2><a name="get">Get</a></h2>
 <h3>Description</h3>
-<p>Gets a file from a URL. When the verbose option is &quot;on&quot;, this task
+<p>Gets files from URLs.  When the verbose option is &quot;on&quot;, this task
 displays a '.' for every 100 Kb retrieved. Any URL schema supported by
 the runtime is valid here, including http:, ftp: and jar:; 
-https: is only valid if the appropriate support is added to the pre-1.4 Java
-runtimes. 
- 
 </p>
 The <i>usetimestamp</i> option enables you to control downloads so that the remote file is
 only fetched if newer than the local copy. If there is no local copy, the download always takes 
-place. When a file is downloaded, the timestamp of the downloaded file is set to the remote timestamp,
-if  the JVM is Java1.2 or later. 
+place. When a file is downloaded, the timestamp of the downloaded file is set to the remote timestamp. 
 NB: This timestamp facility only works on downloads using the HTTP protocol. 
 <p>
 A username and password can be specified, in which case basic 'slightly encoded
 plain text' authentication is used. This is only secure over an HTTPS link.
 </p>
-<p>
-<b>Proxies</b>. Since Ant1.7, Ant running on Java1.5 or later defaults to 
-    <a href="../proxy.html">using
-    the proxy settings of the operating system</a>. There is also the  
- <a href="../OptionalTasks/setproxy.html">&lt;setproxy&gt;</a> task for
-    earlier Java versions. With proxies turned on, <code>&lt;get&gt;</code> requests against
-    localhost may not work as expected, if the request is relayed to the proxy.
-    The <code>-noproxy</code> option can be used to turn this feature off.
-</p>
+
+<p><b>Proxies</b>. Since Ant 1.7.0, Ant running on Java1.5 or later can
+  <a href="../proxy.html">use the proxy settings of the operating
+    system</a> if enabled with the
+  <code>-autoproxy</code> option. There is also the
+  <a href="../OptionalTasks/setproxy.html">&lt;setproxy&gt;</a> task
+  for earlier Java versions. With proxies turned
+  on, <code>&lt;get&gt;</code> requests against localhost may not work
+  as expected, if the request is relayed to the proxy.</p>
  
 <h3>Parameters</h3>
 <table border="1" cellpadding="2" cellspacing="0">
@@ -62,11 +58,12 @@ plain text' authentication is used. This is only secure over an HTTPS link.
   <tr>
     <td valign="top">src</td>
     <td valign="top">the URL from which to retrieve a file.</td>
-    <td align="center" valign="top">Yes</td>
+    <td align="center" valign="top">Yes or a nested resource collection</td>
   </tr>
   <tr>
     <td valign="top">dest</td>
-    <td valign="top">the file where to store the retrieved file.</td>
+    <td valign="top">the file or directory where to store the
+      retrieved file(s).</td>
     <td align="center" valign="top">Yes</td>
   </tr>
   <tr>
@@ -97,15 +94,15 @@ plain text' authentication is used. This is only secure over an HTTPS link.
   </tr>  
   <tr>
     <td valign="top">maxtime</td>
-    <td valign="top">Maximum time in seconds the download may take,
-    otherwise it will be interrupted and treated like a download
+    <td valign="top">Maximum time in seconds a single download may take,
+      otherwise it will be interrupted and treated like a download
       error.  <em>Since Ant 1.8.0</em></td>
     <td align="center" valign="top">No: default 0 which means no
       maximum time</td>
   </tr>  
   <tr>
     <td valign="top">retries</td>
-    <td valign="top">the number of retries on error<br/>
+    <td valign="top">the per download number of retries on error<br/>
       <em>since Ant 1.8.0</em></td>
     <td align="center" valign="top">No; default "3"</td>
   </tr>
@@ -125,6 +122,29 @@ plain text' authentication is used. This is only secure over an HTTPS link.
     <td align="center" valign="top">No; default "true"</td>
   </tr>
 </table>
+<h3>Parameters specified as nested elements</h3>
+<h4>any resource collection</h4>
+
+<p><a href="../CoreTypes/resources.html#collection">Resource
+    Collection</a>s are used to select groups of URLs to download.  If
+    the collection contains more than one resource, the dest attribute
+    must point to a directory if it exists or a directory will be
+    created if it doesn't exist.  The destination file name use the
+    last part of the path of the source URL unless you also specify a
+    mapper.</p>
+
+<h4>mapper</h4>
+
+<p>You can define name transformations by using a
+  nested <a href="../CoreTypes/mapper.html">mapper</a> element.  You
+  can also use any filenamemapper type in place of the mapper
+  element.</p>
+
+<p>The mapper will receive the resource's name as argument.  Any
+  resource for which the mapper returns no or more than one mapped
+  name will be skipped.  If the returned name is a relative path, it
+  will be considered relative to the <em>dest</em> attribute.</p>
+
 <h3>Examples</h3>
 <pre>  &lt;get src=&quot;http://ant.apache.org/&quot; dest=&quot;help/index.html&quot;/&gt;</pre>
 <p>Gets the index page of http://ant.apache.org/, and stores it in the file <code>help/index.html</code>.</p>
@@ -175,6 +195,15 @@ the <a href="input.html">input task</a> to query for a password.</p>
   checksum (assuming a certain naming convention for the checksum
   file, of course) and validate the checksum on the fly.</p>
 
+<pre>
+&lt;get dest=&quot;downloads&quot;&gt;
+  &lt;url url=&quot;http://ant.apache.org/index.html&quot;/&gt; 
+  &lt;url url=&quot;http://ant.apache.org/faq.html&quot;/&gt;
+&lt;/get&gt;
+</pre>
+<p>Gets the index and FAQ pages of http://ant.apache.org/, and stores
+  them in the directory <code>downloads</code> which will be created if
+  necessary.</p>
 </body>
 </html>
 

src/main/org/apache/tools/ant/taskdefs/Get.java

@@ -30,11 +30,19 @@ import java.net.URL;
 import java.net.URLConnection;
 import java.util.Date;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.Set;
 
 import org.apache.tools.ant.BuildException;
 import org.apache.tools.ant.Project;
 import org.apache.tools.ant.Task;
+import org.apache.tools.ant.types.Mapper;
+import org.apache.tools.ant.types.Resource;
+import org.apache.tools.ant.types.ResourceCollection;
+import org.apache.tools.ant.types.resources.Resources;
+import org.apache.tools.ant.types.resources.URLProvider;
+import org.apache.tools.ant.types.resources.URLResource;
+import org.apache.tools.ant.util.FileNameMapper;
 import org.apache.tools.ant.util.FileUtils;
 
 /**
@@ -53,12 +61,12 @@ public class Get extends Task {
     private static final int BIG_BUFFER_SIZE = 100 * 1024;
     private static final FileUtils FILE_UTILS = FileUtils.getFileUtils();
     private static final int REDIRECT_LIMIT = 25;
-    
+
     private static final String HTTP = "http";
     private static final String HTTPS = "https";
 
-    private URL source; // required
-    private File dest; // required
+    private Resources sources = new Resources();
+    private File destination; // required
     private boolean verbose = false;
     private boolean useTimestamp = false; //off by default
     private boolean ignoreErrors = false;
@@ -68,6 +76,7 @@ public class Get extends Task {
     private int numberRetries = NUMBER_RETRIES;
     private boolean skipExisting = false;
     private boolean httpUseCaches = true; // on by default
+    private Mapper mapperElement = null;
 
     /**
      * Does the work.
@@ -75,6 +84,36 @@ public class Get extends Task {
      * @exception BuildException Thrown in unrecoverable error.
      */
     public void execute() throws BuildException {
+        checkAttributes();
+
+        for (Iterator iter = sources.iterator(); iter.hasNext(); ) {
+            Resource r = (Resource) iter.next();
+            URLProvider up = (URLProvider) r.as(URLProvider.class);
+            URL source = up.getURL();
+
+            File dest = destination;
+            if (destination.isDirectory()) {
+                if (mapperElement != null) {
+                    String path = source.getPath();
+                    if (path.endsWith("/")) {
+                        path = path.substring(0, path.length() - 1);
+                    }
+                    int slash = path.lastIndexOf("/");
+                    if (slash > -1) {
+                        path = path.substring(slash + 1);
+                    }
+                    dest = new File(destination, path);
+                } else {
+                    FileNameMapper mapper = mapperElement.getImplementation();
+                    String[] d = mapper.mapFileName(r.getName());
+                    if (d == null || d.length != 1) {
+                        log("skipping " + r + " - mapper can't handle it",
+                            Project.MSG_WARN);
+                        continue;
+                    }
+                    dest = new File(destination, d[0]);
+                }
+            }
 
         //set up logging
         int logLevel = Project.MSG_INFO;
@@ -85,13 +124,14 @@ public class Get extends Task {
 
         //execute the get
         try {
-            doGet(logLevel, progress);
+            doGet(source, dest, logLevel, progress);
         } catch (IOException ioe) {
             log("Error getting " + source + " to " + dest);
             if (!ignoreErrors) {
                 throw new BuildException(ioe, getLocation());
             }
         }
+        }
     }
 
     /**
@@ -106,10 +146,41 @@ public class Get extends Task {
      * @throws IOException for network trouble
      * @throws BuildException for argument errors, or other trouble when ignoreErrors
      * is false.
+     * @deprecated only gets the first configured resource
      */
     public boolean doGet(int logLevel, DownloadProgress progress)
             throws IOException {
         checkAttributes();
+        for (Iterator iter = sources.iterator(); iter.hasNext(); ) {
+            Resource r = (Resource) iter.next();
+            URLProvider up = (URLProvider) r.as(URLProvider.class);
+            URL source = up.getURL();
+            return doGet(source, destination, logLevel, progress);
+        }
+        /*NOTREACHED*/
+        return false;
+    }
+
+    /**
+     * make a get request, with the supplied progress and logging info.
+     *
+     * All the other config parameters like ignoreErrors are set at
+     * the task level.
+     * @param source the URL to get
+     * @param dest the target file
+     * @param logLevel level to log at, see {@link Project#log(String, int)}
+     * @param progress progress callback; null for no-callbacks
+     * @return true for a successful download, false otherwise.
+     * The return value is only relevant when {@link #ignoreErrors} is true, as
+     * when false all failures raise BuildExceptions.
+     * @throws IOException for network trouble
+     * @throws BuildException for argument errors, or other trouble when ignoreErrors
+     * is false.
+     * @since Ant 1.8.0
+     */
+    public boolean doGet(URL source, File dest, int logLevel,
+                         DownloadProgress progress)
+        throws IOException {
 
         if (dest.exists() && skipExisting) {
             log("Destination already exists (skipping): "
@@ -137,7 +208,8 @@ public class Get extends Task {
             hasTimestamp = true;
         }
 
-        GetThread getThread = new GetThread(hasTimestamp, timestamp, progress,
+        GetThread getThread = new GetThread(source, dest,
+                                            hasTimestamp, timestamp, progress,
                                             logLevel);
         getThread.setDaemon(true);
         getProject().registerThreadTask(getThread, this);
@@ -169,32 +241,55 @@ public class Get extends Task {
      * Check the attributes.
      */
     private void checkAttributes() {
-        if (source == null) {
-            throw new BuildException("src attribute is required", getLocation());
+        if (sources.size() == 0) {
+            throw new BuildException("at least one source is required",
+                                     getLocation());
+        }
+        for (Iterator iter = sources.iterator(); iter.hasNext(); ) {
+            Object up = ((Resource) iter.next()).as(URLProvider.class);
+            if (up == null) {
+                throw new BuildException("Only URLProvider resources are"
+                                         + " supported", getLocation());
+            }
         }
 
-        if (dest == null) {
+        if (destination == null) {
             throw new BuildException("dest attribute is required", getLocation());
         }
 
-        if (dest.exists() && dest.isDirectory()) {
-            throw new BuildException("The specified destination is a directory",
-                    getLocation());
+        if (destination.exists() && sources.size() > 1
+            && !destination.isDirectory()) {
+            throw new BuildException("The specified destination is not a"
+                                     + " directory",
+                                     getLocation());
         }
 
-        if (dest.exists() && !dest.canWrite()) {
-            throw new BuildException("Can't write to " + dest.getAbsolutePath(),
-                    getLocation());
+        if (destination.exists() && !destination.canWrite()) {
+            throw new BuildException("Can't write to "
+                                     + destination.getAbsolutePath(),
+                                     getLocation());
+        }
+
+        if (sources.size() > 1 && !destination.exists()) {
+            destination.mkdirs();
         }
     }
 
     /**
-     * Set the URL to get.
+     * Set an URL to get.
      *
      * @param u URL for the file.
      */
     public void setSrc(URL u) {
-        this.source = u;
+        add(new URLResource(u));
+    }
+
+    /**
+     * Adds URLs to get.
+     * @since Ant 1.8.0
+     */
+    public void add(ResourceCollection rc) {
+        sources.add(rc);
     }
 
     /**
@@ -203,7 +298,7 @@ public class Get extends Task {
      * @param dest Path to file.
      */
     public void setDest(File dest) {
-        this.dest = dest;
+        this.destination = dest;
     }
 
     /**
@@ -265,13 +360,6 @@ public class Get extends Task {
         this.pword = p;
     }
 
-    /**
-     * Provide this for Backward Compatibility.
-     */
-    protected static class Base64Converter
-        extends org.apache.tools.ant.util.Base64Converter {
-    }
-
     /**
      * The time in seconds the download is allowed to take before
      * being terminated.
@@ -318,6 +406,37 @@ public class Get extends Task {
         this.httpUseCaches = httpUseCache;
     }
 
+    /**
+     * Define the mapper to map source to destination files.
+     * @return a mapper to be configured.
+     * @exception BuildException if more than one mapper is defined.
+     * @since Ant 1.8.0
+     */
+    public Mapper createMapper() throws BuildException {
+        if (mapperElement != null) {
+            throw new BuildException("Cannot define more than one mapper",
+                                     getLocation());
+        }
+        mapperElement = new Mapper(getProject());
+        return mapperElement;
+    }
+
+    /**
+     * Add a nested filenamemapper.
+     * @param fileNameMapper the mapper to add.
+     * @since Ant 1.8.0
+     */
+    public void add(FileNameMapper fileNameMapper) {
+        createMapper().add(fileNameMapper);
+    }
+
+    /**
+     * Provide this for Backward Compatibility.
+     */
+    protected static class Base64Converter
+        extends org.apache.tools.ant.util.Base64Converter {
+    }
+
     /**
      * Interface implemented for reporting
      * progess of downloading.
@@ -414,6 +533,8 @@ public class Get extends Task {
 
     private class GetThread extends Thread {
 
+        private final URL source;
+        private final File dest;
         private final boolean hasTimestamp;
         private final long timestamp;
         private final DownloadProgress progress;
@@ -427,7 +548,10 @@ public class Get extends Task {
         private URLConnection connection;
         private int redirections = 0;
 
-        GetThread(boolean h, long t, DownloadProgress p, int l) {
+        GetThread(URL source, File dest,
+                  boolean h, long t, DownloadProgress p, int l) {
+            this.source = source;
+            this.dest = dest;
             hasTimestamp = h;
             timestamp = t;
             progress = p;
@@ -445,7 +569,7 @@ public class Get extends Task {
         }
 
         private boolean get() throws IOException, BuildException {
-            
+
             connection = openConnection(source);
 
             if (connection == null)
@@ -461,7 +585,7 @@ public class Get extends Task {
             if (downloadSucceeded && useTimestamp)  {
                 updateTimeStamp();
             }
-            
+
             return downloadSucceeded;
         }
 
@@ -493,7 +617,7 @@ public class Get extends Task {
                 }
             }
 
-            
+
             return true;
         }
 
@@ -640,7 +764,7 @@ public class Get extends Task {
                 FILE_UTILS.setFileLastModified(dest, remoteTimestamp);
             }
         }
-        
+
         /**
          * Has the download completed successfully?
          *