#20767 - adding totalproperty and todir attributes to checksum task - submitted by Aslak Hellesoy

git-svn-id: https://svn.apache.org/repos/asf/ant/core/trunk@274702 13f79535-47bb-0310-9956-ffa450edef68

docs/manual/CoreTasks/checksum.html

@@ -26,6 +26,13 @@ perform checksum verifications.
     <td valign="top" align="center">One of either <var>file</var> or
     at least one nested fileset element.</td>
   </tr>
+  <tr>
+    <td valign="top">todir</td>
+    <td valign="top">The root directory where checksums should be written.</td>
+    <td valign="top" align="center">No. If not specified, checksum files
+      will be written to the same directory as the files themselves.
+    </td>
+  </tr>
   <tr>
     <td valign="top">algorithm</td>
     <td valign="top">Specifies the algorithm to be used to
@@ -64,6 +71,17 @@ perform checksum verifications.
     </td>
     <td valign="top" align="center">No</td>
   </tr>
+  <tr>
+    <td valign="top">totalproperty</td>
+    <td valign="top">If specified, this attribute specifies the name of
+      the property that will hold a checksum of all the checksums and
+      file paths. The individual checksums and the relative paths to
+      the files within the filesets they are defined in will be used to
+      compute this checksum. (The file separators in the paths will be
+      converted to '/' before computation to ensure platform portability).
+    </td>
+    <td valign="top" align="center">No</td>
+  </tr>
   <tr>
     <td valign="top">forceoverwrite</td>
     <td valign="top">Overwrite existing files even if the destination

src/etc/testcases/taskdefs/checksum.xml

@@ -2,38 +2,44 @@
 <project default="cleanup" basedir=".">
 
   <target name="cleanup">
-    <delete file="../asf-logo.gif.md5" />
+    <delete file="../asf-logo.gif.MD5" />
+    <delete>
+      <fileset dir="checksum">
+        <include name="**/*.MD5"/>
+      </fileset>
+    </delete>
+    <delete dir="checksum/checksums" />
   </target>
 
   <target name="createMd5">
-    <checksum file="../asf-logo.gif" fileext=".md5" />
+    <checksum file="../asf-logo.gif" fileext=".MD5" />
   </target>
 
   <target name="setProperty">
-    <checksum file="../asf-logo.gif" property="logo.md5" />
+    <checksum file="../asf-logo.gif" property="logo.MD5" />
   </target>
 
   <target name="verifyAsTask">
-    <copy file="expected/asf-logo.gif.md5" todir=".." />
-    <checksum file="../asf-logo.gif" fileext=".md5" 
-              verifyproperty="logo.md5" />
+    <copy file="expected/asf-logo.gif.MD5" todir=".." />
+    <checksum file="../asf-logo.gif" fileext=".MD5"
+              verifyproperty="logo.MD5" />
 
-    <copy file="checksum.xml" tofile="../asf-logo.gif.md5" 
+    <copy file="checksum.xml" tofile="../asf-logo.gif.MD5"
           overwrite="true" />
-    <checksum file="../asf-logo.gif" fileext=".md5" 
-              verifyproperty="no.logo.md5" />
+    <checksum file="../asf-logo.gif" fileext=".MD5"
+              verifyproperty="no.logo.MD5" />
   </target>
 
   <target name="verifyAsCondition">
-    <copy file="expected/asf-logo.gif.md5" todir=".." />
-    <condition property="logo.md5">
-      <checksum file="../asf-logo.gif" fileext=".md5" />
+    <copy file="expected/asf-logo.gif.MD5" todir=".." />
+    <condition property="logo.MD5">
+      <checksum file="../asf-logo.gif" fileext=".MD5" />
     </condition>
 
-    <copy file="checksum.xml" tofile="../asf-logo.gif.md5" 
+    <copy file="checksum.xml" tofile="../asf-logo.gif.MD5"
           overwrite="true" />
-    <condition property="no.logo.md5">
-      <checksum file="../asf-logo.gif" fileext=".md5" />
+    <condition property="no.logo.MD5">
+      <checksum file="../asf-logo.gif" fileext=".MD5" />
     </condition>
   </target>
 
@@ -42,4 +48,20 @@
     <checksum property="${checksum}" file="checksum.xml"
               verifyproperty="verify"/>
   </target>
+
+  <target name="verifyTotal">
+    <checksum totalproperty="total">
+      <fileset dir="${basedir}/checksum">
+        <exclude name="**/*.MD5"/>
+      </fileset>
+    </checksum>
+  </target>
+
+  <target name="verifyChecksumdir">
+    <checksum totalproperty="total" todir="${basedir}/checksum/checksums">
+      <fileset dir="${basedir}/checksum">
+        <exclude name="**/*.MD5"/>
+      </fileset>
+    </checksum>
+  </target>
 </project>

src/main/org/apache/tools/ant/taskdefs/Checksum.java

@@ -53,19 +53,25 @@
  */
 package org.apache.tools.ant.taskdefs;
 
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
 import java.security.DigestInputStream;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.security.NoSuchProviderException;
-import java.util.Enumeration;
-import java.util.Hashtable;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.util.HashMap;
+import java.util.Map;
 import java.util.Vector;
+import java.util.Hashtable;
+import java.util.Enumeration;
+import java.util.Set;
+import java.util.Arrays;
+
 import org.apache.tools.ant.BuildException;
 import org.apache.tools.ant.DirectoryScanner;
 import org.apache.tools.ant.Project;
@@ -76,16 +82,26 @@ import org.apache.tools.ant.types.FileSet;
  * Used to create or verify file checksums.
  *
  * @author Magesh Umasankar
+ * @author Aslak Hellesoy
  *
  * @since Ant 1.5
  *
  * @ant.task category="control"
  */
 public class Checksum extends MatchingTask implements Condition {
+
     /**
      * File for which checksum is to be calculated.
      */
     private File file = null;
+
+    /**
+     * Root directory in which the checksu files will be written.
+     * If not specified, the checksum files will be written
+     * in the same directory as each file.
+     */
+    private File todir;
+
     /**
      * MessageDigest algorithm to be used.
      */
@@ -103,6 +119,23 @@ public class Checksum extends MatchingTask implements Condition {
      * Holds generated checksum and gets set as a Project Property.
      */
     private String property;
+    /**
+     * Holds checksums for all files (both calculated and cached on disk).
+     * Key:   java.util.File (source file)
+     * Value: java.lang.String (digest)
+     */
+    private Map allDigests = new HashMap();
+    /**
+     * Holds relative file names for all files (always with a forward slash).
+     * This is used to calculate the total hash.
+     * Key:   java.util.File (source file)
+     * Value: java.lang.String (relative file name)
+     */
+    private Map relativeFilePaths = new HashMap();
+    /**
+     * Property where totalChecksum gets set.
+     */
+    private String totalproperty;
     /**
      * Whether or not to create a new file.
      * Defaults to <code>false</code>.
@@ -140,6 +173,14 @@ public class Checksum extends MatchingTask implements Condition {
         this.file = file;
     }
 
+    /**
+     * Sets the root directory where checksum files will be
+     * written/read
+     */
+    public void setTodir(File todir) {
+        this.todir = todir;
+    }
+
     /**
      * Specifies the algorithm to be used to compute the checksum.
      * Defaults to "MD5". Other popular algorithms like "SHA" may be used as well.
@@ -171,6 +212,14 @@ public class Checksum extends MatchingTask implements Condition {
         this.property = property;
     }
 
+    /**
+     * Sets the property to hold the generated total checksum
+     * for all files.
+     */
+    public void setTotalproperty(String totalproperty) {
+        this.totalproperty = totalproperty;
+    }
+
     /**
      * Sets the verify property.  This project property holds
      * the result of a checksum verification - "true" or "false"
@@ -241,6 +290,11 @@ public class Checksum extends MatchingTask implements Condition {
                 "Checksum cannot be generated for directories");
         }
 
+        if (file != null && totalproperty != null) {
+            throw new BuildException(
+                "File and Totalproperty cannot co-exist.");
+        }
+
         if (property != null && fileext != null) {
             throw new BuildException(
                 "Property and FileExt cannot co-exist.");
@@ -309,8 +363,6 @@ public class Checksum extends MatchingTask implements Condition {
         }
 
         try {
-            addToIncludeFileMap(file);
-
             int sizeofFileSet = filesets.size();
             for (int i = 0; i < sizeofFileSet; i++) {
                 FileSet fs = (FileSet) filesets.elementAt(i);
@@ -318,10 +370,19 @@ public class Checksum extends MatchingTask implements Condition {
                 String[] srcFiles = ds.getIncludedFiles();
                 for (int j = 0; j < srcFiles.length; j++) {
                     File src = new File(fs.getDir(getProject()), srcFiles[j]);
+                    if (totalproperty != null) {
+                        // Use '/' to calculate digest based on file name.
+                        // This is required in order to get the same result
+                        // on different platforms.
+                        String relativePath = srcFiles[j].replace(File.separatorChar, '/');
+                        relativeFilePaths.put(src, relativePath);
+                    }
                     addToIncludeFileMap(src);
                 }
             }
 
+            addToIncludeFileMap(file);
+
             return generateChecksums();
         } finally {
             fileext = savedFileExt;
@@ -337,14 +398,25 @@ public class Checksum extends MatchingTask implements Condition {
         if (file != null) {
             if (file.exists()) {
                 if (property == null) {
-                    File dest
-                        = new File(file.getParent(), file.getName() + fileext);
+                    File checksumFile = getChecksumFile(file);
                     if (forceOverwrite || isCondition ||
-                        (file.lastModified() > dest.lastModified())) {
-                        includeFileMap.put(file, dest);
+                        (file.lastModified() > checksumFile.lastModified())) {
+                        includeFileMap.put(file, checksumFile);
                     } else {
-                        log(file + " omitted as " + dest + " is up to date.",
+                        log(file + " omitted as " + checksumFile + " is up to date.",
                             Project.MSG_VERBOSE);
+                        if (totalproperty != null) {
+                            // Read the checksum from disk.
+                            String checksum = null;
+                            try {
+                                BufferedReader diskChecksumReader = new BufferedReader(new FileReader(checksumFile));
+                                checksum = diskChecksumReader.readLine();
+                            } catch (IOException e) {
+                                throw new BuildException("Couldn't read checksum file " + checksumFile, e);
+                            }
+                            byte[] digest = decodeHex(checksum.toCharArray());
+                            allDigests.put(file,digest );
+                        }
                     }
                 } else {
                     includeFileMap.put(file, property);
@@ -359,6 +431,23 @@ public class Checksum extends MatchingTask implements Condition {
         }
     }
 
+    private File getChecksumFile(File file) {
+        File directory;
+        if (todir != null) {
+            // A separate directory was explicitly declared
+            String path = (String) relativeFilePaths.get(file);
+            directory = new File(todir, path).getParentFile();
+            // Create the directory, as it might not exist.
+            directory.mkdirs();
+        } else {
+            // Just use the same directory as the file itself.
+            // This directory will exist
+            directory = file.getParentFile();
+        }
+        File checksumFile = new File(directory, file.getName() + fileext);
+        return checksumFile;
+    }
+
     /**
      * Generate checksum(s) using the message digest created earlier.
      */
@@ -384,15 +473,10 @@ public class Checksum extends MatchingTask implements Condition {
                 fis.close();
                 fis = null;
                 byte[] fileDigest = messageDigest.digest ();
-                StringBuffer checksumSb = new StringBuffer();
-                for (int i = 0; i < fileDigest.length; i++) {
-                    String hexStr = Integer.toHexString(0x00ff & fileDigest[i]);
-                    if (hexStr.length() < 2) {
-                        checksumSb.append("0");
-                    }
-                    checksumSb.append(hexStr);
+                if (totalproperty != null) {
+                    allDigests.put(src,fileDigest);
                 }
-                String checksum = checksumSb.toString();
+                String checksum = createDigestString(fileDigest);
                 //can either be a property name string or a file
                 Object destination = includeFileMap.get(src);
                 if (destination instanceof java.lang.String) {
@@ -429,6 +513,29 @@ public class Checksum extends MatchingTask implements Condition {
                     }
                 }
             }
+            if (totalproperty != null) {
+                // Calculate the total checksum
+                // Convert the keys (source files) into a sorted array.
+                Set keys = allDigests.keySet();
+                Object[] keyArray = keys.toArray();
+                // File is Comparable, so sorting is trivial
+                Arrays.sort(keyArray);
+                // Loop over the checksums and generate a total hash.
+                messageDigest.reset();
+                for (int i = 0; i < keyArray.length; i++) {
+                    File src = (File) keyArray[i];
+
+                    // Add the digest for the file content
+                    byte[] digest = (byte[]) allDigests.get(src);
+                    messageDigest.update(digest);
+
+                    // Add the file path
+                    String fileName = (String) relativeFilePaths.get(src);
+                    messageDigest.update(fileName.getBytes());
+                }
+                String totalChecksum = createDigestString(messageDigest.digest());
+                getProject().setNewProperty(totalproperty, totalChecksum);
+            }
         } catch (Exception e) {
             throw new BuildException(e, getLocation());
         } finally {
@@ -445,4 +552,44 @@ public class Checksum extends MatchingTask implements Condition {
         }
         return checksumMatches;
     }
+
+    private String createDigestString(byte[] fileDigest) {
+        StringBuffer checksumSb = new StringBuffer();
+        for (int i = 0; i < fileDigest.length; i++) {
+            String hexStr = Integer.toHexString(0x00ff & fileDigest[i]);
+            if (hexStr.length() < 2) {
+                checksumSb.append("0");
+            }
+            checksumSb.append(hexStr);
+        }
+        return checksumSb.toString();
+    }
+
+    /**
+     * Converts an array of characters representing hexidecimal values into an
+     * array of bytes of those same values. The returned array will be half the
+     * length of the passed array, as it takes two characters to represent any
+     * given byte. An exception is thrown if the passed char array has an odd
+     * number of elements.
+     *
+     * NOTE: This code is copied from jakarta-commons codec.
+     */
+    public static byte[] decodeHex(char[] data) throws BuildException {
+        int l = data.length;
+
+        if ((l & 0x01) != 0) {
+            throw new BuildException("odd number of characters.");
+        }
+
+        byte[] out = new byte[l >> 1];
+
+        // two characters form the hex value.
+        for (int i = 0, j = 0; j < l; i++) {
+            int f = Character.digit(data[j++], 16) << 4;
+            f = f | Character.digit(data[j++], 16);
+            out[i] = (byte) (f & 0xFF);
+        }
+
+        return out;
+    }
 }

src/testcases/org/apache/tools/ant/taskdefs/ChecksumTest.java

@@ -58,9 +58,11 @@ import org.apache.tools.ant.BuildFileTest;
 import org.apache.tools.ant.util.FileUtils;
 
 import java.io.IOException;
+import java.io.File;
 
 /**
  * @author Stefan Bodewig
+ * @author Aslak Hellesoy
  * @version $Revision$
  */
 public class ChecksumTest extends BuildFileTest {
@@ -80,26 +82,42 @@ public class ChecksumTest extends BuildFileTest {
     public void testCreateMd5() throws IOException {
         FileUtils fileUtils = FileUtils.newFileUtils();
         executeTarget("createMd5");
-        assertTrue(fileUtils.contentEquals(project.resolveFile("expected/asf-logo.gif.md5"),
-                                           project.resolveFile("../asf-logo.gif.md5")));
+        assertTrue(fileUtils.contentEquals(project.resolveFile("expected/asf-logo.gif.MD5"),
+                                           project.resolveFile("../asf-logo.gif.MD5")));
     }
 
     public void testSetProperty() {
         executeTarget("setProperty");
         assertEquals("0541d3df42520911f268abc730f3afe0",
-                     project.getProperty("logo.md5"));
+                     project.getProperty("logo.MD5"));
         assertTrue(!project.resolveFile("../asf-logo.gif.MD5").exists());
     }
 
+    public void testVerifyTotal() {
+        executeTarget("verifyTotal");
+        assertEquals("ef8f1477fcc9bf93832c1a74f629c626",
+                     project.getProperty("total"));
+    }
+
+    public void testVerifyChecksumdir() {
+        executeTarget("verifyChecksumdir");
+        assertEquals("ef8f1477fcc9bf93832c1a74f629c626",
+                     project.getProperty("total"));
+        File shouldExist = project.resolveFile("checksum/checksums/foo/zap/Eenie.MD5");
+        File shouldNotExist = project.resolveFile("checksum/foo/zap/Eenie.MD5");
+        assertTrue( "Checksums should be written to " + shouldExist.getAbsolutePath(), shouldExist.exists());
+        assertTrue( "Checksums should not be written to " + shouldNotExist.getAbsolutePath(), !shouldNotExist.exists());
+    }
+
     public void testVerifyAsTask() {
         testVerify("verifyAsTask");
-        assertNotNull(project.getProperty("no.logo.md5"));
-        assertEquals("false", project.getProperty("no.logo.md5"));
+        assertNotNull(project.getProperty("no.logo.MD5"));
+        assertEquals("false", project.getProperty("no.logo.MD5"));
     }
 
     public void testVerifyAsCondition() {
         testVerify("verifyAsCondition");
-        assertNull(project.getProperty("no.logo.md5"));
+        assertNull(project.getProperty("no.logo.MD5"));
     }
 
     public void testVerifyFromProperty() {
@@ -108,11 +126,11 @@ public class ChecksumTest extends BuildFileTest {
     }
 
     private void testVerify(String target) {
-        assertNull(project.getProperty("logo.md5"));
-        assertNull(project.getProperty("no.logo.md5"));
+        assertNull(project.getProperty("logo.MD5"));
+        assertNull(project.getProperty("no.logo.MD5"));
         executeTarget(target);
-        assertNotNull(project.getProperty("logo.md5"));
-        assertEquals("true", project.getProperty("logo.md5"));
+        assertNotNull(project.getProperty("logo.MD5"));
+        assertEquals("true", project.getProperty("logo.MD5"));
     }
 
 }