diff --git a/src/main/org/apache/tools/ant/taskdefs/Javadoc.java b/src/main/org/apache/tools/ant/taskdefs/Javadoc.java
index 9d9ba787b..83b204eec 100644
--- a/src/main/org/apache/tools/ant/taskdefs/Javadoc.java
+++ b/src/main/org/apache/tools/ant/taskdefs/Javadoc.java
@@ -2083,14 +2083,14 @@ public class Javadoc extends Task {
             }
             if (rc instanceof FileSet) {
                 FileSet fs = (FileSet) rc;
-            if (!fs.hasPatterns() && !fs.hasSelectors()) {
-                fs = (FileSet) fs.clone();
-                fs.createInclude().setName("**/*.java");
-                if (includeNoSourcePackages) {
-                    fs.createInclude().setName("**/package.html");
+                if (!fs.hasPatterns() && !fs.hasSelectors()) {
+                    fs = (FileSet) fs.clone();
+                    fs.createInclude().setName("**/*.java");
+                    if (includeNoSourcePackages) {
+                        fs.createInclude().setName("**/package.html");
+                    }
                 }
             }
-            }
             Iterator iter = rc.iterator();
             while (iter.hasNext()) {
                 sf.addElement(new SourceFile(((FileResource) iter.next())