[#195] Ignore the namespace quota on root directory

paypal · Mar 5, 2019 · dd7ab78 · dd7ab78
1 parent d1b880c
commit dd7ab78
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 7 deletions.
diff --git a/docs/Query_Parameters/Filters.md b/docs/Query_Parameters/Filters.md
@@ -28,9 +28,9 @@ You can always find the full list of available filters by going to `/filters` RE
 22. `accessDate` Functions like `accessTime` but the `value` can be a calendar date, like: `01/01/1989`.
 23. `isUnderConstruction` - Usable by files. Filters the working INode set by some condition of whether the file is under construction.
 24. `isWithSnapshot` - Usable by files and dirs. Filters the working INode set by some condition of whether the file or directory is part of a [Snapshot](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HdfsSnapshots.html).
-25. `hasAcl` - Usable by files and dirs. Filters the working INode set by some condition of whether the file or directory is has a native HDFS ACL.
-26. `hasQuota` - Usable by dirs. Filters the working INode set by some condition of whether the directory has either a namespace or disk space quota assigned.
-27. `isUnderNsQuota` - Usable by files and dirs. Filters the working INode set by checking whether each INode has any parent up to the root that has a namespace quota assigned.
-28. `isUnderDsQuota` - Usable by files and dirs. Filters the working INode set by checking whether each INode has any parent up to the root that has a disk space quota assigned.
+25. `hasAcl` - Usable by files and dirs. Filters the working INode set by some condition of whether the file or directory is has a native HDFS [ACL](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/FileSystemShell.html#setfacl).
+26. `hasQuota` - Usable by dirs. Filters the working INode set by some condition of whether the directory has either a namespace or disk space [quota](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HdfsQuotaAdminGuide.html) assigned.
+27. `isUnderNsQuota` - Usable by files and dirs. Filters the working INode set by checking whether each INode has any parent up to the root that has a namespace [quota](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HdfsQuotaAdminGuide.html) assigned. Ignores any quota set on root directory, `/`.
+28. `isUnderDsQuota` - Usable by files and dirs. Filters the working INode set by checking whether each INode has any parent up to the root that has a disk space [quota](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HdfsQuotaAdminGuide.html) assigned. Ignores any quota set on root directory, `/`.
 
 All memoryConsumed calculations are estimations performed as documented [here](https://www.cloudera.com/documentation/enterprise/5-8-x/topics/admin_nn_memory_config.html).
diff --git a/src/main/java/org/apache/hadoop/hdfs/server/namenode/AbstractQueryEngine.java b/src/main/java/org/apache/hadoop/hdfs/server/namenode/AbstractQueryEngine.java
@@ -153,19 +153,25 @@ public Function<INode, Boolean> getFilterFunctionToBooleanForINode(String filter
         return node -> (node.getAclFeature() != null);
       case "isUnderNsQuota":
         return node -> {
+          if (node.isRoot()) {
+            return false;
+          }
           for (INodeDirectory p = node.getParent(); p != null; p = node.getParent()) {
             node = p;
-            if (versionLoader.getNsQuota(node) >= 0) {
+            if (!node.isRoot() && versionLoader.getNsQuota(node) >= 0) {
               return true;
             }
           }
           return false;
         };
       case "isUnderDsQuota":
         return node -> {
+          if (node.isRoot()) {
+            return false;
+          }
           for (INodeDirectory p = node.getParent(); p != null; p = node.getParent()) {
             node = p;
-            if (versionLoader.getDsQuota(node) >= 0) {
+            if (!node.isRoot() && versionLoader.getDsQuota(node) >= 0) {
               return true;
             }
           }

diff --git a/src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestWithMiniClusterBase.java b/src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestWithMiniClusterBase.java
@@ -26,6 +26,7 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import org.apache.commons.io.FileUtils;
@@ -219,6 +220,15 @@ public void testUpdateSeen() throws Exception {
     } while (checkCount == startingCount);
 
     assertThat(checkCount, is(greaterThan(startingCount)));
+
+    // Test NsQuota histogram is not empty.
+    HttpGet histogram =
+        new HttpGet(
+            "http://localhost:4567/histogram?set=files&filters=isUnderNsQuota:eq:true&parentDirDepth=3&sum=count&type=parentDir&histogramOutput=csv");
+    HttpResponse checkRes = client.execute(hostPort, histogram);
+    assertThat(checkRes.getStatusLine().getStatusCode(), is(200));
+    List<String> checkContent = IOUtils.readLines(checkRes.getEntity().getContent());
+    assertThat(checkContent.size(), is(greaterThan(0)));
   }
 
   protected void addFiles(int numOfFiles, long sleepBetweenMs) throws Exception {
@@ -271,7 +281,7 @@ protected void addFiles(int numOfFiles, long sleepBetweenMs) throws Exception {
           break;
       }
       if (dirNumber1 == 1) {
-        fileSystem.setQuota(filePath.getParent(), -1L, 100000000000000L);
+        fileSystem.setQuota(filePath.getParent(), 9999999L, 100000000000000L);
       }
       int user = RANDOM.nextInt(3);
       switch (user) {