diff --git a/docs/Query_Parameters/Filters.md b/docs/Query_Parameters/Filters.md index 9361498a..2b72e780 100644 --- a/docs/Query_Parameters/Filters.md +++ b/docs/Query_Parameters/Filters.md @@ -28,9 +28,9 @@ You can always find the full list of available filters by going to `/filters` RE 22. `accessDate` Functions like `accessTime` but the `value` can be a calendar date, like: `01/01/1989`. 23. `isUnderConstruction` - Usable by files. Filters the working INode set by some condition of whether the file is under construction. 24. `isWithSnapshot` - Usable by files and dirs. Filters the working INode set by some condition of whether the file or directory is part of a [Snapshot](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HdfsSnapshots.html). -25. `hasAcl` - Usable by files and dirs. Filters the working INode set by some condition of whether the file or directory is has a native HDFS ACL. -26. `hasQuota` - Usable by dirs. Filters the working INode set by some condition of whether the directory has either a namespace or disk space quota assigned. -27. `isUnderNsQuota` - Usable by files and dirs. Filters the working INode set by checking whether each INode has any parent up to the root that has a namespace quota assigned. -28. `isUnderDsQuota` - Usable by files and dirs. Filters the working INode set by checking whether each INode has any parent up to the root that has a disk space quota assigned. +25. `hasAcl` - Usable by files and dirs. Filters the working INode set by some condition of whether the file or directory is has a native HDFS [ACL](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/FileSystemShell.html#setfacl). +26. `hasQuota` - Usable by dirs. Filters the working INode set by some condition of whether the directory has either a namespace or disk space [quota](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HdfsQuotaAdminGuide.html) assigned. +27. `isUnderNsQuota` - Usable by files and dirs. Filters the working INode set by checking whether each INode has any parent up to the root that has a namespace [quota](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HdfsQuotaAdminGuide.html) assigned. Ignores any quota set on root directory, `/`. +28. `isUnderDsQuota` - Usable by files and dirs. Filters the working INode set by checking whether each INode has any parent up to the root that has a disk space [quota](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HdfsQuotaAdminGuide.html) assigned. Ignores any quota set on root directory, `/`. All memoryConsumed calculations are estimations performed as documented [here](https://www.cloudera.com/documentation/enterprise/5-8-x/topics/admin_nn_memory_config.html). \ No newline at end of file diff --git a/src/main/java/org/apache/hadoop/hdfs/server/namenode/AbstractQueryEngine.java b/src/main/java/org/apache/hadoop/hdfs/server/namenode/AbstractQueryEngine.java index e3a0fcfc..b9a785df 100644 --- a/src/main/java/org/apache/hadoop/hdfs/server/namenode/AbstractQueryEngine.java +++ b/src/main/java/org/apache/hadoop/hdfs/server/namenode/AbstractQueryEngine.java @@ -153,9 +153,12 @@ public Function getFilterFunctionToBooleanForINode(String filter return node -> (node.getAclFeature() != null); case "isUnderNsQuota": return node -> { + if (node.isRoot()) { + return false; + } for (INodeDirectory p = node.getParent(); p != null; p = node.getParent()) { node = p; - if (versionLoader.getNsQuota(node) >= 0) { + if (!node.isRoot() && versionLoader.getNsQuota(node) >= 0) { return true; } } @@ -163,9 +166,12 @@ public Function getFilterFunctionToBooleanForINode(String filter }; case "isUnderDsQuota": return node -> { + if (node.isRoot()) { + return false; + } for (INodeDirectory p = node.getParent(); p != null; p = node.getParent()) { node = p; - if (versionLoader.getDsQuota(node) >= 0) { + if (!node.isRoot() && versionLoader.getDsQuota(node) >= 0) { return true; } } diff --git a/src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestWithMiniClusterBase.java b/src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestWithMiniClusterBase.java index 6240fd2d..4a191276 100644 --- a/src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestWithMiniClusterBase.java +++ b/src/test/java/org/apache/hadoop/hdfs/server/namenode/analytics/TestWithMiniClusterBase.java @@ -26,6 +26,7 @@ import java.io.File; import java.io.IOException; +import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; import org.apache.commons.io.FileUtils; @@ -219,6 +220,15 @@ public void testUpdateSeen() throws Exception { } while (checkCount == startingCount); assertThat(checkCount, is(greaterThan(startingCount))); + + // Test NsQuota histogram is not empty. + HttpGet histogram = + new HttpGet( + "http://localhost:4567/histogram?set=files&filters=isUnderNsQuota:eq:true&parentDirDepth=3&sum=count&type=parentDir&histogramOutput=csv"); + HttpResponse checkRes = client.execute(hostPort, histogram); + assertThat(checkRes.getStatusLine().getStatusCode(), is(200)); + List checkContent = IOUtils.readLines(checkRes.getEntity().getContent()); + assertThat(checkContent.size(), is(greaterThan(0))); } protected void addFiles(int numOfFiles, long sleepBetweenMs) throws Exception { @@ -271,7 +281,7 @@ protected void addFiles(int numOfFiles, long sleepBetweenMs) throws Exception { break; } if (dirNumber1 == 1) { - fileSystem.setQuota(filePath.getParent(), -1L, 100000000000000L); + fileSystem.setQuota(filePath.getParent(), 9999999L, 100000000000000L); } int user = RANDOM.nextInt(3); switch (user) {