[#165] Allow blockSize as a filter field and find field

paypal · Nov 30, 2018 · 6ed729f · 6ed729f
1 parent 4c0e4ea
commit 6ed729f
Show file tree

Hide file tree

Showing 4 changed files with 38 additions and 0 deletions.
diff --git a/src/main/java/org/apache/hadoop/hdfs/server/namenode/AbstractQueryEngine.java b/src/main/java/org/apache/hadoop/hdfs/server/namenode/AbstractQueryEngine.java
@@ -58,6 +58,8 @@ public Function<INode, Long> getFilterFunctionToLongForINode(String filter) {
         return node -> node.asFile().computeFileSize() * node.asFile().getFileReplication();
       case "fileReplica":
         return node -> ((long) node.asFile().getFileReplication());
+      case "blockSize":
+        return node -> ((long) node.asFile().getPreferredBlockSize());
       case "numBlocks":
         return node -> ((long) node.asFile().numBlocks());
       case "numReplicas":

diff --git a/src/main/java/org/apache/hadoop/hdfs/server/namenode/Constants.java b/src/main/java/org/apache/hadoop/hdfs/server/namenode/Constants.java
@@ -214,6 +214,7 @@ enum FindField {
     accessTime,
     modTime,
     fileSize,
+    blockSize,
     diskspaceConsumed,
     memoryConsumed
   }
@@ -430,6 +431,7 @@ enum Endpoint {
       EnumSet.of(
           FindField.accessTime,
           FindField.modTime,
+          FindField.blockSize,
           FindField.diskspaceConsumed,
           FindField.fileSize,
           FindField.memoryConsumed);

diff --git a/src/main/java/org/apache/hadoop/hdfs/server/namenode/JavaCollectionQEngine.java b/src/main/java/org/apache/hadoop/hdfs/server/namenode/JavaCollectionQEngine.java
@@ -93,6 +93,8 @@ public class JavaCollectionQEngine extends AbstractQueryEngine {
           "memoryConsumed", node -> getFilterFunctionToLongForINode("memoryConsumed").apply(node));
   private final SimpleAttribute<INode, Long> fileReplica =
       attribute("fileReplica", node -> getFilterFunctionToLongForINode("fileReplica").apply(node));
+  private final SimpleAttribute<INode, Long> blockSize =
+      attribute("blockSize", node -> getFilterFunctionToLongForINode("blockSize").apply(node));
   private final SimpleAttribute<INode, Long> numBlocks =
       attribute("numBlocks", node -> getFilterFunctionToLongForINode("numBlocks").apply(node));
   private final SimpleAttribute<INode, Long> numReplicas =
@@ -590,6 +592,8 @@ private Attribute<INode, Long> getLongAttributeForINode(String filter) {
         return memoryConsumed;
       case "fileReplica":
         return fileReplica;
+      case "blockSize":
+        return blockSize;
       case "numBlocks":
         return numBlocks;
       case "numReplicas":

diff --git a/src/test/java/com/paypal/nnanalytics/TestNNAnalyticsBase.java b/src/test/java/com/paypal/nnanalytics/TestNNAnalyticsBase.java
@@ -479,6 +479,36 @@ public void testAccessTimeHistogramTop10() throws IOException {
     assertThat(res.getStatusLine().getStatusCode(), is(200));
   }
 
+  @Test
+  public void testBlockSizeHistogramByUser() throws IOException {
+    HttpGet get =
+        new HttpGet("http://localhost:4567/histogram?set=files&filters=blockSize:gt:0&type=user");
+    HttpResponse res = client.execute(hostPort, get);
+    List<String> strings = IOUtils.readLines(res.getEntity().getContent());
+    strings.clear();
+    assertThat(res.getStatusLine().getStatusCode(), is(200));
+  }
+
+  @Test
+  public void testBlockSizeMaxFindHistogramByUser() throws IOException {
+    HttpGet get =
+        new HttpGet("http://localhost:4567/histogram?set=files&type=user&find=max:blockSize");
+    HttpResponse res = client.execute(hostPort, get);
+    List<String> strings = IOUtils.readLines(res.getEntity().getContent());
+    strings.clear();
+    assertThat(res.getStatusLine().getStatusCode(), is(200));
+  }
+
+  @Test
+  public void testBlockSizeMinFindHistogramByUser() throws IOException {
+    HttpGet get =
+        new HttpGet("http://localhost:4567/histogram?set=files&type=user&find=min:blockSize");
+    HttpResponse res = client.execute(hostPort, get);
+    List<String> strings = IOUtils.readLines(res.getEntity().getContent());
+    strings.clear();
+    assertThat(res.getStatusLine().getStatusCode(), is(200));
+  }
+
   @Test
   public void testModTimeHistogram() throws IOException {
     HttpGet get = new HttpGet("http://localhost:4567/histogram?set=all&type=modTime");