Structured metadata: Refactor into new variable

src/Components/IndexScene/IndexScene.tsx

@@ -27,6 +27,7 @@ import {
   VAR_LEVELS,
   VAR_LINE_FILTER,
   VAR_LOGS_FORMAT,
+  VAR_METADATA,
   VAR_PATTERNS,
 } from 'services/variables';
 
@@ -224,6 +225,21 @@ function getVariableSet(initialDatasourceUid: string, initialFilters?: AdHocVari
     return operators;
   };
 
+  const metadataVariable = new AdHocFiltersVariable({
+    name: VAR_METADATA,
+    label: 'Metadata',
+    applyMode: 'manual',
+    layout: 'vertical',
+    getTagKeysProvider: () => Promise.resolve({ replace: true, values: [] }),
+    getTagValuesProvider: () => Promise.resolve({ replace: true, values: [] }),
+    expressionBuilder: renderLogQLMetadataFilters,
+    hide: VariableHide.hideLabel,
+  });
+
+  metadataVariable._getOperators = () => {
+    return operators;
+  };
+
   const levelsVariable = new AdHocFiltersVariable({
     name: VAR_LEVELS,
     label: 'Filters',
@@ -258,7 +274,7 @@ function getVariableSet(initialDatasourceUid: string, initialFilters?: AdHocVari
         labelVariable,
         fieldsVariable,
         levelsVariable,
-        // @todo where is patterns being added to the url? Why do we have var-patterns and patterns?
+        metadataVariable,
         new CustomVariable({
           name: VAR_PATTERNS,
           value: '',

src/Components/ServiceScene/Breakdowns/AddToFiltersButton.tsx

@@ -4,7 +4,7 @@ import { AdHocVariableFilter, BusEventBase, DataFrame } from '@grafana/data';
 import { SceneComponentProps, SceneObject, SceneObjectBase, SceneObjectState } from '@grafana/scenes';
 import { VariableHide } from '@grafana/schema';
 import { reportAppInteraction, USER_EVENTS_ACTIONS, USER_EVENTS_PAGES } from 'services/analytics';
-import { LEVEL_VARIABLE_VALUE, VAR_FIELDS, VAR_LABELS, VAR_LEVELS } from 'services/variables';
+import { LEVEL_VARIABLE_VALUE, VAR_FIELDS, VAR_LABELS, VAR_LEVELS, VAR_METADATA } from 'services/variables';
 import { FilterButton } from 'Components/FilterButton';
 import { FilterOp } from 'services/filters';
 import { getDetectedLabelsFrame } from '../ServiceScene';
@@ -36,7 +36,7 @@ export function addAdHocFilter(filter: AdHocVariableFilter, scene: SceneObject,
   addToFilters(filter.key, filter.value, type, scene, variableType);
 }
 
-export type VariableFilterType = typeof VAR_LABELS | typeof VAR_FIELDS | typeof VAR_LEVELS;
+export type VariableFilterType = typeof VAR_LABELS | typeof VAR_FIELDS | typeof VAR_LEVELS | typeof VAR_METADATA;
 
 export function addToFilters(
   key: string,

src/Components/ServiceScene/Breakdowns/FieldValuesBreakdownScene.tsx

@@ -19,9 +19,9 @@ import { LayoutSwitcher } from './LayoutSwitcher';
 import { getQueryRunner } from '../../../services/panel';
 import { ByFrameRepeater } from './ByFrameRepeater';
 import { Alert, DrawStyle, LoadingPlaceholder } from '@grafana/ui';
-import { buildFieldsQueryString, getFilterBreakdownValueScene } from '../../../services/fields';
+import { buildFieldsQueryString, getFilterBreakdownValueScene, getParserForField } from '../../../services/fields';
 import { getLabelValue } from './SortByScene';
-import { VAR_FIELDS } from '../../../services/variables';
+import { VAR_FIELDS, VAR_METADATA } from '../../../services/variables';
 import React from 'react';
 import { FIELDS_BREAKDOWN_GRID_TEMPLATE_COLUMNS, FieldsBreakdownScene } from './FieldsBreakdownScene';
 import { AddFilterEvent } from './AddToFiltersButton';
@@ -158,13 +158,15 @@ export class FieldValuesBreakdownScene extends SceneObjectBase<FieldValuesBreakd
 
   private build(query: LokiQuery) {
     const groupByVariable = getFieldGroupByVariable(this);
-    const tagKey = String(groupByVariable.state.value);
+    const optionValue = String(groupByVariable.state.value);
 
     const { sortBy, direction } = getSortByPreference('fields', DEFAULT_SORT_BY, 'desc');
 
     const fieldsBreakdownScene = sceneGraph.getAncestor(this, FieldsBreakdownScene);
     const getFilter = () => fieldsBreakdownScene.state.search.state.filter ?? '';
 
+    const parserForThisField = getParserForField(optionValue, this);
+
     return new LayoutSwitcher({
       options: [
         { value: 'single', label: 'Single' },
@@ -178,7 +180,7 @@ export class FieldValuesBreakdownScene extends SceneObjectBase<FieldValuesBreakd
           children: [
             new SceneFlexItem({
               minHeight: 300,
-              body: PanelBuilders.timeseries().setTitle(tagKey).build(),
+              body: PanelBuilders.timeseries().setTitle(optionValue).build(),
             }),
           ],
         }),
@@ -198,7 +200,7 @@ export class FieldValuesBreakdownScene extends SceneObjectBase<FieldValuesBreakd
           getLayoutChild: getFilterBreakdownValueScene(
             getLabelValue,
             query?.expr.includes('count_over_time') ? DrawStyle.Bars : DrawStyle.Line,
-            VAR_FIELDS,
+            parserForThisField === 'structuredMetadata' ? VAR_METADATA : VAR_FIELDS,
             sceneGraph.getAncestor(this, FieldsBreakdownScene).state.sort
           ),
           sortBy,
@@ -221,7 +223,7 @@ export class FieldValuesBreakdownScene extends SceneObjectBase<FieldValuesBreakd
           getLayoutChild: getFilterBreakdownValueScene(
             getLabelValue,
             query?.expr.includes('count_over_time') ? DrawStyle.Bars : DrawStyle.Line,
-            VAR_FIELDS,
+            parserForThisField === 'structuredMetadata' ? VAR_METADATA : VAR_FIELDS,
             sceneGraph.getAncestor(this, FieldsBreakdownScene).state.sort
           ),
           sortBy,

src/Components/ServiceScene/Breakdowns/FieldsBreakdownScene.test.ts

@@ -1,5 +1,5 @@
 import { AdHocFiltersVariable } from '@grafana/scenes';
-import { VAR_FIELDS } from '../../../services/variables';
+import { VAR_FIELDS, VAR_METADATA } from '../../../services/variables';
 import { createDataFrame, DataFrame, Field, FieldType } from '@grafana/data';
 import {
   DETECTED_FIELDS_CARDINALITY_NAME,
@@ -47,7 +47,7 @@ describe('buildFieldsQueryString', () => {
 
     const result = buildFieldsQueryString('caller', filterVariable, detectedFieldsFrame);
     expect(result).toEqual(
-      `sum by (caller) (count_over_time({\${filters}}  \${levels} \${patterns} \${lineFilter} | logfmt | caller!="" \${fields} [$__auto]))`
+      `sum by (caller) (count_over_time({\${filters}}  \${metadata} \${levels} \${patterns} \${lineFilter} | logfmt | caller!="" \${fields} [$__auto]))`
     );
   });
   test('should build json-parser query', () => {
@@ -87,7 +87,7 @@ describe('buildFieldsQueryString', () => {
 
     const result = buildFieldsQueryString('caller', filterVariable, detectedFieldsFrame);
     expect(result).toEqual(
-      `sum by (caller) (count_over_time({\${filters}}  \${levels} \${patterns} \${lineFilter} | json | drop __error__, __error_details__ | caller!="" \${fields} [$__auto]))`
+      `sum by (caller) (count_over_time({\${filters}}  \${metadata} \${levels} \${patterns} \${lineFilter} | json | drop __error__, __error_details__ | caller!="" \${fields} [$__auto]))`
     );
   });
   test('should build mixed-parser query', () => {
@@ -127,7 +127,47 @@ describe('buildFieldsQueryString', () => {
 
     const result = buildFieldsQueryString('caller', filterVariable, detectedFieldsFrame);
     expect(result).toEqual(
-      `sum by (caller) (count_over_time({\${filters}}  \${levels} \${patterns} \${lineFilter} | json | logfmt | drop __error__, __error_details__ | caller!="" \${fields} [$__auto]))`
+      `sum by (caller) (count_over_time({\${filters}}  \${metadata} \${levels} \${patterns} \${lineFilter} | json | logfmt | drop __error__, __error_details__ | caller!="" \${fields} [$__auto]))`
+    );
+  });
+  test('should build metadata query', () => {
+    const metadataVariable = new AdHocFiltersVariable({
+      name: VAR_METADATA,
+      filters: [],
+    });
+
+    const nameField: Field = {
+      name: DETECTED_FIELDS_NAME_FIELD,
+      type: FieldType.string,
+      values: ['caller'],
+      config: {},
+    };
+    const cardinalityField: Field = {
+      name: DETECTED_FIELDS_CARDINALITY_NAME,
+      type: FieldType.number,
+      values: [5],
+      config: {},
+    };
+    const parserField: Field = {
+      name: DETECTED_FIELDS_PARSER_NAME,
+      type: FieldType.string,
+      values: [''],
+      config: {},
+    };
+    const typeField: Field = {
+      name: DETECTED_FIELDS_TYPE_NAME,
+      type: FieldType.string,
+      values: ['string'],
+      config: {},
+    };
+
+    const detectedFieldsFrame: DataFrame = createDataFrame({
+      fields: [nameField, cardinalityField, parserField, typeField],
+    });
+
+    const result = buildFieldsQueryString('caller', metadataVariable, detectedFieldsFrame);
+    expect(result).toEqual(
+      `sum by (caller) (count_over_time({\${filters}} | caller!="" \${metadata} \${levels} \${patterns} \${lineFilter}  \${fields} [$__auto]))`
     );
   });
 });

src/Components/ServiceScene/Breakdowns/LabelBreakdownScene.test.ts

@@ -12,7 +12,7 @@ describe('buildLabelsQuery', () => {
 
     const result = buildLabelsQuery({} as SceneObject, VAR_LABEL_GROUP_BY_EXPR, 'cluster');
     expect(result).toMatchObject({
-      expr: `sum(count_over_time({\${filters} ,cluster != ""}  \${levels} \${patterns} \${lineFilter}  \${fields} [$__auto])) by (${VAR_LABEL_GROUP_BY_EXPR})`,
+      expr: `sum(count_over_time({\${filters} ,cluster != ""}  \${metadata} \${levels} \${patterns} \${lineFilter}  \${fields} [$__auto])) by (${VAR_LABEL_GROUP_BY_EXPR})`,
     });
   });
   test('should build no-parser query with structured metadata filters', () => {
@@ -35,7 +35,7 @@ describe('buildLabelsQuery', () => {
 
     const result = buildLabelsQuery({} as SceneObject, VAR_LABEL_GROUP_BY_EXPR, 'cluster');
     expect(result).toMatchObject({
-      expr: `sum(count_over_time({\${filters} ,cluster != ""}  \${levels} \${patterns} \${lineFilter}  \${fields} [$__auto])) by (${VAR_LABEL_GROUP_BY_EXPR})`,
+      expr: `sum(count_over_time({\${filters} ,cluster != ""}  \${metadata} \${levels} \${patterns} \${lineFilter}  \${fields} [$__auto])) by (${VAR_LABEL_GROUP_BY_EXPR})`,
     });
   });
   test('should build logfmt-parser query with structured metadata filters', () => {
@@ -58,7 +58,7 @@ describe('buildLabelsQuery', () => {
 
     const result = buildLabelsQuery({} as SceneObject, VAR_LABEL_GROUP_BY_EXPR, 'cluster');
     expect(result).toMatchObject({
-      expr: `sum(count_over_time({\${filters} ,cluster != ""}  \${levels} \${patterns} \${lineFilter} | logfmt  \${fields} [$__auto])) by (${VAR_LABEL_GROUP_BY_EXPR})`,
+      expr: `sum(count_over_time({\${filters} ,cluster != ""}  \${metadata} \${levels} \${patterns} \${lineFilter} | logfmt  \${fields} [$__auto])) by (${VAR_LABEL_GROUP_BY_EXPR})`,
     });
   });
   test('should build json-parser query with structured metadata filters', () => {
@@ -81,7 +81,7 @@ describe('buildLabelsQuery', () => {
 
     const result = buildLabelsQuery({} as SceneObject, VAR_LABEL_GROUP_BY_EXPR, 'cluster');
     expect(result).toMatchObject({
-      expr: `sum(count_over_time({\${filters} ,cluster != ""}  \${levels} \${patterns} \${lineFilter} | json | drop __error__, __error_details__  \${fields} [$__auto])) by (${VAR_LABEL_GROUP_BY_EXPR})`,
+      expr: `sum(count_over_time({\${filters} ,cluster != ""}  \${metadata} \${levels} \${patterns} \${lineFilter} | json | drop __error__, __error_details__  \${fields} [$__auto])) by (${VAR_LABEL_GROUP_BY_EXPR})`,
     });
   });
   test('should build mixed-parser query with structured metadata filters', () => {
@@ -112,7 +112,7 @@ describe('buildLabelsQuery', () => {
 
     const result = buildLabelsQuery({} as SceneObject, VAR_LABEL_GROUP_BY_EXPR, 'cluster');
     expect(result).toMatchObject({
-      expr: `sum(count_over_time({\${filters} ,cluster != ""}  \${levels} \${patterns} \${lineFilter} | json | logfmt | drop __error__, __error_details__  \${fields} [$__auto])) by (${VAR_LABEL_GROUP_BY_EXPR})`,
+      expr: `sum(count_over_time({\${filters} ,cluster != ""}  \${metadata} \${levels} \${patterns} \${lineFilter} | json | logfmt | drop __error__, __error_details__  \${fields} [$__auto])) by (${VAR_LABEL_GROUP_BY_EXPR})`,
     });
   });
 });

src/Components/ServiceScene/LogsPanelScene.tsx

@@ -14,11 +14,10 @@ import React from 'react';
 import { LogsListScene } from './LogsListScene';
 import { LoadingPlaceholder } from '@grafana/ui';
 import { addToFilters, FilterType } from './Breakdowns/AddToFiltersButton';
-import { getFilterTypeFromLabelType, getLabelTypeFromFrame } from '../../services/fields';
-import { VAR_FIELDS, VAR_LABELS, VAR_LEVELS } from '../../services/variables';
+import { getVariableForLabel } from '../../services/fields';
+import { VAR_FIELDS, VAR_LABELS, VAR_LEVELS, VAR_METADATA } from '../../services/variables';
 import { reportAppInteraction, USER_EVENTS_ACTIONS, USER_EVENTS_PAGES } from '../../services/analytics';
-import { getAdHocFiltersVariable } from '../../services/variableGetters';
-import { LabelType } from '../../services/fieldsTypes';
+import { getAdHocFiltersVariable, getValueFromFieldsFilter } from '../../services/variableGetters';
 
 interface LogsPanelSceneState extends SceneObjectState {
   body?: VizPanel;
@@ -134,14 +133,36 @@ export class LogsPanelScene extends SceneObjectBase<LogsPanelSceneState> {
     const labels = getAdHocFiltersVariable(VAR_LABELS, this);
     const fields = getAdHocFiltersVariable(VAR_FIELDS, this);
     const levels = getAdHocFiltersVariable(VAR_LEVELS, this);
+    const metadata = getAdHocFiltersVariable(VAR_METADATA, this);
+
+    const hasKeyValueFilter = (filter: AdHocFiltersVariable | null) => {
+      return (
+        filter &&
+        filter.state.filters.findIndex(
+          (filter) => filter.operator === '=' && filter.key === key && filter.value === value
+        ) >= 0
+      );
+    };
 
-    const hasKeyValueFilter = (filter: AdHocFiltersVariable | null) =>
-      filter &&
-      filter.state.filters.findIndex(
-        (filter) => filter.operator === '=' && filter.key === key && filter.value === value
-      ) >= 0;
+    // Fields have json encoded values unlike the other variables, get the value for the matching filter and parse it before comparing
+    const hasKeyValueFilterField = (filter: AdHocFiltersVariable | null) => {
+      if (filter) {
+        const fieldFilter = filter.state.filters.find((filter) => filter.operator === '=' && filter.key === key);
 
-    return hasKeyValueFilter(labels) || hasKeyValueFilter(fields) || hasKeyValueFilter(levels);
+        if (fieldFilter) {
+          const fieldValue = getValueFromFieldsFilter(fieldFilter, key);
+          return fieldValue.value === value;
+        }
+      }
+      return false;
+    };
+
+    return (
+      hasKeyValueFilter(labels) ||
+      hasKeyValueFilterField(fields) ||
+      hasKeyValueFilter(levels) ||
+      hasKeyValueFilter(metadata)
+    );
   };
 
   private handleFilterStringClick = (value: string) => {
@@ -160,15 +181,15 @@ export class LogsPanelScene extends SceneObjectBase<LogsPanelSceneState> {
   };
 
   private handleLabelFilter(key: string, value: string, frame: DataFrame | undefined, operator: FilterType) {
-    const labelType = frame ? getLabelTypeFromFrame(key, frame) : LabelType.Parsed;
-    const variableName = getFilterTypeFromLabelType(labelType, key, value);
-    addToFilters(key, value, operator, this, variableName);
+    const variableType = getVariableForLabel(frame, key, this);
+
+    addToFilters(key, value, operator, this, variableType);
 
     reportAppInteraction(
       USER_EVENTS_PAGES.service_details,
       USER_EVENTS_ACTIONS.service_details.logs_detail_filter_applied,
       {
-        filterType: variableName,
+        filterType: variableType,
         key,
         action: operator,
       }

src/Components/ServiceScene/LogsTableScene.tsx

@@ -9,8 +9,7 @@ import { css } from '@emotion/css';
 import { addAdHocFilter } from './Breakdowns/AddToFiltersButton';
 import { areArraysEqual } from '../../services/comparison';
 import { getLogsPanelFrame } from './ServiceScene';
-import { getFilterTypeFromLabelType, getLabelTypeFromFrame } from '../../services/fields';
-import { LabelType } from '../../services/fieldsTypes';
+import { getVariableForLabel } from '../../services/fields';
 
 export class LogsTableScene extends SceneObjectBase {
   public static Component = ({ model }: SceneComponentProps<LogsTableScene>) => {
@@ -28,10 +27,7 @@ export class LogsTableScene extends SceneObjectBase {
 
     // Define callback function to update filters in react
     const addFilter = (filter: AdHocVariableFilter) => {
-      const { key, value } = filter;
-
-      const labelType = dataFrame ? getLabelTypeFromFrame(key, dataFrame) : LabelType.Parsed;
-      const variableType = getFilterTypeFromLabelType(labelType, key, value);
+      const variableType = getVariableForLabel(dataFrame, filter.key, model);
       addAdHocFilter(filter, parentModel, variableType);
     };
 

src/services/expressions.ts

@@ -6,6 +6,7 @@ import {
   VAR_FIELDS_EXPR,
   VAR_LABELS_EXPR,
   VAR_LINE_FILTER_EXPR,
+  VAR_METADATA_EXPR,
   VAR_PATTERNS_EXPR,
 } from './variables';
 import { SceneObject } from '@grafana/scenes';
@@ -36,14 +37,14 @@ export function getTimeSeriesExpr(sceneRef: SceneObject, streamSelectorName: str
   // if we have fields, we also need to add parsers
   if (fieldFilters.length) {
     if (parser === 'mixed') {
-      return `sum(count_over_time({${VAR_LABELS_EXPR}} ${metadataExpressionToAdd} ${VAR_LINE_FILTER_EXPR} ${VAR_PATTERNS_EXPR} ${MIXED_FORMAT_EXPR} ${VAR_FIELDS_EXPR} [$__auto])) by (${streamSelectorName})`;
+      return `sum(count_over_time({${VAR_LABELS_EXPR}} ${metadataExpressionToAdd} ${VAR_METADATA_EXPR} ${VAR_LINE_FILTER_EXPR} ${VAR_PATTERNS_EXPR} ${MIXED_FORMAT_EXPR} ${VAR_FIELDS_EXPR} [$__auto])) by (${streamSelectorName})`;
     }
     if (parser === 'json') {
-      return `sum(count_over_time({${VAR_LABELS_EXPR}} ${metadataExpressionToAdd} ${VAR_LINE_FILTER_EXPR} ${VAR_PATTERNS_EXPR} ${JSON_FORMAT_EXPR} ${VAR_FIELDS_EXPR} [$__auto])) by (${streamSelectorName})`;
+      return `sum(count_over_time({${VAR_LABELS_EXPR}} ${metadataExpressionToAdd} ${VAR_METADATA_EXPR} ${VAR_LINE_FILTER_EXPR} ${VAR_PATTERNS_EXPR} ${JSON_FORMAT_EXPR} ${VAR_FIELDS_EXPR} [$__auto])) by (${streamSelectorName})`;
     }
     if (parser === 'logfmt') {
-      return `sum(count_over_time({${VAR_LABELS_EXPR}} ${metadataExpressionToAdd} ${VAR_LINE_FILTER_EXPR} ${VAR_PATTERNS_EXPR} ${LOGS_FORMAT_EXPR} ${VAR_FIELDS_EXPR} [$__auto])) by (${streamSelectorName})`;
+      return `sum(count_over_time({${VAR_LABELS_EXPR}} ${metadataExpressionToAdd} ${VAR_METADATA_EXPR} ${VAR_LINE_FILTER_EXPR} ${VAR_PATTERNS_EXPR} ${LOGS_FORMAT_EXPR} ${VAR_FIELDS_EXPR} [$__auto])) by (${streamSelectorName})`;
     }
   }
-  return `sum(count_over_time({${VAR_LABELS_EXPR}} ${metadataExpressionToAdd} ${VAR_LINE_FILTER_EXPR} ${VAR_PATTERNS_EXPR} ${VAR_FIELDS_EXPR} [$__auto])) by (${streamSelectorName})`;
+  return `sum(count_over_time({${VAR_LABELS_EXPR}} ${metadataExpressionToAdd} ${VAR_METADATA_EXPR} ${VAR_LINE_FILTER_EXPR} ${VAR_PATTERNS_EXPR} ${VAR_FIELDS_EXPR} [$__auto])) by (${streamSelectorName})`;
 }