feat: wip - add support for detected_field/.../values

docker-compose.local.yaml

@@ -4,7 +4,7 @@ services:
   grafana:
     container_name: 'grafana-logsapp'
     environment:
-      - GF_FEATURE_TOGGLES_ENABLE=accessControlOnCall lokiLogsDataplane
+      - GF_FEATURE_TOGGLES_ENABLE=accessControlOnCall lokiLogsDataplane exploreLogsAggregatedMetrics
       - GF_LOG_FRONTEND_ENABLED=true
       - GF_LOG_FRONTEND_CUSTOM_ENDPOINT=http://localhost:12347/collect
       - GF_TRACING_OPENTELEMETRY_JAEGER_ADDRESS=http://host.docker.internal:14268/api/traces
@@ -22,7 +22,7 @@ services:
     extra_hosts:
       - 'host.docker.internal:host-gateway'
   loki:
-    image: grafana/loki:3.2.0
+    image: grafana/loki:main-12cd12a
     environment:
       LOG_CLUSTER_DEPTH: '8'
       LOG_SIM_TH: '0.3'

src/Components/IndexScene/IndexScene.tsx

@@ -7,6 +7,7 @@ import {
   DataSourceVariable,
   SceneComponentProps,
   SceneControlsSpacer,
+  sceneGraph,
   SceneObject,
   SceneObjectBase,
   SceneObjectState,
@@ -19,8 +20,12 @@ import {
   VariableValueSelectors,
 } from '@grafana/scenes';
 import {
+  DETECTED_FIELD_VALUES_EXPR,
+  DETECTED_LEVELS_VALUES_EXPR,
+  DETECTED_METADATA_VALUES_EXPR,
   EXPLORATION_DS,
   MIXED_FORMAT_EXPR,
+  PENDING_FIELDS_EXPR,
   VAR_DATASOURCE,
   VAR_FIELDS,
   VAR_LABELS,
@@ -50,11 +55,15 @@ import { CustomConstantVariable } from '../../services/CustomConstantVariable';
 import {
   getFieldsVariable,
   getLevelsVariable,
+  getMetadataVariable,
   getPatternsVariable,
   getUrlParamNameForVariable,
+  getValueFromFieldsFilter,
 } from '../../services/variableGetters';
 import { ToolbarScene } from './ToolbarScene';
 import { OptionalRouteMatch } from '../Pages';
+import { AdHocFilterWithLabels, getDetectedFieldValuesTagValuesProvider } from '../../services/TagValuesProvider';
+import { lokiRegularEscape } from '../../services/fields';
 
 export interface AppliedPattern {
   pattern: string;
@@ -121,6 +130,7 @@ export class IndexScene extends SceneObjectBase<IndexSceneState> {
 
   public onActivate() {
     const stateUpdate: Partial<IndexSceneState> = {};
+    this.setVariableTagValuesProviders();
 
     if (!this.state.contentScene) {
       stateUpdate.contentScene = getContentScene(this.state.routeMatch?.params.breakdownLabel);
@@ -139,6 +149,56 @@ export class IndexScene extends SceneObjectBase<IndexSceneState> {
     );
   }
 
+  private setVariableTagValuesProviders() {
+    const fieldsVariable = getFieldsVariable(this);
+    const levelsVariable = getLevelsVariable(this);
+    const metadataVariable = getMetadataVariable(this);
+
+    fieldsVariable.setState({
+      getTagValuesProvider: this.getFieldsTagValuesProvider(VAR_FIELDS),
+    });
+
+    levelsVariable.setState({
+      getTagValuesProvider: this.getFieldsTagValuesProvider(VAR_LEVELS),
+    });
+
+    metadataVariable.setState({
+      getTagValuesProvider: this.getFieldsTagValuesProvider(VAR_METADATA),
+    });
+  }
+
+  private getFieldsTagValuesProvider(variableType: typeof VAR_FIELDS | typeof VAR_METADATA | typeof VAR_LEVELS) {
+    return (variable: AdHocFiltersVariable, filter: AdHocFilterWithLabels) => {
+      const filters = variable.state.filters.filter((f) => f.key !== filter.key);
+      const values = filters.map((f) => {
+        const parsed = variableType === VAR_FIELDS ? getValueFromFieldsFilter(f, variableType) : { value: f.value };
+        return `${f.key}${f.operator}\`${lokiRegularEscape(parsed.value)}\``;
+      });
+      const otherFiltersString = values.length ? '| ' + values.join(' |') : '';
+      const uninterpolatedExpression = this.getFieldsTagValuesExpression(variableType);
+      const expr = uninterpolatedExpression.replace(PENDING_FIELDS_EXPR, otherFiltersString);
+      const interpolated = sceneGraph.interpolate(this, expr);
+      return getDetectedFieldValuesTagValuesProvider(
+        filter,
+        interpolated,
+        this,
+        sceneGraph.getTimeRange(this).state.value,
+        variableType
+      );
+    };
+  }
+
+  private getFieldsTagValuesExpression(variableType: typeof VAR_FIELDS | typeof VAR_METADATA | typeof VAR_LEVELS) {
+    switch (variableType) {
+      case VAR_FIELDS:
+        return DETECTED_FIELD_VALUES_EXPR;
+      case VAR_METADATA:
+        return DETECTED_METADATA_VALUES_EXPR;
+      case VAR_LEVELS:
+        return DETECTED_LEVELS_VALUES_EXPR;
+    }
+  }
+
   /**
    * @todo why do we need to manually sync fields and levels, but not other ad hoc variables?
    * @param variable

src/services/TagValuesProvider.ts

@@ -0,0 +1,76 @@
+import { SceneObject } from '@grafana/scenes';
+import { AdHocVariableFilter, MetricFindValue, ScopedVars, TimeRange } from '@grafana/data';
+import { DataSourceWithBackend, getDataSourceSrv } from '@grafana/runtime';
+import { getDataSource } from './scenes';
+import { logger } from './logger';
+import { LokiQuery } from './lokiQuery';
+import { getValueFromFieldsFilter } from './variableGetters';
+import { VAR_FIELDS, VAR_LEVELS, VAR_METADATA } from './variables';
+
+//@todo export from scenes
+export interface AdHocFilterWithLabels extends AdHocVariableFilter {
+  keyLabel?: string;
+  valueLabels?: string[];
+}
+
+type FetchDetectedLabelValuesOptions = {
+  expr?: string;
+  timeRange?: TimeRange;
+  limit?: number;
+  scopedVars?: ScopedVars;
+};
+
+interface LokiLanguageProvider {
+  fetchDetectedLabelValues: (labelName: string, options?: FetchDetectedLabelValuesOptions) => Promise<string[]>;
+}
+
+export const getDetectedFieldValuesTagValuesProvider = async (
+  filter: AdHocFilterWithLabels,
+  expr: string,
+  sceneRef: SceneObject,
+  timeRange: TimeRange,
+  variable: typeof VAR_FIELDS | typeof VAR_METADATA | typeof VAR_LEVELS
+): Promise<{
+  replace?: boolean;
+  values: MetricFindValue[];
+}> => {
+  const datasource_ = await getDataSourceSrv().get(getDataSource(sceneRef));
+  if (!(datasource_ instanceof DataSourceWithBackend)) {
+    logger.error(new Error('getTagValuesProvider: Invalid datasource!'));
+    throw new Error('Invalid datasource!');
+  }
+  const datasource = datasource_ as DataSourceWithBackend<LokiQuery>;
+
+  const languageProvider = datasource.languageProvider as LokiLanguageProvider;
+
+  if (languageProvider) {
+    // Filter out other values for this key so users can include other values for this label
+    const options: FetchDetectedLabelValuesOptions = {
+      expr,
+      limit: 1000,
+      timeRange,
+    };
+
+    let results = await languageProvider.fetchDetectedLabelValues(filter.key, options);
+    // If the variable has a parser in the value, make sure we extract it and carry it over
+    // @todo can the parser ever change?
+    if (variable === VAR_FIELDS) {
+      const valueDecoded = getValueFromFieldsFilter(filter, variable);
+      return {
+        replace: true,
+        values: results.map((v) => ({
+          text: v,
+          value: JSON.stringify({
+            value: v,
+            parser: valueDecoded.parser,
+          }),
+        })),
+      };
+    }
+
+    return { replace: true, values: results.map((v) => ({ text: v })) };
+  } else {
+    logger.error(new Error('getTagValuesProvider: missing or invalid languageProvider!'));
+    return { replace: true, values: [] };
+  }
+};

src/services/fields.ts

@@ -315,3 +315,11 @@ export function buildFieldsQueryString(
 
   return buildFieldsQuery(optionValue, options);
 }
+
+// copied from /grafana/grafana/public/app/plugins/datasource/loki/datasource.ts:1204
+export function lokiRegularEscape<T>(value: T) {
+  if (typeof value === 'string') {
+    return value.replace(/'/g, "\\\\'");
+  }
+  return value;
+}

src/services/variableGetters.ts

@@ -32,6 +32,7 @@ import {
   VAR_PRIMARY_LABEL,
   VAR_PRIMARY_LABEL_SEARCH,
   VAR_METADATA_EXPR,
+  VAR_METADATA,
 } from './variables';
 import { AdHocVariableFilter } from '@grafana/data';
 import { logger } from './logger';
@@ -68,6 +69,10 @@ export function getLabelsVariable(scene: SceneObject) {
   return getAdHocFiltersVariable(VAR_LABELS, scene);
 }
 
+export function getMetadataVariable(scene: SceneObject) {
+  return getAdHocFiltersVariable(VAR_METADATA, scene);
+}
+
 export function getFieldsVariable(scene: SceneObject) {
   return getAdHocFiltersVariable(VAR_FIELDS, scene);
 }

src/services/variables.ts

@@ -23,6 +23,7 @@ export const VAR_LABELS = 'filters';
 export const VAR_LABELS_EXPR = '${filters}';
 export const VAR_FIELDS = 'fields';
 export const VAR_FIELDS_EXPR = '${fields}';
+export const PENDING_FIELDS_EXPR = '${pendingFields}';
 export const VAR_METADATA = 'metadata';
 export const VAR_METADATA_EXPR = '${metadata}';
 export const VAR_PATTERNS = 'patterns';
@@ -47,6 +48,10 @@ export const VAR_LOGS_FORMAT_EXPR = '${logsFormat}';
 export const VAR_LINE_FILTER = 'lineFilter';
 export const VAR_LINE_FILTER_EXPR = '${lineFilter}';
 export const LOG_STREAM_SELECTOR_EXPR = `{${VAR_LABELS_EXPR}} ${VAR_PATTERNS_EXPR} ${VAR_METADATA_EXPR} ${VAR_LINE_FILTER_EXPR} ${VAR_LEVELS_EXPR} ${VAR_LOGS_FORMAT_EXPR} ${VAR_FIELDS_EXPR}`;
+// Same as the LOG_STREAM_SELECTOR_EXPR, but without the fields as they will need to be built manually to exclude the current filter value
+export const DETECTED_FIELD_VALUES_EXPR = `{${VAR_LABELS_EXPR}} ${VAR_PATTERNS_EXPR} ${VAR_METADATA_EXPR} ${VAR_LINE_FILTER_EXPR} ${VAR_LEVELS_EXPR} ${VAR_LOGS_FORMAT_EXPR} ${PENDING_FIELDS_EXPR}`;
+export const DETECTED_METADATA_VALUES_EXPR = `{${VAR_LABELS_EXPR}} ${VAR_PATTERNS_EXPR} ${PENDING_FIELDS_EXPR} ${VAR_LINE_FILTER_EXPR} ${VAR_LEVELS_EXPR} ${VAR_LOGS_FORMAT_EXPR} ${VAR_FIELDS_EXPR}`;
+export const DETECTED_LEVELS_VALUES_EXPR = `{${VAR_LABELS_EXPR}} ${VAR_PATTERNS_EXPR} ${VAR_METADATA_EXPR} ${VAR_LINE_FILTER_EXPR} ${PENDING_FIELDS_EXPR} ${VAR_LOGS_FORMAT_EXPR} ${VAR_FIELDS_EXPR}`;
 export const PATTERNS_SAMPLE_SELECTOR_EXPR = `{${VAR_LABELS_EXPR}} ${VAR_METADATA_EXPR} ${VAR_PATTERNS_EXPR} ${VAR_LOGS_FORMAT_EXPR}`;
 export const EXPLORATION_DS = { uid: VAR_DATASOURCE_EXPR };
 export const ALL_VARIABLE_VALUE = '$__all';