diff --git a/clickhouse/src/queries/click-house-log-query/ClickHouseLogQuery.tsx b/clickhouse/src/queries/click-house-log-query/ClickHouseLogQuery.tsx index b1235da44..705d0d1e4 100644 --- a/clickhouse/src/queries/click-house-log-query/ClickHouseLogQuery.tsx +++ b/clickhouse/src/queries/click-house-log-query/ClickHouseLogQuery.tsx @@ -11,6 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +import { QueryDefinition } from '@perses-dev/core'; import { parseVariables } from '@perses-dev/plugin-system'; import { getClickHouseLogData } from './get-click-house-log-data'; import { ClickHouseLogQueryEditor } from './ClickHouseLogQueryEditor'; @@ -28,4 +29,18 @@ export const ClickHouseLogQuery: LogQueryPlugin = { variables: allVariables, }; }, + createVolumeQuery: (spec: ClickHouseLogQuerySpec): QueryDefinition | null => { + // Only create volume query if there's a valid query + if (!spec.query || !spec.query.trim()) { + return null; + } + + // For ClickHouse, we need to transform the SQL query into a volume aggregation + // This is a simplified approach that may need refinement based on actual query patterns + // For now, returning null to indicate volume queries need more sophisticated parsing + // TODO: Implement proper SQL parsing and transformation for volume queries + // Example target: SELECT toStartOfInterval(timestamp, INTERVAL 1 minute) as time, level, count() FROM ... GROUP BY time, level + + return null; + }, }; diff --git a/clickhouse/src/queries/click-house-log-query/log-query-plugin-interface.ts b/clickhouse/src/queries/click-house-log-query/log-query-plugin-interface.ts index 3ae031699..5da34609b 100644 --- a/clickhouse/src/queries/click-house-log-query/log-query-plugin-interface.ts +++ b/clickhouse/src/queries/click-house-log-query/log-query-plugin-interface.ts @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { AbsoluteTimeRange, UnknownSpec, LogData } from '@perses-dev/core'; +import { AbsoluteTimeRange, UnknownSpec, LogData, QueryDefinition } from '@perses-dev/core'; import { DatasourceStore, Plugin, VariableStateMap } from '@perses-dev/plugin-system'; export interface LogQueryResult { @@ -35,4 +35,10 @@ type LogQueryPluginDependencies = { export interface LogQueryPlugin extends Plugin { getLogData: (spec: Spec, ctx: ClickHouseQueryContext) => Promise; dependsOn?: (spec: Spec, ctx: ClickHouseQueryContext) => LogQueryPluginDependencies; + /** + * Optional method to create a TimeSeriesQuery for log volume visualization. + * Returns a QueryDefinition that aggregates log volumes over time, typically grouped by log level. + * Returns null if volume queries are not supported or cannot be generated from the given spec. + */ + createVolumeQuery?: (spec: Spec, ctx: ClickHouseQueryContext) => QueryDefinition | null; } diff --git a/logexplorer/.cjs.swcrc b/logexplorer/.cjs.swcrc new file mode 100644 index 000000000..2ed65083d --- /dev/null +++ b/logexplorer/.cjs.swcrc @@ -0,0 +1,20 @@ +{ + "$schema": "https://json.schemastore.org/swcrc", + "jsc": { + "parser": { + "syntax": "typescript", + "tsx": true + }, + "target": "es2022", + "transform": { + "react": { + "runtime": "automatic", + "useBuiltins": true + } + } + }, + "module": { + "type": "commonjs" + }, + "exclude": ["\\.(stories|test)\\."] +} diff --git a/logexplorer/.gitignore b/logexplorer/.gitignore new file mode 100644 index 000000000..fe8baa8ba --- /dev/null +++ b/logexplorer/.gitignore @@ -0,0 +1,21 @@ +.idea/ + +# Local +.DS_Store +*.local +*.log* + +# Dist +node_modules +dist/ + +# IDE +.vscode/* +!.vscode/extensions.json +.idea + +# generated archives +*.tar.gz + +# external CUE dependencies +/*/cue.mod/pkg/ diff --git a/logexplorer/.swcrc b/logexplorer/.swcrc new file mode 100644 index 000000000..feaf67637 --- /dev/null +++ b/logexplorer/.swcrc @@ -0,0 +1,21 @@ +{ + "$schema": "https://json.schemastore.org/swcrc", + "jsc": { + "parser": { + "syntax": "typescript", + "tsx": true + }, + "target": "es2022", + "transform": { + "react": { + "runtime": "automatic", + "useBuiltins": true + } + } + }, + "module": { + "type": "es6" + }, + "sourceMaps": true, + "exclude": ["\\.(stories|test)\\."] +} diff --git a/logexplorer/README.md b/logexplorer/README.md new file mode 100644 index 000000000..58090454c --- /dev/null +++ b/logexplorer/README.md @@ -0,0 +1,41 @@ +# Plugin Module: log-explorer + +### How to install + +This plugin requires react and react-dom 18 + +Install peer dependencies: + +```bash +npm install react@18 react-dom@18 +``` + +Install the plugin: + +```bash +npm install @my-org/log-explorer +``` + +## Development + +### Setup + +Install dependencies: + +```bash +npm install +``` + +### Get Started + +Start the dev server: + +```bash +npm run dev +``` + +Build the plugin for distribution: + +```bash +npm run build +``` diff --git a/logexplorer/cue.mod/module.cue b/logexplorer/cue.mod/module.cue new file mode 100644 index 000000000..5ae153349 --- /dev/null +++ b/logexplorer/cue.mod/module.cue @@ -0,0 +1,7 @@ +module: "github.com/perses-dev/log-explorer@v0" +language: { + version: "v0.12.0" +} +source: { + kind: "git" +} diff --git a/logexplorer/jest.config.ts b/logexplorer/jest.config.ts new file mode 100644 index 000000000..13276c6cd --- /dev/null +++ b/logexplorer/jest.config.ts @@ -0,0 +1,26 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the \"License\"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an \"AS IS\" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import type { Config } from "@jest/types"; +import shared from "../jest.shared"; + +const jestConfig: Config.InitialOptions = { + ...shared, + + setupFilesAfterEnv: [ + ...(shared.setupFilesAfterEnv ?? []), + "/src/setup-tests.ts", + ], +}; + +export default jestConfig; diff --git a/logexplorer/package.json b/logexplorer/package.json new file mode 100644 index 000000000..d9c19b2b1 --- /dev/null +++ b/logexplorer/package.json @@ -0,0 +1,72 @@ +{ + "name": "@perses-dev/log-explorer", + "version": "0.1.0", + "homepage": "https://github.com/perses/plugins/blob/main/README.md", + "repository": { + "type": "git", + "url": "git+https://github.com/perses/plugins.git" + }, + "bugs": { + "url": "https://github.com/perses/plugins/issues" + }, + "scripts": { + "dev": "rsbuild dev", + "build": "npm run build-mf && concurrently \"npm:build:*\"", + "build-mf": "rsbuild build", + "build:cjs": "swc ./src -d dist/lib/cjs --strip-leading-paths --config-file .cjs.swcrc", + "build:esm": "swc ./src -d dist/lib --strip-leading-paths --config-file .swcrc", + "build:types": "tsc --project tsconfig.build.json", + "lint": "eslint src --ext .ts,.tsx", + "test": "cross-env LC_ALL=C TZ=UTC jest --passWithNoTests", + "type-check": "tsc --noEmit" + }, + "main": "lib/cjs/index.js", + "module": "lib/index.js", + "types": "lib/index.d.ts", + "devDependencies": { + "@types/qs": "^6.9.18" + }, + "peerDependencies": { + "@emotion/react": "^11.7.1", + "@emotion/styled": "^11.6.0", + "@hookform/resolvers": "^3.2.0", + "@perses-dev/components": "^0.53.0-rc.2", + "@perses-dev/core": "^0.53.0-rc.0", + "@perses-dev/dashboards": "^0.53.0-rc.2", + "@perses-dev/explore": "^0.53.0-rc.2", + "@perses-dev/plugin-system": "^0.53.0-rc.2", + "@tanstack/react-query": "^4.39.1", + "date-fns": "^4.1.0", + "date-fns-tz": "^3.2.0", + "echarts": "5.5.0", + "immer": "^10.1.1", + "lodash": "^4.17.21", + "react": "^17.0.2 || ^18.0.0", + "react-dom": "^17.0.2 || ^18.0.0", + "react-hook-form": "^7.52.2", + "react-router-dom": "^5 || ^6 || ^7", + "use-resize-observer": "^9.0.0" + }, + "files": [ + "lib/**/*", + "__mf/**/*", + "mf-manifest.json", + "mf-stats.json" + ], + "perses": { + "moduleName": "LogExplorer", + "moduleOrg": "perses.dev", + "schemasPath": "schemas", + "plugins": [ + { + "kind": "Explore", + "spec": { + "display": { + "name": "Log Explorer" + }, + "name": "LogExplorer" + } + } + ] + } +} diff --git a/logexplorer/rsbuild.config.ts b/logexplorer/rsbuild.config.ts new file mode 100644 index 000000000..a46647113 --- /dev/null +++ b/logexplorer/rsbuild.config.ts @@ -0,0 +1,46 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the \"License\"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an \"AS IS\" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { pluginReact } from '@rsbuild/plugin-react'; +import { createConfigForPlugin } from '../rsbuild.shared'; + +export default createConfigForPlugin({ + name: 'LogExplorer', + rsbuildConfig: { + server: { port: 3009 }, + plugins: [pluginReact()], + }, + moduleFederation: { + exposes: { + './LogExplorer': './src/explore/log-explorer', + }, + shared: { + react: { requiredVersion: '18.2.0', singleton: true }, + 'react-dom': { requiredVersion: '18.2.0', singleton: true }, + echarts: { singleton: true }, + 'date-fns': { singleton: true }, + 'date-fns-tz': { singleton: true }, + lodash: { singleton: true }, + '@perses-dev/components': { singleton: true }, + '@perses-dev/plugin-system': { singleton: true }, + '@perses-dev/explore': { singleton: true }, + '@perses-dev/dashboards': { singleton: true }, + '@emotion/react': { requiredVersion: '^11.11.3', singleton: true }, + '@emotion/styled': { singleton: true }, + '@hookform/resolvers': { singleton: true }, + '@tanstack/react-query': { singleton: true }, + 'react-hook-form': { singleton: true }, + 'react-router-dom': { singleton: true }, + }, + }, +}); diff --git a/logexplorer/src/bootstrap.tsx b/logexplorer/src/bootstrap.tsx new file mode 100644 index 000000000..cabf17b20 --- /dev/null +++ b/logexplorer/src/bootstrap.tsx @@ -0,0 +1,18 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the \"License\"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an \"AS IS\" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import React from 'react'; +import ReactDOM from 'react-dom/client'; + +const root = ReactDOM.createRoot(document.getElementById('root')!); +root.render(); diff --git a/logexplorer/src/env.d.ts b/logexplorer/src/env.d.ts new file mode 100644 index 000000000..e216e17b1 --- /dev/null +++ b/logexplorer/src/env.d.ts @@ -0,0 +1,14 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the \"License\"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an \"AS IS\" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/// diff --git a/logexplorer/src/explore/index.ts b/logexplorer/src/explore/index.ts new file mode 100644 index 000000000..617b5ed84 --- /dev/null +++ b/logexplorer/src/explore/index.ts @@ -0,0 +1,14 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the \"License\"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an \"AS IS\" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +export * from './log-explorer'; diff --git a/logexplorer/src/explore/log-explorer/LogExplorer.tsx b/logexplorer/src/explore/log-explorer/LogExplorer.tsx new file mode 100644 index 000000000..bf2077e93 --- /dev/null +++ b/logexplorer/src/explore/log-explorer/LogExplorer.tsx @@ -0,0 +1,205 @@ +// Copyright 2025 The Perses Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { Box, Stack } from '@mui/material'; +import { + DataQueriesProvider, + MultiQueryEditor, + useListPluginMetadata, + usePluginRegistry, + useTimeRange, +} from '@perses-dev/plugin-system'; +import { ReactElement, useEffect, useMemo, useState } from 'react'; +import { QueryDefinition } from '@perses-dev/core'; +import useResizeObserver from 'use-resize-observer'; +import { Panel } from '@perses-dev/dashboards'; +import { useExplorerManagerContext } from '@perses-dev/explore'; + +interface LogExplorerQueryParams { + queries?: QueryDefinition[]; +} + +const PANEL_PREVIEW_HEIGHT = 700; +const HISTOGRAM_HEIGHT = 200; + +function LogsTablePanel({ queries }: { queries: QueryDefinition[] }): ReactElement { + const { ref: boxRef } = useResizeObserver(); + const height = PANEL_PREVIEW_HEIGHT; + + // map QueryDefinition to Definition + const definitions = useMemo( + () => + queries.length + ? queries.map((query) => ({ + kind: query.spec.plugin.kind, + spec: query.spec.plugin.spec, + })) + : [], + [queries] + ); + + return ( + + + + + + ); +} + +function VolumeHistogramPanel({ queries }: { queries: QueryDefinition[] }): ReactElement { + const { ref: boxRef } = useResizeObserver(); + + // map QueryDefinition to Definition + const definitions = useMemo( + () => + queries.length + ? queries.map((query) => ({ + kind: query.spec.plugin.kind, + spec: query.spec.plugin.spec, + })) + : [], + [queries] + ); + + return ( + + + + + + ); +} + +export function LogExplorer(): ReactElement { + const { + data: { queries = [] }, + setData, + } = useExplorerManagerContext(); + + const { getPlugin } = usePluginRegistry(); + const { absoluteTimeRange } = useTimeRange(); + + // Get all datasource plugins that support LogQuery + const { data: datasourcePlugins } = useListPluginMetadata(['Datasource']); + + const logDatasourcePlugins = useMemo( + () => + datasourcePlugins + ?.filter((plugin) => { + // Check if plugin spec has the supportedQueryTypes property + const pluginSpec = plugin.spec as { supportedQueryTypes?: string[] }; + return pluginSpec?.supportedQueryTypes?.includes('LogQuery'); + }) + .map((p) => p.kind) ?? [], + [datasourcePlugins] + ); + + // State for volume queries + const [volumeQueries, setVolumeQueries] = useState([]); + + // Generate volume queries from log queries + useEffect(() => { + const generateVolumeQueries = async () => { + if (queries.length === 0) { + setVolumeQueries([]); + return; + } + + const volumeQueryPromises = queries.map(async (query) => { + if (query.kind !== 'LogQuery') { + return null; + } + + try { + const pluginKind = query.spec.plugin.kind; + const plugin = await getPlugin('LogQuery', pluginKind); + + // Check if plugin has createVolumeQuery method + if (plugin && 'createVolumeQuery' in plugin && typeof plugin.createVolumeQuery === 'function') { + // Pass timeRange context for dynamic interval calculation + const context = { + timeRange: { + start: absoluteTimeRange.start, + end: absoluteTimeRange.end, + }, + }; + const volumeQuery = plugin.createVolumeQuery(query.spec.plugin.spec, context); + return volumeQuery; + } + } catch (error) { + console.error(`[LogExplorer] Failed to create volume query for ${query.spec.plugin.kind}:`, error); + } + + return null; + }); + + const results = await Promise.all(volumeQueryPromises); + const validVolumeQueries = results.filter((q: QueryDefinition | null): q is QueryDefinition => q !== null); + setVolumeQueries(validVolumeQueries); + }; + + generateVolumeQueries(); + }, [queries, getPlugin, absoluteTimeRange]); + + return ( + + setData({ queries: state })} + onQueryRun={() => { + // Query run is handled automatically via onChange + }} + /> + {volumeQueries.length > 0 && } + + + ); +} diff --git a/logexplorer/src/explore/log-explorer/index.ts b/logexplorer/src/explore/log-explorer/index.ts new file mode 100644 index 000000000..45f264f12 --- /dev/null +++ b/logexplorer/src/explore/log-explorer/index.ts @@ -0,0 +1,14 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the \"License\"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an \"AS IS\" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +export * from './LogExplorer'; diff --git a/logexplorer/src/getPluginModule.ts b/logexplorer/src/getPluginModule.ts new file mode 100644 index 000000000..ee2f25a74 --- /dev/null +++ b/logexplorer/src/getPluginModule.ts @@ -0,0 +1,30 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the \"License\"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an \"AS IS\" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import { PluginModuleResource, PluginModuleSpec } from '@perses-dev/plugin-system'; +import packageJson from '../package.json'; + +/** + * Returns the plugin module information from package.json + */ +export function getPluginModule(): PluginModuleResource { + const { name, version, perses } = packageJson; + return { + kind: 'PluginModule', + metadata: { + name, + version, + }, + spec: perses as PluginModuleSpec, + }; +} diff --git a/logexplorer/src/index-federation.ts b/logexplorer/src/index-federation.ts new file mode 100644 index 000000000..36f748007 --- /dev/null +++ b/logexplorer/src/index-federation.ts @@ -0,0 +1,14 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the \"License\"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an \"AS IS\" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import('./bootstrap'); diff --git a/logexplorer/src/index.ts b/logexplorer/src/index.ts new file mode 100644 index 000000000..5d31f9b0f --- /dev/null +++ b/logexplorer/src/index.ts @@ -0,0 +1,15 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the \"License\"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an \"AS IS\" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +export { getPluginModule } from './getPluginModule'; +export * from './explore'; diff --git a/logexplorer/src/setup-tests.ts b/logexplorer/src/setup-tests.ts new file mode 100644 index 000000000..c4b091083 --- /dev/null +++ b/logexplorer/src/setup-tests.ts @@ -0,0 +1,17 @@ +// Copyright The Perses Authors +// Licensed under the Apache License, Version 2.0 (the \"License\"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an \"AS IS\" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +import '@testing-library/jest-dom'; + +// Always mock e-charts during tests since we don't have a proper canvas in jsdom +jest.mock('echarts/core'); diff --git a/logexplorer/tsconfig.build.json b/logexplorer/tsconfig.build.json new file mode 100644 index 000000000..fc0aafe27 --- /dev/null +++ b/logexplorer/tsconfig.build.json @@ -0,0 +1,9 @@ +{ + "extends": "./tsconfig.json", + "exclude": ["**/*.stories.*", "**/*.test.*", "**/*.map"], + "compilerOptions": { + "emitDeclarationOnly": true, + "declaration": true, + "preserveWatchOutput": true + } +} diff --git a/logexplorer/tsconfig.json b/logexplorer/tsconfig.json new file mode 100644 index 000000000..d8471c931 --- /dev/null +++ b/logexplorer/tsconfig.json @@ -0,0 +1,23 @@ +{ + "compilerOptions": { + "outDir": "./dist/lib", + "rootDir": "./src", + "target": "es2022", + "lib": ["dom", "dom.iterable", "esnext"], + "module": "esnext", + "jsx": "react-jsx", + "skipLibCheck": true, + "esModuleInterop": true, + "allowSyntheticDefaultImports": true, + "strict": true, + "forceConsistentCasingInFileNames": true, + "moduleResolution": "node", + "resolveJsonModule": true, + "isolatedModules": true, + "noUncheckedIndexedAccess": true, + "declaration": true, + "declarationMap": true, + "pretty": true + }, + "include": ["src"] +} diff --git a/loki/src/queries/loki-log-query/LokiLogQuery.tsx b/loki/src/queries/loki-log-query/LokiLogQuery.tsx index 5ceeb473f..54ee526e5 100644 --- a/loki/src/queries/loki-log-query/LokiLogQuery.tsx +++ b/loki/src/queries/loki-log-query/LokiLogQuery.tsx @@ -11,12 +11,67 @@ // See the License for the specific language governing permissions and // limitations under the License. +import { QueryDefinition } from '@perses-dev/core'; import { parseVariables } from '@perses-dev/plugin-system'; import { getLokiLogData } from './get-loki-log-data'; import { LokiLogQueryEditor } from './LokiLogQueryEditor'; import { LokiLogQuerySpec } from './loki-log-query-types'; import { LogQueryPlugin } from './log-query-plugin-interface'; +// Target number of bars for log volume histogram +const TARGET_HISTOGRAM_BARS = 40; + +// Standard intervals for histogram calculations (in milliseconds) +const STANDARD_INTERVALS = [ + { ms: 1000, label: '1s' }, + { ms: 2000, label: '2s' }, + { ms: 5000, label: '5s' }, + { ms: 10000, label: '10s' }, + { ms: 15000, label: '15s' }, + { ms: 30000, label: '30s' }, + { ms: 60000, label: '1m' }, + { ms: 120000, label: '2m' }, + { ms: 300000, label: '5m' }, + { ms: 600000, label: '10m' }, + { ms: 900000, label: '15m' }, + { ms: 1800000, label: '30m' }, + { ms: 3600000, label: '1h' }, + { ms: 7200000, label: '2h' }, + { ms: 21600000, label: '6h' }, + { ms: 43200000, label: '12h' }, + { ms: 86400000, label: '1d' }, + { ms: 604800000, label: '7d' }, + { ms: 2592000000, label: '30d' }, +] as const; + +/** + * Calculates an appropriate interval for log volume histograms based on time range. + * Uses standard round intervals for better alignment with time-series data. + */ +function calculateVolumeInterval(timeRangeMs: number): string { + // Prefer smallest interval that produces 20-100 bars (optimal range) + for (const interval of STANDARD_INTERVALS) { + const barCount = timeRangeMs / interval.ms; + if (barCount >= 20 && barCount <= 100) { + return interval.label; + } + } + + // Fallback: find closest to target if no interval fits optimal range + let bestInterval = STANDARD_INTERVALS[STANDARD_INTERVALS.length - 1]!; + let bestDistance = Infinity; + for (const interval of STANDARD_INTERVALS) { + const barCount = timeRangeMs / interval.ms; + const distance = Math.abs(barCount - TARGET_HISTOGRAM_BARS); + if (distance < bestDistance) { + bestDistance = distance; + bestInterval = interval; + } + } + + return bestInterval.label; +} + export const LokiLogQuery: LogQueryPlugin = { getLogData: getLokiLogData, OptionsEditorComponent: LokiLogQueryEditor, @@ -28,4 +83,29 @@ export const LokiLogQuery: LogQueryPlugin = { variables: allVariables, }; }, + createVolumeQuery: (spec: LokiLogQuerySpec, context?): QueryDefinition | null => { + if (!spec.query || !spec.query.trim()) { + return null; + } + + const interval = context?.timeRange + ? calculateVolumeInterval(context.timeRange.end.getTime() - context.timeRange.start.getTime()) + : '1m'; + + const volumeQuery = `sum by (level, detected_level) (count_over_time(${spec.query}[${interval}]))`; + + return { + kind: 'TimeSeriesQuery', + spec: { + plugin: { + kind: 'LokiTimeSeriesQuery', + spec: { + query: volumeQuery, + datasource: spec.datasource, + step: interval, + }, + }, + }, + }; + }, }; diff --git a/loki/src/queries/loki-log-query/log-query-plugin-interface.ts b/loki/src/queries/loki-log-query/log-query-plugin-interface.ts index e67d21167..9b7a2bd01 100644 --- a/loki/src/queries/loki-log-query/log-query-plugin-interface.ts +++ b/loki/src/queries/loki-log-query/log-query-plugin-interface.ts @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { LogData, AbsoluteTimeRange, UnknownSpec } from '@perses-dev/core'; +import { LogData, AbsoluteTimeRange, UnknownSpec, QueryDefinition } from '@perses-dev/core'; import { DatasourceStore, Plugin, VariableStateMap } from '@perses-dev/plugin-system'; export interface LogQueryResult { @@ -35,4 +35,10 @@ type LogQueryPluginDependencies = { export interface LogQueryPlugin extends Plugin { getLogData: (spec: Spec, ctx: LogQueryContext) => Promise; dependsOn?: (spec: Spec, ctx: LogQueryContext) => LogQueryPluginDependencies; + /** + * Optional method to create a TimeSeriesQuery for log volume visualization. + * Returns a QueryDefinition that aggregates log volumes over time, typically grouped by log level. + * Returns null if volume queries are not supported or cannot be generated from the given spec. + */ + createVolumeQuery?: (spec: Spec, ctx: LogQueryContext) => QueryDefinition | null; } diff --git a/package-lock.json b/package-lock.json index 69bf86e96..bd35da926 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "gaugechart", "heatmapchart", "histogramchart", + "logexplorer", "logstable", "loki", "markdown", @@ -216,6 +217,34 @@ "react-dom": "^17.0.2 || ^18.0.0" } }, + "logexplorer": { + "name": "@perses-dev/log-explorer", + "version": "0.1.0", + "devDependencies": { + "@types/qs": "^6.9.18" + }, + "peerDependencies": { + "@emotion/react": "^11.7.1", + "@emotion/styled": "^11.6.0", + "@hookform/resolvers": "^3.2.0", + "@perses-dev/components": "^0.53.0-rc.2", + "@perses-dev/core": "^0.53.0-rc.0", + "@perses-dev/dashboards": "^0.53.0-rc.2", + "@perses-dev/explore": "^0.53.0-rc.2", + "@perses-dev/plugin-system": "^0.53.0-rc.2", + "@tanstack/react-query": "^4.39.1", + "date-fns": "^4.1.0", + "date-fns-tz": "^3.2.0", + "echarts": "5.5.0", + "immer": "^10.1.1", + "lodash": "^4.17.21", + "react": "^17.0.2 || ^18.0.0", + "react-dom": "^17.0.2 || ^18.0.0", + "react-hook-form": "^7.52.2", + "react-router-dom": "^5 || ^6 || ^7", + "use-resize-observer": "^9.0.0" + } + }, "logstable": { "name": "@perses-dev/logs-table-plugin", "version": "0.2.0-rc.1", @@ -3838,6 +3867,10 @@ "resolved": "histogramchart", "link": true }, + "node_modules/@perses-dev/log-explorer": { + "resolved": "logexplorer", + "link": true + }, "node_modules/@perses-dev/logs-table-plugin": { "resolved": "logstable", "link": true diff --git a/package.json b/package.json index 9df78a804..61f1f4518 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "gaugechart", "heatmapchart", "histogramchart", + "logexplorer", "logstable", "loki", "markdown", diff --git a/victorialogs/src/queries/victorialogs-log-query/VictoriaLogsLogQuery.tsx b/victorialogs/src/queries/victorialogs-log-query/VictoriaLogsLogQuery.tsx index d9e021363..990664211 100644 --- a/victorialogs/src/queries/victorialogs-log-query/VictoriaLogsLogQuery.tsx +++ b/victorialogs/src/queries/victorialogs-log-query/VictoriaLogsLogQuery.tsx @@ -11,12 +11,67 @@ // See the License for the specific language governing permissions and // limitations under the License. +import { QueryDefinition } from '@perses-dev/core'; import { parseVariables } from '@perses-dev/plugin-system'; import { getVictoriaLogsLogData } from './query'; import { VictoriaLogsLogQueryEditor } from './VictoriaLogsLogQueryEditor'; import { VictoriaLogsLogQuerySpec } from './types'; import { LogQueryPlugin } from './interface'; +// Target number of bars for log volume histogram +const TARGET_HISTOGRAM_BARS = 40; + +// Standard intervals for histogram calculations (in milliseconds) +const STANDARD_INTERVALS = [ + { ms: 1000, label: '1s' }, + { ms: 2000, label: '2s' }, + { ms: 5000, label: '5s' }, + { ms: 10000, label: '10s' }, + { ms: 15000, label: '15s' }, + { ms: 30000, label: '30s' }, + { ms: 60000, label: '1m' }, + { ms: 120000, label: '2m' }, + { ms: 300000, label: '5m' }, + { ms: 600000, label: '10m' }, + { ms: 900000, label: '15m' }, + { ms: 1800000, label: '30m' }, + { ms: 3600000, label: '1h' }, + { ms: 7200000, label: '2h' }, + { ms: 21600000, label: '6h' }, + { ms: 43200000, label: '12h' }, + { ms: 86400000, label: '1d' }, + { ms: 604800000, label: '7d' }, + { ms: 2592000000, label: '30d' }, +] as const; + +/** + * Calculates an appropriate interval for log volume histograms based on time range. + * Uses standard round intervals for better alignment with time-series data. + */ +function calculateVolumeInterval(timeRangeMs: number): string { + // Prefer smallest interval that produces 20-100 bars (optimal range) + for (const interval of STANDARD_INTERVALS) { + const barCount = timeRangeMs / interval.ms; + if (barCount >= 20 && barCount <= 100) { + return interval.label; + } + } + + // Fallback: find closest to target if no interval fits optimal range + let bestInterval = STANDARD_INTERVALS[STANDARD_INTERVALS.length - 1]!; + let bestDistance = Infinity; + for (const interval of STANDARD_INTERVALS) { + const barCount = timeRangeMs / interval.ms; + const distance = Math.abs(barCount - TARGET_HISTOGRAM_BARS); + if (distance < bestDistance) { + bestDistance = distance; + bestInterval = interval; + } + } + + return bestInterval.label; +} + export const VictoriaLogsLogQuery: LogQueryPlugin = { getLogData: getVictoriaLogsLogData, OptionsEditorComponent: VictoriaLogsLogQueryEditor, @@ -28,4 +83,29 @@ export const VictoriaLogsLogQuery: LogQueryPlugin = { variables: allVariables, }; }, + createVolumeQuery: (spec: VictoriaLogsLogQuerySpec, context?): QueryDefinition | null => { + if (!spec.query || !spec.query.trim()) { + return null; + } + + const interval = context?.timeRange + ? calculateVolumeInterval(context.timeRange.end.getTime() - context.timeRange.start.getTime()) + : '1m'; + + const volumeQuery = `${spec.query} | stats by (_time:${interval}, _stream) count() as volume`; + + return { + kind: 'TimeSeriesQuery', + spec: { + plugin: { + kind: 'VictoriaLogsTimeSeriesQuery', + spec: { + query: volumeQuery, + datasource: spec.datasource, + step: interval, + }, + }, + }, + }; + }, }; diff --git a/victorialogs/src/queries/victorialogs-log-query/interface.ts b/victorialogs/src/queries/victorialogs-log-query/interface.ts index e67d21167..9b7a2bd01 100644 --- a/victorialogs/src/queries/victorialogs-log-query/interface.ts +++ b/victorialogs/src/queries/victorialogs-log-query/interface.ts @@ -11,7 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -import { LogData, AbsoluteTimeRange, UnknownSpec } from '@perses-dev/core'; +import { LogData, AbsoluteTimeRange, UnknownSpec, QueryDefinition } from '@perses-dev/core'; import { DatasourceStore, Plugin, VariableStateMap } from '@perses-dev/plugin-system'; export interface LogQueryResult { @@ -35,4 +35,10 @@ type LogQueryPluginDependencies = { export interface LogQueryPlugin extends Plugin { getLogData: (spec: Spec, ctx: LogQueryContext) => Promise; dependsOn?: (spec: Spec, ctx: LogQueryContext) => LogQueryPluginDependencies; + /** + * Optional method to create a TimeSeriesQuery for log volume visualization. + * Returns a QueryDefinition that aggregates log volumes over time, typically grouped by log level. + * Returns null if volume queries are not supported or cannot be generated from the given spec. + */ + createVolumeQuery?: (spec: Spec, ctx: LogQueryContext) => QueryDefinition | null; } diff --git a/victorialogs/src/queries/victorialogs-time-series-query/query.ts b/victorialogs/src/queries/victorialogs-time-series-query/query.ts index 0f62ada05..abd60b295 100644 --- a/victorialogs/src/queries/victorialogs-time-series-query/query.ts +++ b/victorialogs/src/queries/victorialogs-time-series-query/query.ts @@ -79,9 +79,12 @@ function formatStepForVictoriaLogs(stepSeconds: number): string { function convertMatrixToTimeSeries(matrix: VictoriaLogsMatrixResult[]): TimeSeries[] { return matrix.map((series) => { - const { _stream, ...labels } = series.metric; - if (_stream) { - const match = _stream.match(/{([^}]+)}/); + const labels = { ...series.metric }; + delete labels._stream; + delete labels.__name__; + + if (series.metric._stream) { + const match = series.metric._stream.match(/{([^}]+)}/); if (match && match[1]) { match[1].split(',').forEach((labelPair) => { const [key, val] = labelPair.split('=').map((s) => s.trim().replace(/^"|"$/g, ''));