From 41ac5e8b4f652621f7f8b5e8d48ddbf7a4caf7de Mon Sep 17 00:00:00 2001 From: Marc Parisi Date: Mon, 20 Jan 2025 17:06:30 -0500 Subject: [PATCH 1/4] commit --- .gcp.env | 2 +- .../sessions/SessionAnalyticsAgent.java | 40 +++---- .../src/main/resources/application.properties | 3 +- .../db/migration/V13__command_categorizer.sql | 9 ++ core/pom.xml | 6 +- .../model/categorization/CommandCategory.java | 29 +++++ .../repository/CommandCategoryRepository.java | 15 +++ .../categorization/CommandCategorizer.java | 113 ++++++++++++++++++ .../openai/categorization/CommandTrie.java | 70 +++++++++++ .../openai/categorization/TrieNode.java | 12 ++ .../sso/genai/LLMCommandCategorizer.java | 86 +++++++++++++ docker/keycloak/realms/sentrius-realm.json | 21 ++++ ops-scripts/gcp/deploy-helm.sh | 17 ++- pom.xml | 7 +- .../templates/keycloak-deployment.yaml | 4 + sentrius-gcp-chart/values.yaml | 2 + 16 files changed, 405 insertions(+), 31 deletions(-) create mode 100644 api/src/main/resources/db/migration/V13__command_categorizer.sql create mode 100644 core/src/main/java/io/sentrius/sso/core/model/categorization/CommandCategory.java create mode 100644 core/src/main/java/io/sentrius/sso/core/repository/CommandCategoryRepository.java create mode 100644 core/src/main/java/io/sentrius/sso/core/services/openai/categorization/CommandCategorizer.java create mode 100644 core/src/main/java/io/sentrius/sso/core/services/openai/categorization/CommandTrie.java create mode 100644 core/src/main/java/io/sentrius/sso/core/services/openai/categorization/TrieNode.java create mode 100644 core/src/main/java/io/sentrius/sso/genai/LLMCommandCategorizer.java diff --git a/.gcp.env b/.gcp.env index 62781601..ca7e436f 100644 --- a/.gcp.env +++ b/.gcp.env @@ -1,4 +1,4 @@ SENTRIUS_VERSION=1.0.34 SENTRIUS_SSH_VERSION=1.0.3 -SENTRIUS_KEYCLOAK_VERSION=1.0.4 +SENTRIUS_KEYCLOAK_VERSION=1.0.5 SENTRIUS_AGENT_VERSION=1.0.15 \ No newline at end of file diff --git a/analyagents/src/main/java/io/sentrius/agent/analysis/agents/sessions/SessionAnalyticsAgent.java b/analyagents/src/main/java/io/sentrius/agent/analysis/agents/sessions/SessionAnalyticsAgent.java index eb8f0a89..96645d8f 100644 --- a/analyagents/src/main/java/io/sentrius/agent/analysis/agents/sessions/SessionAnalyticsAgent.java +++ b/analyagents/src/main/java/io/sentrius/agent/analysis/agents/sessions/SessionAnalyticsAgent.java @@ -1,40 +1,35 @@ package io.sentrius.agent.analysis.agents.sessions; +import java.nio.charset.StandardCharsets; import java.sql.Timestamp; import java.util.ArrayList; -import java.util.Arrays; +import java.util.Base64; import java.util.List; -import java.util.Map; import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; -import com.fasterxml.jackson.core.JsonProcessingException; +import io.sentrius.sso.core.model.categorization.CommandCategory; import io.sentrius.sso.core.model.metadata.AnalyticsTracking; import io.sentrius.sso.core.model.metadata.TerminalBehaviorMetrics; import io.sentrius.sso.core.model.metadata.TerminalCommand; import io.sentrius.sso.core.model.metadata.TerminalRiskIndicator; import io.sentrius.sso.core.model.metadata.TerminalSessionMetadata; import io.sentrius.sso.core.model.metadata.UserExperienceMetrics; -import io.sentrius.sso.core.model.security.IntegrationSecurityToken; import io.sentrius.sso.core.model.sessions.TerminalLogs; import io.sentrius.sso.core.repository.AnalyticsTrackingRepository; import io.sentrius.sso.core.services.IntegrationSecurityTokenService; -import io.sentrius.sso.core.services.PluggableServices; import io.sentrius.sso.core.services.SessionService; import io.sentrius.sso.core.services.metadata.TerminalBehaviorMetricsService; import io.sentrius.sso.core.services.metadata.TerminalCommandService; import io.sentrius.sso.core.services.metadata.TerminalRiskIndicatorService; import io.sentrius.sso.core.services.metadata.TerminalSessionMetadataService; import io.sentrius.sso.core.services.metadata.UserExperienceMetricsService; -import io.sentrius.sso.core.utils.JsonUtil; -import io.sentrius.sso.integrations.external.ExternalIntegrationDTO; -import io.sentrius.sso.security.ApiKey; +import io.sentrius.sso.core.services.openai.categorization.CommandCategorizer; import jakarta.transaction.Transactional; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.ApplicationContext; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; @@ -51,6 +46,7 @@ public class SessionAnalyticsAgent { private final UserExperienceMetricsService experienceMetricsService; private final AnalyticsTrackingRepository trackingRepository; private final SessionService sessionService; + private final CommandCategorizer commandCategorizer; final IntegrationSecurityTokenService integrationSecurityTokenService; @@ -61,6 +57,7 @@ public void processSessions() { // Fetch already processed session IDs in bulk Set processedSessionIds = trackingRepository.findAllSessionIds(); + log.info("Found {} processed sessions", processedSessionIds.size()); List unprocessedSessions = sessionMetadataService.getSessionsByState("CLOSED").stream() .filter(session -> !processedSessionIds.contains(session.getId())) .collect(Collectors.toList()); @@ -69,13 +66,13 @@ public void processSessions() { try { processSession(session); // ACTIVE -> INACTIVE -> CLOSED -> PROCESSED - // saveToTracking(session.getId(), "PROCESSED"); + saveToTracking(session.getId(), "PROCESSED"); } catch (Exception e) { log.error("Error processing session {}: {}", session.getId(), e.getMessage(), e); saveToTracking(session.getId(), "ERROR"); } - // session.setSessionStatus("PROCESSED"); - // sessionMetadataService.saveSession(session); + session.setSessionStatus("PROCESSED"); + sessionMetadataService.saveSession(session); } log.info("Finished processing sessions"); @@ -185,7 +182,6 @@ public static String extractCommand(TerminalLogs previousLog, String logLine) { return matcher.group(1).trim(); } else { if (null != previousLog) { - log.info("Previous log: {}", previousLog.getOutput()); // it could be that we are at the beginning of the log set. String lastLogLine = getLastLogLine(previousLog); if (!lastLogLine.isEmpty()) { @@ -215,26 +211,20 @@ private static String getLastLogLine(TerminalLogs previousLog) { } private TerminalCommand createTerminalCommand(String command, TerminalLogs terminalLog, TerminalSessionMetadata sessionMetadata) { + String encodedString = Base64.getEncoder().encodeToString(command.trim().getBytes(StandardCharsets.UTF_8)); + TerminalCommand terminalCommand = new TerminalCommand(); - terminalCommand.setCommand(command.trim()); + terminalCommand.setCommand(encodedString); terminalCommand.setSession(sessionMetadata); terminalCommand.setExecutionTime(new Timestamp(System.currentTimeMillis())); terminalCommand.setExecutionStatus("SUCCESS"); terminalCommand.setOutput(""); // Assume no output initially - terminalCommand.setCommandCategory(categorizeCommand(command)); + terminalCommand.setCommandCategory(categorizeCommand(command).getCategoryName()); return terminalCommand; } - private String categorizeCommand(String command) { - // probably need to define externally - if (command.startsWith("sudo")) { - return "PRIVILEGED"; - } else if (command.contains("rm")) { - return "DESTRUCTIVE"; - } else if (command.contains("ls") || command.contains("cat")) { - return "INFORMATIONAL"; - } - return "GENERAL"; + private CommandCategory categorizeCommand(String command) { + return commandCategorizer.categorizeCommand(command); } } diff --git a/analyagents/src/main/resources/application.properties b/analyagents/src/main/resources/application.properties index af5a66b9..648a6a7c 100644 --- a/analyagents/src/main/resources/application.properties +++ b/analyagents/src/main/resources/application.properties @@ -61,4 +61,5 @@ spring.security.oauth2.client.provider.keycloak.issuer-uri=http://192.168.1.162: # for testing analytics agents agents.session-analytics.enabled=true management.endpoints.web.exposure.include=health -management.endpoint.health.show-details=always \ No newline at end of file +management.endpoint.health.show-details=always + diff --git a/api/src/main/resources/db/migration/V13__command_categorizer.sql b/api/src/main/resources/db/migration/V13__command_categorizer.sql new file mode 100644 index 00000000..2c4418a0 --- /dev/null +++ b/api/src/main/resources/db/migration/V13__command_categorizer.sql @@ -0,0 +1,9 @@ +CREATE TABLE command_categories ( + id SERIAL PRIMARY KEY, + category_name VARCHAR(50) NOT NULL, + pattern TEXT NOT NULL, -- Store regex patterns + priority INT NOT NULL DEFAULT 0 -- Optional: for matching precedence +); + + +CREATE INDEX idx_pattern ON command_categories (pattern); \ No newline at end of file diff --git a/core/pom.xml b/core/pom.xml index 01abb972..b075b2d7 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -139,7 +139,11 @@ quartz ${quartz-version} - + + com.github.ben-manes.caffeine + caffeine + ${caffeine-version} + com.google.protobuf protobuf-java diff --git a/core/src/main/java/io/sentrius/sso/core/model/categorization/CommandCategory.java b/core/src/main/java/io/sentrius/sso/core/model/categorization/CommandCategory.java new file mode 100644 index 00000000..f26d7326 --- /dev/null +++ b/core/src/main/java/io/sentrius/sso/core/model/categorization/CommandCategory.java @@ -0,0 +1,29 @@ +package io.sentrius.sso.core.model.categorization; + +import jakarta.persistence.Entity; +import jakarta.persistence.GeneratedValue; +import jakarta.persistence.GenerationType; +import jakarta.persistence.Id; +import jakarta.persistence.Table; +import lombok.AllArgsConstructor; +import lombok.Builder; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.ToString; + +@Getter +@Builder +@NoArgsConstructor +@ToString +@AllArgsConstructor +@Entity +@Table(name = "command_categories") +public class CommandCategory { + @Id + @GeneratedValue(strategy = GenerationType.IDENTITY) + private Long id; + private String categoryName; + private String pattern; + private int priority; + +} diff --git a/core/src/main/java/io/sentrius/sso/core/repository/CommandCategoryRepository.java b/core/src/main/java/io/sentrius/sso/core/repository/CommandCategoryRepository.java new file mode 100644 index 00000000..60077628 --- /dev/null +++ b/core/src/main/java/io/sentrius/sso/core/repository/CommandCategoryRepository.java @@ -0,0 +1,15 @@ +package io.sentrius.sso.core.repository; + +import java.util.List; +import io.sentrius.sso.core.model.categorization.CommandCategory; +import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Query; +import org.springframework.stereotype.Repository; + +@Repository +public interface CommandCategoryRepository extends JpaRepository { + @Query("SELECT c FROM CommandCategory c ORDER BY c.priority ASC") + List findAllOrderedByPriority(); + + List findByPattern(String pattern); +} diff --git a/core/src/main/java/io/sentrius/sso/core/services/openai/categorization/CommandCategorizer.java b/core/src/main/java/io/sentrius/sso/core/services/openai/categorization/CommandCategorizer.java new file mode 100644 index 00000000..12651e96 --- /dev/null +++ b/core/src/main/java/io/sentrius/sso/core/services/openai/categorization/CommandCategorizer.java @@ -0,0 +1,113 @@ +package io.sentrius.sso.core.services.openai.categorization; + +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.regex.Pattern; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; +import io.sentrius.sso.core.model.categorization.CommandCategory; +import io.sentrius.sso.core.repository.CommandCategoryRepository; +import io.sentrius.sso.core.services.IntegrationSecurityTokenService; +import io.sentrius.sso.core.utils.JsonUtil; +import io.sentrius.sso.genai.GenerativeAPI; +import io.sentrius.sso.genai.GeneratorConfiguration; +import io.sentrius.sso.genai.LLMCommandCategorizer; +import io.sentrius.sso.integrations.external.ExternalIntegrationDTO; +import io.sentrius.sso.security.ApiKey; +import jakarta.annotation.PostConstruct; +import jakarta.transaction.Transactional; +import jdk.jfr.TransitionTo; +import lombok.AllArgsConstructor; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +@Slf4j +@Service +@RequiredArgsConstructor +public class CommandCategorizer { + + private final IntegrationSecurityTokenService integrationSecurityTokenService; + + private final CommandCategoryRepository commandCategoryRepository; + + + private final CommandTrie commandTrie = new CommandTrie(); + + private final Cache commandCache = Caffeine.newBuilder() + .maximumSize(10000) + .expireAfterWrite(24, TimeUnit.HOURS) + .build(); + + @PostConstruct + public void initializeTrie() { + List categories = commandCategoryRepository.findAll(); + for (CommandCategory category : categories) { + log.info("Adding command category {} to trie", category); + commandTrie.insert(category.getPattern(), category); + } + } + + @Transactional + public CommandCategory categorizeCommand(String command) { + return commandCache.get(command, this::categorizeWithRulesOrML); + } + + + protected List getDBCommandCategory(String command){ + return commandCategoryRepository.findByPattern(command); + } + + @Transactional + protected void addCommandCategory(String command, CommandCategory category) { + commandTrie.insert(command, category); + commandCategoryRepository.save(category); + } + + + @Transactional + protected CommandCategory categorizeWithRulesOrML(String command) { + CommandCategory category = commandTrie.searchByPrefix(command); + if (category != null) { + log.info("Found command category {} for {} ", category, command); + return category; + } + + var openaiService = integrationSecurityTokenService.findByConnectionType("openai").stream().findFirst().orElse(null); + + if (null != openaiService){ + log.info("OpenAI service is available"); + ExternalIntegrationDTO externalIntegrationDTO = null; + try { + externalIntegrationDTO = JsonUtil.MAPPER.readValue(openaiService.getConnectionInfo(), + ExternalIntegrationDTO.class); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + ApiKey key = + ApiKey.builder().apiKey(externalIntegrationDTO.getApiToken()).principal(externalIntegrationDTO.getUsername()).build(); + + var commandCategorizer = new LLMCommandCategorizer(key, new GenerativeAPI(key), GeneratorConfiguration.builder().build()); + + try { + category = commandCategorizer.generate(command); + + addCommandCategory(category.getPattern(), category); + + log.info("Categorized command: {}", category); + return category; + } catch (Exception e) { + e.printStackTrace(); + log.error("Error categorizing command", e); + } + + } else { + log.info("OpenAI service is not enabled"); + } + + log.info("Finished processing terminal commands"); + + return CommandCategory.builder().build(); + } +} diff --git a/core/src/main/java/io/sentrius/sso/core/services/openai/categorization/CommandTrie.java b/core/src/main/java/io/sentrius/sso/core/services/openai/categorization/CommandTrie.java new file mode 100644 index 00000000..acfb44f4 --- /dev/null +++ b/core/src/main/java/io/sentrius/sso/core/services/openai/categorization/CommandTrie.java @@ -0,0 +1,70 @@ +package io.sentrius.sso.core.services.openai.categorization; + +import io.sentrius.sso.core.model.categorization.CommandCategory; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class CommandTrie { + private final TrieNode root = new TrieNode(); + + private String normalizePath(String path) { + return path.replaceAll("/+$", ""); // Remove trailing slashes + } + + public void insert(String command, CommandCategory category) { + String[] parts = command.split(" "); + TrieNode current = root; + for (String part : parts) { + part = normalizePath(part); // Normalize each part + current = current.children.computeIfAbsent(part, k -> new TrieNode()); + } + current.isEndOfCommand = true; + if (category.getPattern().endsWith("*")) { + current.isWildcard = true; + } + else { + current.isWildcard = false; + } + current.commandCategory = category; + } + + public CommandCategory search(String command) { + String[] parts = command.split(" "); + TrieNode current = root; + for (String part : parts) { + current = current.children.get(part); + if (current == null) { + return null; // Command not found + } + } + return current.isEndOfCommand ? current.commandCategory : null; + } + + public CommandCategory searchByPrefix(String command) { + String[] parts = command.split(" "); + TrieNode current = root; + CommandCategory lastCategory = null; + + for (String part : parts) { + part = normalizePath(part); // Normalize each part + log.info("Searching for part: {}", part); + + // Check if the part exists in the children + if (current.children.containsKey(part)) { + current = current.children.get(part); + if (current.isEndOfCommand) { + lastCategory = current.commandCategory; + } + } else { + // If no exact match, check for wildcard match (e.g., "cat /etc/") + if (current.isEndOfCommand) { + log.info("Partial match found at: {}", part); + lastCategory = current.commandCategory; + } + break; // No further match possible + } + } + + return lastCategory; + } +} diff --git a/core/src/main/java/io/sentrius/sso/core/services/openai/categorization/TrieNode.java b/core/src/main/java/io/sentrius/sso/core/services/openai/categorization/TrieNode.java new file mode 100644 index 00000000..2404424b --- /dev/null +++ b/core/src/main/java/io/sentrius/sso/core/services/openai/categorization/TrieNode.java @@ -0,0 +1,12 @@ +package io.sentrius.sso.core.services.openai.categorization; + +import java.util.HashMap; +import java.util.Map; +import io.sentrius.sso.core.model.categorization.CommandCategory; + +public class TrieNode { + Map children = new HashMap<>(); + CommandCategory commandCategory; // Store the CommandCategory at the end node + boolean isEndOfCommand = false; + boolean isWildcard = false; // Marks this node as a wildcard +} diff --git a/core/src/main/java/io/sentrius/sso/genai/LLMCommandCategorizer.java b/core/src/main/java/io/sentrius/sso/genai/LLMCommandCategorizer.java new file mode 100644 index 00000000..fe4ab83f --- /dev/null +++ b/core/src/main/java/io/sentrius/sso/genai/LLMCommandCategorizer.java @@ -0,0 +1,86 @@ +package io.sentrius.sso.genai; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.sentrius.sso.core.model.categorization.CommandCategory; +import io.sentrius.sso.core.utils.JsonUtil; +import io.sentrius.sso.genai.model.endpoints.ChatApiEndpointRequest; +import io.sentrius.sso.integrations.exceptions.HttpException; +import io.sentrius.sso.security.TokenProvider; +import lombok.extern.slf4j.Slf4j; + +/** + * The ComplianceScorer class contains methods for generating compliance scores. + * The generate() method returns a double that represents the compliance score. + * + * This class can be used to efficiently score compliance in various domains, including but not limited to + * healthcare, finance, and government regulations. + * + * It is recommended to initialize the ComplianceScorer with relevant settings and parameters for a specific + * compliance scenario. The generate() method can then be called repeatedly on incoming data to obtain + * compliance scores in real-time. + * + * Note: This class does not handle data storage, retrieval or manipulation. It is only intended for + * calculating compliance scores based on input data. + */ +@Slf4j +public class LLMCommandCategorizer extends DataGenerator { + + public LLMCommandCategorizer(TokenProvider token, GenerativeAPI generator, GeneratorConfiguration config) { + super(token, generator, config); + } + + /** + * Parses queries from the response. + * + * @return List of queries. + */ + @Override + public CommandCategory generate(String on) throws HttpException, JsonProcessingException { + var ipt = generateInput(on); + log.info("Input: {}", ipt); + ChatApiEndpointRequest request = ChatApiEndpointRequest.builder().userInput(ipt).build(); + request.setTemperature(0.5f); + Response hello = api.sample(request, Response.class); + var resp = hello.concatenateResponses(); + log.info("Response: {}", resp); + var objectNode = JsonUtil.MAPPER.readValue(resp, ObjectNode.class); + return CommandCategory.builder().categoryName(objectNode.get("category").asText()).pattern(objectNode.get( + "pattern").asText()).priority(objectNode.get("priority").asInt()).build(); + } + + /** + * Generates input for the generative AI endpoint. + * + * @return Question to be asked to the generative AI endpoint. + */ + @Override + public String generateInput(String on) { + return """ + Categorize the following command with a generalized pattern, defined as a **regex**, that captures the intent and considers risk factors. Include specific arguments or paths in the regex only if they significantly impact the risk level. If no regex is predefined for the command, generate one that appropriately generalizes the command's behavior while retaining any risk-relevant components. + + For example: + - 'cat /etc/passwd' is risky due to sensitive user data, so it should be included as-is with the regex '^cat /etc/passwd$'. + - 'cat /etc/hosts' has low risk, so it should be generalized to '^cat /etc/.*$' to cover all files in the `/etc` directory. + - 'sudo rm -rf /important_dir' should include '/important_dir' if it's sensitive, but otherwise generalized to '^sudo rm -rf .*'. + + Command: "%s" + + **Categories to choose from:** + - PRIVILEGED: Commands that require elevated permissions or pose a risk if misused. + - DESTRUCTIVE: Commands that can delete or alter critical files or system configurations. + - INFORMATIONAL: Commands that retrieve information without altering the system state. + - GENERAL: Commands that do not fit into the above categories. + + Respond in **JSON format** as follows: + { + "category": "", + "priority": , + "pattern": "", + "rationale": "" + } + + """.formatted(on); + } + +} diff --git a/docker/keycloak/realms/sentrius-realm.json b/docker/keycloak/realms/sentrius-realm.json index 32164353..64b1e511 100644 --- a/docker/keycloak/realms/sentrius-realm.json +++ b/docker/keycloak/realms/sentrius-realm.json @@ -65,5 +65,26 @@ ], "realmRoles": ["user"] } + ], + "identityProviders": [ + { + "alias": "google", + "displayName": "Google", + "providerId": "google", + "enabled": true, + "trustEmail": true, + "storeToken": true, + "addReadTokenRoleOnCreate": false, + "authenticateByDefault": false, + "linkOnly": false, + "config": { + "clientId": "${GOOGLE_CLIENT_ID}", + "clientSecret": "${GOOGLE_CLIENT_SECRET}", + "defaultScope": "openid email profile", + "authorizationUrl": "https://accounts.google.com/o/oauth2/auth", + "tokenUrl": "https://oauth2.googleapis.com/token", + "userInfoUrl": "https://openidconnect.googleapis.com/v1/userinfo" + } + } ] } diff --git a/ops-scripts/gcp/deploy-helm.sh b/ops-scripts/gcp/deploy-helm.sh index a32c60f7..8e217d60 100755 --- a/ops-scripts/gcp/deploy-helm.sh +++ b/ops-scripts/gcp/deploy-helm.sh @@ -7,9 +7,21 @@ source ${SCRIPT_DIR}/base.sh source ${SCRIPT_DIR}/../../.gcp.env TENANT=$1 +KEYCLOAK_CLIENT_ID=$2 +KEYCLOAK_CLIENT_SECRET=$3 if [[ -z "$TENANT" ]]; then - echo "Must provide single argument for tenant name" 1>&2 + echo "Must provide first argument for tenant name" 1>&2 + exit 1 +fi + +if [[ -z "$KEYCLOAK_CLIENT_ID" ]]; then + echo "Must provide second argument for tenant name" 1>&2 + exit 1 +fi + +if [[ -z "$KEYCLOAK_CLIENT_SECRET" ]]; then + echo "Must provide second argument for tenant name" 1>&2 exit 1 fi @@ -21,7 +33,6 @@ if [[ $? -ne 0 ]]; then fi - helm upgrade --install sentrius ./sentrius-gcp-chart --namespace ${TENANT} \ --set tenant=${TENANT} \ --set subdomain=${TENANT}.sentrius.cloud \ @@ -31,6 +42,8 @@ helm upgrade --install sentrius ./sentrius-gcp-chart --namespace ${TENANT} \ --set ssh.image.tag=${SENTRIUS_SSH_VERSION} \ --set keycloak.image.repository=us-central1-docker.pkg.dev/sentrius-project/sentrius-repo/sentrius-keycloak \ --set keycloak.image.tag=${SENTRIUS_KEYCLOAK_VERSION} \ + --set keycloak.clientId=${KEYCLOAK_CLIENT_ID} \ + --set keycloak.clientSecret=${KEYCLOAK_CLIENT_SECRET} \ --set sentriusagent.image.repository=us-central1-docker.pkg.dev/sentrius-project/sentrius-repo/sentrius-agent \ --set sentriusagent.image.tag=${SENTRIUS_AGENT_VERSION} || { echo "Failed to deploy Sentrius with Helm"; exit 1; } diff --git a/pom.xml b/pom.xml index 8792f1ad..b01b9c0a 100644 --- a/pom.xml +++ b/pom.xml @@ -35,6 +35,7 @@ 5.10.1 5.10.0 25.0.3 + 3.2.0 @@ -60,7 +61,11 @@ provided ${lombok.version} - + + com.github.ben-manes.caffeine + caffeine + ${caffeine-version} + com.google.guava guava diff --git a/sentrius-gcp-chart/templates/keycloak-deployment.yaml b/sentrius-gcp-chart/templates/keycloak-deployment.yaml index d2b76ee9..51cfbfcb 100644 --- a/sentrius-gcp-chart/templates/keycloak-deployment.yaml +++ b/sentrius-gcp-chart/templates/keycloak-deployment.yaml @@ -58,5 +58,9 @@ spec: value: "false" - name: KC_HTTP_ENABLED value: "true" + - name: GOOGLE_CLIENT_ID + value: {{ .Values.keycloak.clientId }} + - name: GOOGLE_CLIENT_SECRET + value: {{ .Values.keycloak.clientSecret }} command: [ "/opt/keycloak/bin/kc.sh" ] args: [ "start-dev", "--proxy=edge", "--import-realm", "--health-enabled=true"] diff --git a/sentrius-gcp-chart/values.yaml b/sentrius-gcp-chart/values.yaml index 31955b83..b153b10e 100644 --- a/sentrius-gcp-chart/values.yaml +++ b/sentrius-gcp-chart/values.yaml @@ -91,6 +91,8 @@ keycloak: adminUser: admin adminPassword: admin port: 8081 + clientId: sentrius-api + clientSecret: nGkEukexSWTvDzYjSkDmeUlM0FJ5Jhh0 db: image: postgres:15 user: keycloak From 9a58c610d958a59dbcbead46ca49a01ccb752f3c Mon Sep 17 00:00:00 2001 From: Marc Parisi Date: Tue, 21 Jan 2025 10:27:17 -0500 Subject: [PATCH 2/4] Fixup issue with regexes --- .../V14__command_categorizer_gin.sql | 3 ++ .../repository/CommandCategoryRepository.java | 5 ++ .../categorization/CommandCategorizer.java | 48 +++++++++++-------- .../sso/genai/LLMCommandCategorizer.java | 44 ++++++++--------- 4 files changed, 59 insertions(+), 41 deletions(-) create mode 100644 api/src/main/resources/db/migration/V14__command_categorizer_gin.sql diff --git a/api/src/main/resources/db/migration/V14__command_categorizer_gin.sql b/api/src/main/resources/db/migration/V14__command_categorizer_gin.sql new file mode 100644 index 00000000..8d0a7b45 --- /dev/null +++ b/api/src/main/resources/db/migration/V14__command_categorizer_gin.sql @@ -0,0 +1,3 @@ +CREATE EXTENSION IF NOT EXISTS pg_trgm; + +CREATE INDEX idx_command_pattern_trgm ON command_categories USING gin (pattern gin_trgm_ops); diff --git a/core/src/main/java/io/sentrius/sso/core/repository/CommandCategoryRepository.java b/core/src/main/java/io/sentrius/sso/core/repository/CommandCategoryRepository.java index 60077628..9a33a54f 100644 --- a/core/src/main/java/io/sentrius/sso/core/repository/CommandCategoryRepository.java +++ b/core/src/main/java/io/sentrius/sso/core/repository/CommandCategoryRepository.java @@ -4,6 +4,7 @@ import io.sentrius.sso.core.model.categorization.CommandCategory; import org.springframework.data.jpa.repository.JpaRepository; import org.springframework.data.jpa.repository.Query; +import org.springframework.data.repository.query.Param; import org.springframework.stereotype.Repository; @Repository @@ -12,4 +13,8 @@ public interface CommandCategoryRepository extends JpaRepository findAllOrderedByPriority(); List findByPattern(String pattern); + + @Query(value = "SELECT * FROM command_categories WHERE :command ~ pattern", nativeQuery = true) + List findMatchingCategories(@Param("command") String command); + } diff --git a/core/src/main/java/io/sentrius/sso/core/services/openai/categorization/CommandCategorizer.java b/core/src/main/java/io/sentrius/sso/core/services/openai/categorization/CommandCategorizer.java index 12651e96..3b411027 100644 --- a/core/src/main/java/io/sentrius/sso/core/services/openai/categorization/CommandCategorizer.java +++ b/core/src/main/java/io/sentrius/sso/core/services/openai/categorization/CommandCategorizer.java @@ -1,8 +1,10 @@ package io.sentrius.sso.core.services.openai.categorization; +import java.util.Comparator; import java.util.List; import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; import com.fasterxml.jackson.core.JsonProcessingException; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; @@ -33,42 +35,44 @@ public class CommandCategorizer { private final CommandCategoryRepository commandCategoryRepository; - private final CommandTrie commandTrie = new CommandTrie(); - private final Cache commandCache = Caffeine.newBuilder() - .maximumSize(10000) - .expireAfterWrite(24, TimeUnit.HOURS) + .maximumSize(1000) + .expireAfterWrite(1, TimeUnit.HOURS) .build(); - @PostConstruct - public void initializeTrie() { - List categories = commandCategoryRepository.findAll(); - for (CommandCategory category : categories) { - log.info("Adding command category {} to trie", category); - commandTrie.insert(category.getPattern(), category); - } + + + private CommandCategory fetchFromDatabase(String command) { + List matchingCategories = commandCategoryRepository.findMatchingCategories(command); + return matchingCategories.stream() + .min(Comparator.comparingInt(CommandCategory::getPriority)) + .orElse(null); } + @Transactional public CommandCategory categorizeCommand(String command) { return commandCache.get(command, this::categorizeWithRulesOrML); } - protected List getDBCommandCategory(String command){ return commandCategoryRepository.findByPattern(command); } - @Transactional - protected void addCommandCategory(String command, CommandCategory category) { - commandTrie.insert(command, category); - commandCategoryRepository.save(category); + + public boolean isValidRegex(String regex) { + try { + Pattern.compile(regex); + return true; // Valid regex + } catch (PatternSyntaxException e) { + return false; // Invalid regex + } } @Transactional protected CommandCategory categorizeWithRulesOrML(String command) { - CommandCategory category = commandTrie.searchByPrefix(command); + CommandCategory category = fetchFromDatabase(command); if (category != null) { log.info("Found command category {} for {} ", category, command); return category; @@ -93,8 +97,9 @@ protected CommandCategory categorizeWithRulesOrML(String command) { try { category = commandCategorizer.generate(command); - addCommandCategory(category.getPattern(), category); - + if (isValidRegex(category.getPattern())) { + addCommandCategory(category.getPattern(), category); + } log.info("Categorized command: {}", category); return category; } catch (Exception e) { @@ -110,4 +115,9 @@ protected CommandCategory categorizeWithRulesOrML(String command) { return CommandCategory.builder().build(); } + + private void addCommandCategory(String pattern, CommandCategory category) { + commandCategoryRepository.save(category); + commandCache.put(pattern, category); + } } diff --git a/core/src/main/java/io/sentrius/sso/genai/LLMCommandCategorizer.java b/core/src/main/java/io/sentrius/sso/genai/LLMCommandCategorizer.java index fe4ab83f..9d787f34 100644 --- a/core/src/main/java/io/sentrius/sso/genai/LLMCommandCategorizer.java +++ b/core/src/main/java/io/sentrius/sso/genai/LLMCommandCategorizer.java @@ -58,28 +58,28 @@ public CommandCategory generate(String on) throws HttpException, JsonProcessingE public String generateInput(String on) { return """ Categorize the following command with a generalized pattern, defined as a **regex**, that captures the intent and considers risk factors. Include specific arguments or paths in the regex only if they significantly impact the risk level. If no regex is predefined for the command, generate one that appropriately generalizes the command's behavior while retaining any risk-relevant components. - - For example: - - 'cat /etc/passwd' is risky due to sensitive user data, so it should be included as-is with the regex '^cat /etc/passwd$'. - - 'cat /etc/hosts' has low risk, so it should be generalized to '^cat /etc/.*$' to cover all files in the `/etc` directory. - - 'sudo rm -rf /important_dir' should include '/important_dir' if it's sensitive, but otherwise generalized to '^sudo rm -rf .*'. - - Command: "%s" - - **Categories to choose from:** - - PRIVILEGED: Commands that require elevated permissions or pose a risk if misused. - - DESTRUCTIVE: Commands that can delete or alter critical files or system configurations. - - INFORMATIONAL: Commands that retrieve information without altering the system state. - - GENERAL: Commands that do not fit into the above categories. - - Respond in **JSON format** as follows: - { - "category": "", - "priority": , - "pattern": "", - "rationale": "" - } - + + For example: + - 'cat /etc/passwd' is risky due to sensitive user data, so it should be included as-is with the regex '^cat /etc/passwd$'. + - 'cat /etc/hosts' has low risk, so it should be generalized to '^cat /etc/.*$' to cover all files in the `/etc` directory. + - 'sudo rm -rf /important_dir' should include '/important_dir' if it's sensitive, but otherwise generalized to '^sudo rm -rf .*'. + + Command: "%s" + + **Categories to choose from:** + - PRIVILEGED: Commands that require elevated permissions or pose a risk if misused. + - DESTRUCTIVE: Commands that can delete or alter critical files or system configurations. + - INFORMATIONAL: Commands that retrieve information without altering the system state. + - GENERAL: Commands that do not fit into the above categories. + + Respond in **JSON format** as follows: + { + "category": "", + "priority": , + "pattern": "", + "rationale": "" + } + """.formatted(on); } From 6c47570992bda1de717ada74ec0394b541a282d7 Mon Sep 17 00:00:00 2001 From: Marc Parisi Date: Wed, 22 Jan 2025 14:38:46 -0500 Subject: [PATCH 3/4] update --- .gcp.env | 8 ++++---- api/src/main/resources/templates/sso/dashboard.html | 10 +++------- .../openai/categorization/CommandCategorizer.java | 3 --- 3 files changed, 7 insertions(+), 14 deletions(-) diff --git a/.gcp.env b/.gcp.env index ca7e436f..0218842b 100644 --- a/.gcp.env +++ b/.gcp.env @@ -1,4 +1,4 @@ -SENTRIUS_VERSION=1.0.34 -SENTRIUS_SSH_VERSION=1.0.3 -SENTRIUS_KEYCLOAK_VERSION=1.0.5 -SENTRIUS_AGENT_VERSION=1.0.15 \ No newline at end of file +SENTRIUS_VERSION=1.0.37 +SENTRIUS_SSH_VERSION=1.0.4 +SENTRIUS_KEYCLOAK_VERSION=1.0.6 +SENTRIUS_AGENT_VERSION=1.0.16 \ No newline at end of file diff --git a/api/src/main/resources/templates/sso/dashboard.html b/api/src/main/resources/templates/sso/dashboard.html index b499f50a..f30c2ef7 100755 --- a/api/src/main/resources/templates/sso/dashboard.html +++ b/api/src/main/resources/templates/sso/dashboard.html @@ -428,18 +428,14 @@
User Operations
-
User Operations
+
AI Admin Operations
- Delete Users + Build Automation Add User - Your Settings - View JITs + data-bs-target="#userFormModal">Usage Patterns
diff --git a/core/src/main/java/io/sentrius/sso/core/services/openai/categorization/CommandCategorizer.java b/core/src/main/java/io/sentrius/sso/core/services/openai/categorization/CommandCategorizer.java index 3b411027..bb9752d5 100644 --- a/core/src/main/java/io/sentrius/sso/core/services/openai/categorization/CommandCategorizer.java +++ b/core/src/main/java/io/sentrius/sso/core/services/openai/categorization/CommandCategorizer.java @@ -17,10 +17,7 @@ import io.sentrius.sso.genai.LLMCommandCategorizer; import io.sentrius.sso.integrations.external.ExternalIntegrationDTO; import io.sentrius.sso.security.ApiKey; -import jakarta.annotation.PostConstruct; import jakarta.transaction.Transactional; -import jdk.jfr.TransitionTo; -import lombok.AllArgsConstructor; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; From 5d5e5c24351d9cafa94cdc849460b35d939c91ae Mon Sep 17 00:00:00 2001 From: Marc Parisi Date: Thu, 23 Jan 2025 06:52:06 -0500 Subject: [PATCH 4/4] Closes #23 to ensure we consistently name Host Groups Host Enclaves --- .../startup/ConfigurationApplicationTask.java | 20 +++++++++---------- api/src/main/resources/static/js/rules.js | 4 ++-- .../templates/fragments/assign_rule.html | 10 +++++----- .../fragments/dashboard/dashboard_cards.html | 2 +- .../templates/fragments/ssh_server_card.html | 2 +- .../resources/templates/fragments/topbar.html | 2 +- .../templates/sso/ssh/list_servers.html | 2 +- .../specification/HostGroupSpecification.java | 2 +- .../sso/core/services/HostGroupService.java | 6 +++--- .../sso/core/services/UserService.java | 4 ++-- 10 files changed, 27 insertions(+), 27 deletions(-) diff --git a/api/src/main/java/io/sentrius/sso/startup/ConfigurationApplicationTask.java b/api/src/main/java/io/sentrius/sso/startup/ConfigurationApplicationTask.java index 86b0940e..eaca42be 100644 --- a/api/src/main/java/io/sentrius/sso/startup/ConfigurationApplicationTask.java +++ b/api/src/main/java/io/sentrius/sso/startup/ConfigurationApplicationTask.java @@ -276,7 +276,7 @@ protected List createHostGroups(List sideEffects, Map createHostGroups(List sideEffects, Map createHostGroups(List sideEffects, Map createHostGroups(List sideEffects, Map createUsers( sideEffects.add(SideEffect.builder().sideEffectDescription( - "Assigning user " + userDTO.getUsername() + " to Host Group " + + "Assigning user " + userDTO.getUsername() + " to Host Enclave " + hostGroup.getName()).type( SideEffectType.UPDATE_DATABASE).asset("Users").build()); - log.info("Assigning user {} to Host Group {}", userDTO.getUsername(), + log.info("Assigning user {} to Host Enclave {}", userDTO.getUsername(), hostGroup.getId()); user.getHostGroups().add(hostGroup); } @@ -473,15 +473,15 @@ protected List createUsers( for (var profile : userDTO.getHostGroups()) { for (HostGroup hostGroup : profiles) { if (hostGroup.getName().equals(profile.getDisplayName())) { - log.info("Assigning user {} to Host Group {}", userDTO.getUsername(), + log.info("Assigning user {} to Host Enclave {}", userDTO.getUsername(), hostGroup.getId()); if (null == hostGroup.getId() || !userRepository.isAssignedToHostGroups(user.getId(), List.of(hostGroup.getId()))) { sideEffects.add(SideEffect.builder().sideEffectDescription( - "Assigning user " + userDTO.getUsername() + " to Host Group " + + "Assigning user " + userDTO.getUsername() + " to Host Enclave " + hostGroup.getName()).type( SideEffectType.UPDATE_DATABASE).asset("Users").build()); - log.info("Assigning user {} to Host Group {}", userDTO.getUsername(), + log.info("Assigning user {} to Host Enclave {}", userDTO.getUsername(), hostGroup.getId() ); } diff --git a/api/src/main/resources/static/js/rules.js b/api/src/main/resources/static/js/rules.js index 752343dd..5a325512 100644 --- a/api/src/main/resources/static/js/rules.js +++ b/api/src/main/resources/static/js/rules.js @@ -110,7 +110,7 @@ $(document).ready(function () { if (row.canEdit) { buttons += - ` + ` `; } @@ -165,7 +165,7 @@ $(document).ready(function () { if (response.ok) { $('#rule-table').DataTable().ajax.reload(null, false); } else { - alert("Failed to assign host groups."); + alert("Failed to assign host enclaves."); } const modal = bootstrap.Modal.getInstance(document.getElementById("assignHostGroupsModal")); modal.hide(); diff --git a/api/src/main/resources/templates/fragments/assign_rule.html b/api/src/main/resources/templates/fragments/assign_rule.html index 6710917e..ce8dde8d 100644 --- a/api/src/main/resources/templates/fragments/assign_rule.html +++ b/api/src/main/resources/templates/fragments/assign_rule.html @@ -1,10 +1,10 @@ - +
-

Host Groups

+

Host Enclaves

Assigned Systems: diff --git a/api/src/main/resources/templates/fragments/topbar.html b/api/src/main/resources/templates/fragments/topbar.html index 2747269d..a8f5e30b 100755 --- a/api/src/main/resources/templates/fragments/topbar.html +++ b/api/src/main/resources/templates/fragments/topbar.html @@ -23,7 +23,7 @@
  • diff --git a/api/src/main/resources/templates/sso/ssh/list_servers.html b/api/src/main/resources/templates/sso/ssh/list_servers.html index 7188e777..2a47c684 100755 --- a/api/src/main/resources/templates/sso/ssh/list_servers.html +++ b/api/src/main/resources/templates/sso/ssh/list_servers.html @@ -89,7 +89,7 @@

    Assigned SSH Servers

    System Name Hostname - Host Group + Host Enclave SSH User Port Operations diff --git a/core/src/main/java/io/sentrius/sso/core/data/specification/HostGroupSpecification.java b/core/src/main/java/io/sentrius/sso/core/data/specification/HostGroupSpecification.java index 392c14ea..faae6366 100644 --- a/core/src/main/java/io/sentrius/sso/core/data/specification/HostGroupSpecification.java +++ b/core/src/main/java/io/sentrius/sso/core/data/specification/HostGroupSpecification.java @@ -18,7 +18,7 @@ public static Specification findByUserIdAndOptionalFilters(@NonNull L return (root, query, criteriaBuilder) -> { List predicates = new ArrayList<>(); - // Join the users to filter host groups by user ID + // Join the users to filter host enclaves by user ID Join userJoin = root.join("users", JoinType.INNER); predicates.add(criteriaBuilder.equal(userJoin.get("id"), userId)); diff --git a/core/src/main/java/io/sentrius/sso/core/services/HostGroupService.java b/core/src/main/java/io/sentrius/sso/core/services/HostGroupService.java index 41544e11..cb9dd4dc 100644 --- a/core/src/main/java/io/sentrius/sso/core/services/HostGroupService.java +++ b/core/src/main/java/io/sentrius/sso/core/services/HostGroupService.java @@ -43,7 +43,7 @@ public HostGroup getHostGroup(Long hostGroupId) { @Transactional public Optional getHostGroupWithHostSystems(User user, Long hostGroupId) { HostGroup hostGroup = hostGroupRepository.findByIdWithUsers(hostGroupId) - .orElseThrow(() -> new EntityNotFoundException("Host Group not found")); + .orElseThrow(() -> new EntityNotFoundException("Host Enclave not found")); boolean userIsMember = hostGroup.getUsers().stream() @@ -69,7 +69,7 @@ public List getAssignedHostsForUser(User user) { userRepository.findHostGroupsByUserId(user.getId()).forEach(hostGroup -> { HostGroup sys = hostGroupRepository.findById(hostGroup.getId()) - .orElseThrow(() -> new EntityNotFoundException("Host Group not found")); + .orElseThrow(() -> new EntityNotFoundException("Host Enclave not found")); log.info("HostGroup: {}", sys.getId()); if (null != sys && null != sys.getHostSystemList()) { Hibernate.initialize(sys.getHostSystemList()); @@ -93,7 +93,7 @@ public List getAssignedHostsForUserAndId(User user, Long groupId) { return hostGroup.getId().equals(groupId); }).forEach(hostGroup -> { HostGroup sys = hostGroupRepository.findById(hostGroup.getId()) - .orElseThrow(() -> new EntityNotFoundException("Host Group not found")); + .orElseThrow(() -> new EntityNotFoundException("Host Enclave not found")); log.info("HostGroup: {}", sys.getId()); if (null != sys && null != sys.getHostSystemList()) { Hibernate.initialize(sys.getHostSystemList()); diff --git a/core/src/main/java/io/sentrius/sso/core/services/UserService.java b/core/src/main/java/io/sentrius/sso/core/services/UserService.java index 867bf983..dd6ad9f7 100644 --- a/core/src/main/java/io/sentrius/sso/core/services/UserService.java +++ b/core/src/main/java/io/sentrius/sso/core/services/UserService.java @@ -95,8 +95,8 @@ public User getOperatingUser(HttpServletRequest request, log.info("Creating new user: {}", operatingUser); save(operatingUser); HostGroup newHg = - HostGroup.builder().name("Host Group for " + operatingUser.getUsername()).description( - operatingUser.getUsername() + "'s Host Group").build(); + HostGroup.builder().name("Host Enclave for " + operatingUser.getUsername()).description( + operatingUser.getUsername() + "'s Host Enclave").build(); ProfileDB.save(newHg); operatingUser.getHostGroups().add(newHg);