From 79329c34d5330cc2a351ac0cb67cf810b67d5f69 Mon Sep 17 00:00:00 2001 From: Juan Date: Thu, 2 Apr 2026 18:22:03 +0200 Subject: [PATCH 1/7] Add AI-assisted text generation to log_text Made-with: Cursor --- tests/test_api_client.py | 76 +++++++++++++++++++++++++++++- validmind/api_client.py | 99 ++++++++++++++++++++++++++++++++++++---- 2 files changed, 166 insertions(+), 9 deletions(-) diff --git a/tests/test_api_client.py b/tests/test_api_client.py index 8ea5e4228..d0512066f 100644 --- a/tests/test_api_client.py +++ b/tests/test_api_client.py @@ -16,10 +16,11 @@ import validmind.api_client as api_client from validmind.__version__ import __version__ from validmind.errors import ( + APIRequestError, MissingAPICredentialsError, MissingModelIdError, - APIRequestError, ) +from validmind.utils import md_to_html from validmind.vm_models.figure import Figure @@ -245,6 +246,79 @@ def test_log_test_result(self, mock_post): mock_post.assert_called_with(url, data=json.dumps(result)) + @patch("requests.post") + @patch("aiohttp.ClientSession.post") + def test_log_text_generates_text_and_logs_metadata( + self, mock_aiohttp_post, mock_requests_post + ): + mock_requests_post.return_value = Mock(status_code=200) + mock_requests_post.return_value.json.return_value = { + "content": "## Generated Summary\nGenerated content." + } + mock_aiohttp_post.return_value = MockAsyncResponse( + 200, + json={ + "content_id": "dataset_summary_text", + "text": md_to_html("## Generated Summary\nGenerated content.", mathml=True), + }, + ) + + api_client.log_text( + content_id="dataset_summary_text", + prompt="Summarize the dataset.", + context={"content_ids": ["train_dataset", "target_description_text"]}, + ) + + mock_requests_post.assert_called_once_with( + url=f"{os.environ['VM_API_HOST']}/ai/generate/qualitative_text_generation", + headers={ + "X-API-KEY": os.environ["VM_API_KEY"], + "X-API-SECRET": os.environ["VM_API_SECRET"], + "X-MODEL-CUID": os.environ["VM_API_MODEL"], + "X-MONITORING": "False", + "X-LIBRARY-VERSION": __version__, + }, + json={ + "content_id": "dataset_summary_text", + "generate": True, + "prompt": "Summarize the dataset.", + "context": { + "content_ids": ["train_dataset", "target_description_text"] + }, + }, + ) + mock_aiohttp_post.assert_called_once_with( + f"{os.environ['VM_API_HOST']}/log_metadata", + data=json.dumps( + { + "content_id": "dataset_summary_text", + "text": md_to_html( + "## Generated Summary\nGenerated content.", mathml=True + ), + } + ), + ) + + def test_log_text_rejects_prompt_when_text_is_provided(self): + with self.assertRaisesRegex( + ValueError, "`prompt` is only supported when `text` is omitted" + ): + api_client.log_text( + content_id="dataset_summary_text", + text="Hello world", + prompt="Ignore the provided text.", + ) + + def test_log_text_rejects_invalid_context(self): + with self.assertRaisesRegex( + ValueError, + "`context\\['content_ids'\\]` must contain only non-empty strings", + ): + api_client.log_text( + content_id="dataset_summary_text", + context={"content_ids": ["valid", ""]}, + ) + if __name__ == "__main__": unittest.main() diff --git a/validmind/api_client.py b/validmind/api_client.py index affacb80e..2bdcf279a 100644 --- a/validmind/api_client.py +++ b/validmind/api_client.py @@ -445,16 +445,74 @@ def log_input(input_id: str, type: str, metadata: Dict[str, Any]) -> Dict[str, A return run_async(alog_input, input_id, type, metadata) -def log_text(content_id: str, text: str, _json: Optional[Dict[str, Any]] = None) -> str: - """Logs free-form text to ValidMind API. +def _validate_log_text_context( + context: Optional[Dict[str, Any]], +) -> Optional[Dict[str, List[str]]]: + """Validate supported AI generation context for ``log_text``.""" + if context is None: + return None + + if not isinstance(context, dict): + raise ValueError("`context` must be a dictionary or None") + + allowed_keys = {"content_ids"} + unknown_keys = set(context.keys()) - allowed_keys + if unknown_keys: + raise ValueError( + "Unsupported `context` keys: " + f"{', '.join(sorted(unknown_keys))}. Only `content_ids` is supported." + ) + + content_ids = context.get("content_ids") + if content_ids is None: + raise ValueError("`context` must include `content_ids` when provided") + if not isinstance(content_ids, list) or not content_ids: + raise ValueError("`context['content_ids']` must be a non-empty list") + if any(not isinstance(content_id, str) or not content_id for content_id in content_ids): + raise ValueError( + "`context['content_ids']` must contain only non-empty strings" + ) + + return {"content_ids": content_ids} + + +def generate_qualitative_text(text_generation_data: Dict[str, Any]) -> Dict[str, Any]: + """Generate qualitative text using the ValidMind AI API.""" + r = requests.post( + url=_get_url("ai/generate/qualitative_text_generation"), + headers=_get_api_headers(), + json=text_generation_data, + ) + + if r.status_code != 200: + raise_api_error(r.text) + + return r.json() + + +def log_text( + content_id: str, + text: Optional[str] = None, + prompt: Optional[str] = None, + context: Optional[Dict[str, Any]] = None, + _json: Optional[Dict[str, Any]] = None, +) -> str: + """Logs or generates free-form text to ValidMind API. Args: content_id (str): Unique content identifier for the text. - text (str): The text to log. Will be converted to HTML with MathML support. + text (str, optional): The text to log. Will be converted to HTML with + MathML support when Markdown is provided. If omitted, text is + generated using the qualitative text generation backend. + prompt (str, optional): Custom prompt used for AI-assisted text + generation. Only supported when `text` is omitted. + context (dict, optional): Context object for AI-assisted text + generation. When omitted, the full document is used as context. + Currently only supports `{"content_ids": [, ...]}`. _json (dict, optional): Additional metadata to associate with the text. Defaults to None. Raises: - ValueError: If content_id or text are empty or not strings. + ValueError: If arguments are invalid or use incompatible combinations. Exception: If the API call fails. Returns: @@ -462,11 +520,36 @@ def log_text(content_id: str, text: str, _json: Optional[Dict[str, Any]] = None) """ if not content_id or not isinstance(content_id, str): raise ValueError("`content_id` must be a non-empty string") - if not text or not isinstance(text, str): - raise ValueError("`text` must be a non-empty string") - if not is_html(text): - text = md_to_html(text, mathml=True) + if text is not None: + if not isinstance(text, str) or not text: + raise ValueError("`text` must be a non-empty string") + if prompt is not None: + raise ValueError("`prompt` is only supported when `text` is omitted") + if context is not None: + raise ValueError("`context` is only supported when `text` is omitted") + + if not is_html(text): + text = md_to_html(text, mathml=True) + else: + validated_context = _validate_log_text_context(context) + request_data = { + "content_id": content_id, + "generate": True, + } + if prompt is not None: + if not isinstance(prompt, str) or not prompt: + raise ValueError("`prompt` must be a non-empty string") + request_data["prompt"] = prompt + if validated_context is not None: + request_data["context"] = validated_context + + text = generate_qualitative_text(request_data)["content"] + + if not text or not isinstance(text, str): + raise ValueError("Generated text must be a non-empty string") + if not is_html(text): + text = md_to_html(text, mathml=True) log_text = run_async(alog_metadata, content_id, text, _json) From 8cb15b982b0769722ad18d89a4ebb0939402ee2d Mon Sep 17 00:00:00 2001 From: Juan Date: Thu, 2 Apr 2026 22:16:04 +0200 Subject: [PATCH 2/7] Add content ID helper for section-based text generation Made-with: Cursor --- .../qualitative_text_generation.ipynb | 531 ++++++++++++++++++ tests/test_client.py | 39 ++ validmind/__init__.py | 2 + validmind/client.py | 22 +- validmind/template.py | 45 +- 5 files changed, 637 insertions(+), 2 deletions(-) create mode 100644 notebooks/how_to/qualitative_text/qualitative_text_generation.ipynb diff --git a/notebooks/how_to/qualitative_text/qualitative_text_generation.ipynb b/notebooks/how_to/qualitative_text/qualitative_text_generation.ipynb new file mode 100644 index 000000000..a5dcc6dc0 --- /dev/null +++ b/notebooks/how_to/qualitative_text/qualitative_text_generation.ipynb @@ -0,0 +1,531 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "9a900020", + "metadata": {}, + "source": [ + "# Qualitative text generation\n", + "\n", + "This notebook shows how to to generate qualitative documentation content directly from the ValidMind library. You will learn how to provide a custom prompt, optionally scope generation to specific document sections for context, and populate documentation text without manually writing markdown or HTML.\n", + "\n", + "By the end, you will see how AI-assisted text generation can be used to draft an entire customer churn document from within a notebook, bringing the library experience in line with the qualitative text generation available in the UI." + ] + }, + { + "cell_type": "markdown", + "id": "23020a1b", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Setting up" + ] + }, + { + "cell_type": "markdown", + "id": "6202d6dc", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Install the ValidMind Library\n", + "\n", + "To install the library:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "045b05a6", + "metadata": {}, + "outputs": [], + "source": [ + "%pip install -q validmind" + ] + }, + { + "cell_type": "markdown", + "id": "b3231d8e", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Initialize the ValidMind Library" + ] + }, + { + "cell_type": "markdown", + "id": "56592217", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Register sample model\n", + "\n", + "Let's first register a sample model for use with this notebook:\n", + "\n", + "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n", + "\n", + "2. In the left sidebar, navigate to **Inventory** and click **+ Register Model**.\n", + "\n", + "3. Enter the model details and click **Next >** to continue to assignment of model stakeholders. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/register-models-in-inventory.html))\n", + "\n", + "4. Select your own name under the **MODEL OWNER** drop-down.\n", + "\n", + "5. Click **Register Model** to add the model to your inventory." + ] + }, + { + "cell_type": "markdown", + "id": "43ed3d0c", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Apply documentation template\n", + "\n", + "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", + "\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", + "\n", + "2. Under **TEMPLATE**, select `Binary classification`.\n", + "\n", + "3. Click **Use Template** to apply the template." + ] + }, + { + "cell_type": "markdown", + "id": "9b9203be", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Get your code snippet\n", + "\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", + "\n", + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "690dc368", + "metadata": {}, + "outputs": [], + "source": [ + "# Load your model identifier credentials from an `.env` file\n", + "\n", + "%load_ext dotenv\n", + "%dotenv .env\n", + "\n", + "# Or replace with your code snippet\n", + "\n", + "import validmind as vm\n", + "\n", + "vm.init(\n", + " api_host=\"http://localhost:5000/api/v1/tracking\",\n", + " api_key=\"..\",\n", + " api_secret=\"..\",\n", + " document=\"documentation\", # requires library >=2.12.0\n", + " model=\"..\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3fa2d9de", + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "\n", + "import xgboost as xgb\n", + "from validmind.utils import display" + ] + }, + { + "cell_type": "markdown", + "id": "40c9eb24", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Preview the documentation template\n", + "\n", + "Let's verify that you have connected the ValidMind Library to the ValidMind Platform and that the appropriate *template* is selected for your model.\n", + "\n", + "You will upload documentation and test results unique to your model based on this template later on. For now, **take a look at the default structure that the template provides with [the `vm.preview_template()` function](https://docs.validmind.ai/validmind/validmind.html#preview_template)** from the ValidMind library and note the empty sections:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "62842e84", + "metadata": {}, + "outputs": [], + "source": [ + "vm.preview_template()" + ] + }, + { + "cell_type": "markdown", + "id": "3d7ad25a", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Load the Demo Dataset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8ea8188e", + "metadata": {}, + "outputs": [], + "source": [ + "# You can also import taiwan_credit like this:\n", + "# from validmind.datasets.classification import taiwan_credit as demo_dataset\n", + "from validmind.datasets.classification import customer_churn as demo_dataset\n", + "\n", + "df = demo_dataset.load_data()" + ] + }, + { + "cell_type": "markdown", + "id": "a5642e73", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Prepocess the raw dataset" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9d2bec58", + "metadata": {}, + "outputs": [], + "source": [ + "train_df, validation_df, test_df = demo_dataset.preprocess(df)" + ] + }, + { + "cell_type": "markdown", + "id": "a5d573a1", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Train a model for testing\n", + "\n", + "We train a simple customer churn model for our test." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0e7f62d3", + "metadata": {}, + "outputs": [], + "source": [ + "x_train = train_df.drop(demo_dataset.target_column, axis=1)\n", + "y_train = train_df[demo_dataset.target_column]\n", + "x_val = validation_df.drop(demo_dataset.target_column, axis=1)\n", + "y_val = validation_df[demo_dataset.target_column]\n", + "\n", + "model = xgb.XGBClassifier(early_stopping_rounds=10)\n", + "model.set_params(\n", + " eval_metric=[\"error\", \"logloss\", \"auc\"],\n", + ")\n", + "model.fit(\n", + " x_train,\n", + " y_train,\n", + " eval_set=[(x_val, y_val)],\n", + " verbose=False,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "c2a6b492", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Initialize ValidMind objects\n", + "\n", + "We initize the objects required to run test suites using the ValidMind Library." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "081548ae", + "metadata": {}, + "outputs": [], + "source": [ + "vm_dataset = vm.init_dataset(\n", + " input_id=\"raw_dataset\",\n", + " dataset=df,\n", + " target_column=demo_dataset.target_column,\n", + " class_labels=demo_dataset.class_labels,\n", + ")\n", + "\n", + "vm_train_ds = vm.init_dataset(\n", + " input_id=\"train_dataset\",\n", + " dataset=train_df,\n", + " type=\"generic\",\n", + " target_column=demo_dataset.target_column,\n", + ")\n", + "\n", + "vm_test_ds = vm.init_dataset(\n", + " input_id=\"test_dataset\",\n", + " dataset=test_df,\n", + " type=\"generic\",\n", + " target_column=demo_dataset.target_column,\n", + ")\n", + "\n", + "vm_model = vm.init_model(model, input_id=\"model\")" + ] + }, + { + "cell_type": "markdown", + "id": "3cfbce05", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Assign predictions to the datasets\n", + "\n", + "We can now use the `assign_predictions()` method from the `Dataset` object to link existing predictions to any model. If no prediction values are passed, the method will compute predictions automatically:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "922baa9d", + "metadata": {}, + "outputs": [], + "source": [ + "vm_train_ds.assign_predictions(\n", + " model=vm_model,\n", + ")\n", + "vm_test_ds.assign_predictions(\n", + " model=vm_model,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "a7e46a18", + "metadata": {}, + "source": [ + "## Log text in Markdown or HTML\n", + "\n", + "In this section, we focus on using `vm.log_text()` with the current supported input formats. The examples show how to log qualitative text by passing either Markdown or HTML content and update an existing text block in the model documentation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dedc29a2", + "metadata": {}, + "outputs": [], + "source": [ + "dataset_summary_text = \"\"\"\n", + "## Dataset Summary\n", + "\n", + "The customer churn dataset contains customer-level subscription, billing, and service usage attributes used to support model development and evaluation.\n", + "\n", + "### Key Characteristics\n", + "- Records represent individual customers.\n", + "- Features include demographic, account, and service-related variables.\n", + "- The target captures whether a customer churned.\n", + "\n", + "### Intended Use\n", + "This dataset is used to train and assess a churn prediction model that helps identify customers at higher risk of attrition.\n", + "\"\"\"\n", + "\n", + "html =vm.log_text(\n", + " content_id=\"dataset_summary_text\",\n", + " text=dataset_summary_text,\n", + ")\n", + "\n", + "display(html)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "46ccb3f5", + "metadata": {}, + "outputs": [], + "source": [ + "dataset_summary_text = \"\"\"\n", + "

Dataset Summary

\n", + "

\n", + "The customer churn dataset contains customer-level subscription, billing, and service\n", + "usage attributes used to support model development and evaluation.\n", + "

\n", + "

Key Characteristics

\n", + "
    \n", + "
  • Records represent individual customers.
    • \n", + "
  • Features include demographic, account, and service-related variables.
    • \n", + "
  • The target captures whether a customer churned.
    • \n", + "
\n", + "

Intended Use

\n", + "

\n", + "This dataset is used to train and assess a churn prediction model that helps identify\n", + "customers at higher risk of attrition.\n", + "

\n", + "\"\"\"\n", + "html = vm.log_text(\n", + " content_id=\"dataset_summary_text\",\n", + " text=dataset_summary_text,\n", + ")\n", + "display(html)" + ] + }, + { + "cell_type": "markdown", + "id": "899c8553", + "metadata": {}, + "source": [ + "## Generate text with default settings\n", + "\n", + "In this section, we focus on generating qualitative text with the default behavior of `vm.log_text()`. When no additional configuration is provided, ValidMind uses the current document as context to generate content and update the corresponding section in the model documentation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "26fcddf9", + "metadata": {}, + "outputs": [], + "source": [ + "html = vm.log_text(\n", + " content_id=\"dataset_summary_text\",\n", + ")\n", + "display(html)" + ] + }, + { + "cell_type": "markdown", + "id": "caff6490", + "metadata": {}, + "source": [ + "## Generate text with a custom prompt\n", + "\n", + "In this section, we focus on generating qualitative text using a custom prompt. This allows you to guide both what content should be generated and why it should be written in a particular way for the model documentation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fbf10ad9", + "metadata": {}, + "outputs": [], + "source": [ + "html = vm.log_text(\n", + " content_id=\"dataset_summary_text\",\n", + " prompt=\"Generate a summary of the dataset using bullet points.\",\n", + ")\n", + "display(html)" + ] + }, + { + "cell_type": "markdown", + "id": "99a0740e", + "metadata": {}, + "source": [ + "## Generate text with contextual content IDs\n", + "\n", + "In this section, we focus on generating qualitative text using specific content_ids as context. This allows you to guide the generation with selected parts of the document instead of relying on the full document context" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1e1a919e", + "metadata": {}, + "outputs": [], + "source": [ + "html = vm.log_text(\n", + " content_id=\"dataset_summary_text\",\n", + " context={\"content_ids\": [\"validmind.data_validation.DescriptiveStatistics\"]},\n", + ")\n", + "display(html)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "faba886b", + "metadata": {}, + "outputs": [], + "source": [ + "section_ids = [\"data_description\", \"model_development\"]\n", + "content_ids = vm.get_content_ids(section_ids)\n", + "content_ids" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4a7523ce", + "metadata": {}, + "outputs": [], + "source": [ + "html = vm.log_text(\n", + " content_id=\"dataset_summary_text\",\n", + " context={\"content_ids\": content_ids},\n", + ")\n", + "display(html)" + ] + }, + { + "cell_type": "markdown", + "id": "copyright-18d82030e09942c4953248e9bf432249", + "metadata": {}, + "source": [ + "\n", + "\n", + "\n", + "\n", + "***\n", + "\n", + "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "ValidMind Library", + "language": "python", + "name": "validmind" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/tests/test_client.py b/tests/test_client.py index 72a9579f5..a1810c6e6 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -12,6 +12,7 @@ import validmind from validmind import ( + get_content_ids, init_dataset, init_model, get_test_suite, @@ -163,6 +164,44 @@ def test_get_default_config(self): self.assertIn("params", config) +class TestGetContentIds(TestCase): + @mock.patch( + "validmind.client_config.client_config.documentation_template", + MockedConfig.documentation_template, + ) + def test_get_all_content_ids(self): + content_ids = get_content_ids() + self.assertEqual( + content_ids, + [ + "validmind.data_validation.ClassImbalance", + "validmind.data_validation.DatasetSplit", + ], + ) + + @mock.patch( + "validmind.client_config.client_config.documentation_template", + MockedConfig.documentation_template, + ) + def test_get_content_ids_for_single_section(self): + content_ids = get_content_ids("test_section_1") + self.assertEqual(content_ids, ["validmind.data_validation.ClassImbalance"]) + + @mock.patch( + "validmind.client_config.client_config.documentation_template", + MockedConfig.documentation_template, + ) + def test_get_content_ids_for_multiple_sections(self): + content_ids = get_content_ids(["test_section_1", "test_section_2"]) + self.assertEqual( + content_ids, + [ + "validmind.data_validation.ClassImbalance", + "validmind.data_validation.DatasetSplit", + ], + ) + + # TODO: Fix this test # class TestPreviewTemplate(TestCase): # @mock.patch( diff --git a/validmind/__init__.py b/validmind/__init__.py index 93cc0aaca..2af45dc86 100644 --- a/validmind/__init__.py +++ b/validmind/__init__.py @@ -52,6 +52,7 @@ from .__version__ import __version__ # noqa: E402 from .api_client import init, log_metric, log_test_result, log_text, reload from .client import ( # noqa: E402 + get_content_ids, get_test_suite, init_dataset, init_model, @@ -117,6 +118,7 @@ def check_version(): "init_model", "init_r_model", "get_test_suite", + "get_content_ids", "log_metric", "preview_template", "print_env", diff --git a/validmind/client.py b/validmind/client.py index c0fe8b2ca..910719b17 100644 --- a/validmind/client.py +++ b/validmind/client.py @@ -31,7 +31,7 @@ from .logging import get_logger from .models.metadata import MetadataModel from .models.r_model import RModel -from .template import get_template_test_suite +from .template import get_template_content_ids, get_template_test_suite from .template import preview_template as _preview_template from .test_suites import get_by_id as get_test_suite_by_id from .utils import get_dataset_info, get_model_info @@ -433,6 +433,26 @@ def preview_template() -> None: _preview_template(client_config.documentation_template) +def get_content_ids( + section_ids: Optional[Union[str, List[str]]] = None +) -> List[str]: + """Get content IDs for one or more documentation template sections. + + Args: + section_ids: Section ID or list of section IDs. If omitted, all content + IDs in the current documentation template are returned. + + Returns: + A list of content IDs in template order. + """ + if client_config.documentation_template is None: + raise MissingDocumentationTemplate( + "No documentation template found. Please run `vm.init()`" + ) + + return get_template_content_ids(client_config.documentation_template, section_ids) + + def run_documentation_tests( section: Optional[str] = None, send: bool = True, diff --git a/validmind/template.py b/validmind/template.py index d4aec2a0e..03f699edd 100644 --- a/validmind/template.py +++ b/validmind/template.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial import uuid -from typing import Any, Dict, List, Optional, Type +from typing import Any, Dict, List, Optional, Type, Union from .html_templates.content_blocks import ( failed_content_block_html, @@ -224,6 +224,49 @@ def preview_template(template: str) -> None: display(html_content) +def _get_section_content_ids(section: Dict[str, Any]) -> List[str]: + """Get all content IDs in a section and its subsections.""" + content_ids = [ + content["content_id"] + for content in section.get("contents", []) + if content.get("content_id") + ] + + for sub_section in section.get("sections", []): + content_ids.extend(_get_section_content_ids(sub_section)) + + return content_ids + + +def get_template_content_ids( + template: Dict[str, Any], section_ids: Optional[Union[str, List[str]]] = None +) -> List[str]: + """Get content IDs for one or more template sections. + + Args: + template: A valid flat template. + section_ids: Section ID or list of section IDs. If omitted, all content + IDs in the template are returned. + + Returns: + A list of content IDs, preserving template order. + """ + sections = [section_ids] if isinstance(section_ids, str) else section_ids + section_trees = ( + _convert_sections_to_section_tree(template["sections"], start_section_id=s) + for s in sections + ) if sections else [_convert_sections_to_section_tree(template["sections"])] + + content_ids = [] + for section_tree in section_trees: + for section_node in section_tree: + for content_id in _get_section_content_ids(section_node): + if content_id not in content_ids: + content_ids.append(content_id) + + return content_ids + + def _get_section_tests(section: Dict[str, Any]) -> List[str]: """ Get all the tests in a section and its subsections. From 1c47bc71d902b07734b01a700b13aaf29ec514cc Mon Sep 17 00:00:00 2001 From: Juan Date: Thu, 2 Apr 2026 22:20:55 +0200 Subject: [PATCH 3/7] fix lint --- validmind/api_client.py | 94 +++++++++++++++++++++++++------------ validmind/client.py | 4 +- validmind/models/r_model.py | 3 +- validmind/template.py | 10 ++-- 4 files changed, 72 insertions(+), 39 deletions(-) diff --git a/validmind/api_client.py b/validmind/api_client.py index 2bdcf279a..5274c5d1c 100644 --- a/validmind/api_client.py +++ b/validmind/api_client.py @@ -468,10 +468,10 @@ def _validate_log_text_context( raise ValueError("`context` must include `content_ids` when provided") if not isinstance(content_ids, list) or not content_ids: raise ValueError("`context['content_ids']` must be a non-empty list") - if any(not isinstance(content_id, str) or not content_id for content_id in content_ids): - raise ValueError( - "`context['content_ids']` must contain only non-empty strings" - ) + if any( + not isinstance(content_id, str) or not content_id for content_id in content_ids + ): + raise ValueError("`context['content_ids']` must contain only non-empty strings") return {"content_ids": content_ids} @@ -490,6 +490,63 @@ def generate_qualitative_text(text_generation_data: Dict[str, Any]) -> Dict[str, return r.json() +def _normalize_logged_text(text: str, field_name: str) -> str: + """Validate text content and convert Markdown to HTML when needed.""" + if not isinstance(text, str) or not text: + raise ValueError(f"`{field_name}` must be a non-empty string") + + if not is_html(text): + return md_to_html(text, mathml=True) + + return text + + +def _validate_manual_log_text_args( + text: str, prompt: Optional[str], context: Optional[Dict[str, Any]] +) -> str: + """Validate manual log_text arguments.""" + if prompt is not None: + raise ValueError("`prompt` is only supported when `text` is omitted") + if context is not None: + raise ValueError("`context` is only supported when `text` is omitted") + + return _normalize_logged_text(text, "text") + + +def _build_log_text_generation_request( + content_id: str, + prompt: Optional[str], + context: Optional[Dict[str, Any]], +) -> Dict[str, Any]: + """Build the request payload for AI-assisted text generation.""" + request_data = { + "content_id": content_id, + "generate": True, + } + + if prompt is not None: + if not isinstance(prompt, str) or not prompt: + raise ValueError("`prompt` must be a non-empty string") + request_data["prompt"] = prompt + + validated_context = _validate_log_text_context(context) + if validated_context is not None: + request_data["context"] = validated_context + + return request_data + + +def _generate_log_text( + content_id: str, + prompt: Optional[str], + context: Optional[Dict[str, Any]], +) -> str: + """Generate text for log_text and normalize it to HTML.""" + request_data = _build_log_text_generation_request(content_id, prompt, context) + generated_text = generate_qualitative_text(request_data)["content"] + return _normalize_logged_text(generated_text, "generated text") + + def log_text( content_id: str, text: Optional[str] = None, @@ -522,34 +579,9 @@ def log_text( raise ValueError("`content_id` must be a non-empty string") if text is not None: - if not isinstance(text, str) or not text: - raise ValueError("`text` must be a non-empty string") - if prompt is not None: - raise ValueError("`prompt` is only supported when `text` is omitted") - if context is not None: - raise ValueError("`context` is only supported when `text` is omitted") - - if not is_html(text): - text = md_to_html(text, mathml=True) + text = _validate_manual_log_text_args(text, prompt, context) else: - validated_context = _validate_log_text_context(context) - request_data = { - "content_id": content_id, - "generate": True, - } - if prompt is not None: - if not isinstance(prompt, str) or not prompt: - raise ValueError("`prompt` must be a non-empty string") - request_data["prompt"] = prompt - if validated_context is not None: - request_data["context"] = validated_context - - text = generate_qualitative_text(request_data)["content"] - - if not text or not isinstance(text, str): - raise ValueError("Generated text must be a non-empty string") - if not is_html(text): - text = md_to_html(text, mathml=True) + text = _generate_log_text(content_id, prompt, context) log_text = run_async(alog_metadata, content_id, text, _json) diff --git a/validmind/client.py b/validmind/client.py index 910719b17..522ff7c77 100644 --- a/validmind/client.py +++ b/validmind/client.py @@ -433,9 +433,7 @@ def preview_template() -> None: _preview_template(client_config.documentation_template) -def get_content_ids( - section_ids: Optional[Union[str, List[str]]] = None -) -> List[str]: +def get_content_ids(section_ids: Optional[Union[str, List[str]]] = None) -> List[str]: """Get content IDs for one or more documentation template sections. Args: diff --git a/validmind/models/r_model.py b/validmind/models/r_model.py index cf36e308c..e6caaede1 100644 --- a/validmind/models/r_model.py +++ b/validmind/models/r_model.py @@ -126,8 +126,7 @@ def predict(self, new_data, return_probs=False): Converts the predicted probabilities to classes """ try: - from rpy2.robjects import conversion, default_converter - from rpy2.robjects import pandas2ri + from rpy2.robjects import conversion, default_converter, pandas2ri except ImportError: raise MissingRExtrasError() diff --git a/validmind/template.py b/validmind/template.py index 03f699edd..7352e8b26 100644 --- a/validmind/template.py +++ b/validmind/template.py @@ -253,9 +253,13 @@ def get_template_content_ids( """ sections = [section_ids] if isinstance(section_ids, str) else section_ids section_trees = ( - _convert_sections_to_section_tree(template["sections"], start_section_id=s) - for s in sections - ) if sections else [_convert_sections_to_section_tree(template["sections"])] + ( + _convert_sections_to_section_tree(template["sections"], start_section_id=s) + for s in sections + ) + if sections + else [_convert_sections_to_section_tree(template["sections"])] + ) content_ids = [] for section_tree in section_trees: From 892e6919f4a01b22c96172163f5a98cd59a61a5e Mon Sep 17 00:00:00 2001 From: Juan Date: Fri, 3 Apr 2026 10:35:51 +0200 Subject: [PATCH 4/7] Add text generation result workflow Made-with: Cursor --- .../qualitative_text_generation.ipynb | 88 ++++++++++++++++--- tests/test_client.py | 29 +++++- tests/test_results.py | 28 ++++++ validmind/__init__.py | 2 + validmind/api_client.py | 52 +++++++---- validmind/client.py | 37 ++++++++ validmind/experimental/agents.py | 3 + validmind/vm_models/result/result.py | 15 ++-- 8 files changed, 223 insertions(+), 31 deletions(-) diff --git a/notebooks/how_to/qualitative_text/qualitative_text_generation.ipynb b/notebooks/how_to/qualitative_text/qualitative_text_generation.ipynb index a5dcc6dc0..a0b5d1ba9 100644 --- a/notebooks/how_to/qualitative_text/qualitative_text_generation.ipynb +++ b/notebooks/how_to/qualitative_text/qualitative_text_generation.ipynb @@ -410,10 +410,20 @@ "metadata": {}, "outputs": [], "source": [ - "html = vm.log_text(\n", + "result = vm.run_text_generation(\n", " content_id=\"dataset_summary_text\",\n", ")\n", - "display(html)" + "result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "659578ad", + "metadata": {}, + "outputs": [], + "source": [ + "result.log()" ] }, { @@ -433,11 +443,31 @@ "metadata": {}, "outputs": [], "source": [ - "html = vm.log_text(\n", + "result = vm.run_text_generation(\n", " content_id=\"dataset_summary_text\",\n", " prompt=\"Generate a summary of the dataset using bullet points.\",\n", ")\n", - "display(html)" + "result" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e700454a", + "metadata": {}, + "outputs": [], + "source": [ + "result.prompt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a510dc40", + "metadata": {}, + "outputs": [], + "source": [ + "result.log()" ] }, { @@ -457,11 +487,30 @@ "metadata": {}, "outputs": [], "source": [ - "html = vm.log_text(\n", + "result = vm.run_text_generation(\n", " content_id=\"dataset_summary_text\",\n", " context={\"content_ids\": [\"validmind.data_validation.DescriptiveStatistics\"]},\n", - ")\n", - "display(html)" + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "257a9c81", + "metadata": {}, + "outputs": [], + "source": [ + "result.context" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b622c91d", + "metadata": {}, + "outputs": [], + "source": [ + "result.log()" ] }, { @@ -483,11 +532,30 @@ "metadata": {}, "outputs": [], "source": [ - "html = vm.log_text(\n", + "result = vm.run_text_generation(\n", " content_id=\"dataset_summary_text\",\n", " context={\"content_ids\": content_ids},\n", - ")\n", - "display(html)" + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8ad5702c", + "metadata": {}, + "outputs": [], + "source": [ + "result.context" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7a4d1a20", + "metadata": {}, + "outputs": [], + "source": [ + "result.log()" ] }, { diff --git a/tests/test_client.py b/tests/test_client.py index a1810c6e6..952145256 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -16,9 +16,11 @@ init_dataset, init_model, get_test_suite, + run_text_generation, run_documentation_tests, ) from validmind.errors import UnsupportedModelError +from validmind.vm_models.result import TextGenerationResult @dataclass @@ -116,7 +118,7 @@ def test_init_model_invalid_metadata_dict(self): "key": "value", "foo": "bar", } - with self.assertRaises(UnsupportedModelError) as context: + with self.assertRaises(UnsupportedModelError): init_model(attributes=metadata, __log=False) def test_init_model_metadata_dict(self): @@ -202,6 +204,31 @@ def test_get_content_ids_for_multiple_sections(self): ) +class TestRunTextGeneration(TestCase): + @mock.patch( + "validmind.client.api_client._generate_log_text", + return_value="

Generated text

", + ) + def test_run_text_generation(self, mock_generate_text): + result = run_text_generation( + content_id="dataset_summary_text", + prompt="Summarize the dataset.", + context={"content_ids": ["train_dataset"]}, + show=False, + ) + + self.assertIsInstance(result, TextGenerationResult) + self.assertEqual(result.content_id, "dataset_summary_text") + self.assertEqual(result.prompt, "Summarize the dataset.") + self.assertEqual(result.context, {"content_ids": ["train_dataset"]}) + self.assertEqual(result.description, "

Generated text

") + mock_generate_text.assert_called_once_with( + "dataset_summary_text", + "Summarize the dataset.", + {"content_ids": ["train_dataset"]}, + ) + + # TODO: Fix this test # class TestPreviewTemplate(TestCase): # @mock.patch( diff --git a/tests/test_results.py b/tests/test_results.py index 0eafb0679..16b011d54 100644 --- a/tests/test_results.py +++ b/tests/test_results.py @@ -186,6 +186,34 @@ def test_text_generation_result(self): self.assertIsInstance(html, str) self.assertIn("Generated text", html) + @patch("validmind.vm_models.result.result.api_client.alog_text") + async def test_text_generation_result_log_async(self, mock_log_text): + """Test async logging of TextGenerationResult through alog_text""" + text_result = TextGenerationResult( + result_id="text_1", + content_id="dataset_summary_text", + description="Generated text", + ) + + await text_result.log_async() + + mock_log_text.assert_called_once_with( + content_id="dataset_summary_text", + text="Generated text", + ) + + async def test_text_generation_result_log_async_requires_content_id(self): + """Test TextGenerationResult requires a content_id when logging""" + text_result = TextGenerationResult( + result_id="text_1", + description="Generated text", + ) + + with self.assertRaisesRegex( + ValueError, "`content_id` must be provided to log generated text" + ): + await text_result.log_async() + def test_validate_log_config(self): """Test validation of log configuration""" test_result = TestResult(result_id="test_1") diff --git a/validmind/__init__.py b/validmind/__init__.py index 2af45dc86..c6bbfa2c4 100644 --- a/validmind/__init__.py +++ b/validmind/__init__.py @@ -60,6 +60,7 @@ preview_template, run_documentation_tests, run_test_suite, + run_text_generation, ) from .experimental import agents as experimental_agent from .tests.decorator import scorer as scorer_decorator @@ -123,6 +124,7 @@ def check_version(): "preview_template", "print_env", "reload", + "run_text_generation", "run_documentation_tests", # log metric function (for direct/bulk/retroactive logging of metrics) # test suite functions (less common) diff --git a/validmind/api_client.py b/validmind/api_client.py index 5274c5d1c..cb614617e 100644 --- a/validmind/api_client.py +++ b/validmind/api_client.py @@ -547,6 +547,35 @@ def _generate_log_text( return _normalize_logged_text(generated_text, "generated text") +def _render_logged_text(logged_text: Dict[str, Any]) -> str: + """Render logged text as notebook-friendly HTML.""" + from .vm_models.html_renderer import StatefulHTMLRenderer + + return StatefulHTMLRenderer.render_accordion( + items=[logged_text["text"]], + titles=[f"Text Block: '{logged_text['content_id']}'"], + ) + + +async def alog_text( + content_id: str, + text: Optional[str] = None, + prompt: Optional[str] = None, + context: Optional[Dict[str, Any]] = None, + _json: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + """Async variant of ``log_text`` that logs or generates text.""" + if not content_id or not isinstance(content_id, str): + raise ValueError("`content_id` must be a non-empty string") + + if text is not None: + text = _validate_manual_log_text_args(text, prompt, context) + else: + text = _generate_log_text(content_id, prompt, context) + + return await alog_metadata(content_id, text, _json) + + def log_text( content_id: str, text: Optional[str] = None, @@ -575,22 +604,15 @@ def log_text( Returns: str: HTML string containing the logged text in an accordion format. """ - if not content_id or not isinstance(content_id, str): - raise ValueError("`content_id` must be a non-empty string") - - if text is not None: - text = _validate_manual_log_text_args(text, prompt, context) - else: - text = _generate_log_text(content_id, prompt, context) - - log_text = run_async(alog_metadata, content_id, text, _json) - - from .vm_models.html_renderer import StatefulHTMLRenderer - - return StatefulHTMLRenderer.render_accordion( - items=[log_text["text"]], - titles=[f"Text Block: '{log_text['content_id']}'"], + logged_text = run_async( + alog_text, + content_id=content_id, + text=text, + prompt=prompt, + context=context, + _json=_json, ) + return _render_logged_text(logged_text) async def alog_metric( diff --git a/validmind/client.py b/validmind/client.py index 522ff7c77..338dad02c 100644 --- a/validmind/client.py +++ b/validmind/client.py @@ -17,6 +17,7 @@ except Exception: torch = None # type: ignore # noqa: F401 +from . import api_client from .api_client import log_input as log_input from .client_config import client_config from .errors import ( @@ -43,6 +44,7 @@ get_model_class, is_model_metadata, ) +from .vm_models.result import TextGenerationResult pd.option_context("format.precision", 2) @@ -451,6 +453,41 @@ def get_content_ids(section_ids: Optional[Union[str, List[str]]] = None) -> List return get_template_content_ids(client_config.documentation_template, section_ids) +def run_text_generation( + content_id: str, + prompt: Optional[str] = None, + context: Optional[Dict[str, Any]] = None, + show: bool = True, +) -> TextGenerationResult: + """Generate qualitative text and return a loggable result object. + + Args: + content_id: Content ID to generate text for. + prompt: Custom prompt for text generation. + context: Optional context object for text generation. + show: Whether to display the generated result. + + Returns: + A TextGenerationResult containing the generated text. + """ + description = api_client._generate_log_text(content_id, prompt, context) + result = TextGenerationResult( + result_id=content_id, + result_type="qualitative_text_generation", + content_id=content_id, + title=f"Text Generation: {content_id}", + doc="Generated qualitative text", + description=description, + prompt=prompt, + context=context, + ) + + if show: + result.show() + + return result + + def run_documentation_tests( section: Optional[str] = None, send: bool = True, diff --git a/validmind/experimental/agents.py b/validmind/experimental/agents.py index 3b52fa1b5..ac7513cab 100644 --- a/validmind/experimental/agents.py +++ b/validmind/experimental/agents.py @@ -54,10 +54,13 @@ def run_task( # Create a test result with the generated text result = TextGenerationResult( + content_id=input.get("content_id"), result_type=f"{task}", description=generated_text, title=f"Text Generation: {task}", doc=f"Generated {task}", + prompt=input.get("prompt"), + context=input.get("context"), ) if show: result.show() diff --git a/validmind/vm_models/result/result.py b/validmind/vm_models/result/result.py index ce7e502f8..f0ba78d97 100644 --- a/validmind/vm_models/result/result.py +++ b/validmind/vm_models/result/result.py @@ -693,11 +693,14 @@ class TextGenerationResult(Result): name: str = "Text Generation Result" ref_id: str = None + content_id: Optional[str] = None title: Optional[str] = None doc: Optional[str] = None description: Optional[Union[str, DescriptionFuture]] = None params: Optional[Dict[str, Any]] = None metadata: Optional[Dict[str, Any]] = None + prompt: Optional[str] = None + context: Optional[Dict[str, Any]] = None _was_description_generated: bool = False def __post_init__(self): @@ -770,11 +773,13 @@ async def log_async( self, content_id: str = None, ): - return await asyncio.gather( - update_metadata( - content_id=f"{content_id}", - text=self.description, - ) + resolved_content_id = content_id or self.content_id + if not resolved_content_id or not isinstance(resolved_content_id, str): + raise ValueError("`content_id` must be provided to log generated text") + + return await api_client.alog_text( + content_id=resolved_content_id, + text=self.description, ) def log( From 4dc7fa4095cb55a352a9a1e1136d2efc1a6b913b Mon Sep 17 00:00:00 2001 From: Juan Date: Fri, 3 Apr 2026 11:28:11 +0200 Subject: [PATCH 5/7] Update text generation result metadata Made-with: Cursor --- .../qualitative_text_generation.ipynb | 106 ++++++++++++++---- tests/test_client.py | 3 + tests/test_results.py | 2 + validmind/client.py | 6 +- validmind/experimental/agents.py | 7 +- validmind/tests/run.py | 2 +- validmind/vm_models/result/result.py | 9 +- 7 files changed, 107 insertions(+), 28 deletions(-) diff --git a/notebooks/how_to/qualitative_text/qualitative_text_generation.ipynb b/notebooks/how_to/qualitative_text/qualitative_text_generation.ipynb index a0b5d1ba9..1e370dad5 100644 --- a/notebooks/how_to/qualitative_text/qualitative_text_generation.ipynb +++ b/notebooks/how_to/qualitative_text/qualitative_text_generation.ipynb @@ -131,7 +131,7 @@ " api_key=\"..\",\n", " api_secret=\"..\",\n", " document=\"documentation\", # requires library >=2.12.0\n", - " model=\"..\",\n", + " model=\"...\",\n", ")" ] }, @@ -145,7 +145,29 @@ "%matplotlib inline\n", "\n", "import xgboost as xgb\n", - "from validmind.utils import display" + "\n", + "from validmind.utils import display\n", + "from validmind.tests import run_test" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "cc006f12", + "metadata": {}, + "outputs": [], + "source": [ + "def print_public_attrs(obj):\n", + " for name in sorted(attr for attr in dir(obj) if not attr.startswith(\"_\")):\n", + " try:\n", + " value = getattr(obj, name)\n", + " except Exception as exc:\n", + " value = f\"\"\n", + "\n", + " if callable(value):\n", + " continue\n", + "\n", + " print(f\"{name}: {value}\")" ] }, { @@ -322,6 +344,49 @@ ")" ] }, + { + "cell_type": "markdown", + "id": "2fb45208", + "metadata": {}, + "source": [ + "## Run tests" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7b0b30e8", + "metadata": {}, + "outputs": [], + "source": [ + "result = run_test(\n", + " \"validmind.data_validation.DatasetDescription\",\n", + " inputs={\n", + " \"dataset\": vm_dataset,\n", + " },\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3c963384", + "metadata": {}, + "outputs": [], + "source": [ + "print_public_attrs(result)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "153a5861", + "metadata": {}, + "outputs": [], + "source": [ + "result.log()" + ] + }, { "cell_type": "markdown", "id": "a7e46a18", @@ -412,8 +477,17 @@ "source": [ "result = vm.run_text_generation(\n", " content_id=\"dataset_summary_text\",\n", - ")\n", - "result" + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "043fb74b", + "metadata": {}, + "outputs": [], + "source": [ + "print_public_attrs(result)" ] }, { @@ -446,18 +520,17 @@ "result = vm.run_text_generation(\n", " content_id=\"dataset_summary_text\",\n", " prompt=\"Generate a summary of the dataset using bullet points.\",\n", - ")\n", - "result" + ")" ] }, { "cell_type": "code", "execution_count": null, - "id": "e700454a", + "id": "202d7a1c", "metadata": {}, "outputs": [], "source": [ - "result.prompt" + "print_public_attrs(result)" ] }, { @@ -500,7 +573,7 @@ "metadata": {}, "outputs": [], "source": [ - "result.context" + "print_public_attrs(result)" ] }, { @@ -516,22 +589,13 @@ { "cell_type": "code", "execution_count": null, - "id": "faba886b", + "id": "4a7523ce", "metadata": {}, "outputs": [], "source": [ "section_ids = [\"data_description\", \"model_development\"]\n", "content_ids = vm.get_content_ids(section_ids)\n", - "content_ids" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4a7523ce", - "metadata": {}, - "outputs": [], - "source": [ + "\n", "result = vm.run_text_generation(\n", " content_id=\"dataset_summary_text\",\n", " context={\"content_ids\": content_ids},\n", @@ -545,7 +609,7 @@ "metadata": {}, "outputs": [], "source": [ - "result.context" + "print_public_attrs(result)" ] }, { diff --git a/tests/test_client.py b/tests/test_client.py index 952145256..8af37224f 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -222,6 +222,9 @@ def test_run_text_generation(self, mock_generate_text): self.assertEqual(result.prompt, "Summarize the dataset.") self.assertEqual(result.context, {"content_ids": ["train_dataset"]}) self.assertEqual(result.description, "

Generated text

") + self.assertIn("validmind", result.metadata) + self.assertIn("timestamp", result.metadata) + self.assertIn("duration_seconds", result.metadata) mock_generate_text.assert_called_once_with( "dataset_summary_text", "Summarize the dataset.", diff --git a/tests/test_results.py b/tests/test_results.py index 16b011d54..f0d943bb6 100644 --- a/tests/test_results.py +++ b/tests/test_results.py @@ -181,6 +181,8 @@ def test_text_generation_result(self): self.assertEqual(text_result.name, "Text Generation Result") self.assertEqual(text_result.title, "Text Test") self.assertEqual(text_result.description, "Generated text") + self.assertIsNone(text_result.doc) + self.assertIsNone(text_result.test_name) html = text_result.to_html() self.assertIsInstance(html, str) diff --git a/validmind/client.py b/validmind/client.py index 338dad02c..78cece897 100644 --- a/validmind/client.py +++ b/validmind/client.py @@ -6,6 +6,7 @@ Client interface for all data and model validation functions """ +import time from typing import Any, Callable, Dict, List, Optional, Union import numpy as np @@ -35,6 +36,7 @@ from .template import get_template_content_ids, get_template_test_suite from .template import preview_template as _preview_template from .test_suites import get_by_id as get_test_suite_by_id +from .tests.run import _get_run_metadata from .utils import get_dataset_info, get_model_info from .vm_models import TestSuite, TestSuiteRunner from .vm_models.dataset import DataFrameDataset, PolarsDataset, TorchDataset, VMDataset @@ -470,14 +472,16 @@ def run_text_generation( Returns: A TextGenerationResult containing the generated text. """ + start_time = time.perf_counter() description = api_client._generate_log_text(content_id, prompt, context) + metadata = _get_run_metadata(duration_seconds=time.perf_counter() - start_time) result = TextGenerationResult( result_id=content_id, result_type="qualitative_text_generation", content_id=content_id, title=f"Text Generation: {content_id}", - doc="Generated qualitative text", description=description, + metadata=metadata, prompt=prompt, context=context, ) diff --git a/validmind/experimental/agents.py b/validmind/experimental/agents.py index ac7513cab..b2fd431ee 100644 --- a/validmind/experimental/agents.py +++ b/validmind/experimental/agents.py @@ -6,9 +6,12 @@ Agent interface for all text generation tasks """ +import time + import requests from validmind.api_client import _get_api_headers, _get_url, raise_api_error +from validmind.tests.run import _get_run_metadata from validmind.utils import is_html, md_to_html from validmind.vm_models.result import TextGenerationResult @@ -35,6 +38,8 @@ def run_task( ValueError: If an unsupported task is provided requests.exceptions.RequestException: If the API request fails """ + start_time = time.perf_counter() + if task == "code_explainer" or task == "qualitative_text_generation": r = requests.post( url=_get_url(f"ai/generate/{task}"), @@ -58,7 +63,7 @@ def run_task( result_type=f"{task}", description=generated_text, title=f"Text Generation: {task}", - doc=f"Generated {task}", + metadata=_get_run_metadata(duration_seconds=time.perf_counter() - start_time), prompt=input.get("prompt"), context=input.get("context"), ) diff --git a/validmind/tests/run.py b/validmind/tests/run.py index 7e316c177..194a80bec 100644 --- a/validmind/tests/run.py +++ b/validmind/tests/run.py @@ -11,7 +11,7 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Union from uuid import uuid4 -from validmind import __version__ +from validmind.__version__ import __version__ from validmind.ai.test_descriptions import get_result_description from validmind.errors import MissingRequiredTestInputError from validmind.input_registry import input_registry diff --git a/validmind/vm_models/result/result.py b/validmind/vm_models/result/result.py index f0ba78d97..cb0696d3b 100644 --- a/validmind/vm_models/result/result.py +++ b/validmind/vm_models/result/result.py @@ -737,17 +737,18 @@ def __getattribute__(self, name): return super().__getattribute__(name) @property - def test_name(self) -> str: - """Get the test name, using custom title if available.""" - return self.title or test_id_to_name(self.result_id) + def test_name(self) -> None: + """Text generation results do not expose a test-style name.""" + return None def to_html(self): """Generate HTML that persists in saved notebooks.""" html_parts = [StatefulHTMLRenderer.get_base_css()] + display_title = self.title or "" html_parts.append( StatefulHTMLRenderer.render_result_header( - test_name=self.test_name, passed=None + test_name=display_title, passed=None ) ) From 1a6c9d3b58fd76e4c37f108302b5a427cf12ab4e Mon Sep 17 00:00:00 2001 From: Juan Date: Fri, 3 Apr 2026 17:18:46 +0200 Subject: [PATCH 6/7] Update qualitative text generation notebook Made-with: Cursor --- .../qualitative_text_generation.ipynb | 1464 +++++++++-------- 1 file changed, 803 insertions(+), 661 deletions(-) diff --git a/notebooks/how_to/qualitative_text/qualitative_text_generation.ipynb b/notebooks/how_to/qualitative_text/qualitative_text_generation.ipynb index 1e370dad5..1b4207101 100644 --- a/notebooks/how_to/qualitative_text/qualitative_text_generation.ipynb +++ b/notebooks/how_to/qualitative_text/qualitative_text_generation.ipynb @@ -1,663 +1,805 @@ { - "cells": [ - { - "cell_type": "markdown", - "id": "9a900020", - "metadata": {}, - "source": [ - "# Qualitative text generation\n", - "\n", - "This notebook shows how to to generate qualitative documentation content directly from the ValidMind library. You will learn how to provide a custom prompt, optionally scope generation to specific document sections for context, and populate documentation text without manually writing markdown or HTML.\n", - "\n", - "By the end, you will see how AI-assisted text generation can be used to draft an entire customer churn document from within a notebook, bringing the library experience in line with the qualitative text generation available in the UI." - ] - }, - { - "cell_type": "markdown", - "id": "23020a1b", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Setting up" - ] - }, - { - "cell_type": "markdown", - "id": "6202d6dc", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Install the ValidMind Library\n", - "\n", - "To install the library:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "045b05a6", - "metadata": {}, - "outputs": [], - "source": [ - "%pip install -q validmind" - ] - }, - { - "cell_type": "markdown", - "id": "b3231d8e", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Initialize the ValidMind Library" - ] - }, - { - "cell_type": "markdown", - "id": "56592217", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Register sample model\n", - "\n", - "Let's first register a sample model for use with this notebook:\n", - "\n", - "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n", - "\n", - "2. In the left sidebar, navigate to **Inventory** and click **+ Register Model**.\n", - "\n", - "3. Enter the model details and click **Next >** to continue to assignment of model stakeholders. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/register-models-in-inventory.html))\n", - "\n", - "4. Select your own name under the **MODEL OWNER** drop-down.\n", - "\n", - "5. Click **Register Model** to add the model to your inventory." - ] - }, - { - "cell_type": "markdown", - "id": "43ed3d0c", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Apply documentation template\n", - "\n", - "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", - "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", - "\n", - "2. Under **TEMPLATE**, select `Binary classification`.\n", - "\n", - "3. Click **Use Template** to apply the template." - ] - }, - { - "cell_type": "markdown", - "id": "9b9203be", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Get your code snippet\n", - "\n", - "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", - "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", - "2. Click **Copy snippet to clipboard**.\n", - "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "690dc368", - "metadata": {}, - "outputs": [], - "source": [ - "# Load your model identifier credentials from an `.env` file\n", - "\n", - "%load_ext dotenv\n", - "%dotenv .env\n", - "\n", - "# Or replace with your code snippet\n", - "\n", - "import validmind as vm\n", - "\n", - "vm.init(\n", - " api_host=\"http://localhost:5000/api/v1/tracking\",\n", - " api_key=\"..\",\n", - " api_secret=\"..\",\n", - " document=\"documentation\", # requires library >=2.12.0\n", - " model=\"...\",\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3fa2d9de", - "metadata": {}, - "outputs": [], - "source": [ - "%matplotlib inline\n", - "\n", - "import xgboost as xgb\n", - "\n", - "from validmind.utils import display\n", - "from validmind.tests import run_test" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "cc006f12", - "metadata": {}, - "outputs": [], - "source": [ - "def print_public_attrs(obj):\n", - " for name in sorted(attr for attr in dir(obj) if not attr.startswith(\"_\")):\n", - " try:\n", - " value = getattr(obj, name)\n", - " except Exception as exc:\n", - " value = f\"\"\n", - "\n", - " if callable(value):\n", - " continue\n", - "\n", - " print(f\"{name}: {value}\")" - ] - }, - { - "cell_type": "markdown", - "id": "40c9eb24", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Preview the documentation template\n", - "\n", - "Let's verify that you have connected the ValidMind Library to the ValidMind Platform and that the appropriate *template* is selected for your model.\n", - "\n", - "You will upload documentation and test results unique to your model based on this template later on. For now, **take a look at the default structure that the template provides with [the `vm.preview_template()` function](https://docs.validmind.ai/validmind/validmind.html#preview_template)** from the ValidMind library and note the empty sections:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "62842e84", - "metadata": {}, - "outputs": [], - "source": [ - "vm.preview_template()" - ] - }, - { - "cell_type": "markdown", - "id": "3d7ad25a", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Load the Demo Dataset" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8ea8188e", - "metadata": {}, - "outputs": [], - "source": [ - "# You can also import taiwan_credit like this:\n", - "# from validmind.datasets.classification import taiwan_credit as demo_dataset\n", - "from validmind.datasets.classification import customer_churn as demo_dataset\n", - "\n", - "df = demo_dataset.load_data()" - ] - }, - { - "cell_type": "markdown", - "id": "a5642e73", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Prepocess the raw dataset" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9d2bec58", - "metadata": {}, - "outputs": [], - "source": [ - "train_df, validation_df, test_df = demo_dataset.preprocess(df)" - ] - }, - { - "cell_type": "markdown", - "id": "a5d573a1", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Train a model for testing\n", - "\n", - "We train a simple customer churn model for our test." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "0e7f62d3", - "metadata": {}, - "outputs": [], - "source": [ - "x_train = train_df.drop(demo_dataset.target_column, axis=1)\n", - "y_train = train_df[demo_dataset.target_column]\n", - "x_val = validation_df.drop(demo_dataset.target_column, axis=1)\n", - "y_val = validation_df[demo_dataset.target_column]\n", - "\n", - "model = xgb.XGBClassifier(early_stopping_rounds=10)\n", - "model.set_params(\n", - " eval_metric=[\"error\", \"logloss\", \"auc\"],\n", - ")\n", - "model.fit(\n", - " x_train,\n", - " y_train,\n", - " eval_set=[(x_val, y_val)],\n", - " verbose=False,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "c2a6b492", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Initialize ValidMind objects\n", - "\n", - "We initize the objects required to run test suites using the ValidMind Library." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "081548ae", - "metadata": {}, - "outputs": [], - "source": [ - "vm_dataset = vm.init_dataset(\n", - " input_id=\"raw_dataset\",\n", - " dataset=df,\n", - " target_column=demo_dataset.target_column,\n", - " class_labels=demo_dataset.class_labels,\n", - ")\n", - "\n", - "vm_train_ds = vm.init_dataset(\n", - " input_id=\"train_dataset\",\n", - " dataset=train_df,\n", - " type=\"generic\",\n", - " target_column=demo_dataset.target_column,\n", - ")\n", - "\n", - "vm_test_ds = vm.init_dataset(\n", - " input_id=\"test_dataset\",\n", - " dataset=test_df,\n", - " type=\"generic\",\n", - " target_column=demo_dataset.target_column,\n", - ")\n", - "\n", - "vm_model = vm.init_model(model, input_id=\"model\")" - ] - }, - { - "cell_type": "markdown", - "id": "3cfbce05", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Assign predictions to the datasets\n", - "\n", - "We can now use the `assign_predictions()` method from the `Dataset` object to link existing predictions to any model. If no prediction values are passed, the method will compute predictions automatically:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "922baa9d", - "metadata": {}, - "outputs": [], - "source": [ - "vm_train_ds.assign_predictions(\n", - " model=vm_model,\n", - ")\n", - "vm_test_ds.assign_predictions(\n", - " model=vm_model,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "2fb45208", - "metadata": {}, - "source": [ - "## Run tests" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7b0b30e8", - "metadata": {}, - "outputs": [], - "source": [ - "result = run_test(\n", - " \"validmind.data_validation.DatasetDescription\",\n", - " inputs={\n", - " \"dataset\": vm_dataset,\n", - " },\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "3c963384", - "metadata": {}, - "outputs": [], - "source": [ - "print_public_attrs(result)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "153a5861", - "metadata": {}, - "outputs": [], - "source": [ - "result.log()" - ] - }, - { - "cell_type": "markdown", - "id": "a7e46a18", - "metadata": {}, - "source": [ - "## Log text in Markdown or HTML\n", - "\n", - "In this section, we focus on using `vm.log_text()` with the current supported input formats. The examples show how to log qualitative text by passing either Markdown or HTML content and update an existing text block in the model documentation" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "dedc29a2", - "metadata": {}, - "outputs": [], - "source": [ - "dataset_summary_text = \"\"\"\n", - "## Dataset Summary\n", - "\n", - "The customer churn dataset contains customer-level subscription, billing, and service usage attributes used to support model development and evaluation.\n", - "\n", - "### Key Characteristics\n", - "- Records represent individual customers.\n", - "- Features include demographic, account, and service-related variables.\n", - "- The target captures whether a customer churned.\n", - "\n", - "### Intended Use\n", - "This dataset is used to train and assess a churn prediction model that helps identify customers at higher risk of attrition.\n", - "\"\"\"\n", - "\n", - "html =vm.log_text(\n", - " content_id=\"dataset_summary_text\",\n", - " text=dataset_summary_text,\n", - ")\n", - "\n", - "display(html)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "46ccb3f5", - "metadata": {}, - "outputs": [], - "source": [ - "dataset_summary_text = \"\"\"\n", - "

Dataset Summary

\n", - "

\n", - "The customer churn dataset contains customer-level subscription, billing, and service\n", - "usage attributes used to support model development and evaluation.\n", - "

\n", - "

Key Characteristics

\n", - "
    \n", - "
  • Records represent individual customers.
    • \n", - "
  • Features include demographic, account, and service-related variables.
    • \n", - "
  • The target captures whether a customer churned.
    • \n", - "
\n", - "

Intended Use

\n", - "

\n", - "This dataset is used to train and assess a churn prediction model that helps identify\n", - "customers at higher risk of attrition.\n", - "

\n", - "\"\"\"\n", - "html = vm.log_text(\n", - " content_id=\"dataset_summary_text\",\n", - " text=dataset_summary_text,\n", - ")\n", - "display(html)" - ] - }, - { - "cell_type": "markdown", - "id": "899c8553", - "metadata": {}, - "source": [ - "## Generate text with default settings\n", - "\n", - "In this section, we focus on generating qualitative text with the default behavior of `vm.log_text()`. When no additional configuration is provided, ValidMind uses the current document as context to generate content and update the corresponding section in the model documentation." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "26fcddf9", - "metadata": {}, - "outputs": [], - "source": [ - "result = vm.run_text_generation(\n", - " content_id=\"dataset_summary_text\",\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "043fb74b", - "metadata": {}, - "outputs": [], - "source": [ - "print_public_attrs(result)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "659578ad", - "metadata": {}, - "outputs": [], - "source": [ - "result.log()" - ] - }, - { - "cell_type": "markdown", - "id": "caff6490", - "metadata": {}, - "source": [ - "## Generate text with a custom prompt\n", - "\n", - "In this section, we focus on generating qualitative text using a custom prompt. This allows you to guide both what content should be generated and why it should be written in a particular way for the model documentation." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fbf10ad9", - "metadata": {}, - "outputs": [], - "source": [ - "result = vm.run_text_generation(\n", - " content_id=\"dataset_summary_text\",\n", - " prompt=\"Generate a summary of the dataset using bullet points.\",\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "202d7a1c", - "metadata": {}, - "outputs": [], - "source": [ - "print_public_attrs(result)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a510dc40", - "metadata": {}, - "outputs": [], - "source": [ - "result.log()" - ] - }, - { - "cell_type": "markdown", - "id": "99a0740e", - "metadata": {}, - "source": [ - "## Generate text with contextual content IDs\n", - "\n", - "In this section, we focus on generating qualitative text using specific content_ids as context. This allows you to guide the generation with selected parts of the document instead of relying on the full document context" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1e1a919e", - "metadata": {}, - "outputs": [], - "source": [ - "result = vm.run_text_generation(\n", - " content_id=\"dataset_summary_text\",\n", - " context={\"content_ids\": [\"validmind.data_validation.DescriptiveStatistics\"]},\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "257a9c81", - "metadata": {}, - "outputs": [], - "source": [ - "print_public_attrs(result)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b622c91d", - "metadata": {}, - "outputs": [], - "source": [ - "result.log()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4a7523ce", - "metadata": {}, - "outputs": [], - "source": [ - "section_ids = [\"data_description\", \"model_development\"]\n", - "content_ids = vm.get_content_ids(section_ids)\n", - "\n", - "result = vm.run_text_generation(\n", - " content_id=\"dataset_summary_text\",\n", - " context={\"content_ids\": content_ids},\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8ad5702c", - "metadata": {}, - "outputs": [], - "source": [ - "print_public_attrs(result)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7a4d1a20", - "metadata": {}, - "outputs": [], - "source": [ - "result.log()" - ] - }, - { - "cell_type": "markdown", - "id": "copyright-18d82030e09942c4953248e9bf432249", - "metadata": {}, - "source": [ - "\n", - "\n", - "\n", - "\n", - "***\n", - "\n", - "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", - "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "ValidMind Library", - "language": "python", - "name": "validmind" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.11" - } - }, - "nbformat": 4, - "nbformat_minor": 5 + "cells": [ + { + "cell_type": "markdown", + "id": "9a900020", + "metadata": {}, + "source": [ + "# Generate qualitative text with the ValidMind library\n", + "\n", + "This notebook shows how to generate qualitative documentation content directly from the ValidMind library using `vm.run_text_generation()`. Instead of switching to the UI to write text manually or trigger generation one section at a time, you can generate content for documentation text blocks programmatically from within a notebook and log it back to the corresponding sections of the model document.\n", + "\n", + "After building an example model and documenting its quantitative results, we’ll show how to generate text for individual content blocks, customize the output with prompts, control the context used for generation, and apply the same pattern across the full template. By the end, you’ll have an end-to-end example of how quantitative test results and AI-generated qualitative content can work together to populate a full model document from Python, giving you a more automated documentation workflow directly in the library." + ] + }, + { + "cell_type": "markdown", + "id": "cd48db57", + "metadata": {}, + "source": [ + "::: {.content-hidden when-format=\"html\"}\n", + "## Contents \n", + "- [About ValidMind](#toc1__) \n", + " - [Before you begin](#toc1_1__) \n", + " - [New to ValidMind?](#toc1_2__) \n", + " - [Key concepts](#toc1_3__) \n", + "- [Setting up](#toc2__) \n", + " - [Install the ValidMind Library](#toc2_1__) \n", + " - [Initialize the ValidMind Library](#toc2_2__) \n", + " - [Register sample model](#toc2_2_1__) \n", + " - [Apply documentation template](#toc2_2_2__) \n", + " - [Get your code snippet](#toc2_2_3__) \n", + " - [Initialize the Python environment](#toc2_3__) \n", + "- [Getting to know ValidMind](#toc3__) \n", + " - [Preview the documentation template](#toc3_1__) \n", + " - [View model documentation in the ValidMind Platform](#toc3_2__) \n", + "- [Build the Example Model](#toc4__) \n", + " - [Import the sample dataset](#toc4_1__) \n", + " - [Preprocessing the raw dataset](#toc4_2__) \n", + " - [Training an XGBoost classifier model](#toc4_3__) \n", + "- [Initialize the ValidMind Inputs](#toc5__) \n", + "- [Document Test Results](#toc6__) \n", + "- [Document Qualitative Sections](#toc7__) \n", + " - [Generate Text for a Single Content Block](#toc7_1__) \n", + " - [Customize the Prompt](#toc7_2__) \n", + " - [Pass Section-Specific Context](#toc7_3__) \n", + " - [Populate All Text Blocks](#toc7_4__) \n", + "\n", + ":::\n", + "\n", + "" + ] + }, + { + "cell_type": "markdown", + "id": "a67217b3", + "metadata": {}, + "source": [ + "\n", + "\n", + "## About ValidMind\n", + "\n", + "ValidMind is a suite of tools for managing model risk, including risk associated with AI and statistical models. \n", + "\n", + "You use the ValidMind Library to automate documentation and validation tests, and then use the ValidMind Platform to collaborate on model documentation. Together, these products simplify model risk management, facilitate compliance with regulations and institutional standards, and enhance collaboration between yourself and model validators." + ] + }, + { + "cell_type": "markdown", + "id": "281cfb86", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Before you begin\n", + "\n", + "This notebook assumes you have basic familiarity with Python, including an understanding of how functions work. If you are new to Python, you can still run the notebook but we recommend further familiarizing yourself with the language. \n", + "\n", + "If you encounter errors due to missing modules in your Python environment, install the modules with `pip install`, and then re-run the notebook. For more help, refer to [Installing Python Modules](https://docs.python.org/3/installing/index.html)." + ] + }, + { + "cell_type": "markdown", + "id": "51c11b52", + "metadata": {}, + "source": [ + "\n", + "\n", + "### New to ValidMind?\n", + "\n", + "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n", + "\n", + "
For access to all features available in this notebook, you'll need access to a ValidMind account.\n", + "

\n", + "Register with ValidMind
" + ] + }, + { + "cell_type": "markdown", + "id": "9103cd45", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Key concepts\n", + "\n", + "**Validation report**: A comprehensive and structured assessment of a model’s development and performance, focusing on verifying its integrity, appropriateness, and alignment with its intended use. It includes analyses of model assumptions, data quality, performance metrics, outcomes of testing procedures, and risk considerations. The validation report supports transparency, regulatory compliance, and informed decision-making by documenting the validator’s independent review and conclusions.\n", + "\n", + "**Validation report template**: Serves as a standardized framework for conducting and documenting model validation activities. It outlines the required sections, recommended analyses, and expected validation tests, ensuring consistency and completeness across validation reports. The template helps guide validators through a systematic review process while promoting comparability and traceability of validation outcomes.\n", + "\n", + "**Tests**: A function contained in the ValidMind Library, designed to run a specific quantitative test on the dataset or model. Tests are the building blocks of ValidMind, used to evaluate and document models and datasets.\n", + "\n", + "**Metrics**: A subset of tests that do not have thresholds. In the context of this notebook, metrics and tests can be thought of as interchangeable concepts.\n", + "\n", + "**Custom metrics**: Custom metrics are functions that you define to evaluate your model or dataset. These functions can be registered with the ValidMind Library to be used in the ValidMind Platform.\n", + "\n", + "**Inputs**: Objects to be evaluated and documented in the ValidMind Library. They can be any of the following:\n", + "\n", + " - **model**: A single model that has been initialized in ValidMind with [`vm.init_model()`](https://docs.validmind.ai/validmind/validmind.html#init_model).\n", + " - **dataset**: Single dataset that has been initialized in ValidMind with [`vm.init_dataset()`](https://docs.validmind.ai/validmind/validmind.html#init_dataset).\n", + " - **models**: A list of ValidMind models - usually this is used when you want to compare multiple models in your custom metric.\n", + " - **datasets**: A list of ValidMind datasets - usually this is used when you want to compare multiple datasets in your custom metric. (Learn more: [Run tests with multiple datasets](https://docs.validmind.ai/notebooks/how_to/tests/run_tests/configure_tests/run_tests_that_require_multiple_datasets.html))\n", + "\n", + "**Parameters**: Additional arguments that can be passed when running a ValidMind test, used to pass additional information to a metric, customize its behavior, or provide additional context.\n", + "\n", + "**Outputs**: Custom metrics can return elements like tables or plots. Tables may be a list of dictionaries (each representing a row) or a pandas DataFrame. Plots may be matplotlib or plotly figures." + ] + }, + { + "cell_type": "markdown", + "id": "23020a1b", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Setting up" + ] + }, + { + "cell_type": "markdown", + "id": "6202d6dc", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Install the ValidMind Library\n", + "\n", + "
Recommended Python versions\n", + "

\n", + "Python 3.8 <= x <= 3.14
\n", + "\n", + "To install the library:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "045b05a6", + "metadata": {}, + "outputs": [], + "source": [ + "%pip install -q validmind" + ] + }, + { + "cell_type": "markdown", + "id": "b3231d8e", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Initialize the ValidMind Library" + ] + }, + { + "cell_type": "markdown", + "id": "56592217", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Register sample model\n", + "\n", + "Let's first register a sample model for use with this notebook:\n", + "\n", + "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n", + "\n", + "2. In the left sidebar, navigate to **Inventory** and click **+ Register Model**.\n", + "\n", + "3. Enter the model details and click **Next >** to continue to assignment of model stakeholders. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/register-models-in-inventory.html))\n", + "\n", + "4. Select your own name under the **MODEL OWNER** drop-down.\n", + "\n", + "5. Click **Register Model** to add the model to your inventory." + ] + }, + { + "cell_type": "markdown", + "id": "43ed3d0c", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Apply documentation template\n", + "\n", + "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", + "\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", + "\n", + "2. Under **TEMPLATE**, select `Customer Churn`.\n", + "\n", + "3. Click **Use Template** to apply the template." + ] + }, + { + "cell_type": "markdown", + "id": "9b9203be", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Get your code snippet\n", + "\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", + "\n", + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "690dc368", + "metadata": {}, + "outputs": [], + "source": [ + "# Load your model identifier credentials from an `.env` file\n", + "\n", + "%load_ext dotenv\n", + "%dotenv .env\n", + "\n", + "# Or replace with your code snippet\n", + "\n", + "import validmind as vm\n", + "\n", + "vm.init(\n", + " api_host=\"http://localhost:5000/api/v1/tracking\",\n", + " api_key=\"..\",\n", + " api_secret=\"..\",\n", + " document=\"documentation\", # requires library >=2.12.0\n", + " model=\"..\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "a68f6031", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Initialize the Python environment\n", + "\n", + "Then, let's import the necessary libraries and set up your Python environment for data analysis:\n", + "\n", + "- Import **Extreme Gradient Boosting** (XGBoost) with an alias so that we can reference its functions in later calls. XGBoost is a powerful machine learning library designed for speed and performance, especially in handling structured or tabular data.\n", + "- Enable **`matplotlib`**, a plotting library used for visualizing data. Ensures that any plots you generate will render inline in our notebook output rather than opening in a separate window." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3fa2d9de", + "metadata": {}, + "outputs": [], + "source": [ + "%matplotlib inline\n", + "\n", + "import xgboost as xgb" + ] + }, + { + "cell_type": "markdown", + "id": "69a37995", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Getting to know ValidMind" + ] + }, + { + "cell_type": "markdown", + "id": "40c9eb24", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Preview the documentation template\n", + "\n", + "Let's verify that you have connected the ValidMind Library to the ValidMind Platform and that the appropriate *template* is selected for your model.\n", + "\n", + "You will upload documentation and test results unique to your model based on this template later on. For now, **take a look at the default structure that the template provides with [the `vm.preview_template()` function](https://docs.validmind.ai/validmind/validmind.html#preview_template)** from the ValidMind library and note the empty sections:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "62842e84", + "metadata": {}, + "outputs": [], + "source": [ + "vm.preview_template()" + ] + }, + { + "cell_type": "markdown", + "id": "6fab1c1c", + "metadata": {}, + "source": [ + "\n", + "\n", + "### View model documentation in the ValidMind Platform\n", + "\n", + "Next, let's head to the ValidMind Platform to see the template in action:\n", + "\n", + "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n", + "\n", + "2. In the left sidebar, navigate to **Inventory** and select the model you registered for this notebook.\n", + "\n", + "3. Click **Development** under Documents for your model and note how the structure of the documentation matches our preview above." + ] + }, + { + "cell_type": "markdown", + "id": "606d932b", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Build the Example Model" + ] + }, + { + "cell_type": "markdown", + "id": "3d7ad25a", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Import the sample dataset\n", + "\n", + "First, let's import the public [Bank Customer Churn Prediction](https://www.kaggle.com/datasets/shantanudhakadd/bank-customer-churn-prediction) dataset from Kaggle so that we have something to work with.\n", + "\n", + "In our below example, note that: \n", + "\n", + "- The target column, `Exited` has a value of `1` when a customer has churned and `0` otherwise.\n", + "- The ValidMind Library provides a wrapper to automatically load the dataset as a [Pandas DataFrame](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html) object. A Pandas Dataframe is a two-dimensional tabular data structure that makes use of rows and columns." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8ea8188e", + "metadata": {}, + "outputs": [], + "source": [ + "from validmind.datasets.classification import customer_churn\n", + "\n", + "print(\n", + " f\"Loaded demo dataset with: \\n\\n\\t• Target column: '{customer_churn.target_column}' \\n\\t• Class labels: {customer_churn.class_labels}\"\n", + ")\n", + "\n", + "raw_df = customer_churn.load_data()\n", + "raw_df.head()" + ] + }, + { + "cell_type": "markdown", + "id": "a5ceef72", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Preprocessing the raw dataset\n", + "\n", + "In this section, we preprocess the raw dataset so it is ready for model training and validation. This includes splitting the data into training, validation, and test subsets to support both model fitting and evaluation on unseen data, and then separating each subset into input features and target labels so the model can learn from customer attributes and predict whether a customer churned." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9d2bec58", + "metadata": {}, + "outputs": [], + "source": [ + "train_df, validation_df, test_df = customer_churn.preprocess(raw_df)\n", + "\n", + "x_train = train_df.drop(customer_churn.target_column, axis=1)\n", + "y_train = train_df[customer_churn.target_column]\n", + "x_val = validation_df.drop(customer_churn.target_column, axis=1)\n", + "y_val = validation_df[customer_churn.target_column]" + ] + }, + { + "cell_type": "markdown", + "id": "3b9edacf", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Training an XGBoost classifier model\n", + "\n", + "In this section, we train an XGBoost classifier to predict customer churn, using early stopping to halt training if performance does not improve after 10 rounds and reduce unnecessary fitting. We configure the model to evaluate performance with three complementary metrics: error for incorrect predictions, logloss for prediction confidence, and auc for class separation. The model is trained on the training split and evaluated against the validation split during fitting, while verbose=False keeps the training output concise." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "658447fc", + "metadata": {}, + "outputs": [], + "source": [ + "model = xgb.XGBClassifier(early_stopping_rounds=10)\n", + "\n", + "model.set_params(\n", + " eval_metric=[\"error\", \"logloss\", \"auc\"],\n", + ")\n", + "\n", + "model.fit(\n", + " x_train,\n", + " y_train,\n", + " eval_set=[(x_val, y_val)],\n", + " verbose=False,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "c2a6b492", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Initialize the ValidMind Inputs\n", + "\n", + "We begin by registering the datasets and trained model as ValidMind inputs so they can be referenced consistently throughout the documentation workflow. For the datasets, this means creating ValidMind Dataset objects for the raw, training, and testing data, each with a unique `input_id` for traceability. Where needed, we also provide supporting metadata such as the target column and class labels so tests can interpret the data correctly." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "081548ae", + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize the raw dataset\n", + "vm_raw_dataset = vm.init_dataset(\n", + " dataset=raw_df,\n", + " input_id=\"raw_dataset\",\n", + " target_column=customer_churn.target_column,\n", + " class_labels=customer_churn.class_labels,\n", + ")\n", + "\n", + "# Initialize the training dataset\n", + "vm_train_ds = vm.init_dataset(\n", + " dataset=train_df,\n", + " input_id=\"train_dataset\",\n", + " target_column=customer_churn.target_column,\n", + ")\n", + "\n", + "# Initialize the testing dataset\n", + "vm_test_ds = vm.init_dataset(\n", + " dataset=test_df,\n", + " input_id=\"test_dataset\",\n", + " target_column=customer_churn.target_column\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "1ebfda19", + "metadata": {}, + "source": [ + "Next, we initialize a ValidMind model object with `vm.init_model()`. This creates a standardized representation of the trained model that can be passed into ValidMind tests and other library functions, making it possible to evaluate the model and connect its results to the documentation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6cc5aff8", + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize the model\n", + "vm_model = vm.init_model(\n", + " model,\n", + " input_id=\"model\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "48d23cf8", + "metadata": {}, + "source": [ + "Finally, we assign predictions from the trained model to the training and testing datasets. The `assign_predictions()` method links predicted classes and probabilities to each dataset, and can also compute predictions automatically if they are not passed explicitly. This step is what allows ValidMind to run performance and diagnostic tests using the model outputs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "922baa9d", + "metadata": {}, + "outputs": [], + "source": [ + "vm_train_ds.assign_predictions(\n", + " model=vm_model,\n", + ")\n", + "vm_test_ds.assign_predictions(\n", + " model=vm_model,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "7c9a174d", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Document Test Results\n", + "\n", + "In this section, we run the documentation tests defined by the applied template to populate the quantitative parts of the model documentation. The `vm.run_documentation_tests()` function discovers each test-driven block in the template, executes the corresponding tests, and uploads the resulting artifacts to the ValidMind Platform.\n", + "\n", + "To run the full suite successfully, ValidMind needs to know which model and dataset inputs should be used for each test. This can be done with a shared `inputs` argument when all tests use the same objects, or with a `config` dictionary when individual tests require specific inputs or parameters. In this example, we use the default test parameters and provide the input configuration needed for the demo model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "47f7e709", + "metadata": {}, + "outputs": [], + "source": [ + "from validmind.utils import preview_test_config\n", + "\n", + "test_config = customer_churn.get_demo_test_config()\n", + "preview_test_config(test_config)" + ] + }, + { + "cell_type": "markdown", + "id": "3f22d37b", + "metadata": {}, + "source": [ + "Once the configuration is prepared, we pass it to `vm.run_documentation_tests()` and execute the full suite. The returned `full_suite` object contains the test results and represents the quantitative documentation that has been generated for the model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "999be7fe", + "metadata": {}, + "outputs": [], + "source": [ + "full_suite = vm.run_documentation_tests(config=test_config)" + ] + }, + { + "cell_type": "markdown", + "id": "5d531744", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Document Qualitative Sections\n", + "\n", + "In addition to documenting quantitative results through tests, ValidMind now supports programmatic generation of qualitative content for the text blocks in a model documentation template through `vm.run_text_generation()`. This function allows you to generate AI-assisted text for a specific content block directly from a notebook and then log it back to the corresponding section of the document. As a result, you can populate qualitative sections without switching to the UI to write text manually or trigger generation one section at a time.\n", + "\n", + "In the next sections, we’ll walk through the main ways to use this functionality. We’ll start by generating text for a single content block with the default behavior, then show how to customize the output with a prompt, how to control the context used for generation by selecting specific sections, and finally how to scale the same pattern across all text blocks in the document." + ] + }, + { + "cell_type": "markdown", + "id": "899c8553", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Generate Text for a Single Content Block\n", + "\n", + "First, we’ll use `vm.run_text_generation()` to generate qualitative text for a single documentation block. By providing a `content_id`, you can target the exact text placeholder you want to populate and let ValidMind generate content using the current document context. The helper `vm.get_content_ids()` is useful for inspecting which content blocks are available in the active template, making it easier to identify the IDs you can use when generating and logging text programmatically." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "85cc552f", + "metadata": {}, + "outputs": [], + "source": [ + "vm.get_content_ids()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "26fcddf9", + "metadata": {}, + "outputs": [], + "source": [ + "vm.run_text_generation(\n", + " content_id=\"model_overview_text\",\n", + ").log()" + ] + }, + { + "cell_type": "markdown", + "id": "caff6490", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Customize the Prompt\n", + "\n", + "Next, we’ll customize the generated output by passing a `prompt` to `vm.run_text_generation()`. This makes it possible to guide not just the subject of the generated text, but also its structure, tone, level of detail, and presentation format. In practice, this allows you to tailor the output for different documentation needs, such as producing a short narrative summary, a more structured section, or content written for a specific audience, while still relying on the same underlying document context for generation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "52165b98", + "metadata": {}, + "outputs": [], + "source": [ + "prompt = \"\"\"\n", + "Use exactly this structure:\n", + "\n", + "

Purpose

\n", + "

Explain in 1-2 sentences what the model predicts and why it is used.

\n", + "\n", + "

Model Summary

\n", + "

Summarize the model type, algorithm, target outcome, and the main input features in 2-3 sentences.

\n", + "\n", + "

Key Strengths

\n", + "
    \n", + "
  • Include 2-3 concise strengths.
    • \n", + "
\n", + "\n", + "

Key Risks and Considerations

\n", + "
    \n", + "
  • Include 2-3 concise limitations, weaknesses, or monitoring concerns.
    • \n", + "
\n", + "\n", + "

Overall Assessment

\n", + "

End with a short balanced conclusion on the model's readiness and reliability.

\n", + "\"\"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fbf10ad9", + "metadata": {}, + "outputs": [], + "source": [ + "vm.run_text_generation(\n", + " content_id=\"model_overview_text\",\n", + " prompt=prompt,\n", + ").log()" + ] + }, + { + "cell_type": "markdown", + "id": "99a0740e", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Pass Section-Specific Context\n", + "\n", + "Then, we’ll control the `context` used for generation by passing a selected set of content IDs to `vm.run_text_generation()`. Rather than relying on the full document, this lets you focus the model on the most relevant parts of the documentation for a given text block. In practice, that means you can generate more targeted qualitative content by choosing which existing test and text blocks should inform the output." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1e1a919e", + "metadata": {}, + "outputs": [], + "source": [ + "vm.run_text_generation(\n", + " content_id=\"dataset_description_text\",\n", + " context={\"content_ids\": vm.get_content_ids(\"data_description\")},\n", + ").log()" + ] + }, + { + "cell_type": "markdown", + "id": "6e032b79", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Populate All Text Blocks\n", + "\n", + "Finally, we’ll scale this pattern across the full document by generating text for all documentation text blocks in a loop. A simple helper maps each target content_id to the most relevant section context, resolves those section IDs into content IDs, and applies `vm.run_text_generation()` consistently across the template. This makes it possible to programmatically populate all qualitative sections of the document from the notebook in a single workflow.\n", + "\n", + "The helper below runs the same generation pattern across multiple text blocks. `_build_context()` takes a list of section IDs and resolves them into the `content_ids` format expected by `vm.run_text_generation()`, while `generate_documentation_text()` loops through a configuration dictionary, runs generation for each target `content_id`, and logs the result. The `config` dictionary maps each text block we want to populate and the section IDs we want to use as context for that block. When a value is set to `None`, the generation falls back to using the full document as context." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "19615186", + "metadata": {}, + "outputs": [], + "source": [ + "def _build_context(section_ids):\n", + " return {\"content_ids\": vm.get_content_ids(section_ids=section_ids)}\n", + "\n", + "def generate_documentation_text(config, show=True):\n", + " results = {}\n", + " for content_id, section_ids in config.items():\n", + " results[content_id] = vm.run_text_generation(\n", + " content_id=content_id,\n", + " prompt=\"Write a single paragraph with no headings, bullets, or tables.\",\n", + " context=_build_context(section_ids),\n", + " show=show,\n", + " ).log()\n", + "\n", + "config = {\n", + " \"model_overview_text\": [\"data_description\", \"model_evaluation\"],\n", + " \"intended_use_text\": None,\n", + " \"model_limitations_text\": [\"data_description\", \"model_evaluation\"],\n", + " \"model_selection_text\": [\"model_evaluation\"],\n", + " \"dataset_description_text\": [\"data_description\"],\n", + " \"data_quality_text\": [\"data_quality\"],\n", + " \"correlations_text\": [\"correlations\"],\n", + " \"feature_selection_text\": [\"data_description\", \"data_quality\", \"correlations\"],\n", + " \"model_training_text\": [\"model_training\"],\n", + " \"model_evaluation_text\": [\"model_evaluation\"],\n", + " \"explainability_text\": [\"explainability\"],\n", + " \"model_diagnosis_text\": [\"model_diagnosis\"],\n", + " \"monitoring_plan_text\": None,\n", + " \"monitoring_implementation_text\": None,\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4209b4d2", + "metadata": {}, + "outputs": [], + "source": [ + "generate_documentation_text(config)" + ] + }, + { + "cell_type": "markdown", + "id": "copyright-18d82030e09942c4953248e9bf432249", + "metadata": {}, + "source": [ + "\n", + "\n", + "\n", + "\n", + "***\n", + "\n", + "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "ValidMind Library", + "language": "python", + "name": "validmind" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 } From 8beb29dc6e0c7471bd32fd2074efe6e241332dd2 Mon Sep 17 00:00:00 2001 From: Juan Date: Fri, 3 Apr 2026 17:49:28 +0200 Subject: [PATCH 7/7] Update notebook --- .../qualitative_text_generation.ipynb | 151 ++++++++++++++++-- 1 file changed, 135 insertions(+), 16 deletions(-) diff --git a/notebooks/how_to/qualitative_text/qualitative_text_generation.ipynb b/notebooks/how_to/qualitative_text/qualitative_text_generation.ipynb index 1b4207101..c095cbf68 100644 --- a/notebooks/how_to/qualitative_text/qualitative_text_generation.ipynb +++ b/notebooks/how_to/qualitative_text/qualitative_text_generation.ipynb @@ -33,17 +33,22 @@ "- [Getting to know ValidMind](#toc3__) \n", " - [Preview the documentation template](#toc3_1__) \n", " - [View model documentation in the ValidMind Platform](#toc3_2__) \n", - "- [Build the Example Model](#toc4__) \n", + "- [Build the example model](#toc4__) \n", " - [Import the sample dataset](#toc4_1__) \n", " - [Preprocessing the raw dataset](#toc4_2__) \n", " - [Training an XGBoost classifier model](#toc4_3__) \n", - "- [Initialize the ValidMind Inputs](#toc5__) \n", - "- [Document Test Results](#toc6__) \n", - "- [Document Qualitative Sections](#toc7__) \n", - " - [Generate Text for a Single Content Block](#toc7_1__) \n", - " - [Customize the Prompt](#toc7_2__) \n", - " - [Pass Section-Specific Context](#toc7_3__) \n", - " - [Populate All Text Blocks](#toc7_4__) \n", + "- [Initialize the ValidMind inputs](#toc5__) \n", + "- [Document test results](#toc6__) \n", + "- [Document qualitative sections](#toc7__) \n", + " - [Generate text for a single content block](#toc7_1__) \n", + " - [Customize the prompt](#toc7_2__) \n", + " - [Pass section-specific context](#toc7_3__) \n", + " - [Populate all text blocks](#toc7_4__) \n", + "- [In summary](#toc8__) \n", + "- [Next steps](#toc9__) \n", + " - [Work with your model documentation](#toc9_1__) \n", + " - [Discover more learning resources](#toc9_2__) \n", + "- [Upgrade ValidMind](#toc10__) \n", "\n", ":::\n", "