diff --git a/.github/workflows/deploy-docs-prod.yaml b/.github/workflows/deploy-docs-prod.yaml index 8491f5f47b..c2e62fb39b 100644 --- a/.github/workflows/deploy-docs-prod.yaml +++ b/.github/workflows/deploy-docs-prod.yaml @@ -38,21 +38,37 @@ jobs: with: repository: validmind/validmind-library path: site/_source/validmind-library - token: ${{ secrets.RELEASE_NOTES_RO_PAT }} + token: ${{ secrets.DOCS_CI_RO_PAT }} - name: Check out installation repository uses: actions/checkout@v4 with: repository: validmind/installation path: site/_source/installation - token: ${{ secrets.INSTALLATION_RO_PAT }} + token: ${{ secrets.DOCS_CI_RO_PAT }} + sparse-checkout: | + site/installation + sparse-checkout-cone-mode: true - name: Check out release-notes repository uses: actions/checkout@v4 with: repository: validmind/release-notes path: site/_source/release-notes - token: ${{ secrets.RELEASE_NOTES_RO_PAT }} + token: ${{ secrets.DOCS_CI_RO_PAT }} + sparse-checkout: | + releases + sparse-checkout-cone-mode: true + + - name: Check out backend repository + uses: actions/checkout@v4 + with: + repository: validmind/backend + path: site/_source/backend + token: ${{ secrets.DOCS_CI_RO_PAT }} + sparse-checkout: | + src/backend/templates/documentation/model_documentation + sparse-checkout-cone-mode: true - name: Set up Quarto uses: quarto-dev/quarto-actions/setup@v2 @@ -73,6 +89,11 @@ jobs: mkdir -p validmind rsync -av --exclude '_build' --exclude 'templates' _source/validmind-library/docs/ validmind/ + - name: Generate template schema docs + run: | + pip install json-schema-for-humans + BACKEND_ROOT=site/_source/backend python scripts/generate_template_schema_docs.py + - name: Populate installation run: cp -r site/_source/installation/site/installation site/installation @@ -96,7 +117,7 @@ jobs: run: aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID_PROD }} && aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY_PROD }} && aws configure set default.region us-east-1 - name: Deploy docs prod site - run: aws s3 sync site/_site s3://validmind-docs-prod/site --delete --exclude "installation/omnibus/*" --exclude "installation/helm-repo/*" && aws cloudfront create-invalidation --distribution-id E2BGG3USKQTR9W --paths "/*" --no-cli-pager + run: aws s3 sync site/_site s3://validmind-docs-prod/site --delete --exclude "installation/omnibus/*" --exclude "installation/helm-repo/*" --exclude "llm/*" && aws cloudfront create-invalidation --distribution-id E2BGG3USKQTR9W --paths "/*" --no-cli-pager # Release headroom and shrink before final lightweight steps & post-job - name: Release reserve & shrink @@ -107,6 +128,7 @@ jobs: remove_paths: | site/_source/installation site/_source/release-notes + site/_source/backend site/render_errors.log site/_freeze dev.env diff --git a/.github/workflows/deploy-docs-staging.yaml b/.github/workflows/deploy-docs-staging.yaml index 4192197a6c..48e76d348f 100644 --- a/.github/workflows/deploy-docs-staging.yaml +++ b/.github/workflows/deploy-docs-staging.yaml @@ -38,22 +38,38 @@ jobs: with: repository: validmind/validmind-library path: site/_source/validmind-library - token: ${{ secrets.RELEASE_NOTES_RO_PAT }} + token: ${{ secrets.DOCS_CI_RO_PAT }} - name: Check out installation repository uses: actions/checkout@v4 with: repository: validmind/installation path: site/_source/installation - token: ${{ secrets.INSTALLATION_RO_PAT }} + token: ${{ secrets.DOCS_CI_RO_PAT }} + sparse-checkout: | + site/installation + sparse-checkout-cone-mode: true - name: Check out release-notes repository uses: actions/checkout@v4 with: repository: validmind/release-notes path: site/_source/release-notes - token: ${{ secrets.RELEASE_NOTES_RO_PAT }} - + token: ${{ secrets.DOCS_CI_RO_PAT }} + sparse-checkout: | + releases + sparse-checkout-cone-mode: true + + - name: Check out backend repository + uses: actions/checkout@v4 + with: + repository: validmind/backend + path: site/_source/backend + token: ${{ secrets.DOCS_CI_RO_PAT }} + sparse-checkout: | + src/backend/templates/documentation/model_documentation + sparse-checkout-cone-mode: true + - name: Set up Quarto uses: quarto-dev/quarto-actions/setup@v2 with: @@ -72,7 +88,12 @@ jobs: rm -rf validmind mkdir -p validmind rsync -av --exclude '_build' --exclude 'templates' _source/validmind-library/docs/ validmind/ - + + - name: Generate template schema docs + run: | + pip install json-schema-for-humans + BACKEND_ROOT=site/_source/backend python scripts/generate_template_schema_docs.py + - name: Populate installation run: cp -r site/_source/installation/site/installation site/installation @@ -99,7 +120,7 @@ jobs: run: aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID_STAGING }} && aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY_STAGING }} && aws configure set default.region us-west-2 - name: Deploy docs staging site - run: aws s3 sync site/_site s3://validmind-docs-staging/site --delete --exclude "installation/helm-repo/*" --exclude "pr_previews/*" && aws cloudfront create-invalidation --distribution-id ESWVTZYFL873V --paths "/*" --no-cli-pager + run: aws s3 sync site/_site s3://validmind-docs-staging/site --delete --exclude "installation/helm-repo/*" --exclude "pr_previews/*" --exclude "llm/*" && aws cloudfront create-invalidation --distribution-id ESWVTZYFL873V --paths "/*" --no-cli-pager # Release headroom and shrink before final lightweight steps & post-job - name: Release reserve & shrink @@ -110,6 +131,7 @@ jobs: remove_paths: | site/_source/installation site/_source/release-notes + site/_source/backend site/render_errors.log site/_freeze dev.env diff --git a/.github/workflows/publish-llm-markdown.yaml b/.github/workflows/publish-llm-markdown.yaml new file mode 100644 index 0000000000..e5500fe568 --- /dev/null +++ b/.github/workflows/publish-llm-markdown.yaml @@ -0,0 +1,109 @@ +name: Publish LLM markdown + +on: + push: + branches: + - main + - prod + workflow_dispatch: + +jobs: + publish: + runs-on: ubuntu-latest + + steps: + - name: Check out documentation repository + uses: actions/checkout@v4 + + - name: Free space + create reserve + uses: ./.github/actions/free-disk-space + with: + remove_dotnet: "true" + remove_android: "true" + remove_haskell: "true" + prune_docker: "true" + apt_cleanup: "true" + create_reserve_gb: "3" + + - run: mkdir -p site/_source + + - name: Check out validmind-library repository + uses: actions/checkout@v4 + with: + repository: validmind/validmind-library + path: site/_source/validmind-library + token: ${{ secrets.DOCS_CI_RO_PAT }} + + - name: Set up Quarto + uses: quarto-dev/quarto-actions/setup@v2 + with: + version: ${{ vars.QUARTO_VERSION }} + + - name: Install Poetry + run: | + curl -sSL https://install.python-poetry.org | python3 - + echo "$HOME/.local/bin" >> $GITHUB_PATH + + - name: Generate Python library docs + run: | + cd _source/validmind-library + make install && make quarto-docs + cd ../../ + rm -rf validmind + mkdir -p validmind + rsync -av --exclude '_build' --exclude 'templates' _source/validmind-library/docs/ validmind/ + working-directory: site + + - name: Install pandoc + run: | + sudo apt-get update + sudo apt-get install -y pandoc + + - name: Render LLM markdown + run: bash llm/render.sh + working-directory: site + + - name: Set archive name + id: archive + run: | + BRANCH="${GITHUB_REF_NAME//\//-}" + echo "name=docs-markdown-${BRANCH}.tar.gz" >> "$GITHUB_OUTPUT" + + - name: Create tar.gz archive + run: | + cd site/llm/_llm-output + tar -czf "/tmp/${{ steps.archive.outputs.name }}" . + + # main → staging bucket (us-west-2), prod → prod bucket (us-east-1) + - name: Configure AWS credentials + run: | + if [ "${{ github.ref_name }}" = "prod" ]; then + aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID_PROD }} + aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY_PROD }} + aws configure set default.region us-east-1 + else + aws configure set aws_access_key_id ${{ secrets.AWS_ACCESS_KEY_ID_STAGING }} + aws configure set aws_secret_access_key ${{ secrets.AWS_SECRET_ACCESS_KEY_STAGING }} + aws configure set default.region us-west-2 + fi + + - name: Upload to S3 + run: | + if [ "${{ github.ref_name }}" = "prod" ]; then + BUCKET="s3://validmind-docs-prod/site/llm/${{ steps.archive.outputs.name }}" + else + BUCKET="s3://validmind-docs-staging/site/llm/${{ steps.archive.outputs.name }}" + fi + aws s3 cp "/tmp/${{ steps.archive.outputs.name }}" "$BUCKET" + echo "Uploaded to $BUCKET" + + - name: Release reserve & shrink + if: always() + uses: ./.github/actions/free-disk-space + with: + release_reserve: "true" + remove_paths: | + site/_source/installation + site/_source/release-notes + site/render_errors.log + site/_freeze diff --git a/.github/workflows/validate-docs-site.yaml b/.github/workflows/validate-docs-site.yaml index 39626e421e..da1d3bc9d1 100644 --- a/.github/workflows/validate-docs-site.yaml +++ b/.github/workflows/validate-docs-site.yaml @@ -33,21 +33,37 @@ jobs: with: repository: validmind/validmind-library path: site/_source/validmind-library - token: ${{ secrets.RELEASE_NOTES_RO_PAT }} - + token: ${{ secrets.DOCS_CI_RO_PAT }} + - name: Check out installation repository uses: actions/checkout@v4 with: repository: validmind/installation path: site/_source/installation - token: ${{ secrets.INSTALLATION_RO_PAT }} + token: ${{ secrets.DOCS_CI_RO_PAT }} + sparse-checkout: | + site/installation + sparse-checkout-cone-mode: true - name: Check out release-notes repository uses: actions/checkout@v4 with: repository: validmind/release-notes path: site/_source/release-notes - token: ${{ secrets.RELEASE_NOTES_RO_PAT }} + token: ${{ secrets.DOCS_CI_RO_PAT }} + sparse-checkout: | + releases + sparse-checkout-cone-mode: true + + - name: Check out backend repository + uses: actions/checkout@v4 + with: + repository: validmind/backend + path: site/_source/backend + token: ${{ secrets.DOCS_CI_RO_PAT }} + sparse-checkout: | + src/backend/templates/documentation/model_documentation + sparse-checkout-cone-mode: true - name: Verify copyright headers run: | @@ -57,7 +73,7 @@ jobs: - name: Set up Quarto uses: quarto-dev/quarto-actions/setup@v2 with: - version: ${{ vars.QUARTO_VERSION }} + version: pre-release - name: Install Poetry run: | @@ -73,6 +89,11 @@ jobs: mkdir -p validmind rsync -av --exclude '_build' --exclude 'templates' _source/validmind-library/docs/ validmind/ + - name: Generate template schema docs + run: | + pip install json-schema-for-humans + BACKEND_ROOT=site/_source/backend python scripts/generate_template_schema_docs.py + - name: Populate installation run: cp -r site/_source/installation/site/installation site/installation @@ -143,6 +164,15 @@ jobs: body: comment }); + - name: Install pandoc + run: | + sudo apt-get update + sudo apt-get install -y pandoc + + - name: Validate LLM markdown render + run: bash llm/render.sh && bash llm/clean.sh + working-directory: site + # Release headroom and shrink before final lightweight steps & post-job - name: Release reserve & shrink if: always() @@ -152,6 +182,7 @@ jobs: remove_paths: | site/_source/installation site/_source/release-notes + site/_source/backend site/render_errors.log site/_freeze dev.env diff --git a/.gitignore b/.gitignore index de8667d222..9684da543c 100644 --- a/.gitignore +++ b/.gitignore @@ -36,6 +36,11 @@ site/validmind-docs.yaml # Python API docs are now generated on the fly site/validmind +# Generated template schema documentation +site/guide/templates/_template-schema-generated.qmd + # Cursor rules .cursor/rules/ -.cursor/skills/ \ No newline at end of file +.cursor/skills/ + +_llm-output/ diff --git a/README.md b/README.md index f87db708af..1734a7b31e 100644 --- a/README.md +++ b/README.md @@ -25,11 +25,17 @@ You need: - To use Cursor to author documentation, the [`create-user-documentation`](https://github.com/validmind/skills/tree/main/create-user-documentation) skill - For Windows operating systems, install the `make` command via [Cygwin](https://cygwin.com/install.html) -### Additional dependencies +### Fetching required repositories -Some interactive tables, such as our breaking changes and dependency history rely you have R and some R packages installed in order for you to be able to preview or render certain pages of the docs site locally. +Before previewing or rendering the docs site, run: + +```bash +cd site +make get-source +``` + +This make action clones required repositories and generates documentation from source. Without this step, Quarto will return warnings or errors related to our installation guides, release notes, library docs, and template schema reference. -**Refer to the [Breaking changes and deprecations](site/releases/breaking-changes/README.md) guide** for more information on how to install R and set up these tables. ## How to contribute @@ -138,6 +144,36 @@ These directories may have sub-directories depending on their size and grouped s - `notebooks` — This is where notebooks retrieved from the [`validmind-library` repo](https://github.com/validmind/validmind-library) live. - `tests` — This is where test descriptions generated from the Python source in the [`validmind-library` repo](https://github.com/validmind/validmind-library) live. +### Generated documentation + +Some documentation content is auto-generated from backend source files. These scripts ensure the docs stay in sync with the codebase. + +#### Template schema documentation + +The template schema reference in `site/guide/templates/customize-document-templates.qmd` is auto-generated from the backend JSON Schema. CI workflows generate this automatically, but you can also regenerate locally: + +```bash +cd site +make get-source # Clones all repos including backend (sparse checkout) +make template-schema-docs # Generates the schema docs +``` + +The backend repo is sparse-cloned as part of `make clone` (only the schema directory is fetched). To use a specific backend branch: + +```bash +make clone BACKEND_BRANCH=feature-branch +make template-schema-docs +``` + +**Requirements:** +- Python 3.9+ +- SSH access to the backend repository + +The script reads from: +- `backend/src/backend/templates/documentation/model_documentation/mdd_template_schema_v5.json` — template schema definition + +Output: `site/guide/templates/_template-schema-generated.qmd` + #### Stylesheet organization (IN PROGRESS) The site uses a modular stylesheet architecture to maintain organized and maintainable styles: diff --git a/scripts/generate_template_schema_docs.py b/scripts/generate_template_schema_docs.py new file mode 100644 index 0000000000..3becbac57d --- /dev/null +++ b/scripts/generate_template_schema_docs.py @@ -0,0 +1,158 @@ +#!/usr/bin/env python3 +# Copyright © 2023-2026 ValidMind Inc. All rights reserved. +# Refer to the LICENSE file in the root of this repository for details. +# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial +""" +Generate template schema documentation from backend JSON Schema. + +Uses json-schema-for-humans to generate HTML documentation from the +template schema used to validate YAML documentation templates. + +Usage: + pip install json-schema-for-humans + python scripts/generate_template_schema_docs.py + + # Or with custom backend path: + BACKEND_ROOT=/path/to/backend python scripts/generate_template_schema_docs.py + +Requirements: + - Backend repo must be cloned (default: ../backend/, or set BACKEND_ROOT) + - json-schema-for-humans package installed +""" +import os +import re +import subprocess +import sys +from pathlib import Path + +SCRIPT_DIR = Path(__file__).parent +REPO_ROOT = SCRIPT_DIR.parent +BACKEND_ROOT = Path(os.environ.get("BACKEND_ROOT", REPO_ROOT.parent / "backend")) + +SCHEMA_FILE = BACKEND_ROOT / "src/backend/templates/documentation/model_documentation/mdd_template_schema_v5_ui.json" +OUTPUT_FILE = REPO_ROOT / "site/guide/templates/_template-schema-generated.qmd" + +# CSS stylesheets to link in the generated HTML +STYLESHEETS = [ + '/styles.css', + '/assets/styles/template-schema.css', +] + + +def main(): + # Verify schema file exists + if not SCHEMA_FILE.exists(): + print(f"Error: Schema file not found: {SCHEMA_FILE}") + print("Make sure the backend repo is cloned at ../backend/ or set BACKEND_ROOT") + sys.exit(1) + + # Check if json-schema-for-humans is installed + try: + subprocess.run( + ["generate-schema-doc", "--help"], + capture_output=True, + check=True + ) + except FileNotFoundError: + print("Error: json-schema-for-humans not installed") + print("Install with: pip install json-schema-for-humans") + sys.exit(1) + + # Generate HTML documentation + print(f"Generating schema documentation from {SCHEMA_FILE}") + + # Create temporary output + temp_output = OUTPUT_FILE.with_suffix(".tmp.html") + + subprocess.run([ + "generate-schema-doc", + "--config", "expand_buttons=true", + str(SCHEMA_FILE), + str(temp_output) + ], check=True) + + # Read generated HTML and inject stylesheet links + with open(temp_output, "r") as f: + html_content = f.read() + + # Strip external CSS/JS that affects global page styling + # These are included by json-schema-for-humans but break the Quarto page layout + html_content = re.sub( + r']*href="https://stackpath\.bootstrapcdn\.com/bootstrap/[^"]*"[^>]*>', + '', + html_content + ) + html_content = re.sub( + r']*href="https://fonts\.googleapis\.com/css\?family=Overpass[^"]*"[^>]*>', + '', + html_content + ) + html_content = re.sub( + r']*src="https://use\.fontawesome\.com/[^"]*"[^>]*>', + '', + html_content + ) + # Strip local schema_doc.css which has unscoped global body styles + html_content = re.sub( + r']*href="schema_doc\.css"[^>]*>', + '', + html_content + ) + + # Build stylesheet link tags + stylesheet_links = '\n'.join( + f'' + for href in STYLESHEETS + ) + + # Insert stylesheet links after the opening tag + if "" in html_content: + html_content = html_content.replace( + "", + f"\n{stylesheet_links}" + ) + + # Wrap body content in a div for CSS scoping + # (Can't use body class because Quarto merges embedded body with its own) + html_content = re.sub( + r'(]*>)', + r'\1
', + html_content + ) + html_content = html_content.replace( + '', + '
' + ) + + # Copyright header to place before the raw HTML block + copyright_header = """""" + + # Wrap HTML in Quarto raw HTML block for .qmd file, with comment before + qmd_content = f"""{copyright_header} + +```{{=html}} +{html_content} +``` +""" + + # Write final output + OUTPUT_FILE.parent.mkdir(parents=True, exist_ok=True) + with open(OUTPUT_FILE, "w") as f: + f.write(qmd_content) + + # Clean up temp file + temp_output.unlink() + + print(f"Generated {OUTPUT_FILE}") + + +if __name__ == "__main__": + main() diff --git a/site/Makefile b/site/Makefile index 150bda6156..6773da52d5 100644 --- a/site/Makefile +++ b/site/Makefile @@ -2,6 +2,7 @@ LIBRARY_BRANCH ?= $(or $(LIBRARY),main) INSTALLATION_BRANCH ?= $(or $(INSTALLATION),main) RELEASE_NOTES_BRANCH ?= $(or $(RELEASE_NOTES),main) +BACKEND_BRANCH ?= $(or $(BACKEND),main) LIBRARY_BRANCH ?= $(or $(BRANCH),main) GIT_BRANCH := $(or $(GITHUB_HEAD_REF),$(GITHUB_REF_NAME),$(shell git rev-parse --abbrev-ref HEAD)) SRC_ROOT := _source @@ -13,7 +14,7 @@ SRC_ROOT := _source SRC_DIR := $(SRC_ROOT)/validmind-library # Define .PHONY target for help section -.PHONY: help add-copyright clean clone copy-installation copy-release-notes delete-demo-branch deploy-demo-branch deploy-prod deploy-staging docker-build docker-serve docker-site docker-site-lite docs-site execute generate-sitemap get-api-json get-source kind-serve kind-stop kind-restart kind-logs notebooks python-docs release-notes test-descriptions verify-copyright yearly-releases +.PHONY: help add-copyright clean clone copy-installation copy-release-notes delete-demo-branch deploy-demo-branch deploy-prod deploy-staging docker-build docker-serve docker-site docker-site-lite docs-site execute generate-sitemap get-api-json get-source kind-serve kind-stop kind-restart kind-logs notebooks python-docs release-notes render-llm template-schema-docs test-descriptions verify-copyright yearly-releases # Help section help: @@ -26,6 +27,7 @@ help: @echo " LIBRARY_BRANCH=" @echo " INSTALLATION_BRANCH=" @echo " RELEASE_NOTES_BRANCH=" + @echo " BACKEND_BRANCH=" @echo " copy-installation Copy customer-managed installation docs into installation/" @echo " copy-release-notes Copy release notes into releases/ and installation/releases/" @echo " delete-demo-branch Delete docs demo site in s3://docs-ci-cd-demo/site/pr_previews/$(GIT_BRANCH)/" @@ -49,6 +51,8 @@ help: @echo " python-docs Copy the Python library docs into validmind/" @echo " release-notes TAG= Clone and copy release notes for the specified tag or repo/tag" @echo " Examples: TAG=cmvm/25.07 or TAG=validmind-library/v2.8.22" + @echo " template-schema-docs Generate template schema documentation from backend" + @echo " render-llm Render site to GFM markdown for LLM ingestion (mirrors CI)" @echo " test-descriptions Copy the ValidMind tests docs into tests/" @echo " verify-copyright Verify that all .qmd and .yml/.yaml files have copyright headers" @echo " yearly-releases Collate releases by year into a listing landing and update releases sidebar" @@ -90,6 +94,12 @@ clone: @echo "\nCloning release-notes repo from $(RELEASE_NOTES_BRANCH) ..."; \ git clone --depth 1 -b $(RELEASE_NOTES_BRANCH) git@github.com:validmind/release-notes.git $(SRC_ROOT)/release-notes + @echo "\nCloning backend repo from $(BACKEND_BRANCH) ..." + @git clone --depth 1 --filter=blob:none --sparse -b $(BACKEND_BRANCH) \ + git@github.com:validmind/backend.git $(SRC_ROOT)/backend + @cd $(SRC_ROOT)/backend && \ + git sparse-checkout set src/backend/templates/documentation/model_documentation + copy-installation: @echo "\nCopying installation source files ..." @mkdir -p installation @@ -274,7 +284,7 @@ get-api-json: @curl -s -o reference/rapidoc-min.js "https://unpkg.com/rapidoc/dist/rapidoc-min.js" || echo "Failed to fetch RapiDoc library" # Get all source files -get-source: clean clone copy-installation copy-release-notes notebooks python-docs test-descriptions +get-source: clean clone copy-installation copy-release-notes notebooks python-docs test-descriptions template-schema-docs # Requires that you've run `make docker-build` kind-serve: @@ -414,6 +424,12 @@ release-notes: echo "Successfully created PR for release notes $(TAG)!"; \ fi +template-schema-docs: + @echo "\nGenerating template schema documentation ..." + @if [ ! -d "$(SRC_ROOT)/backend" ]; then echo "Error: Backend not cloned. Run 'make clone' first."; exit 1; fi + @pip install -q json-schema-for-humans + @BACKEND_ROOT=$(SRC_ROOT)/backend python ../scripts/generate_template_schema_docs.py + test-descriptions: @echo "\nUpdating test descriptions source ..." @cd _source/validmind-library && make install && poetry run python scripts/extract_descriptions.py validmind/tests @@ -434,3 +450,9 @@ yearly-releases: cd ../ git status | grep -v 'release-scripts/' quarto preview + +# Render site to GFM markdown for LLM ingestion +render-llm: + @echo "\nRendering site to GFM markdown for LLM ingestion ..." + @bash llm/render.sh && bash llm/clean.sh + @echo "\nOutput in llm/_llm-output/" diff --git a/site/_quarto-development.yml b/site/_quarto-development.yml index 1787fc9274..60eed47770 100644 --- a/site/_quarto-development.yml +++ b/site/_quarto-development.yml @@ -11,6 +11,6 @@ website: format: html: - include-in-header: + include-in-header: - zendesk-chat.html - - environments/noindex.html \ No newline at end of file + - environments/noindex.html diff --git a/site/_quarto-production.yml b/site/_quarto-production.yml index 754d491141..a0cfccd2ce 100644 --- a/site/_quarto-production.yml +++ b/site/_quarto-production.yml @@ -10,6 +10,6 @@ website: format: html: - include-in-header: + include-in-header: - environments/datadog-production.html - - zendesk-chat.html \ No newline at end of file + - zendesk-chat.html diff --git a/site/_quarto.yml b/site/_quarto.yml index 72fc8bf124..3431441f74 100644 --- a/site/_quarto.yml +++ b/site/_quarto.yml @@ -133,7 +133,7 @@ website: - faq/faq-workflows.qmd - text: "Inventory & activity" file: faq/faq-inventory.qmd - - text: "Documentation & templates" + - text: "Documents & templates" file: faq/faq-documentation.qmd - text: "Validation & artifacts" file: faq/faq-validation.qmd diff --git a/site/about/deployment/deployment-options.qmd b/site/about/deployment/deployment-options.qmd index 9f09fb5321..9dab1db881 100644 --- a/site/about/deployment/deployment-options.qmd +++ b/site/about/deployment/deployment-options.qmd @@ -36,7 +36,7 @@ The {{< var validmind.platform >}} provides: - **Documentation & validation engine** — Automated testing and documentation, with validation processes, ensuring compliance with regulations and internal policies. -- **Template management** — Allows for easy creation, customization, and reuse of documentation templates. +- **Template management** — Allows for easy creation, customization, and reuse of document templates. - **{{< var vm.product >}} dashboard** — A user-friendly interface providing insights, status updates, and governance reporting for model risk. diff --git a/site/about/glossary/_validation-reports.qmd b/site/about/glossary/_validation-reports.qmd index d989ba5e19..da7eea5591 100644 --- a/site/about/glossary/_validation-reports.qmd +++ b/site/about/glossary/_validation-reports.qmd @@ -11,12 +11,12 @@ These instructions update the documentation concept on `preparing-validation-rep --> -{{< include validation_reports/_artifacts.qmd >}} - {{< include validation_reports/_report-intro.qmd >}} {{< include validation_reports/_actions.qmd >}} +{{< include validation_reports/_artifacts.qmd >}} + {{< include validation_reports/_evidence.qmd >}} {{< include validation_reports/_review.qmd >}} diff --git a/site/about/glossary/_validmind-features.qmd b/site/about/glossary/_validmind-features.qmd index 81b24abdd9..b53f80ca4c 100644 --- a/site/about/glossary/_validmind-features.qmd +++ b/site/about/glossary/_validmind-features.qmd @@ -8,7 +8,7 @@ client library, Python client library : Enables the interaction of your development environment with the {{< var validmind.platform >}} as part of the {{< var validmind.developer >}}. content block -: Content blocks provide you with sections that are part of a template, and are used in model documentation, validation reports, and ongoing monitoring plans.^[[Work with content blocks](/guide/model-documentation/work-with-content-blocks.qmd)] +: Content blocks provide you with sections that are part of a template, and are used in model documentation, validation reports, ongoing monitoring reports, and custom document types.^[[Work with content blocks](/guide/model-documentation/work-with-content-blocks.qmd)] documentation automation : A core benefit of {{< var vm.product >}} that allows for the automatic creation of model documentation using predefined templates and test suites. diff --git a/site/about/glossary/key_concepts/_inputs.qmd b/site/about/glossary/key_concepts/_inputs.qmd index e4c579d282..0c45358b5e 100644 --- a/site/about/glossary/key_concepts/_inputs.qmd +++ b/site/about/glossary/key_concepts/_inputs.qmd @@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial --> inputs : Objects to be evaluated and documented in the {{< var validmind.developer >}}. They can be any of the following: - - **model**: A single model that has been initialized in {{< var vm.product >}} with `vm.init_model()`. See the [Model Documentation](/validmind/validmind.qmd#init_model){target="_blank"} or the for more information. - - **dataset**: Single dataset that has been initialized in {{< var vm.product >}} with `vm.init_dataset()`. See the [Dataset Documentation](/validmind/validmind.qmd#init_dataset){target="_blank"} for more information. + - **model**: A single model that has been initialized in {{< var vm.product >}}. Refer to the [`vm.init_model()` function](/validmind/validmind.qmd#init_model){target="_blank"} for more information. + - **dataset**: Single dataset that has been initialized in {{< var vm.product >}}. Refer to the [`vm.init_dataset()` function](/validmind/validmind.qmd#init_dataset){target="_blank"} for more information. - **models**: A list of {{< var vm.product >}} models - usually this is used when you want to compare multiple models in your custom tests. - - **datasets**: A list of {{< var vm.product >}} datasets - usually this is used when you want to compare multiple datasets in your custom tests. (Learn more: [Run tests with multiple datasets(/notebooks/how_to/run_tests_that_require_multiple_datasets.ipynb)]) + - **datasets**: A list of {{< var vm.product >}} datasets - usually this is used when you want to compare multiple datasets in your custom tests. (Learn more: [Run tests with multiple datasets](/notebooks/how_to/tests/run_tests/configure_tests/run_tests_that_require_multiple_datasets.ipynb)) \ No newline at end of file diff --git a/site/about/glossary/key_concepts/_template.qmd b/site/about/glossary/key_concepts/_template.qmd index 658899002a..2648e551fb 100644 --- a/site/about/glossary/key_concepts/_template.qmd +++ b/site/about/glossary/key_concepts/_template.qmd @@ -2,10 +2,25 @@ Refer to the LICENSE file in the root of this repository for details. SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial --> -documentation template -: Functions as a test suite and lays out the structure of model documentation, segmented into various sections and sub-sections. Documentation templates define the structure of your model documentation, specifying the tests that should be run, and how the results should be displayed. +document template +: Lays out the structure of model documents, segmented into various sections and sub-sections, and function as test suites to help automate your development, validation, monitoring, and other risk management processes. Document templates are available for default {{< var vm.product >}} document types[^default-documents] as well as custom document types. -validation report template -: Serves as a standardized framework for conducting and documenting model validation activities. It outlines the required sections, recommended analyses, and expected validation tests, ensuring consistency and completeness across validation reports. The template helps guide validators through a systematic review process while promoting comparability and traceability of validation outcomes. +documentation template^[**Refer also to:** [Model documentation](/about/glossary/glossary.qmd#model-documentation)] +: A default {{< var vm.product >}} document type that serves as a standardized framework for developing and documenting models, including sections designated for model details, data descriptions, test results, and performance metrics. By outlining required documentation and recommended analyses, document templates ensure consistency and completeness across model documentation and help guide developers through a systematic development process while promoting comparability and traceability of development outcomes. + +validation report template^[**Refer also to:** [Validation reports](/about/glossary/glossary.qmd#validation-reports)] +: A default {{< var vm.product >}} document type that serves as a standardized framework for conducting and documenting model validation, including sections designated for attaching test results, evidence, or artifacts (findings). By outlining required documentation, recommended analyses, and expected validation tests, validation report templates ensure consistency and completeness across validation reports and help guide validators through a systematic review process while promoting comparability and traceability of validation outcomes. + +monitoring template, monitoring report template^[**Refer also to:** [Ongoing monitoring](/about/glossary/glossary.qmd#ongoing-monitoring)] +: A default {{< var vm.product >}} document type that serves as a standardized framework for ongoing model monitoring, including sections designated for test results, performance metrics, and drift analyses. By outlining required monitoring checks and expected routine tests, monitoring templates ensure consistency and completeness across monitoring reports and help guide model owners through a systematic monitoring process while promoting early detection of model performance degradation. + + + +[^default-documents]: + + Default {{< var vm.product >}} document type templates: + + - [documentation template](#documentation-template) + - [validation report template](#validation-report-template) + - [monitoring template](#monitoring-template) - {{< var vm.product >}} templates come with pre-defined sections, similar to test placeholders, including boilerplates and spaces designated for test results, evidence, or artifacts. \ No newline at end of file diff --git a/site/about/glossary/validation_reports/_artifacts.qmd b/site/about/glossary/validation_reports/_artifacts.qmd index d2e79be543..4bfdaa2030 100644 --- a/site/about/glossary/validation_reports/_artifacts.qmd +++ b/site/about/glossary/validation_reports/_artifacts.qmd @@ -2,5 +2,5 @@ Refer to the LICENSE file in the root of this repository for details. SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial --> -artifacts (findings) -: Observations or issues identified during model validation, including any deviations from expected performance or standards. \ No newline at end of file +artifacts (previously findings) +: Observations or issues identified during model validation, including any deviations from expected performance or standards. Artifacts are organized by type — default types include Validation Issue, Policy Exception, and Model Limitation. Custom artifact types such as Change Management Record can be created to track other categories relevant to your organization. \ No newline at end of file diff --git a/site/about/overview-model-documentation.qmd b/site/about/overview-model-documentation.qmd index a57598481b..59e462e4aa 100644 --- a/site/about/overview-model-documentation.qmd +++ b/site/about/overview-model-documentation.qmd @@ -146,7 +146,7 @@ How the {{< var validmind.developer >}} works: [^1]: [Model risk management](overview-model-risk-management.qmd) -[^2]: [Supported models](/developer/supported-models.qmd) +[^2]: [Supported models and frameworks](/developer/supported-models-and-frameworks.qmd) [^3]: [Customize document templates](/guide/templates/customize-document-templates.qmd) diff --git a/site/about/overview-model-risk-management.qmd b/site/about/overview-model-risk-management.qmd index 7cb371e8f6..5be5ae05f0 100644 --- a/site/about/overview-model-risk-management.qmd +++ b/site/about/overview-model-risk-management.qmd @@ -37,7 +37,7 @@ With the {{< var vm.platform >}}, you can: - **Track your model inventory**[^1] — Manage the model lifecycle, track the workflow status for models, plan for upcoming validation dates, and more. - **Work on validation initiatives**[^2] — Collaborate with developers and validators to review documentation, add artifacts, keep track of review statuses, and generate validation reports. - **Configure workflows**[^3] — Set up the {{< var validmind.platform >}} to follow your existing model risk management processes, manage statuses for different parts of the workflow, and get an end-to-end view of workflows and who is involved. -- **Use, create, or edit tests, test suites, and templates**[^4] — Create and/or configure required validation tests, test suites, and documentation templates for specific model use cases, tailoring it to your own specific needs. +- **Use, create, or edit tests, test suites, and templates**[^4] — Create and/or configure required validation tests, test suites, and document templates for specific model use cases, tailoring it to your own specific needs. - **Integrate with your stack**[^5] — Import and export model documentation and validation reports. ::: diff --git a/site/about/use-cases/ai-governance.qmd b/site/about/use-cases/ai-governance.qmd index 439b61ead0..d4fb96419e 100644 --- a/site/about/use-cases/ai-governance.qmd +++ b/site/about/use-cases/ai-governance.qmd @@ -106,7 +106,7 @@ Workflows : Manages review cycles, approval paths, and assigned tasks. Documents & Templates -: Provides standard forms, reports, and documentation templates. +: Provides standard forms, reports, and document templates. Regulations & Policies : Includes internal policies, external regulations, and compliance frameworks. diff --git a/site/about/use-cases/e-23.qmd b/site/about/use-cases/e-23.qmd index a288345a37..1351be8a51 100644 --- a/site/about/use-cases/e-23.qmd +++ b/site/about/use-cases/e-23.qmd @@ -36,7 +36,7 @@ Enhanced requirements for model interpretability ensure stakeholders can underst #### Steps -1. Configure documentation templates[^3] to capture explainability information. +1. Configure document templates[^3] to capture explainability information. 2. Run interpretability tests to generate explanations.[^4] @@ -119,7 +119,7 @@ Management of continuously updating models. Use this checklist to track your E-23 implementation progress: - [x] Model inventory established with AI/ML classification -- [x] Documentation templates configured for AI-specific requirements +- [x] Document templates configured for AI-specific requirements - [x] Validation workflows include AI/ML considerations - [x] Explainability requirements documented by risk tier - [x] Bias assessment processes implemented diff --git a/site/about/use-cases/eu-ai-act.qmd b/site/about/use-cases/eu-ai-act.qmd index 2f6504ffa5..fa4da61a3c 100644 --- a/site/about/use-cases/eu-ai-act.qmd +++ b/site/about/use-cases/eu-ai-act.qmd @@ -86,7 +86,7 @@ Generate and customize required model documentation to meet Article 11 requireme ### Steps -1. Work with documentation templates:[^6] +1. Work with document templates:[^6] - Select or create templates aligned to EU AI Act requirements. - Configure sections for system description, design, and compliance. diff --git a/site/about/use-cases/sr-11-7.qmd b/site/about/use-cases/sr-11-7.qmd index a5462a32c0..6ca28f27af 100644 --- a/site/about/use-cases/sr-11-7.qmd +++ b/site/about/use-cases/sr-11-7.qmd @@ -56,7 +56,7 @@ Document models according to SR 11-7 development standards. ### Steps -1. Configure documentation templates:[^4] +1. Configure document templates:[^4] - Select or create templates aligned to SR 11-7. - Include sections for purpose, design, theory, and logic. @@ -187,7 +187,7 @@ Establish governance framework aligned to SR 11-7. - [x] Model definition criteria established - [x] Model inventory configured and populated -- [x] Documentation templates aligned to SR 11-7 +- [x] Document templates aligned to SR 11-7 - [x] Testing framework implemented - [x] Validation workflows configured - [x] Ongoing monitoring established diff --git a/site/about/use-cases/ss1-23.qmd b/site/about/use-cases/ss1-23.qmd index dff082011e..dd2b1b4918 100644 --- a/site/about/use-cases/ss1-23.qmd +++ b/site/about/use-cases/ss1-23.qmd @@ -86,7 +86,7 @@ Ensure models are developed and used per MRM framework guidelines. ### Steps -1. Configure documentation templates aligned to SS1/23:[^6] +1. Configure document templates aligned to SS1/23:[^6] - Select or create templates covering SS1/23 requirements. - Include sections for intended purpose and use boundaries. @@ -196,7 +196,7 @@ Document your proportionality assessment and how it influences your MRM framewor - [x] Risk classification and tiering configured - [x] SMF responsibilities documented - [x] Roles and permissions configured -- [x] Documentation templates aligned to SS1/23 +- [x] Document templates aligned to SS1/23 - [x] Validation workflows established - [x] Findings tracking implemented - [x] Ongoing monitoring configured diff --git a/site/assets/styles/template-schema.css b/site/assets/styles/template-schema.css new file mode 100644 index 0000000000..6e5b04e5a5 --- /dev/null +++ b/site/assets/styles/template-schema.css @@ -0,0 +1,148 @@ +/* Template schema documentation styles + Used by scripts/generate_template_schema_docs.py + Scoped to .template-schema-docs for CSS specificity */ + +/* Override font to match site (Inter instead of Overpass) */ +.template-schema-docs { + font-family: Inter, sans-serif; + margin-top: 0.5rem; + margin-bottom: 2rem; +} + +/* Button container alignment - float to avoid taking up vertical space */ +.template-schema-docs .text-right { + float: right; +} + +/* Remove body padding from embedded HTML (schema_doc.css sets padding: 40px) */ +.template-schema-docs body, +body .template-schema-docs { + padding: 0; +} + +/* Card styling (Bootstrap CSS was removed to prevent global style pollution) */ +.template-schema-docs .card { + position: relative; + display: flex; + flex-direction: column; + min-width: 0; + word-wrap: break-word; + background-color: #fff; + background-clip: border-box; + border: 1px solid rgba(0, 0, 0, 0.125); + border-radius: 0.25rem; +} + +.template-schema-docs .card-header { + padding: 0.75rem 1.25rem; + margin-bottom: 0; + background-color: rgba(0, 0, 0, 0.03); + border-bottom: 1px solid rgba(0, 0, 0, 0.125); +} + +.template-schema-docs .card-header:first-child { + border-radius: calc(0.25rem - 1px) calc(0.25rem - 1px) 0 0; +} + +.template-schema-docs .breadcrumb { + background-color: white; + padding: 0px; +} + +.template-schema-docs .accordion > .card .card-header { + margin-bottom: -1px; + background-color: #f7f7f7; +} + +.template-schema-docs .card-body { + -ms-flex: 1 1 auto; + flex: 1 1 auto; + padding: 1.25rem +} + +.template-schema-docs .pl-5, +.template-schema-docs .px-5 { + padding-left: 3rem !important +} + +.template-schema-docs .badge { + display: inline-block; + padding: .4em .4em .3em .4em; + font-size: 100%; + font-weight: 700; + line-height: 1; + text-align: center; + white-space: nowrap; + vertical-align: baseline; + border-radius: .25rem; + margin-bottom: 0.5rem; + margin-top: 0.5rem; + transition: color .15s ease-in-out, background-color .15s ease-in-out, border-color .15s ease-in-out, box-shadow .15s ease-in-out +} + +.template-schema-docs .badge.value-type { + font-size: 120%; + margin-right: 5px; + margin-bottom: 10px; +} + +.template-schema-docs .badge-warning { + color: #ffffff; + background-color: #de257e; +} + +.template-schema-docs .badge-info { + color: #ffffff; + background-color: #de257e; +} + +.template-schema-docs .badge.badge-dark { + color: #ffffff; + background-color: #083E44; +} + +.template-schema-docs .btn.btn-link { + font-size: 18px; + user-select: text; + text-decoration: none; + color: #de257e; +} + +.template-schema-docs .btn.btn-link:hover { + color: #083E44; +} + +.template-schema-docs .btn.btn-primary { + display: inline-block; + padding: 0.375rem 0.75rem; + font-size: 1rem; + font-weight: 400; + line-height: 1.5; + background-color: #de257e; + border: 1px solid #de257e; + border-radius: 0.25rem; + color: #ffffff; + cursor: pointer; + margin: 0.25rem; +} + +.template-schema-docs .btn.btn-primary:hover { + background-color: #083E44; + border-color: #083E44; +} + +.template-schema-docs a { + color: #de257e; + text-decoration: none; +} + +.template-schema-docs a:hover { + color: #DE257E; + text-decoration: underline 2px solid #083E44; +} + +/* Hide the json-schema-for-humans generated footer */ +.template-schema-docs footer, +.template-schema-docs .generated-by-footer { + display: none; +} diff --git a/site/developer/_sidebar.yaml b/site/developer/_sidebar.yaml index 1215f1cf41..d96b29d0f3 100644 --- a/site/developer/_sidebar.yaml +++ b/site/developer/_sidebar.yaml @@ -10,13 +10,18 @@ website: # USING THE VARIABLE IN THE LINK TEXT MESSES UP THE MOBILE VIEW - text: "ValidMind Library" file: developer/validmind-library.qmd - - developer/supported-models.qmd + - developer/supported-models-and-frameworks.qmd - text: "---" - text: "Quickstart" - notebooks/quickstart/quickstart_model_documentation.ipynb - notebooks/quickstart/quickstart_model_validation.ipynb - - text: "Install and initialize ValidMind Library" - file: developer/model-documentation/install-and-initialize-validmind-library.qmd + # USING THE VARIABLE IN THE LINK TEXT MESSES UP THE MOBILE VIEW & BREADCRUMB + - section: "Install and initialize ValidMind" + contents: + - text: "Install and initialize the library" + file: developer/model-documentation/install-and-initialize-validmind-library.qmd + - text: "Install and initialize the library for R" + file: developer/model-documentation/install-and-initialize-validmind-for-r.qmd - developer/model-documentation/store-credentials-in-env-file.qmd - text: "---" - text: "End-to-End Tutorials" diff --git a/site/developer/how-to/feature-overview.qmd b/site/developer/how-to/feature-overview.qmd index 7f9536acfd..cf222ffbdb 100644 --- a/site/developer/how-to/feature-overview.qmd +++ b/site/developer/how-to/feature-overview.qmd @@ -9,6 +9,7 @@ listing: - id: tests type: grid grid-columns: 2 + page-size: 4 image-placeholder: "jupyter-logo-rectangle.svg" max-description-length: 350 image-height: "100%" @@ -17,6 +18,7 @@ listing: - id: data-and-datasets type: grid grid-columns: 2 + page-size: 4 image-placeholder: "jupyter-logo-rectangle.svg" max-description-length: 350 image-height: "100%" @@ -25,6 +27,7 @@ listing: - id: metrics type: grid grid-columns: 2 + page-size: 4 image-placeholder: "jupyter-logo-rectangle.svg" max-description-length: 350 image-height: "100%" @@ -33,6 +36,7 @@ listing: - id: scoring type: grid grid-columns: 2 + page-size: 4 image-placeholder: "jupyter-logo-rectangle.svg" max-description-length: 350 image-height: "100%" diff --git a/site/developer/how-to/testing-overview.qmd b/site/developer/how-to/testing-overview.qmd index 658aa37ac0..7d1511f138 100644 --- a/site/developer/how-to/testing-overview.qmd +++ b/site/developer/how-to/testing-overview.qmd @@ -73,6 +73,70 @@ listing: {{< var vm.product >}} provides many built-in tests and test suites, which help you produce documentation during stages of the model lifecycle, where you need to ensure that your work satisfies regulatory and risk management requirements. +::: {.callout-important collapse="false" appearance="minimal"} +## Quick reference + +::: {.panel-tabset} + +## Running a test + +![](testing-overview_files/figure-commonmark/dot-figure-1.png) + +**Test ID** +: Each test has a unique identifier like `validmind.data_validation.ClassImbalance`. Use `vm.tests.list_tests()` to browse available tests. + +**Result ID** +: When running the same test multiple times with different parameters, add a `:suffix` to distinguish results, such as `validmind.data_validation.ClassImbalance:custom_threshold`. + +**Inputs** +: The datasets and models your test analyzes. Pass these via `inputs={}`: + + - `dataset` — A VM dataset object + - `model` — A VM model object + - `datasets` / `models` — For tests comparing multiple datasets or models + +**Parameters** +: Configuration that customizes how the test runs. Pass these via `params={}`: + + - Thresholds, such as `min_percent_threshold`, `disparity_tolerance` + - Metrics to compute, such as `metrics=["fnr", "fpr", "tpr"]` + - Any setting that isn't a dataset or model + +Use `vm.tests.describe_test("test_id")` or check the [test descriptions](/developer/test-descriptions.qmd) page to see what parameters a test accepts. + +**Outputs** +: Tests return results like tables and plots. Tables can be lists of dictionaries or pandas DataFrames; plots can be matplotlib or plotly figures. + +**Logging results** +: Call `.log()` on a test result to send it to the {{< var validmind.platform >}}. + +## Running a test suite + +![](testing-overview_files/figure-commonmark/dot-figure-2.png) + +**Test suite ID** +: Test suites are collections of tests that run together. Each has an identifier like `classifier_full_suite`. Use `vm.tests.list_test_suites()` to browse available test suites. + +**Inputs** +: The datasets and models the test suite analyzes. Pass these via `inputs={}`: + + - `dataset` — A VM dataset object + - `model` — A VM model object + - `datasets` / `models` — For suites comparing multiple datasets or models + +**Config** +: A dictionary of parameters to pass to the tests in the suite. Use `vm.get_test_suite().get_default_config()` to see the default configuration for all tests. + +**Outputs** +: Test suites return results from all tests — tables and plots. Results are automatically logged to the platform unless `send=False`. + +**Logging results** +: By default, `run_test_suite()` logs all results automatically. Set `send=False` to run without logging. + +::: + +::: + ## Explore tests Start by exploring the {{< var validmind.developer >}}'s available tests and tests suites: @@ -166,4 +230,5 @@ We provide instructions with code examples that you can adapt: Learn more about using the other features of the {{< var validmind.developer >}} and browse our code samples by use case: :::{#testing-next} -::: \ No newline at end of file +::: + diff --git a/site/developer/how-to/testing-overview_files/figure-commonmark/dot-figure-1.png b/site/developer/how-to/testing-overview_files/figure-commonmark/dot-figure-1.png new file mode 100644 index 0000000000..63ffd193bc Binary files /dev/null and b/site/developer/how-to/testing-overview_files/figure-commonmark/dot-figure-1.png differ diff --git a/site/developer/how-to/testing-overview_files/figure-commonmark/dot-figure-2.png b/site/developer/how-to/testing-overview_files/figure-commonmark/dot-figure-2.png new file mode 100644 index 0000000000..1d05d2a2fa Binary files /dev/null and b/site/developer/how-to/testing-overview_files/figure-commonmark/dot-figure-2.png differ diff --git a/site/developer/model-documentation/_code-snippet.qmd b/site/developer/model-documentation/_code-snippet.qmd new file mode 100644 index 0000000000..b98da91b48 --- /dev/null +++ b/site/developer/model-documentation/_code-snippet.qmd @@ -0,0 +1,5 @@ + + +Initialize the {{< var validmind.developer >}} with the _code snippet_ unique to each model per document to connect your environment to the {{< var validmind.platform >}}, ensuring test results are uploaded to the correct model and automatically populated in the right document. \ No newline at end of file diff --git a/site/developer/model-documentation/install-and-initialize-validmind-for-r.qmd b/site/developer/model-documentation/install-and-initialize-validmind-for-r.qmd new file mode 100644 index 0000000000..e7ebfa5075 --- /dev/null +++ b/site/developer/model-documentation/install-and-initialize-validmind-for-r.qmd @@ -0,0 +1,227 @@ +--- +# Copyright © 2023-2026 ValidMind Inc. All rights reserved. +# Refer to the LICENSE file in the root of this repository for details. +# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial +title: "Install and initialize ValidMind for R" +date: last-modified +listing: + - id: whats-next + type: grid + max-description-length: 250 + sort: false + fields: [title, description] + grid-columns: 2 + contents: + - path: https://github.com/validmind/validmind-library/blob/main/notebooks/code_sharing/r/quickstart_model_documentation.Rmd + title: "Quickstart for model documentation" + description: "End-to-end model documentation workflow in R: load data, preprocess, train a GLM model, and run the full documentation test suite." + - path: https://github.com/validmind/validmind-library/blob/main/notebooks/code_sharing/r/quickstart_model_validation.Rmd + title: "Quickstart for model validation" + description: "End-to-end model validation workflow in R: load data, run data quality tests, train a champion GLM model, and run model evaluation tests." +--- + +Use the ValidMind R package to document and validate models built in R. The package interfaces with the {{< var validmind.developer >}} through `reticulate`, giving you access to the full Python API from R. + +::: {.attn} + +## Prerequisites + +- [x] {{< var link.login >}} +- [x] The model you want to connect to is registered in the model inventory.[^1] +- [x] The model document has a template applied.[^2] +- [x] You are the model owner, model validator, or another model stakeholder with write access to the model's documents.[^3] + +::: + +## Install R + +Download and install R from [CRAN](https://cran.r-project.org/): + +::: {.panel-tabset} +### macOS + +Install via Homebrew: + +```bash +brew install r +``` + +Or download the `.pkg` installer from [CRAN](https://cran.r-project.org/bin/macosx/). + +### Windows + +Download and run the installer from [CRAN](https://cran.r-project.org/bin/windows/base/). + +### Linux + +On Debian/Ubuntu: + +```bash +sudo apt update +sudo apt install r-base +``` + +On Fedora/RHEL: + +```bash +sudo dnf install R +``` + +::: + +## Install Python dependencies + +Install the {{< var validmind.developer >}} and `rpy2` so Python can interface with R models: + +```bash +pip install validmind rpy2 +``` + +::: {.callout} +On macOS, if `rpy2` fails to find R libraries, rebuild it from source against your installed R: + +```bash +R_HOME=$(Rscript -e 'cat(R.home())') pip install --no-binary :all: --force-reinstall rpy2 +``` + +::: + +## Install R packages + +Open R and install the required packages: + +```r +install.packages(c("reticulate", "dplyr", "caTools", "knitr", "glue", "plotly", "htmltools", "rmarkdown", "DT", "base64enc")) +``` + +## Install the ValidMind R package + +::: {.panel-tabset} +### From CRAN + +```r +install.packages("validmind") +``` + +### From GitHub + +```r +devtools::install_github("validmind/validmind-library", subdir="r/validmind") +``` + +### From source + +Clone the repository and run from the `r/validmind` directory: + +```r +devtools::install() +``` + +::: + +## Initialize ValidMind + +Connect to your ValidMind profile by providing your credentials. Point `python_version` to your Python binary — for example, the one in your project's `.venv`: + +```r +vm_r <- vm( + api_key = "YOUR_API_KEY", + api_secret = "YOUR_API_SECRET", + model = "YOUR_MODEL_ID", + python_version = "path/to/python", + api_host = "{{< var api.host >}}/tracking", + document = "documentation" +) +``` + +The `document` parameter specifies which document type to associate with the session: + +- `"documentation"` — For model development +- `"validation-report"` — For model validation + +## Key APIs + +Since the R package returns the full Python `validmind` module, you can call any Python API directly via `reticulate`: + +```r +# Preview the documentation template +vm_r$preview_template() + +# Initialize datasets +vm_dataset <- vm_r$init_dataset( + dataset = df, + input_id = "my_dataset", + target_column = "target" +) + +# Initialize R models +model_path <- save_model(model) +vm_model <- vm_r$init_r_model(model_path = model_path, input_id = "model") + +# Assign predictions +vm_dataset$assign_predictions(model = vm_model) + +# Run the full documentation test suite +vm_r$run_documentation_tests(config = test_config) + +# Run individual tests +vm_r$tests$run_test( + "validmind.data_validation.ClassImbalance", + inputs = list(dataset = vm_dataset) +)$log() + +# List available tests +vm_r$tests$list_tests(tags = list("data_quality"), task = "classification") +vm_r$tests$list_tasks_and_tags() +``` + +## Troubleshooting + +::: {.panel-tabset} + +### libc++ not found on macOS + +When calling `vm()` you may see an error like: + +``` +OSError: dlopen(...libllvmlite.dylib...): Library not loaded: @rpath/libc++.1.dylib +``` + +This typically means the `libc++` library is installed but R cannot find it. Create a symlink: + +```bash +# Find the path to libc++.1.dylib +sudo find / -name "libc++.1.dylib" 2>/dev/null + +# Create a symlink (adjust the source path based on the find result) +sudo ln -s /opt/homebrew/Cellar/llvm/.../libc++.1.dylib /Library/Frameworks/R.framework/Resources/lib/libc++.1.dylib +``` + +### Numba circular import error + +If you see: + +``` +ImportError: cannot import name 'NumbaTypeError' from partially initialized module 'numba.core.errors' +``` + +Reinstall Numba and restart the R session: + +```bash +pip install -U numba +``` + +::: + +## What's next + +:::{#whats-next} +::: + + + +[^1]: [Register models in the inventory](/guide/model-inventory/register-models-in-inventory.qmd) + +[^2]: [Working with templates](/guide/templates/working-with-document-templates.qmd) + +[^3]: [Manage model stakeholders](/guide/model-inventory/edit-model-inventory-fields.qmd#manage-model-stakeholders) diff --git a/site/developer/model-documentation/install-and-initialize-validmind-library.qmd b/site/developer/model-documentation/install-and-initialize-validmind-library.qmd index bd3ce6b84a..8238cf968a 100644 --- a/site/developer/model-documentation/install-and-initialize-validmind-library.qmd +++ b/site/developer/model-documentation/install-and-initialize-validmind-library.qmd @@ -10,46 +10,69 @@ aliases: - /guide/install-and-initialize-validmind-library.html - /guide/install-and-initialize-developer-framework.html - install-and-initialize-client-library.html +listing: + - id: whats-next + type: grid + max-description-length: 250 + sort: false + fields: [title, description] + grid-columns: 2 + contents: + - store-credentials-in-env-file.qmd + - ../how-to/testing-overview.qmd --- -{{< var vm.product >}} generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the {{< var validmind.developer >}} with this code snippet, ensuring that your documentation and tests are uploaded to the correct model. +{{< include _code-snippet.qmd >}} ::: {.attn} ## Prerequisites - [x] {{< var link.login >}} -- [x] You are the model owner.[^1] -- [x] You are a [{{< fa code >}} Developer]{.bubble} or assigned another role with sufficient permissions to perform the tasks in this guide.[^2] +- [x] The model you want to connect to is registered in the model inventory.[^1] +- [x] The model document has a template applied.[^2] +- [x] You are the model owner, model validator, or another model stakeholder with write access to the model's documents.[^3] -In order to integrate the {{< var vm.developer >}} and to be able to upload to the {{< var vm.platform >}}, you must provide the following information through a code snippet that you copy from the {{< var validmind.platform >}}: +::: {.callout} +The {{< var validmind.developer >}} requires access to the data sources where relevant datasets and model files are stored in order to help you run tests. + +::: + + +::: + +## Installing the {{< var validmind.developer >}} + +In order to upload test results from the {{< var validmind.developer >}} to the {{< var validmind.platform >}}, provide the following information through a code snippet copied from the {{< var vm.platform >}}: | Argument | Description | |---------------|------------------------------------| | `api_host` | The location of the {{< var vm.product >}} API | | `api_key` | The account API key | | `api_secret` | The account secret key | +| `document` | The model document identifier key | | `model` | The model identifier | -If you do not have a suitable model with a code snippet to use, you can register a model first.[^3] +### Get your code snippet -The {{< var vm.developer >}} also requires access to the data sources where data sets used for training, testing, and trained model files are stored. This access is needed to run model documentation and validation tests, and to upload to the {{< var vm.platform >}} to populate the model documentation and validation reports. +Retrieve your code snippet for your model's selected document from the {{< var validmind.platform >}}: -::: +1. In the left sidebar, click **{{< fa cubes >}} Inventory**. -## Install {{< var vm.product >}} +2. Select a model by clicking on it or find your model by applying a filter or searching for it.[^4] -### Locate the {{< var vm.developer >}} integration instructions +3. In the left sidebar that appears for your model, click **{{< fa rocket >}} Getting Started**. -For existing models, this information can be found in the {{< var validmind.platform >}}: +4. Select the **[document]{.smallcaps}** you want to automatically upload test results to.[^5] -1. In the left sidebar, click **{{< fa cubes >}} Inventory**. +5. Click **{{< fa regular copy >}} Copy snippet to clipboard**. -2. Select a model by clicking on it or find your model by applying a filter or searching for it.[^4] +::: {.callout} +## Selecting a **[document]{.smallcaps}** to connect to requires {{< var validmind.developer >}} version >=2.12.0.[^6] -3. In the left sidebar that appears for your model, click **{{< fa rocket >}} Getting Started**. +A template must already be applied to your selected document to populate test results in the {{< var validmind.platform >}}. Attempting to initialize the {{< var vm.developer >}} with a document that does not have a template applied will result in an error. -4. Locate the code snippet and click **{{< fa regular copy >}} Copy snippet to clipboard**. +::: ### Install the {{< var vm.developer >}} @@ -70,21 +93,27 @@ vm.init( api_host = "{{< var api.host >}}/tracking", api_key = "API_KEY", api_secret = "API_SECRET", + document="document-key", # requires library >=2.12.0 model = "MODEL_IDENTIFIER" ) ``` :::{.callout} -To also enable monitoring, add `monitoring=True` to the `vm.init` method in your code snippet. [^5] +To also enable monitoring, add `monitoring=True` to the `vm.init` method in your code snippet. [^7] ::: -After you have pasted the code snippet into your development source code and run your code, the {{< var validmind.developer >}} will connect and register with the {{< var validmind.platform >}}. You can now use the {{< var vm.developer >}} to document and test your models, and to upload model documentation and test results to the {{< var vm.platform >}}. +::: {.feature} +#### **Automate with {{< var vm.product >}}** + +After you run the code snippet in your environment, the {{< var validmind.developer >}} will connect to your model and selected document in the {{< var validmind.platform >}}. Automate your workflow by using the {{< var vm.developer >}} to run tests, then seamlessly upload your test results to the {{< var vm.platform >}}. + +::: -## Upgrade {{< var vm.product >}} +## Upgrade the {{< var validmind.developer >}} -After installing {{< var vm.product >}},[^6] you'll want to periodically make sure you are on the latest version to access any new features and other enhancements: +After installing the {{< var validmind.developer >}},[^8] you'll want to periodically make sure you are on the latest version to access any new features and other enhancements: -1. In your Jupyter Notebook or developer environment, retrieve the information for the currently installed version of {{< var vm.product >}}: +1. In your Jupyter Notebook or developer environment, retrieve the information for the currently installed version of the {{< var vm.developer >}}: ```python %pip show validmind @@ -94,29 +123,39 @@ After installing {{< var vm.product >}},[^6] you'll want to periodically make su **Example output:** ```bash Name: validmind -Version: 2.5.15 +Version: 2.11.0 ... ``` ::: -2. If the version returned is lower than the version indicated in our production open-source code,[^7] run the following command: +2. If the version returned is lower than the version indicated in our production open-source code,[^9] run the following command: ```python %pip install --upgrade validmind ``` +## What's next + +:::{#whats-next} +::: + + -[^1]: [Manage model stakeholders](/guide/model-inventory/edit-model-inventory-fields.qmd#manage-model-stakeholders) +[^1]: [Register models in the inventory](/guide/model-inventory/register-models-in-inventory.qmd) -[^2]: [Manage permissions](/guide/configuration/manage-permissions.qmd) +[^2]: [Working with templates](/guide/templates/working-with-document-templates.qmd) -[^3]: [Register models in the inventory](/guide/model-inventory/register-models-in-inventory.qmd) +[^3]: [Manage model stakeholders](/guide/model-inventory/edit-model-inventory-fields.qmd#manage-model-stakeholders) [^4]: [Working with the model inventory](/guide/model-inventory/working-with-model-inventory.qmd#search-filter-and-sort-models) -[^5]: [Ongoing monitoring](/guide/monitoring/ongoing-monitoring.qmd) +[^5]: [Working with model documents](/guide/templates/working-with-model-documents.qmd) + +[^6]: [Upgrade the {{< var validmind.developer >}}](#upgrade-the) + +[^7]: [Ongoing monitoring](/guide/monitoring/ongoing-monitoring.qmd) -[^6]: [Install {{< var vm.product >}}](#install-validmind) +[^8]: [Install {{< var vm.product >}}](#install-validmind) -[^7]: **ValidMind GitHub:** [validmind-library/validmind/__version__.py](https://github.com/validmind/validmind-library/blob/prod/validmind/__version__.py) \ No newline at end of file +[^9]: **ValidMind GitHub:** [validmind-library/validmind/__version__.py](https://github.com/validmind/validmind-library/blob/prod/validmind/__version__.py) \ No newline at end of file diff --git a/site/developer/model-documentation/store-credentials-in-env-file.qmd b/site/developer/model-documentation/store-credentials-in-env-file.qmd index a9a7c243fc..48dd9cc500 100644 --- a/site/developer/model-documentation/store-credentials-in-env-file.qmd +++ b/site/developer/model-documentation/store-credentials-in-env-file.qmd @@ -6,16 +6,31 @@ title: "Store model credentials in `.env` files" date: last-modified aliases: - /guide/store-credentials-in-env-file.html +listing: + - id: whats-next + type: grid + max-description-length: 250 + sort: false + fields: [title, description] + grid-columns: 2 + contents: + - ../how-to/testing-overview.qmd + - ../how-to/feature-overview.qmd --- -Learn how to store model identifier credentials in a `.env` file instead of using inline credentials. This topic is relevant for model developers who want to follow best practices for security when running notebooks. +Learn how to store model identifier credentials in an `.env` file instead of using inline credentials, allowing you to follow best practices for security when running Jupyter Notebooks. -## Why is this recommended? +
-Storing credentials in a `.env` file is considered a best practice for security. Embedding credentials directly within the code makes them more susceptible to accidental exposure when sharing code or collaborating on models. +::: {.callout-button .pl4 .nt4} +::: {.callout collapse="true" appearance="minimal"} +### Why is it recommended to store credentials in an `.env` file? + +Storing credentials in a `.env` file is considered a best practice for security. Embedding credentials directly within the code makes them more susceptible to accidental exposure when sharing code or collaborating on models. + +Keeping model credentials in a separate file also allows for precise access control and ensures that sensitive credentials are not publicly accessible. -::: {.callout-important} -Keeing model credentials in a separate file also allows for precise access control and ensures that sensitive credentials are not publically accessible. +::: ::: ::: {.attn} @@ -24,25 +39,41 @@ Keeing model credentials in a separate file also allows for precise access contr - [x] {{< var link.login >}} - [x] The model is registered in the model inventory.[^1] -- [x] You are a [{{< fa code >}} Developer]{.bubble} or assigned another role with sufficient permissions to perform the tasks in this guide.[^2] +- [x] You are the model owner, model validator, or another model stakeholder with sufficient permissions to perform the tasks in this guide.[^2] + +::: {.callout} +Before you follow the steps on this page, we recommend that you first complete our introductory guide on how to [Install and initialize the {{< var validmind.developer >}}](/developer/model-documentation/install-and-initialize-validmind-library.qmd). ::: -## Steps +::: -1. Create a new file in the same folder as your notebook and name it `.env`. +## 1. Store credentials in an `.env` file - This is a hidden file, so you may need to change your settings to view it. +::: {.panel-tabset} + +### a. Get your code snippet + +In the {{< var validmind.platform >}}, retrieve the code snippet for your model: + +i. In the left sidebar, click **{{< fa cubes >}} Inventory**. + +ii. Select a model by clicking on it or find your model by applying a filter or searching for it.[^3] + +iii. In the left sidebar that appears for your model, click **{{< fa rocket >}} Getting Started**. + +iv. Select the **[document]{.smallcaps}** you want to automatically upload test results to.[^4] + +v. Click **{{< fa regular copy >}} Copy snippet to clipboard**. -2. Locate the code snippet for your model: +### b. Create an `.env` file - a. In the left sidebar, click **{{< fa cubes >}} Inventory**. - b. Select a model by clicking on it or find your model by applying a filter or searching for it.[^3] - c. In the left sidebar that appears for your model, click **{{< fa rocket >}} Getting Started**. - d. Locate the code snippet and click **{{< fa regular copy >}} Copy snippet to clipboard**. +i. Create a new file in the same folder as your notebook and name it `.env`. + + This is a hidden file, so you may need to change your settings to view it. + +ii. Use the code snippet from your clipboard to build the credentials in your `.env` file in the following format:[^5] -3. Paste the credentials into your `.env` file in the following format: - ```yaml VM_API_HOST= VM_API_KEY= @@ -50,83 +81,115 @@ Keeing model credentials in a separate file also allows for precise access contr VM_API_MODEL= ``` -::: {.column-margin} -For example, if your credentials look like this: - -```python -import validmind as vm - -vm.init( -api_host = "https://api.prod.validmind.ai/api/v1/tracking", -api_key = "key123", -api_secret = "secret456", -model = "model789" -) -``` - -Then your `.env` file should look like this: +::: {.callout} +To define the document to connect the {{< var validmind.developer >}} to, or to enable monitoring, include these in your `vm.init()` snippet within your Jupyter Notebook.[^6] -```bash -VM_API_HOST=https://api.prod.validmind.ai/api/v1/tracking -VM_API_KEY=key123 -VM_API_SECRET=secret456 -VM_API_MODEL=model789 -``` +::: ::: +## 2. Use credentials in your Jupyter Notebook + +a. After installing the {{< var validmind.developer >}} within your Jupyter Notebook,[^7] insert this code snippet in a code cell above your model identifier credentials: -4. Insert this code snippet above your model identifier credentials: - ```python %load_ext dotenv %dotenv dev.env ``` -5. Remove the inline credentials from `vm.init()`.[^4] +b. Remove the inline credentials from `vm.init()` after importing the {{< var vm.developer >}}: + + ::: {.panel-tabset} + + ### Without ongoing monitoring + + Example updated code cell: - The updated code cell should now look like this: - ```python %load_ext dotenv %dotenv .env - + import validmind as vm - + vm.init( + document="document-key", ) ``` -::: {.column-margin} -To enable monitoring when you are storing credentials in an `.env` file:^[[Enable monitoring](/guide/monitoring/enable-monitoring.qmd)] + ### With ongoing monitoring -```python -%load_ext dotenv -%dotenv .env + Example updated code cell including ongoing monitoring:[^8] -import validmind as vm + ```python + %load_ext dotenv + %dotenv .env -vm.init( - monitoring=True -) -``` + import validmind as vm -::: + vm.init( + document="monitoring", + monitoring=True + ) + ``` + + ::: + + ::: {.callout} + ## A template must already be applied to your selected document to populate test results in the {{< var validmind.platform >}}. + + Selecting a document to connect to requires {{< var validmind.developer >}} version >=2.12.0.[^9] Attempting to initialize the {{< var vm.developer >}} with a document that does not have a template applied will result in an error. -6. Run the cell. Instead of using inline credentials, this cell will now load your model credentials from a `.env` file. + ::: +c. Run the cell. + + Instead of using inline credentials, this code cell will now load your model credentials from a `.env` file. ## What's next -- [Working with model documentation](/guide/model-documentation/working-with-model-documentation.qmd) -- [{{< var validmind.developer >}}](/developer/validmind-library.qmd) +:::{#whats-next} +::: [^1]: [Register models in the inventory](/guide/model-inventory/register-models-in-inventory.qmd) -[^2]: [Manage permissions](/guide/configuration/manage-permissions.qmd) +[^2]: [Manage model stakeholders](/guide/model-inventory/edit-model-inventory-fields.qmd#manage-model-stakeholders) [^3]: [Working with the model inventory](/guide/model-inventory/working-with-model-inventory.qmd#search-filter-and-sort-models) +[^4]: [Working with model documents](/guide/templates/working-with-model-documents.qmd) + +[^5]: + + For example, if your credentials look like this: + + ```python + import validmind as vm + + vm.init( + api_host = "https://api.prod.validmind.ai/api/v1/tracking", + api_key = "key123", + api_secret = "secret456", + document = "document-key", + model = "model789" + ) + ``` + + Then your `.env` file should look like this: + + ```bash + VM_API_HOST=https://api.prod.validmind.ai/api/v1/tracking + VM_API_KEY=key123 + VM_API_SECRET=secret456 + VM_API_MODEL=model789 + ``` + +[^6]: [Use credentials in your Jupyter Notebook](#use-credentials-in-your-jupyter-notebook) + +[^7]: [Install and initialize the {{< var validmind.developer >}}](/developer/model-documentation/install-and-initialize-validmind-library.qmd#install-the) + +[^8]: [Enable monitoring](/guide/monitoring/enable-monitoring.qmd) + +[^9]: [Install and initialize the {{< var validmind.developer >}}](/developer/model-documentation/install-and-initialize-validmind-library.qmd#install-the) diff --git a/site/developer/samples-jupyter-notebooks.qmd b/site/developer/samples-jupyter-notebooks.qmd index 11a5aa88c7..63786d527b 100644 --- a/site/developer/samples-jupyter-notebooks.qmd +++ b/site/developer/samples-jupyter-notebooks.qmd @@ -8,6 +8,7 @@ listing: - id: agents type: grid grid-columns: 2 + page-size: 4 image-placeholder: "jupyter-logo-rectangle.svg" max-description-length: 350 image-height: "100%" @@ -16,6 +17,7 @@ listing: - id: capital-markets type: grid grid-columns: 2 + page-size: 4 image-placeholder: "jupyter-logo-rectangle.svg" max-description-length: 350 image-height: "100%" @@ -24,6 +26,7 @@ listing: - id: code-explainer type: grid grid-columns: 2 + page-size: 4 image-placeholder: "jupyter-logo-rectangle.svg" max-description-length: 350 image-height: "100%" @@ -32,6 +35,7 @@ listing: - id: credit-risk type: grid grid-columns: 2 + page-size: 4 image-placeholder: "jupyter-logo-rectangle.svg" max-description-length: 350 image-height: "100%" @@ -40,6 +44,7 @@ listing: - id: model-validation type: grid grid-columns: 2 + page-size: 4 image-placeholder: "jupyter-logo-rectangle.svg" max-description-length: 350 image-height: "100%" @@ -48,6 +53,7 @@ listing: - id: nlp-and-llm type: grid grid-columns: 2 + page-size: 4 image-placeholder: "jupyter-logo-rectangle.svg" max-description-length: 350 image-height: "100%" @@ -56,6 +62,7 @@ listing: - id: ongoing-monitoring type: grid grid-columns: 2 + page-size: 4 image-placeholder: "jupyter-logo-rectangle.svg" max-description-length: 350 image-height: "100%" @@ -64,6 +71,7 @@ listing: - id: regression type: grid grid-columns: 2 + page-size: 4 image-placeholder: "jupyter-logo-rectangle.svg" max-description-length: 350 image-height: "100%" @@ -72,6 +80,7 @@ listing: - id: time-series type: grid grid-columns: 2 + page-size: 4 image-placeholder: "jupyter-logo-rectangle.svg" max-description-length: 350 image-height: "100%" diff --git a/site/developer/supported-models-and-frameworks.qmd b/site/developer/supported-models-and-frameworks.qmd new file mode 100644 index 0000000000..d1b25d81af --- /dev/null +++ b/site/developer/supported-models-and-frameworks.qmd @@ -0,0 +1,498 @@ +--- +# Copyright © 2023-2026 ValidMind Inc. All rights reserved. +# Refer to the LICENSE file in the root of this repository for details. +# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial +title: "Supported models and frameworks" +date: last-modified +aliases: + - /guide/supported-models.html + - /developer/model-documentation/supported-models.html + - /developer/supported-models.html + - /developer/supported-model-frameworks.html +listing: + - id: next-models + type: grid + grid-columns: 2 + max-description-length: 250 + sort: false + fields: [title, description] + contents: + - /how-to/testing-overview.qmd + - test-descriptions.qmd + - /how-to/feature-overview.qmd + - samples-jupyter-notebooks.qmd +--- + +Learn about the wide range of models and frameworks for testing and documentation supported by the {{< var validmind.developer >}}. Understand which frameworks are supported, what your model needs to provide for tests to run, and how to work with models that don't fit standard patterns. + +::: {.attn} + +## What does _supported_ mean? + +_Supported_ here means the {{< var validmind.developer >}} provides dedicated wrappers, install options, and guidance for these models and frameworks. + +You can also use other code with the {{< var vm.developer >}} for models that don't fit standard framework patterns. For example, you can wrap any Python callable with `FunctionModel` or pass precomputed predictions when you don't have a model object. + +:::: {.columns} + +::: {.column width="50%" .pr3} +Framework-agnostic support +: Compatible with any Python object that implements a `predict()` method. Framework-specific wrappers provide extra features and convenience, but they are not required to use generic Python models. +::: + +::: {.column width="50%" .pl3} +Vendor model compatibility +: Works with both first-party models you create and third-party models from external vendors, enabling flexible integration with both proprietary and external sources. +::: + +:::: + +::: + +## AI systems + +::: {.panel-tabset} +## LLM classifiers +LLM-based classification models via OpenAI-compatible APIs. + +- Test suite: `LLMClassifierFullSuite` +- Includes prompt validation, text data quality, classifier metrics + +## NLP classifiers +Hugging Face and other NLP classification models. + +- Test suite: `NLPClassifierFullSuite` +- Text classification, sentiment analysis, named entity recognition + +## Summarization models +Models that generate concise summaries from longer texts. + +- Test suite: `SummarizationMetrics` +- BERT, BLEU, METEOR, ROUGE score evaluation + +## Embeddings models +Text and vector embedding models. + +- Test suite: `EmbeddingsFullSuite` +- Similarity metrics, stability analysis, visualization + +## RAG systems +Retrieval-augmented generation pipelines. + +- Tests: RAGAS integration +- Refer to [RAG evaluation](#rag-evaluation) for dataset requirements and available tests +::: + +## Machine learning models + +::: {.panel-tabset} +## Hugging Face-compatible +NLP and tabular models using Hugging Face Transformers. + +- Test suite: `NLPClassifierFullSuite` (for NLP), `ClassifierFullSuite` or `RegressionFullSuite` (for tabular) +- Text classification, tabular classification/regression + +## PyTorch +Any PyTorch neural network architecture, including LSTM, RNN, CNN, transformers, and custom architectures. + +- Test suite: `ClassifierFullSuite` or `RegressionFullSuite` (depending on task) +- Model performance, feature importance, diagnosis + +## Tree-based +Gradient boosting and ensemble tree methods (XGBoost, CatBoost, random forest). + +- Test suite: `ClassifierFullSuite` or `RegressionFullSuite` +- Confusion matrix, ROC/AUC, feature importance, SHAP values + +## K-nearest neighbors (KNN) +Distance-based classification and regression. + +- Test suite: `ClassifierFullSuite` or `RegressionFullSuite` +- Performance metrics, model comparison + +## Clustering +Any sklearn-compatible clustering algorithm, including K-means, DBSCAN, and hierarchical clustering. + +- Test suite: `ClusterFullSuite` +- Silhouette score, homogeneity, completeness, cluster distribution +::: + + +## Traditional statistical models + +::: {.panel-tabset} +## Linear regression +Models the relationship between a scalar response and one or more explanatory variables. + +- Test suite: `RegressionFullSuite` +- R² score, regression errors, feature importance + +## Logistic regression +Models the probability of a binary outcome based on one or more predictor variables. + +- Test suite: `ClassifierFullSuite` +- Confusion matrix, ROC/AUC, precision/recall, feature importance + +## Timeseries and forecasting +ARIMA, VAR, and other statsmodels time series models. + +- Test suite: `TimeSeriesModelValidation` +- Stationarity tests, autocorrelation analysis, forecast evaluation +::: + + +## Frameworks + +The {{< var validmind.developer >}} provides wrapper classes for common frameworks. When you call `vm.init_model()`, the library automatically detects your model's framework from its module, such as `sklearn`, `torch`, or `transformers`, and selects the appropriate wrapper. You don't need to specify the wrapper class manually. + +::: {.panel-tabset} +## CatBoost + +Gradient boosting on decision trees with categorical feature support. + +- **Wrapper:** `CatBoostModel` +- **Install:** `pip install validmind` + +## Hugging Face Transformers + +Pre-trained models for text classification, summarization, translation, and generation from the Hugging Face Transformers library. + +- **Wrapper:** `HFModel` +- **Install:** `pip install validmind[huggingface]` +- **Supported tasks:** `text_classification`, `text2text_generation` (summarization), `feature_extraction` (embeddings) + +## LLMs via API + +Access to OpenAI and Azure OpenAI models for text generation and analysis via OpenAI-compatible APIs. + +- **Wrapper:** `FoundationModel` +- **Install:** `pip install validmind[llm]` + +## PyTorch + +An open-source machine learning library for computer vision and natural language processing. + +- **Wrapper:** `PyTorchModel` +- **Install:** `pip install validmind[pytorch]` + +## R models via rpy2 + +Integrates R's statistical computing with Python via rpy2. + +- **Wrapper:** `RModel` +- **Install:** `pip install validmind rpy2` +- **Supported model types:** `glm` (including logistic regression), `lm`, `xgb.Booster` +- **Initialization:** Use `vm.init_r_model(model_path, input_id)` to load R model objects + +## scikit-learn + +A Python library for machine learning, offering supervised and unsupervised learning algorithms. + +- **Wrapper:** `SKlearnModel` +- **Install:** `pip install validmind` + +## statsmodels + +A Python module for statistical models, tests, and data exploration. + +- **Wrapper:** `StatsModelsModel` +- **Install:** `pip install validmind[stats]` + +## XGBoost + +An optimized gradient boosting library for classification and regression. + +- **Wrapper:** `XGBoostModel` +- **Install:** `pip install validmind[xgboost]` + +## Python callable + +Wrap custom prediction functions or models from otherwise unsupported frameworks. + +- **Wrapper:** `FunctionModel` +- **Install:** `pip install validmind` +::: + +To install all optional dependencies: + +```bash +pip install validmind[all] +``` + +### Test input requirements + +Different tests require different inputs from your model and dataset. Understanding these requirements helps you run the right tests for your use case.[^1] + +::: {.panel-tabset} + +#### When `predict()` is required + +Most model tests call your model's `predict()` method to generate predictions. This includes: + +- Performance metrics (accuracy, precision, recall, F1) +- Error analysis tests +- Robustness tests + +#### When `predict_proba()` is needed + +Classification metrics that evaluate probability outputs require `predict_proba()`: + +- ROC-AUC score +- Precision-recall curves +- Calibration tests +- Probability distribution analysis + +If your model doesn't have `predict_proba()`, these tests will be skipped or return an error. + +::: + +### Test input flow + +```{mermaid} +flowchart LR + subgraph Inputs + Model[Model Object] + Dataset[Dataset] + Precomputed[Precomputed Predictions] + end + + Model --> predict["predict()"] + Model --> predict_proba["predict_proba()"] + Precomputed --> Dataset + + subgraph TestTypes[Test Types] + ModelTests[Model Tests] + DatasetTests[Dataset Tests] + ClassificationTests[Classification Metrics] + end + + predict --> ModelTests + predict --> DatasetTests + predict_proba --> ClassificationTests + Dataset --> DatasetTests +``` + +### Using precomputed predictions + +If you can't provide a model object because your model runs in a separate environment, you can pass precomputed predictions directly to the dataset: + +```python +vm_dataset = vm.init_dataset( + dataset=df, + target_column="target", + prediction_values=predictions, # numpy array of predictions + probability_values=probabilities # optional: for classification +) +``` + +Alternatively, if predictions are already in your dataframe: + +```python +vm_dataset = vm.init_dataset( + dataset=df, + target_column="target", + prediction_column="predicted", # column name in df + probability_column="probability" # optional: for classification +) +``` + +### Dataset-only tests + +Some tests analyze data quality and don't require a model at all: + +- Missing value analysis +- Class imbalance detection +- Feature correlation +- Outlier detection +- Data drift tests + +### Supported dataset formats + +The library accepts multiple data formats when initializing datasets with `vm.init_dataset()`: + +| Format | Notes | +|--------|-------| +| Pandas DataFrame | Primary format, used internally | +| Polars DataFrame | Converted to pandas internally | +| NumPy ndarray | Requires column names to be specified | +| PyTorch TensorDataset | Requires `pip install validmind[pytorch]` | + +## Custom model wrappers + +For models that don't fit standard framework patterns, use these flexible wrappers: + +::: {.panel-tabset} +### FunctionModel + +Wrap any Python callable as a model: + +```python +from validmind.models import FunctionModel + +def my_predict(X): + # Your prediction logic here + return predictions + +vm_model = vm.init_model( + model=FunctionModel(predict_fn=my_predict), + input_id="my_model" +) +``` + +### PipelineModel + +Chain multiple models or processing steps using the `|` operator between `VMModel` instances: + +```python +from validmind.models import PipelineModel + +vm_model = vm.init_model( + model=PipelineModel(vm_preprocessor | vm_model), + input_id="my_pipeline" +) +``` + +Or, pass the pipeline directly: + +```python +vm_model = vm.init_model(vm_preprocessor | vm_model, input_id="my_pipeline") +``` + +### MetadataModel + +Use when you have model metadata but no inference capability — for example, when documenting a model deployed in an external system. Calling `vm.init_model()` with `attributes` creates a `MetadataModel` implicitly: + +```python +vm_model = vm.init_model( + input_id="external_model", + attributes={ + "architecture": "ExternalModel", + "language": "Python", + }, +) +``` + +With a metadata-only model, you can still run dataset-only tests and use precomputed predictions. + +## Setting up for LLMs and GenAI + +This section covers how to configure and test large language models and generative AI systems. + +### FoundationModel + +Use `FoundationModel` for LLM endpoints which wraps a callable `predict_fn` and a `Prompt`: + +```python +import validmind as vm +from openai import OpenAI +from validmind.models import FoundationModel, Prompt + +client = OpenAI() +def call_model(prompt: str) -> str: + return ( + client.chat.completions.create( + model="gpt-4", + messages=[{"role": "user", "content": prompt}], + ) + .choices[0] + .message.content + ) + +vm_model = vm.init_model( + model=FoundationModel( + predict_fn=call_model, + prompt=Prompt( + template="Classify the sentiment: {text}", + variables=["text"], + ), + ), + input_id="sentiment_classifier", +) +``` +::: + +### LLM test suites + +Install LLM dependencies: + +```bash +pip install validmind[llm] +``` + +Available test suites for LLMs include: + +- Prompt injection detection +- Output consistency +- Hallucination detection +- Toxicity analysis +- Bias evaluation + +Tests such as bias, clarity, robustness, and toxicity can be applied to any text-generating model, regardless of type. + +## RAG evaluation + +For retrieval-augmented generation (RAG) systems, the {{< var validmind.developer >}} integrates with [RAGAS](https://docs.ragas.io/) for comprehensive evaluation. + +### Dataset requirements + +RAG evaluation expects these logical fields, but you can map your own column names using test parameters: + +| Field | Type | Description | +|-------|------|-------------| +| `user_input` | str | The user's query | +| `response` | str | Model output | +| `retrieved_contexts` | List[str] | Retrieved context chunks | +| `reference` | str | Ground truth (required for some tests) | + +To map custom column names, pass them as parameters when running the test: + +```python +run_test( + "validmind.model_validation.ragas.Faithfulness", + inputs={"dataset": vm_test_ds}, + params={ + "user_input_column": "question", + "response_column": "rag_model_prediction", + "retrieved_contexts_column": "retrieval_model_prediction", + }, +).log() +``` + +### Available RAG tests + +- **Faithfulness** — Measures how well the response is grounded in retrieved contexts +- **Context Recall** — Evaluates if relevant information was retrieved +- **Context Precision** — Measures relevance of retrieved contexts +- **Answer Relevancy** — Assesses if the response addresses the query + +## Python and dependency compatibility + +The {{< var validmind.developer >}} requires: + +- **Python:** >=3.9, <3.13 +- **Core dependencies:** pandas, numpy, scikit-learn + +Optional dependencies for specific frameworks: + +| Extra | Includes | +|-------|----------| +| `pytorch` | PyTorch, torchvision | +| `llm` | OpenAI, langchain, ragas, deepeval | +| `xgboost` | XGBoost | +| `huggingface` | Transformers, sentencepiece | +| `nlp` | langdetect, nltk, textblob, evaluate, rouge, bert-score | +| `stats` | scipy, statsmodels, arch | +| `explainability` | SHAP | +| `credit_risk` | scorecardpy | +| `datasets` | Hugging Face datasets | +| `pii-detection` | presidio-analyzer, presidio-structured | +| `all` | All optional dependencies | + +## What's next + +:::{#next-models} +::: + + + +[^1]: [How to run tests and test suites](/developer/how-to/testing-overview.qmd) diff --git a/site/developer/supported-models.qmd b/site/developer/supported-models.qmd deleted file mode 100644 index 930db0533f..0000000000 --- a/site/developer/supported-models.qmd +++ /dev/null @@ -1,160 +0,0 @@ ---- -# Copyright © 2023-2026 ValidMind Inc. All rights reserved. -# Refer to the LICENSE file in the root of this repository for details. -# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial -title: "Supported models" -date: last-modified -aliases: - - /guide/supported-models.html - - /developer/model-documentation/supported-models.html -listing: - - id: next-models - type: grid - grid-columns: 2 - max-description-length: 250 - sort: false - fields: [title, description] - contents: - - /how-to/testing-overview.qmd - - test-descriptions.qmd - - /how-to/feature-overview.qmd - - samples-jupyter-notebooks.qmd ---- - -The {{< var validmind.developer >}} provides out-of-the-box support for testing and documentation for an array of model types and modeling packages. - -## What is a supported model? - -A _supported model_ refers to a model for which predefined testing or documentation functions exist in the {{< var validmind.developer >}}, provided that the model you are developing is documented using a supported version of our {{< var vm.developer >}}. These model types cover a very large portion of the models used in commercial and retail banking. - -::: {.callout} -## {{< var vm.product >}} does not limit our users to specific model types. - -- The {{< var validmind.developer >}} is extensible to support future model types or modeling packages to accomodate rapid developments in the AI space, including the advent of large language models (LLMs). -- You always have the flexibility to implement custom tests and integrate external test providers.[^1] -::: - - - -## Supported model types - -::: {.column-margin} -::: {.feature} -Vendor models -: {{< var vm.product >}} offers support for both first-party models and [third-party vendor models](/about/glossary/glossary.qmd#vendor-model). - -::: - -::: - -### Traditional statistical models - -:::: {.flex .flex-wrap .justify-around} - -::: {.w-30-ns} - -#### Linear regression -Models relationship between a scalar response and one or more explanatory variables. - -::: - -::: {.w-30-ns} - -#### Logistic regression -Models relationship between a scalar response and one or more explanatory variables. - -::: - -::: {.w-30-ns} -#### Time series -Analyzes data points collected or sequenced over time. - -::: - -:::: - -### Machine learning models - -:::: {.flex .flex-wrap .justify-around} - -::: {.w-50-ns .pr2} -#### Hugging Face-compatible models -- Natural language processing (NLP) text classification — Categorizes text into predefined classes. -- Tabular classification — Assigns categories to tabular dataset entries. -- Tabular regression — Predicts continuous outcomes from tabular data. - -#### Neural networks -- Long short-term memory (LSTM) — Processes sequences of data, remembering inputs over long periods. -- Recurrent neural network (RNN) — Processes sequences by maintaining a state that reflects the history of processed elements. -- Convolutional neural network (CNN) — Primarily used for processing grid-like data such as images. - - -::: - -::: {.w-50-ns .pl2 .pr2} - -#### Tree-based models
(XGBoost / CatBoost / random forest) -- Classification — Predicts categorical outcomes using decision trees. -- Regression — Predicts continuous outcomes using decision trees. - -#### K-nearest neighbors (KNN) -- Classification — Assigns class by majority vote of the k-nearest neighbors. -- Regression — Predicts value based on the average of the k-nearest neighbors. - -#### Clustering -- K-means — Partitions _n_ observations into _k_ clusters in which each observation belongs to the cluster with the nearest mean. - - -::: - -:::: - -### Generative AI models - -#### Large language models (LLMs) -- Classification — Categorizes input into predefined classes. -- Text summarization — Generates concise summaries from longer texts. - -## Supported modeling libraries and other tools - -:::: {.flex .flex-wrap .justify-around} - -::: {.w-50-ns} - -- **[scikit-learn](https://scikit-learn.org/stable/)** — A Python library for machine learning, offering a range of supervised and unsupervised learning algorithms. - -- **[statsmodels](https://www.statsmodels.org/stable/index.html)** — A Python module that provides classes and functions for the estimation of many different statistical models, as well as for conducting statistical tests and exploring data. - -- **[PyTorch](https://pytorch.org/)** — An open-source machine learning library based on the Torch library, used for applications such as computer vision and natural language processing. - -- **[Hugging Face Transformers](https://huggingface.co/docs/transformers/en/index)** — Provides thousands of pre-trained models to perform tasks on texts such as classification, information extraction, question answering, summarization, translation, and text generation. - -- **[XGBoost](https://xgboost.readthedocs.io/en/stable/)** — An optimized distributed gradient boosting library designed to be highly efficient, flexible, and portable, implementing machine learning algorithms under the Gradient Boosting framework. - -::: - -::: {.w-50-ns} - -- **[CatBoost](https://catboost.ai/)** — An open-source gradient boosting on decision trees library with categorical feature support out of the box, for ranking, classification, regression, and other ML tasks. - -- **[LightGBM](https://lightgbm.readthedocs.io/en/stable/)** — A fast, distributed, high-performance gradient boosting (GBDT, GBRT, GBM, or MART) framework based on decision tree algorithms, used for ranking, classification, and many other machine learning tasks. - -- **R models, via [rpy2 - R in Python](https://rpy2.github.io/)** — Facilitates the integration of R's statistical computing and graphics capabilities with Python, allowing for R models to be called from Python. - -- **Large language models (LLMs), via [OpenAI-compatible APIs](https://platform.openai.com/docs/introduction)** — Access to advanced AI models trained by OpenAI for a variety of natural language tasks, including text generation, translation, and analysis, through a compatible API interface. This support includes both the OpenAI API and the Azure OpenAI Service via API. -::: - -:::: - -## What's next - -:::{#next-models} -::: - - - - -[^1]: - - - [Implement custom tests](/notebooks/how_to/tests/custom_tests/implement_custom_tests.ipynb) - - [Integrate external test providers](/notebooks/how_to/tests/custom_tests/integrate_external_test_providers.ipynb) \ No newline at end of file diff --git a/site/faq/_faq-activity.qmd b/site/faq/_faq-activity.qmd index 3e62b57b02..b950ebeccd 100644 --- a/site/faq/_faq-activity.qmd +++ b/site/faq/_faq-activity.qmd @@ -2,7 +2,7 @@ Refer to the LICENSE file in the root of this repository for details. SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial --> -## Is activity on models, documentation, etc. logged? +## Is activity on models, documents, etc. logged? - Yes, the {{< var validmind.platform >}}^[[Accessing {{< var vm.product >}}](/guide/access/accessing-validmind.qmd)] provides an audit trail functionality, enabling you to track or audit all the events associated with a specific model. -- You can review a full record of comments, workflow status changes, and any other updates made to the model, including modifications to documentation or test results. \ No newline at end of file +- You can review a full record of comments, workflow status changes, and any other updates made to the model, including modifications to documents or test results. \ No newline at end of file diff --git a/site/faq/_faq-attachments.qmd b/site/faq/_faq-attachments.qmd index 6177946554..727ecae416 100644 --- a/site/faq/_faq-attachments.qmd +++ b/site/faq/_faq-attachments.qmd @@ -2,11 +2,11 @@ Refer to the LICENSE file in the root of this repository for details. SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial --> -## Can we attach files to models, artifacts, or documentation? +## Can we attach files to models, artifacts, or documents? Yes, attachment type inventory fields are available for custom use.^[[Manage model inventory fields](/guide/model-inventory/manage-model-inventory-fields.qmd)] Once created, attachment type fields allow you to upload supporting files to your model. - Out-of-the-box functionality is included for attaching files to model artifacts. -- You can also attach images to documentation content blocks and comments. +- You can also attach images to document content blocks and comments. By default, the [{{< fa hand >}} Customer Admin]{.bubble} role has sufficient permissions to manage model inventory fields. \ No newline at end of file diff --git a/site/faq/_faq-exporting.qmd b/site/faq/_faq-exporting.qmd index 8113cdf761..67f93679ac 100644 --- a/site/faq/_faq-exporting.qmd +++ b/site/faq/_faq-exporting.qmd @@ -2,6 +2,6 @@ Refer to the LICENSE file in the root of this repository for details. SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial --> -## Can documentation be exported? +## Can documents be exported? -Yes, {{< var vm.product >}} currently supports exporting model documentation and validation reports in Word (.docx) format.^[Depending on the integration requirements of the systems used by your validation teams, such as connectivity via API, SharePoint, and more, {{< var vm.product >}} can work with you to automate the export and storage of documentation into these systems.] \ No newline at end of file +Yes, {{< var vm.product >}} currently supports exporting documents in Word (.docx) format.^[Depending on the integration requirements of the systems used by your validation teams, such as connectivity via API, SharePoint, and more, {{< var vm.product >}} can work with you to automate the export and storage of documents into these systems.] \ No newline at end of file diff --git a/site/faq/_faq-images.qmd b/site/faq/_faq-images.qmd index 2b640cdd5f..2d36be4758 100644 --- a/site/faq/_faq-images.qmd +++ b/site/faq/_faq-images.qmd @@ -2,9 +2,9 @@ Refer to the LICENSE file in the root of this repository for details. SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial --> -## Do you support including images in model documentation? +## Do you support including images in model documents? -Yes, as long as you can produce the image with Python or open the image from a file, you can include it in your documentation with {{< var vm.product >}}:^[[Implement custom tests](/notebooks/how_to/tests/custom_tests/implement_custom_tests.ipynb#custom-test-images)] +Yes, as long as you can produce the image with Python or open the image from a file, you can include it in your documents with {{< var vm.product >}}:^[[Implement custom tests](/notebooks/how_to/tests/custom_tests/implement_custom_tests.ipynb#custom-test-images)] - If you want to log an image as a test result, you can do so by passing the path to the image as a parameter to the custom test and then opening the file in the test function. - If you are using a plotting library that isn’t directly supported by {{< var vm.product >}}, you can still return the image directly as a bytes-like object. \ No newline at end of file diff --git a/site/faq/_faq-monitoring.qmd b/site/faq/_faq-monitoring.qmd index 5468bd6338..76d6b43827 100644 --- a/site/faq/_faq-monitoring.qmd +++ b/site/faq/_faq-monitoring.qmd @@ -9,4 +9,4 @@ Yes, {{< var vm.product >}} offers ongoing monitoring support to help you regula - You can enable monitoring for both new and existing models.^[[Enable monitoring](/guide/monitoring/enable-monitoring.qmd)] - You use the {{< var validmind.developer >}} to automatically populate the monitoring template for your model with data, providing a comprehensive view of your model’s performance over time. - You then access and examine these results within the {{< var validmind.platform >}}, allowing you to identify any deviations from expected performance and take corrective actions as needed.^[[Review monitoring results](/guide/monitoring/review-monitoring-results.qmd)] -- Once generated via the {{< var validmind.developer >}}, view and add metrics over time to your ongoing monitoring plans in the {{< var validmind.platform >}}.^[[Work with metrics over time](/guide/monitoring/work-with-metrics-over-time.qmd)] \ No newline at end of file +- Once generated via the {{< var validmind.developer >}}, view and add metrics over time to your ongoing monitoring reports in the {{< var validmind.platform >}}.^[[Work with metrics over time](/guide/monitoring/work-with-metrics-over-time.qmd)] \ No newline at end of file diff --git a/site/faq/faq-collaboration.qmd b/site/faq/faq-collaboration.qmd index bcac6628ed..4cdbaaedf0 100644 --- a/site/faq/faq-collaboration.qmd +++ b/site/faq/faq-collaboration.qmd @@ -17,18 +17,18 @@ listing: - ../guide/model-inventory/view-model-activity.qmd - ../guide/model-documentation/collaborate-with-others.qmd - ../guide/workflows/working-with-workflows.qmd -categories: ["real-time collaboration", "model documentation", "model activity", "auditing", "workflows", "model lifecycle", "validmind platform"] +categories: ["real-time collaboration", "documents","documentation", "model activity", "auditing", "workflows", "model lifecycle", "validmind platform"] --- {{< include _faq-activity.qmd >}} ## What real-time collaboration features does {{< var vm.product >}} offer? -- You can simultaneously edit model documentation, leave and respond to comments or suggestions all within the {{< var validmind.platform >}}. -- You can also saved named versions of edits to retain specific revisions, and any changes to model documentation are automatically logged on your model's activity feed. +- You can simultaneously edit model documents, leave and respond to comments or suggestions all within the {{< var validmind.platform >}}. +- You can also saved named versions of edits to retain specific revisions, and any changes to model documents are automatically logged on your model's activity feed. ::: {.callout} -## Multiple users are able to simultaneously edit documentation in the {{< var validmind.platform >}}. +## Multiple users are able to simultaneously edit documents in the {{< var validmind.platform >}}. If two users are editing the same cell within {{< var vm.platform >}}, the most recently saved version of the content will prevail. ::: diff --git a/site/faq/faq-documentation.qmd b/site/faq/faq-documentation.qmd index 31c2a12fa5..52aa81d02b 100644 --- a/site/faq/faq-documentation.qmd +++ b/site/faq/faq-documentation.qmd @@ -2,47 +2,49 @@ # Copyright © 2023-2026 ValidMind Inc. All rights reserved. # Refer to the LICENSE file in the root of this repository for details. # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial -title: "Model documentation and templates" +title: "Model documents and templates" date: last-modified aliases: - /guide/faq-documentation.html listing: - id: faq-documentation type: grid - grid-columns: 3 + grid-columns: 2 max-description-length: 250 sort: false fields: [title, description] contents: - ../guide/templates/working-with-document-templates.qmd - ../guide/model-documentation/working-with-model-documentation.qmd -categories: ["templates", "model documentation", "customization", "images", "validmind platform", "validmind library"] +categories: ["templates", "documents", "documentation","customization", "images", "validmind platform", "validmind library"] --- -## What kind of templates are available through {{< var vm.product >}}? +## What kind of templates are provided by {{< var vm.product >}}? -{{< var vm.product >}} supports the following template types:[^1] +{{< var vm.product >}} provides the following default template types:[^1] -- Documentation -- Validation Report -- Ongoing Monitoring +- Development (model documentation) +- Validation (validation reports) +- Monitoring (ongoing monitoring reports) -## Can templates be configured per use case or to match our own? +You can also create custom document types and associated templates to suit your organization's unique needs. -Yes, the {{< var validmind.platform >}}[^2] allows you to configure versioned templates based on documentation requirements for each model or lifecycle use case. +## Can templates be customized for our use cases? + +Yes, the {{< var validmind.platform >}}[^2] allows you to configure versioned templates based on document requirements for each model or lifecycle use case. - {{< var vm.product >}}'s templates are fully customizable,[^3] and are complemented by the ability to manage validation guidelines. - You can swap between different versions of templates or apply another version of the current template.[^4] - Template outlines can be modified right in the {{< var validmind.platform >}}. We also offer an advanced raw YAML editor, enabling you to make detailed changes. -By default, the [{{< fa hand >}} Customer Admin]{.bubble} role[^5] has sufficient permissions to manage documentation templates, and you must be a [{{< fa code >}} Developer]{.bubble} or [{{< fa circle-check >}} Validator]{.bubble} to be able to swap documentation templates. +By default, the [{{< fa hand >}} Customer Admin]{.bubble} role[^5] has sufficient permissions to manage document templates, and you must be a [{{< fa code >}} Developer]{.bubble} or [{{< fa circle-check >}} Validator]{.bubble} to be able to swap document templates. -## Can documentation be created right in the {{< var validmind.platform >}}? +## Can documents be created right in the {{< var validmind.platform >}}? -Yes, you can write documentation directly in the {{< var validmind.platform >}}, without having to first generate anything using the {{< var validmind.developer >}}.[^6] +Yes, you can work with model documentation, validation reports, ongoing monitoring reports, or any other document type directly in the {{< var validmind.platform >}}, without having to first generate anything using the {{< var validmind.developer >}}.[^6] -1. Add and edit text on any documentation within the {{< var vm.platform >}} using our content editing toolbar.[^7] -2. Using the {{< var vm.developer >}}, execute test suites and generate the corresponding supporting results. These results can then be added to your documentation within the {{< var vm.platform >}}.[^8] +1. Add and edit text on any document within the {{< var vm.platform >}} using our content editing toolbar.[^7] +2. Using the {{< var vm.developer >}}, execute test suites and generate the corresponding supporting results. These results can then be added to your documents within the {{< var vm.platform >}}.[^8] ## Can I run tests and log documentation without a model? @@ -75,9 +77,9 @@ However, tests that need a trained model will not work with "empty" models. - :::: {.content-visible unless-format="revealjs" when-meta="includes.developer"} -a. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select **Documentation**. +a. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select **Development**. -b. Under [create document]{.smallcaps}, select `Binary classification` from the **[template]{.smallcaps}** drop-down menu. +b. Under [create document]{.smallcaps}, choose **From {{< var vm.product >}} Template** and then select `Binary classification` from the **[template]{.smallcaps}** drop-down menu. c. Click **Use Template** to apply the template. @@ -18,9 +18,9 @@ c. Click **Use Template** to apply the template. :::: {.content-visible unless-format="revealjs" unless-meta="includes.developer"} -a. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select **Validation Report**. +a. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select **Validation**. -b. Under [create document]{.smallcaps}, select `Generic Validation Report` from the **[template]{.smallcaps}** drop-down menu. +b. Under [create document]{.smallcaps}, choose **From {{< var vm.product >}} Template** and then select `Generic Validation Report` from the **[template]{.smallcaps}** drop-down menu. c. Click **Use Template** to apply the template. diff --git a/site/get-started/developer/developer-code-snippet.png b/site/get-started/developer/developer-code-snippet.png index 47bc3b6b5a..eea9ac2ef4 100644 Binary files a/site/get-started/developer/developer-code-snippet.png and b/site/get-started/developer/developer-code-snippet.png differ diff --git a/site/get-started/developer/developer-model-details.png b/site/get-started/developer/developer-model-details.png index 9264fe32f5..7b1284f196 100644 Binary files a/site/get-started/developer/developer-model-details.png and b/site/get-started/developer/developer-model-details.png differ diff --git a/site/get-started/developer/developer-new-model.png b/site/get-started/developer/developer-new-model.png index 055035e384..53d1b0e389 100644 Binary files a/site/get-started/developer/developer-new-model.png and b/site/get-started/developer/developer-new-model.png differ diff --git a/site/get-started/developer/documentation-template-selection.png b/site/get-started/developer/documentation-template-selection.png index 538e1af1cf..15ac728186 100644 Binary files a/site/get-started/developer/documentation-template-selection.png and b/site/get-started/developer/documentation-template-selection.png differ diff --git a/site/get-started/developer/quickstart-developer.qmd b/site/get-started/developer/quickstart-developer.qmd index 6467fa259e..8548557241 100644 --- a/site/get-started/developer/quickstart-developer.qmd +++ b/site/get-started/developer/quickstart-developer.qmd @@ -96,14 +96,14 @@ After you successfully run the notebook, return to the {{< var validmind.platfor 2. Locate or search for the model you registered for this quickstart and click on it to select it.[^2] -3. In the left sidebar that appears for the model, click **Documentation** under {{< fa file >}} Documents: +3. In the left sidebar that appears for the model, click **Development** under {{< fa file >}} Documents: - - Your model's documentation is broken down into sections, defined by your documentation templates.[^3] + - Your model's documentation is broken down into sections, defined by your document templates.[^3] - Each section can be assigned a completion status,[^4] allowing you to track the progress of your documentation. 4. Click on any section of the documentation to review the test results and drafts of test result summaries generated by the {{< var validmind.developer >}}. - In any subsection, expand the **{{< var vm.product >}} Insights™** panel to view the Documentation Guidelines[^5] that tell you more about what these sections mean and help you with the task of documenting the model as a developer. + In any subsection, expand the **{{< var vm.product >}} Insights™** panel to view the Guidelines[^5] that tell you more about what these sections mean and help you with the task of documenting the model as a developer. 5. Refine your model documentation by: diff --git a/site/get-started/validator/quickstart-validator.qmd b/site/get-started/validator/quickstart-validator.qmd index 2c5d156898..da751ee363 100644 --- a/site/get-started/validator/quickstart-validator.qmd +++ b/site/get-started/validator/quickstart-validator.qmd @@ -100,7 +100,7 @@ Our companion notebook, **Quickstart for model validation,**[^1] walks you throu ```bash 2025-06-11 22:06:42,610 - INFO(validmind.api_client): 🎉 Connected to {{< var vm.product >}}! 📊 Model: [ValidMind] Model validation quickstart (ID: cmbs6g0pi02li0g874mq9fek4) -📁 Document Type: model_validation +📁 Document Type: validation_report ``` ::: @@ -114,7 +114,7 @@ After you successfully run the notebook, return to the {{< var validmind.platfor 2. Locate or search for the model you registered for this quickstart and click on it to select it.[^2] -3. In the left sidebar that appears for the model, click **Validation Report** under {{< fa file >}} Documents: +3. In the left sidebar that appears for the model, click **Validation** under {{< fa file >}} Documents: - Your model's validation is broken down into sections, defined by your validation report templates.[^3] - The report includes risk assessment summaries at the overall and per-section level, overviews of your compliance assessments provided within your validation report.[^4] diff --git a/site/get-started/validator/validation-report-overview.png b/site/get-started/validator/validation-report-overview.png index ee379882b9..4a2de667d0 100644 Binary files a/site/get-started/validator/validation-report-overview.png and b/site/get-started/validator/validation-report-overview.png differ diff --git a/site/get-started/validator/validation-template-selection.png b/site/get-started/validator/validation-template-selection.png index 39819190f7..49a899c37f 100644 Binary files a/site/get-started/validator/validation-template-selection.png and b/site/get-started/validator/validation-template-selection.png differ diff --git a/site/get-started/validator/validator-code-snippet.png b/site/get-started/validator/validator-code-snippet.png index cd330fe31b..fe40c0e0cf 100644 Binary files a/site/get-started/validator/validator-code-snippet.png and b/site/get-started/validator/validator-code-snippet.png differ diff --git a/site/get-started/validator/validator-new-model.png b/site/get-started/validator/validator-new-model.png index 9f7e1cf32c..e4777667f5 100644 Binary files a/site/get-started/validator/validator-new-model.png and b/site/get-started/validator/validator-new-model.png differ diff --git a/site/guide/_sidebar.yaml b/site/guide/_sidebar.yaml index d8f660b0d3..b5d7d6dcda 100644 --- a/site/guide/_sidebar.yaml +++ b/site/guide/_sidebar.yaml @@ -136,11 +136,13 @@ website: contents: - guide/reporting/view-report-data.qmd - guide/reporting/manage-custom-reports.qmd + - guide/reporting/export-analytics.qmd - file: guide/reporting/generating-exports.qmd contents: - guide/reporting/export-inventory.qmd - guide/reporting/export-documents.qmd - guide/reporting/export-artifacts.qmd + - guide/reporting/export-analytics.qmd - text: "---" - text: "Monitoring" - file: guide/monitoring/ongoing-monitoring.qmd diff --git a/site/guide/configuration/managing-your-organization.qmd b/site/guide/configuration/managing-your-organization.qmd index 7c6f932550..90c1ffb3a9 100644 --- a/site/guide/configuration/managing-your-organization.qmd +++ b/site/guide/configuration/managing-your-organization.qmd @@ -58,19 +58,26 @@ You will see the option to switch between organizations only if you belong to mo By default, the [{{< fa hand >}} Customer Admin]{.bubble} role has these permissions.[^4] ::: -1. In the left sidebar, click **{{< fa gear >}} Settings**. +1. In the left sidebar, click **{{< fa gear >}} Settings**. 2. Under Workspace, select **Organization**. 3. Under **Organization Name**, type in your new name and click **Update** to save your changes. +## Manage document defaults + +1. In the left sidebar, click **{{< fa gear >}} Settings**. + +2. Under Workspace, select **Organization**. + +3. Under Document Defaults, toggle whether you would like to **Enable Track Changes by default**[^5] for all content blocks within documents for your current active organization. + + ## Organization setup :::{#guides-configuration} ::: - - [^1]: Your active organization is displayed under your name and roles in the left sidebar. @@ -83,3 +90,5 @@ By default, the [{{< fa hand >}} Customer Admin]{.bubble} role has these permiss - Your active organization is displayed under your name and roles in the left sidebar. [^4]: [Manage permissions](/guide/configuration/manage-permissions.qmd) + +[^5]: [Collaborate with others](/guide/model-documentation/collaborate-with-others.qmd#tracking-changes) diff --git a/site/guide/guides.qmd b/site/guide/guides.qmd index b845a0cb6d..23f5d7493a 100644 --- a/site/guide/guides.qmd +++ b/site/guide/guides.qmd @@ -178,7 +178,7 @@ Use the {{< var validmind.platform >}} model inventory to thoroughly track your ## Model documentation -First, customize your model documentation templates and reusable blocks: +First, customize your model document templates and reusable blocks: :::{#guides-docs-templates} ::: diff --git a/site/guide/integrations/configure-analytics-exports.qmd b/site/guide/integrations/configure-analytics-exports.qmd index 2ede12355f..c8a3c4dd8b 100644 --- a/site/guide/integrations/configure-analytics-exports.qmd +++ b/site/guide/integrations/configure-analytics-exports.qmd @@ -29,7 +29,7 @@ Configure scheduled exports of analytics datasets to cloud storage so that BI to 4. From the **[datasets]{.smallcaps}** dropdown, choose which analytics datasets to export: - - Artifact - Change Management Record + - Artifact - Change Management Record (custom artifact example)[^3] - Artifact - Model Limitation - Artifact - Policy Exception - Artifact - Validation Issue @@ -156,3 +156,5 @@ Data you exported previously will remain in the cloud storage destination unchan [^1]: [Manage permissions](/guide/configuration/manage-permissions.qmd) [^2]: [Manage secrets](/guide/integrations/manage-secrets.qmd) + +[^3]: [Manage artifact types](/guide/model-validation/manage-artifact-types.qmd) diff --git a/site/guide/model-documentation/_add-test-results.qmd b/site/guide/model-documentation/_add-test-results.qmd index 80ce79e6a2..30f5af8588 100644 --- a/site/guide/model-documentation/_add-test-results.qmd +++ b/site/guide/model-documentation/_add-test-results.qmd @@ -7,7 +7,9 @@ SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial --> 1. Select a model or find your model by applying a filter or searching for it.^[[Working with the model inventory](/guide/model-inventory/working-with-model-inventory.qmd#search-filter-and-sort-models)] -1. In the left sidebar that appears for your model, under {{< fa file >}} Documents select **Documentation**, **Validation Report**, or **Ongoing Monitoring**.^[ [Working with model documents](/guide/templates/working-with-model-documents.qmd)] +1. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select the **Latest** tab.^[[Work with document versions](/guide/model-documentation/work-with-document-versions.qmd)] + +1. Click on the document file you want to add test results to.^[[Working with model documents](/guide/templates/working-with-model-documents.qmd)] 1. Click on a section header to expand that section and add content. @@ -17,9 +19,6 @@ SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial --> 1. Click {{< fa square-plus >}} and then select **Test-Driven**^[[Work with content blocks](/guide/model-documentation/work-with-content-blocks.qmd)] under [from library]{.smallcaps}. - - By default, the [{{< fa code >}} Developer]{.bubble} role can add test-driven blocks within model documentation or ongoing-monitoring plans. - - By default, the [{{< fa circle-check >}} Validator]{.bubble} role can add test-driven blocks within validation reports. - 1. Select test results: - Choose from available **VM Library** (out-of-the-box) or **Custom** tests under [test-driven]{.smallcaps} in the left sidebar of the test selection modal. @@ -41,7 +40,7 @@ With the test results logged, let's head to the model we connected to at the beg 1. From the **{{< fa cubes >}} Inventory** in the {{< var validmind.platform >}}, go to the model you connected to earlier. -1. In the left sidebar that appears for your model, click **Documentation** under {{< fa file >}} Documents. +1. In the left sidebar that appears for your model, click **Development** under {{< fa file >}} Documents. 1. Locate the Data Preparation section and click on **2.3 Correlations and Interactions** to expand that section. diff --git a/site/guide/model-documentation/_check-model-documentation.qmd b/site/guide/model-documentation/_check-model-documentation.qmd index 04aa96ae4c..9d9addae2d 100644 --- a/site/guide/model-documentation/_check-model-documentation.qmd +++ b/site/guide/model-documentation/_check-model-documentation.qmd @@ -9,7 +9,7 @@ SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial --> 1. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select the **Latest** or **Saved Versions** tab.^[[Work with document versions](/guide/model-documentation/work-with-document-versions.qmd)] -1. Click on a Documentation type file.^[[Working with model documents](/guide/templates/working-with-model-documents.qmd)] +1. Click on a Development type file.^[[Working with model documents](/guide/templates/working-with-model-documents.qmd)] 1. Locate the {{< var vm.checker >}} box on the right and click **{{< fa check >}} Check Document**. @@ -58,7 +58,7 @@ Use the feedback provided by the {{< var validmind.checker >}} to review the app 1. Select a model or find your model [by applying a filter or searching for it](/guide/model-inventory/working-with-model-inventory.qmd#search-filter-and-sort-models){target="_blank"}. -1. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select **Documentation**. +1. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select **Development**. 1. Locate the {{< var vm.checker >}} box on the right and click **{{< fa check >}} Check Document**. diff --git a/site/guide/model-documentation/_locate-document-overview.qmd b/site/guide/model-documentation/_locate-document-overview.qmd index f0045ffa0c..7b03b39d41 100644 --- a/site/guide/model-documentation/_locate-document-overview.qmd +++ b/site/guide/model-documentation/_locate-document-overview.qmd @@ -11,7 +11,7 @@ To locate your document overview for a model: 1. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select the **Latest** or **Saved Versions** tab.^[[Work with document versions](/guide/model-documentation/work-with-document-versions.qmd)] -1. Click on a Documentation type file.^[[Working with model documents](/guide/templates/working-with-model-documents.qmd)] +1. Click on a Development type file.^[[Working with model documents](/guide/templates/working-with-model-documents.qmd)] :::: @@ -20,7 +20,7 @@ To locate your document overview for a model: 1. Select a model or [find your model by applying a filter or searching for it](/guide/model-inventory/working-with-model-inventory.qmd#search-filter-and-sort-models){target="blank"}. -1. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select **Documentation**. +1. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select **Development**. :::: diff --git a/site/guide/model-documentation/_view-test-result-metadata.qmd b/site/guide/model-documentation/_view-test-result-metadata.qmd index 2decae1692..eac461556b 100644 --- a/site/guide/model-documentation/_view-test-result-metadata.qmd +++ b/site/guide/model-documentation/_view-test-result-metadata.qmd @@ -2,11 +2,29 @@ Refer to the LICENSE file in the root of this repository for details. SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial --> -1. Locate the test result whose metadata you want to view. +:::: {.content-visible unless-format="revealjs"} +5. Locate the test result whose metadata you want to view. -1. Hover over the test result until the top-right hand corner menu appears and select **{{< fa clock >}} See Timeline**: +6. Hover over the test result until the top-right hand corner menu appears and select **{{< fa clock >}} See Timeline**: - The [Active]{.green-bg} (most recent) test result is highlighted. - On the test result timeline, click on the **{{< fa chevron-down >}}** associated with a test run to expand for details, such as **Model Information** or **Dataset Information**. - - When you are done, you can either click **Cancel** or **{{< fa x >}}** to close the metadata menu. \ No newline at end of file + + When you are done, you can either click **Cancel** or **{{< fa x >}}** to close the metadata menu.. + +:::: + + + + +:::: {.content-hidden unless-format="revealjs"} +4. Locate the test result whose metadata you want to view. + +5. Hover over the test result until the top-right hand corner menu appears and select **{{< fa clock >}} See Timeline**: + + - The [Active]{.green-bg} (most recent) test result is highlighted. + - On the test result timeline, click on the **{{< fa chevron-down >}}** associated with a test run to expand for details, such as **Model Information** or **Dataset Information**. + + When you are done, you can either click **Cancel** or **{{< fa x >}}** to close the metadata menu. + +:::: \ No newline at end of file diff --git a/site/guide/model-documentation/assign-documentation-section-statuses.qmd b/site/guide/model-documentation/assign-documentation-section-statuses.qmd index 69eb03df2b..b0df632ad8 100644 --- a/site/guide/model-documentation/assign-documentation-section-statuses.qmd +++ b/site/guide/model-documentation/assign-documentation-section-statuses.qmd @@ -32,7 +32,7 @@ You can only assign section statuses for PDFs converted to editable documents,[^ 3. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select the **Latest** tab.[^5] -4. Click on a Documentation type file.[^6] +4. Click on a Development type file.[^6] Jump to any section of the model documentation by selecting the relevant section you would like to view. diff --git a/site/guide/model-documentation/collaborate-with-others.qmd b/site/guide/model-documentation/collaborate-with-others.qmd index 99262f2426..251ab76cc3 100644 --- a/site/guide/model-documentation/collaborate-with-others.qmd +++ b/site/guide/model-documentation/collaborate-with-others.qmd @@ -43,7 +43,7 @@ While working with content blocks in documents,[^5] you can suggest changes inst 1. Click the **{{< fa file-pen >}} Track changes** button in the toolbar to turn on suggestion mode. -2. Make your changes to the model document. +2. Make your changes to the model document. When changes tracking is enabled, other contributors can accept or decline the suggested changes. @@ -61,9 +61,16 @@ While working with content blocks in documents,[^5] you can suggest changes inst - **Discard all suggestions** - **Discard all selected suggestions** +::: {.callout} +## Your administrators have the option to enable track changes by default for all content blocks for your organization.[^6] + +If you have write permissions for the document you're editing, you can choose to disable track changes on a per-block, per-editing session basis. + +::: + ## Revision history -While working with content blocks in documents,[^6] you can save named versions of your edits to retain specific revisions you'd like to keep. +While working with content blocks in documents,[^7] you can save named versions of your edits to retain specific revisions you'd like to keep. 1. Click the **{{< fa clock-rotate-left >}} Revision history** button in the toolbar. @@ -157,4 +164,6 @@ To delete a comment in a resolved thread, follow the same steps but click the ** [^5]: [Work with content blocks](work-with-content-blocks.qmd) -[^6]: [Work with content blocks](work-with-content-blocks.qmd) \ No newline at end of file +[^6]: [Managing your organization](/guide/configuration/managing-your-organization.qmd#manage-document-defaults) + +[^7]: [Work with content blocks](work-with-content-blocks.qmd) \ No newline at end of file diff --git a/site/guide/model-documentation/content-editing-toolbar.png b/site/guide/model-documentation/content-editing-toolbar.png index 55ecd31b50..846109487e 100644 Binary files a/site/guide/model-documentation/content-editing-toolbar.png and b/site/guide/model-documentation/content-editing-toolbar.png differ diff --git a/site/guide/model-documentation/content_blocks/_insert-mathematical-formulas.qmd b/site/guide/model-documentation/content_blocks/_insert-mathematical-formulas.qmd new file mode 100644 index 0000000000..ef0401768d --- /dev/null +++ b/site/guide/model-documentation/content_blocks/_insert-mathematical-formulas.qmd @@ -0,0 +1,45 @@ + + + + +:::: {.content-visible unless-format="revealjs" when-meta="includes.blocks"} +1. Click **$f(x)$** in the toolbar while editing a content block. + +1. Enter your LaTeX formula in the text box and confirm that the Equation preview generates as expected:[^equation-preview] + + - To insert the equation inline, leave **Display mode** untoggled. + - To insert the equation on its own line, toggle **Display mode** on. + + ![The two formula display modes](/guide/model-documentation/content_blocks/formula-display-modes.png){fig-alt="A screenshot showing the two formula display modes" .screenshot group="latex"} + +1. Click **[{{< fa check >}}]{.green}** to add the equation to your content block. + +:::: + + + +:::: {.content-visible unless-format="revealjs" unless-meta="includes.blocks"} +a. Click **$f(x)$** in the toolbar while editing a content block. + +b. Enter your LaTeX formula in the text box and confirm that the Equation preview generates as expected:[^equation-preview] + + - To insert the equation inline, leave **Display mode** untoggled. + - To insert the equation on its own line, toggle **Display mode** on. + + ![The two formula display modes](/guide/model-documentation/content_blocks/formula-display-modes.png){fig-alt="A screenshot showing the two formula display modes" .screenshot group="latex"} + +c. Click **[{{< fa check >}}]{.green}** to add the equation to your content block. + +:::: + +[^equation-preview]: ![Equation preview in the formula editor](/guide/model-documentation/content_blocks/equation-preview.png){fig-alt="A screenshot showing the equation preview in the formula editor" .screenshot group="latex"} diff --git a/site/guide/model-documentation/content_blocks/_metric-over-time-block.qmd b/site/guide/model-documentation/content_blocks/_metric-over-time-block.qmd index aa1ed46ca1..49bf8483cc 100644 --- a/site/guide/model-documentation/content_blocks/_metric-over-time-block.qmd +++ b/site/guide/model-documentation/content_blocks/_metric-over-time-block.qmd @@ -3,4 +3,4 @@ Refer to the LICENSE file in the root of this repository for details. SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial --> metric over time block [from library]{.smallercaps .pink} -: Can be added within default model documentation and default ongoing monitoring plans to display one of the supported metric over time results collected by the {{< var vm.developer >}}. \ No newline at end of file +: Can be added within ongoing monitoring reports to display one of the supported metric over time results collected by the {{< var vm.developer >}}. \ No newline at end of file diff --git a/site/guide/model-documentation/content_blocks/_reference-field-values.qmd b/site/guide/model-documentation/content_blocks/_reference-field-values.qmd new file mode 100644 index 0000000000..c5af2218a5 --- /dev/null +++ b/site/guide/model-documentation/content_blocks/_reference-field-values.qmd @@ -0,0 +1,44 @@ + + + + +:::: {.content-visible unless-format="revealjs" when-meta="includes.blocks"} +- The associated model's inventory fields^[[Edit model inventory fields](/guide/model-inventory/edit-model-inventory-fields.qmd)] +- Fields from artifacts logged on the associated model^[[Add and manage artifacts](/guide/model-validation/add-manage-artifacts.qmd)] + +1. Click **{ } Insert Variable {{< fa angle-down >}}** in the toolbar while editing a content block. + +1. Select the variable you want to reference from the drop-down menus: + + - **[model]{.smallcaps}** — Model inventory field values by field group. + - **[{artifact type}]{.smallcaps}** — Artifact field values grouped by artifact type, where `{artifact type}` represents the name of your artifact type. Only types of artifacts already logged on the associated model will appear in this selection. + + ::: {.callout} + Note that while you are able to select fields with empty values within available model inventory or artifact type fields, no value will be displayed in the content block until the field is populated. + + ::: + +1. Click away from the content block to apply your changes, and confirm that the correct variable value is displayed in the content block as expected. + +:::: + + + +:::: {.content-visible unless-format="revealjs" unless-meta="includes.blocks"} +a. Click **{ } Insert Variable {{< fa angle-down >}}** in the toolbar while editing a content block. + +b. Select the **Model {{< fa arrow-right >}}** variable you want to reference from the drop-down menu.^[Model inventory field values: [Edit model inventory fields](/guide/model-inventory/edit-model-inventory-fields.qmd)] + +c. Click away from the content block to apply your changes, and confirm that the correct variable value is displayed in the content block as expected. + +:::: \ No newline at end of file diff --git a/site/guide/model-documentation/content_blocks/_test-driven-block.qmd b/site/guide/model-documentation/content_blocks/_test-driven-block.qmd index ddd5e6c62a..2500c91bcc 100644 --- a/site/guide/model-documentation/content_blocks/_test-driven-block.qmd +++ b/site/guide/model-documentation/content_blocks/_test-driven-block.qmd @@ -3,4 +3,4 @@ Refer to the LICENSE file in the root of this repository for details. SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial --> test-driven block [from library]{.smallercaps .pink} -: Can be added within default model documentation and default ongoing monitoring plans to display one of the supported test results collected by the {{< var validmind.developer >}}. \ No newline at end of file +: Can be added within documents to display one of the supported test results collected by the {{< var validmind.developer >}}. \ No newline at end of file diff --git a/site/guide/model-documentation/equation-preview.png b/site/guide/model-documentation/content_blocks/equation-preview.png similarity index 100% rename from site/guide/model-documentation/equation-preview.png rename to site/guide/model-documentation/content_blocks/equation-preview.png diff --git a/site/guide/model-documentation/formula-display-modes.png b/site/guide/model-documentation/content_blocks/formula-display-modes.png similarity index 100% rename from site/guide/model-documentation/formula-display-modes.png rename to site/guide/model-documentation/content_blocks/formula-display-modes.png diff --git a/site/guide/model-documentation/view-documentation-guidelines.qmd b/site/guide/model-documentation/view-documentation-guidelines.qmd index bd394e98f1..8b088120aa 100644 --- a/site/guide/model-documentation/view-documentation-guidelines.qmd +++ b/site/guide/model-documentation/view-documentation-guidelines.qmd @@ -37,11 +37,11 @@ By default, the [{{< fa hand >}} Customer Admin]{.bubble} role has these permiss 3. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select the **Latest** tab.[^5] -4. Click on a Documentation type file.[^6] +4. Click on a Development type file.[^6] 5. In any section of the documentation for a model, click **{{< var vm.product >}} Insights™** in the top right corner to expand the **{{< var vm.product >}} Insights™** sidebar. - **Documentation Guidelines** are shown for each document section, associated with the template configured by your model validation team. + **Guidelines** are shown for each document section, associated with the template configured by your model validation team. diff --git a/site/guide/model-documentation/work-with-content-blocks.qmd b/site/guide/model-documentation/work-with-content-blocks.qmd index d06fd46eef..e138a779a8 100644 --- a/site/guide/model-documentation/work-with-content-blocks.qmd +++ b/site/guide/model-documentation/work-with-content-blocks.qmd @@ -8,6 +8,7 @@ aliases: - /guide/work-with-content-blocks.html includes: pdf-context: false + blocks: true --- Make edits to your model documents by adding or removing content blocks directly in the online editor. @@ -41,10 +42,11 @@ The content editing toolbar is a rich text editor that enables you to: - Undo or redo changes - Format your text, including adding hyperlinks and code blocks -- Insert LaTex formulas[^2] +- Reference related model inventory and artifact field values with variables[^2] +- Insert LaTex formulas[^3] - Attach images via upload or URL -You can also use the toolbar to suggest changes, save named versions of content, and leave comments.[^3] +You can also use the toolbar to suggest changes, save named versions of content, and leave comments.[^4] ::: {.callout title="Stuck on where to start?"} Use {{< var vm.product >}} to assist you with generating content via AI!^[[Generate content drafts with AI](#generate-content)] @@ -55,9 +57,9 @@ Use {{< var vm.product >}} to assist you with generating content via AI!^[[Gener ## Prerequisites - [x] {{< var link.login >}} -- [x] There are models registered in the model inventory.[^4] -- [x] Documents exist and are completed or are in progress for your model.[^5] -- [x] You are a [{{< fa code >}} Developer]{.bubble} or [{{< fa circle-check >}} Validator]{.bubble}, or assigned another role with sufficient permissions to perform the tasks in this guide.[^6] +- [x] There are models registered in the model inventory.[^5] +- [x] Documents exist and are completed or are in progress for your model.[^6] +- [x] You are a [{{< fa code >}} Developer]{.bubble} or [{{< fa circle-check >}} Validator]{.bubble}, or assigned another role with sufficient permissions to perform the tasks in this guide.[^7] ::: @@ -76,48 +78,40 @@ Use {{< var vm.product >}} to assist you with generating content via AI!^[[Gener #### [from library]{.smallcaps} Text Block - : Inserts a text block from a template in your block library:[^7] + : Inserts a text block from a template in your block library:[^8] a. Select the reusable blocks you want to add. b. Click **Insert # Text Block(s) to Document**. - Test-Driven[^8] + Test-Driven[^9] : Adds a new section with logged test results. - Metric Over Time[^9] + Metric Over Time[^10] : Adds a new section with logged metric over time results. ::: -7. After adding the block to your document, click on the text to make changes or add comments.[^10] +7. After adding the block to your document, click on the text to make changes or add comments.[^11] -### Add mathematical formulas +### Reference field values -While editing a simple text block, you can insert math equations using the formula editor: +While editing a simple text block within documents, you can reference values in the form of variables from: -1. Click **$f(x)$** in the toolbar while editing a content block. +{{< include content_blocks/_reference-field-values.qmd >}} -2. Enter your LaTeX formula in the text box and confirm that the Equation preview generates as expected: +### Insert mathematical formulas - - To insert the equation inline, leave **Display mode** untoggled. - - To insert the equation on its own line, toggle **Display mode** on. +While editing a simple text block within documents, you can insert math equations using the formula editor: -::: {.column-margin} -![Equation preview in the formula editor](equation-preview.png){fig-alt="A screenshot showing the equation preview in the formula editor" .screenshot group="latex"} - -::: - - ![The two formula display modes](formula-display-modes.png){fig-alt="A screenshot showing the two formula display modes" .screenshot group="latex"} - -3. Click **[{{< fa check >}}]{.green}** to add the equation to your content block. +{{< include content_blocks/_insert-mathematical-formulas.qmd >}} ### Generate content drafts with AI -While editing a simple text block, you can have {{< var vm.product >}} assist you with generating content drafts. +While editing a simple text block within documents, you can have {{< var vm.product >}} assist you with generating content drafts. ::: {.callout title="Have you logged your tests?"} -Generating content drafts works best after you've logged tests with the {{< var validmind.developer >}},[^11] as existing test descriptions and results provide more context for the {{< var vm.product >}} AI Content Builder to draw upon. +Generating content drafts works best after you've logged tests with the {{< var validmind.developer >}},[^12] as existing test descriptions and results provide more context for the {{< var vm.product >}} AI Content Builder to draw upon. ::: To generate content drafts: @@ -125,7 +119,7 @@ To generate content drafts: {{< include content_blocks/_generate-with-ai.qmd >}} -When generating content drafts with AI, accepted versions and edits are retained in your {{< fa wifi >}} Model Activity[^12] just like other updates to your documents. +When generating content drafts with AI, accepted versions and edits are retained in your {{< fa wifi >}} Model Activity[^13] just like other updates to your documents.
@@ -155,11 +149,11 @@ Test-driven or metric over time blocks can be re-added later on but **text block 1. In the left sidebar, click **{{< fa cubes >}} Inventory**. -2. Select a model or find your model by applying a filter or searching for it.[^13] +2. Select a model or find your model by applying a filter or searching for it.[^14] -3. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select the **Latest** tab.[^14] +3. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select the **Latest** tab.[^15] -4. Click on the document file you want to remove a block from.[^15] +4. Click on the document file you want to remove a block from.[^16] 5. Click on a section header to expand that section and remove content. @@ -174,30 +168,32 @@ Test-driven or metric over time blocks can be re-added later on but **text block [^1]: [Manage model documents](/guide/templates/manage-model-documents.qmd#add-model-documents) -[^2]: [Add mathematical formulas](#add-mathematical-formulas) +[^2]: [Reference field values](#reference-field-values) + +[^3]: [Insert mathematical formulas](#insert-mathematical-formulas) -[^3]: [Collaborate with others](collaborate-with-others.qmd) +[^4]: [Collaborate with others](collaborate-with-others.qmd) -[^4]: [Register models in the inventory](/guide/model-inventory/register-models-in-inventory.qmd) +[^5]: [Register models in the inventory](/guide/model-inventory/register-models-in-inventory.qmd) -[^5]: [Working with model documents](/guide/templates/working-with-model-documents.qmd) +[^6]: [Working with model documents](/guide/templates/working-with-model-documents.qmd) -[^6]: [Manage permissions](/guide/configuration/manage-permissions.qmd) +[^7]: [Manage permissions](/guide/configuration/manage-permissions.qmd) -[^7]: [Manage text block library](/guide/templates/manage-text-block-library.qmd) +[^8]: [Manage text block library](/guide/templates/manage-text-block-library.qmd) -[^8]: [Work with test results](/guide/model-documentation/work-with-test-results.qmd) +[^9]: [Work with test results](/guide/model-documentation/work-with-test-results.qmd) -[^9]: [Work with metrics over time](/guide/monitoring/work-with-metrics-over-time.qmd) +[^10]: [Work with metrics over time](/guide/monitoring/work-with-metrics-over-time.qmd) -[^10]: [Collaborate with others](/guide/model-documentation/collaborate-with-others.qmd) +[^11]: [Collaborate with others](/guide/model-documentation/collaborate-with-others.qmd) -[^11]: [Run tests and test suites](/developer/how-to/testing-overview.qmd) +[^12]: [Run tests and test suites](/developer/how-to/testing-overview.qmd) -[^12]: [View model activity](/guide/model-inventory/view-model-activity.qmd) +[^13]: [View model activity](/guide/model-inventory/view-model-activity.qmd) -[^13]: [Working with the model inventory](/guide/model-inventory/working-with-model-inventory.qmd#search-filter-and-sort-models) +[^14]: [Working with the model inventory](/guide/model-inventory/working-with-model-inventory.qmd#search-filter-and-sort-models) -[^14]: [Work with document versions](/guide/model-documentation/work-with-document-versions.qmd) +[^15]: [Work with document versions](/guide/model-documentation/work-with-document-versions.qmd) -[^15]: [Working with model documents](/guide/templates/working-with-model-documents.qmd) \ No newline at end of file +[^16]: [Working with model documents](/guide/templates/working-with-model-documents.qmd) \ No newline at end of file diff --git a/site/guide/model-documentation/work-with-test-results.qmd b/site/guide/model-documentation/work-with-test-results.qmd index 651f6ee0da..08c75e83c8 100644 --- a/site/guide/model-documentation/work-with-test-results.qmd +++ b/site/guide/model-documentation/work-with-test-results.qmd @@ -8,7 +8,7 @@ aliases: - /developer/model-documentation/work-with-test-results.html --- -Once generated via the {{< var validmind.developer >}}, view and add the test results to your model documentation in the {{< var validmind.platform >}}. +Once generated via the {{< var validmind.developer >}}, view and add the test results to your documentation in the {{< var validmind.platform >}}. ::: {.callout title="Test results cannot be attached to static PDF documents."} You can only upload and work with test results for PDFs converted to editable documents,[^1] documents generated by the {{< var validmind.developer >}}, or documents created in the {{< var validmind.platform >}}. @@ -22,22 +22,32 @@ You can only upload and work with test results for PDFs converted to editable do - [x] {{< var link.login >}} - [x] The model you want to attach test results to is registered in the model inventory.[^2] - [x] Test results have already been documented via the {{< var validmind.developer >}} for your model.[^3] -- [x] You are a [{{< fa code >}} Developer]{.bubble} or [{{< fa circle-check >}} Validator]{.bubble}, or assigned another role with sufficient permissions to perform the tasks in this guide.[^4] +- [x] You are a assigned a role with sufficient permissions to perform the tasks in this guide.[^4] + +
+ +::: {.callout-button .pl4 .nt4} +::: {.callout collapse="true" appearance="minimal"} +### What are the default permissions for adding test-driven blocks? +- By default, the [{{< fa code >}} Developer]{.bubble} role can add test-driven blocks within model documentation or ongoing-monitoring plans. +- By default, the [{{< fa circle-check >}} Validator]{.bubble} role can add test-driven blocks within validation reports. +- Permissions for custom document types are defined by your organization. + +::: +::: ::: ## Add test results - - {{< include _add-test-results.qmd >}} ### Configure test result displays -Once added to your model documentation, validation reports, and ongoing monitoring reports, configure how the logged test results are displayed: +Once added to your documents, configure how the logged test results are displayed: 1. Locate the test result whose display you want to configure, and hover over it until the result block's toolbar appears in the top right-hand corner. @@ -57,7 +67,9 @@ Once added to your model documentation, validation reports, and ongoing monitori 2. Select a model by clicking on it or find your model by applying a filter or searching for it.[^6] -3. In the left sidebar that appears for your model, under {{< fa file >}} Documents and select **Documentation**, **Validation Report**, or **Ongoing Monitoring**.[^7] +3. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select the **Latest** tab.[^7] + +4. Click on the document file you want to review test result metadata for.[^8] {{< include /guide/model-documentation/_view-test-result-metadata.qmd >}} @@ -108,4 +120,6 @@ Filters can be removed from the list of test result metadata by clicking on the [^6]: [Working with the model inventory](/guide/model-inventory/working-with-model-inventory.qmd#search-filter-and-sort-models) -[^7]: [Working with model documents](/guide/templates/working-with-model-documents.qmd) \ No newline at end of file +[^7]: [Work with document versions](/guide/model-documentation/work-with-document-versions.qmd) + +[^8]: [Working with model documents](/guide/templates/working-with-model-documents.qmd) \ No newline at end of file diff --git a/site/guide/model-inventory/_add-edit-inventory-fields.qmd b/site/guide/model-inventory/_add-edit-inventory-fields.qmd index 127bb57cbe..2317339d86 100644 --- a/site/guide/model-inventory/_add-edit-inventory-fields.qmd +++ b/site/guide/model-inventory/_add-edit-inventory-fields.qmd @@ -23,6 +23,8 @@ a. Include optional information for your field: a. When you are satisfied with the setup of your inventory field, click **Save**. +{{< include /guide/model-inventory/_example-next-review-date-and-days-remaining.qmd >}} + ### Edit inventory fields #### Edit custom inventory fields diff --git a/site/guide/model-inventory/_example-next-review-date-and-days-remaining.qmd b/site/guide/model-inventory/_example-next-review-date-and-days-remaining.qmd new file mode 100644 index 0000000000..187aba895b --- /dev/null +++ b/site/guide/model-inventory/_example-next-review-date-and-days-remaining.qmd @@ -0,0 +1,126 @@ + + +:::: {.content-visible unless-format="revealjs"} + +::: {.callout-button .pl4 .nt2} +::: {.callout collapse="true" appearance="minimal"} + +### Example — Next review date & days remaining + +Combining different inventory field types provides a flexible way to automatically track model review schedules. This example creates fields that calculate the next review date based on approval date and validation frequency, adjusted by risk tier, and then computes the days remaining until that review. + +Common date and time field types available in your formulas include: + +- `vm_today` — Today's date (updates each time the formula runs) +- `date` — Python's `datetime.date` class +- `datetime` — Python's `datetime.datetime` class +- `timedelta` — Duration in days, seconds, or microseconds +- `relativedelta` — Duration in months, years, etc. (from `dateutil`) + +Here, we show you how to use `date`, `relativedelta`, and `vm_today`. + +#### Calculate the next review date + +Determine the next review date based on an approval date and a frequency of validation that adjusts based on the risk tier: + +1. Create an `Approved Date` date field. + +2. Create a `Frequency of Validation` string field with the following options: + + - Weekly + - Monthly + - Yearly + +3. Create a `Risk Tier` calculation field that depends on the frequency of validation: + + ```python + def formula(params): + if params.frequencyOfValidation == "Weekly": + return "Tier 1" + elif params.frequencyOfValidation == "Monthly": + return "Tier 2" + elif params.frequencyOfValidation == "Yearly": + return "Tier 3" + else: + return "N/A" + ``` + +4. Create a `Next Review Date` calculation field that depends on the `Approved Date` and `Risk Tier` fields: + + ```python + def formula(params): + """ + Calculate the next review date based on the approval date and review frequency. + + Args: + params.dmApprovedDate (str): The approval date in 'YYYY-MM-DD' format. + params.dmRiskTier (string): The review tier (Tier 1, Tier 2, or Tier 3). + + Returns: + datetime: The next review date. + """ + # Guard against empty dates + if params.dmApprovedDate == "": + return "N/A" + + # Convert the approved_on date string to a datetime object + approved_date = date.fromtimestamp((int(params.dmApprovedDate))/1000) + + # Define the review frequency mapping + review_frequency = { + "Tier 1": relativedelta(months=3), # Quarterly + "Tier 2": relativedelta(months=6), # Semi-annually + "Tier 3": relativedelta(years=1), # Annually + } + + # Get the appropriate time delta based on the tier + frequency = review_frequency.get(params.dmRiskTier) + if not frequency: + # "Invalid tier. Must be Tier 1, Tier 2, or Tier 3." + return "N/A" + + # Calculate the next review date + next_review_date = approved_date + frequency + return next_review_date.isoformat() + ``` + +![Adding a calculation type field that automatically calculates the next review date](calculation-field-next-review-date.png){fig-alt="A screenshot showing the screen for adding a calculation type field that automatically calculates the next review date" width=90% .screenshot} + +You can now determine the next review date in workflows by making a workflow depend on `Approved Date`. To test, change the `Approved Date` after the fact and see how `Next Review Date` changes. + +#### Calculate the days remaining until the next review + +1. Create a `Days Remaining` calculation field that depends on the `Next Review Date` field: + + ```python + def formula(params): + """ + Calculate days remaining until the next review. + + Args: + params.nextReviewDate (str): The next review date in ISO format. + + Returns: + str: Days remaining until review (example:"45 days remaining"). + """ + # Get next review date (stored as ISO format string) + next_review_date = getattr(params, "nextReviewDate", "") + if not next_review_date: + return "Not applicable" + next_review = date.fromisoformat(next_review_date) + + # Calculate days until review using built-in vm_today + difference = next_review - vm_today + return f"{difference.days} days remaining" + ``` + + +![Adding a calculation type field that automatically calculates the days remaining until the next review](calculation-field-days-remaining.png){fig-alt="A screenshot showing the screen for adding a calculation type field that automatically calculates the days remaining until the next review" width=90% .screenshot} + +You can now check the number of days remaining until the next review. +::: +::: + +:::: diff --git a/site/guide/model-inventory/_field-types.qmd b/site/guide/model-inventory/_field-types.qmd index 3a29d11c5e..677fa30897 100644 --- a/site/guide/model-inventory/_field-types.qmd +++ b/site/guide/model-inventory/_field-types.qmd @@ -22,7 +22,7 @@ Aggregation ::: {.callout-button .pl4 .nt4} ::: {.callout collapse="true" appearance="minimal"} -### Example — Aggregation Use Cases +### Example — Aggregation use cases ##### Open validation issue count @@ -78,7 +78,7 @@ Calculation ::: {.callout-button .pl4 .nt4} ::: {.callout collapse="true" appearance="minimal"} -### Example — Risk Tier Calculation +### Example — Risk tier calculation You have numeric model inventory fields of `materiality` and `complexity`, where a larger value indicates a lower risk. @@ -107,7 +107,6 @@ Your calculated field is grouped under `Model Risk` inventory fields, and can on ::: ::: - :::: :::: {.content-visible when-format="html" when-meta="includes.artifacts"} @@ -120,7 +119,7 @@ Your calculated field is grouped under `Model Risk` inventory fields, and can on ::: {.callout-button .pl4 .nt4} ::: {.callout collapse="true" appearance="minimal"} -### Use artifact types in your formulas to create dynamic calculations. +### Example — Use artifact types in your formulas to create dynamic calculations Use simple dot notation to include [artifact types](/guide/model-validation/manage-artifact-types.qmd){target="_blank"}: diff --git a/site/guide/model-inventory/_reference-field-values.qmd b/site/guide/model-inventory/_reference-field-values.qmd new file mode 100644 index 0000000000..e0629c1109 --- /dev/null +++ b/site/guide/model-inventory/_reference-field-values.qmd @@ -0,0 +1,63 @@ + + + + +:::: {.content-visible unless-format="revealjs" when-meta="includes.inventory"} +::: {.callout-button .pl4 .nt4} +::: {.callout collapse="true" appearance="minimal"} +## Reference inventory field values in your content blocks within documents and library text blocks. + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-50-ns .tc} +[Work with content blocks](/guide/model-documentation/work-with-content-blocks.qmd#reference-field-values){.button .button-green} + +::: + +::: {.w-50-ns .tc} +[Manage text block library](/guide/templates/manage-text-block-library.qmd#add-text-blocks){.button .button-green} + +::: + +:::: +::: +::: + +:::: + + + +:::: {.content-visible unless-format="revealjs" unless-meta="includes.inventory"} +
+ +::: {.callout-button .pl4 .nt4} +::: {.callout collapse="true" appearance="minimal"} +## Reference artifact field values in your content blocks within documents and library text blocks. + +:::: {.flex .flex-wrap .justify-around} + +::: {.w-50-ns .tc} +[Work with content blocks](/guide/model-documentation/work-with-content-blocks.qmd#reference-field-values){.button .button-green} + +::: + +::: {.w-50-ns .tc} +[Manage text block library](/guide/templates/manage-text-block-library.qmd#add-text-blocks){.button .button-green} + +::: + +:::: +::: +::: + +:::: \ No newline at end of file diff --git a/site/guide/model-inventory/_view-model-activity-overview.qmd b/site/guide/model-inventory/_view-model-activity-overview.qmd index 792880d2ee..5d5f44651a 100644 --- a/site/guide/model-inventory/_view-model-activity-overview.qmd +++ b/site/guide/model-inventory/_view-model-activity-overview.qmd @@ -2,26 +2,28 @@ Refer to the LICENSE file in the root of this repository for details. SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial --> -The **{{< fa wifi >}} Model Activity** page shows a record of all activities, including actions performed by users in your organization and updates generated via the {{< var vm.developer >}} such as: +The **{{< fa wifi >}} Model Activity** page shows a record of all activities, including actions performed by users in your organization, activity from automated workflows, and updates generated via the {{< var validmind.developer >}} on a per model basis, such as: -:::: {.content-visible unless-format="revealjs"} -- Inventory fields updated on the model^[[Edit model inventory fields](/guide/model-inventory/edit-model-inventory-fields.qmd)] +:::: {.content-visible unless-format="revealjs"} +- Inventory fields updated^[[Edit model inventory fields](/guide/model-inventory/edit-model-inventory-fields.qmd)] - Model stage transitions^[[Manage model stages](/guide/workflows/manage-model-stages.qmd)] -- Updates to model documentation, validation reports,^[[Working with model documentation](/guide/model-documentation/working-with-model-documentation.qmd)] or ongoing monitoring plans^[[Ongoing monitoring](/guide/monitoring/ongoing-monitoring.qmd)] -- Test results added to your model^[[{{< var validmind.developer >}}](/developer/validmind-library.qmd)] +- Updates to documents: model documentation,^[[Working with model documentation](/guide/model-documentation/working-with-model-documentation.qmd)] validation reports,^[[Preparing validation reports](/guide/model-validation/preparing-validation-reports.qmd)] ongoing monitoring reports^[[Ongoing monitoring](/guide/monitoring/ongoing-monitoring.qmd)], or custom documents +- Test results or metrics added via the {{< var vm.developer >}}^[[{{< var validmind.developer >}}](/developer/validmind-library.qmd)] - Artifacts added, updated, or removed^[[Working with artifacts](/guide/model-validation/working-with-artifacts.qmd)] -- Comment creation and replies^[[Commenting](/guide/model-documentation/collaborate-with-others.qmd#commenting)] -- Model stakeholders added or removed from your model^[[Manage model stakeholder types](/guide/configuration/manage-model-stakeholder-types.qmd#manage-stakeholder-types-on-models)] +- User comment creation and replies^[[Commenting](/guide/model-documentation/collaborate-with-others.qmd#commenting)] +- Model stakeholders added or removed^[[Manage model stakeholder types](/guide/configuration/manage-model-stakeholder-types.qmd#manage-stakeholder-types-on-models)] +- Workflow activity^[[Working with workflows](/guide/workflows/working-with-workflows.qmd)] :::: :::: {.content-hidden unless-format="revealjs"} -- Inventory fields updated on the model +- Inventory fields updated - Model stage transitions -- Updates to model documentation, validation reports, or ongoing monitoring plans -- Test results added to your model +- Updates to documents: model documentation, validation reports, ongoing monitoring reports, or custom documents +- Test results or metrics added via the {{< var vm.developer >}} - Artifacts added, updated, or removed -- Comment creation and replies -- Model stakeholders added or removed from your model +- User comment creation and replies +- Model stakeholders added or removed +- Workflow activity :::: \ No newline at end of file diff --git a/site/guide/model-inventory/calculation-field-days-remaining.png b/site/guide/model-inventory/calculation-field-days-remaining.png new file mode 100644 index 0000000000..b7044e1eec Binary files /dev/null and b/site/guide/model-inventory/calculation-field-days-remaining.png differ diff --git a/site/guide/model-inventory/calculation-field-next-review-date.png b/site/guide/model-inventory/calculation-field-next-review-date.png new file mode 100644 index 0000000000..718c8ce01d Binary files /dev/null and b/site/guide/model-inventory/calculation-field-next-review-date.png differ diff --git a/site/guide/model-inventory/edit-model-inventory-fields.qmd b/site/guide/model-inventory/edit-model-inventory-fields.qmd index eca4ac0097..8c62a34457 100644 --- a/site/guide/model-inventory/edit-model-inventory-fields.qmd +++ b/site/guide/model-inventory/edit-model-inventory-fields.qmd @@ -8,6 +8,7 @@ aliases: - /guide/edit-model-inventory-fields.html includes: pdf-context: false + inventory: true --- Edit individual detail fields on a model to ensure that model details are accurate and up-to-date. @@ -28,13 +29,19 @@ Edit individual detail fields on a model to ensure that model details are accura 2. Select a model or find your model by applying a filter or searching for it.[^4] -3. Hover over a field until the **{{< fa pencil >}}** appears, then click to make your changes.[^5] +3. On the model details page, click on an inventory field to make your changes.[^5] 4. Click **Save** to apply your changes. +Changes you make to model inventory fields are automatically logged to your model's activity.[^6] + +
+ +{{< include /guide/model-inventory/_reference-field-values.qmd >}} + ### Manage model stakeholders -1. On the model's detail page, click on the model stakeholder type[^6] you'd like to add users to. +1. On the model's detail page, click on the model stakeholder type[^7] you'd like to add users to. 2. Select your desired users from the drop-down menu. @@ -48,15 +55,15 @@ Edit individual detail fields on a model to ensure that model details are accura :::: {.content-visible when-format="html" when-meta="includes.pdf-context"} ::: {.callout title="Uploaded files must be less than 50 MB each in size."} -- To work with attachments on models, first add an attachment inventory field.[^7] -- Select model attachments in `.pdf` format as context documents when generating text blocks with AI within your documents.[^8] +- To work with attachments on models, first add an attachment inventory field.[^8] +- Select model attachments in `.pdf` format as context documents when generating text blocks with AI within your documents.[^9] ::: :::: :::: {.content-visible when-format="html" unless-meta="includes.pdf-context"} ::: {.callout title="Uploaded files must be less than 50 MB each in size."} -To work with attachments on models, first add an attachment inventory field.[^7] +To work with attachments on models, first add an attachment inventory field.[^8] ::: :::: @@ -97,12 +104,14 @@ To work with attachments on models, first add an attachment inventory field.[^7] ::: -[^6]: [Manage model stakeholder types](/guide/configuration/manage-model-stakeholder-types.qmd) +[^6]: [View model activity](view-model-activity.qmd) + +[^7]: [Manage model stakeholder types](/guide/configuration/manage-model-stakeholder-types.qmd) -[^7]: [Manage model inventory fields](manage-model-inventory-fields.qmd#inventory-field-types) +[^8]: [Manage model inventory fields](manage-model-inventory-fields.qmd#inventory-field-types) :::: {.content-visible when-format="html" when-meta="includes.pdf-context"} -[^8]: [Work with content blocks](/guide/model-documentation/work-with-content-blocks.qmd#generate-content) +[^9]: [Work with content blocks](/guide/model-documentation/work-with-content-blocks.qmd#generate-content) ::: diff --git a/site/guide/model-inventory/manage-model-inventory-fields.qmd b/site/guide/model-inventory/manage-model-inventory-fields.qmd index 3970ac700f..1119c23448 100644 --- a/site/guide/model-inventory/manage-model-inventory-fields.qmd +++ b/site/guide/model-inventory/manage-model-inventory-fields.qmd @@ -22,6 +22,10 @@ includes: ::: +## Inventory field types + +{{< include /guide/model-inventory/_field-types.qmd >}} + ## View, search, and filter inventory fields 1. In the left sidebar, click **{{< fa gear >}} Settings**. @@ -87,10 +91,6 @@ To rename the field keys for custom inventory fields: {{< include /guide/model-inventory/_rename-field-keys.qmd >}} -#### Inventory field types - -{{< include /guide/model-inventory/_field-types.qmd >}} - ## Add inventory field groups To group model inventory fields, first create an inventory field group: diff --git a/site/guide/model-inventory/view-model-activity.qmd b/site/guide/model-inventory/view-model-activity.qmd index f953be2b52..43c4697cca 100644 --- a/site/guide/model-inventory/view-model-activity.qmd +++ b/site/guide/model-inventory/view-model-activity.qmd @@ -20,43 +20,117 @@ Use the audit trail functionality in the {{< var validmind.platform >}} to track ::: -## Model Activity +## View all model activity {{< include _view-model-activity-overview.qmd >}} {{< include _view-model-activity-steps.qmd >}} -### Filter activity +## Filter all model activity -Use the **{{< fa filter >}} Filter** button to narrow down activity by category or specific values. +Filter to narrow down all model activity per model by category or specific values: -1. Click **{{< fa filter >}} Filter** to open the filter popover. +1. Select a model or find your model by applying a filter or searching for it.[^2] -2. Click **{{< fa plus >}} Add Rule** to add a filter condition. +2. In the expanded sidebar that appears for your model, click **{{< fa wifi >}} Model Activity**. -3. From the dropdown, select a filter option: +3. Click **{{< fa filter >}} Filter** to open the filter popover. - Event Category - : Shows _all events_ in a category. For example, selecting **Artifacts: Validation Issue** shows all validation issue events. Options include artifact types (such as Artifacts: Validation Issue, Artifacts: Policy Exception, or Artifacts: Model Limitation), Model Documents, Model Inventory Fields, Model Lifecycle, Model Metrics, Model Stakeholders, and Workflows. +4. Click **{{< fa plus >}} Add Rule** to add a filter condition. - Filter by Value - : Shows events for _specific items you select_. Pick a category, then choose one or more specific items from the second dropdown. For example, choose a particular validation issue, workflow, or custom field. +5. From the **Select category** dropdown, choose your filter options: -4. (Optional) Add more rules by repeating the previous steps. Multiple rules combine with AND logic — for example, filter by a category and a specific user to see only that user's activity in that category. + ::: {.panel-tabset} -5. Click **Apply Filters**. + ### Event Category -### View active filters + Shows all events in a category: + + - **Artifacts**[^3] — Artifacts logged on the model, by type.[^4] Only types of artifacts logged on the model will appear for selection. + - **Model Metrics**[^5] — Metrics over time logged on the model. + - **Model Stakeholders**[^6] — Stakeholders added or removed from the model. + - **Model Inventory Fields**[^7] — Inventory fields updated on the model. + - **Model Documents**[^8] — Updates made to documents on the model, including any logged test results from the {{< var validmind.developer >}}.[^9] + - **Model Lifecycle**[^10] — Model stage transitions. + - **Workflows**[^11] — Workflow activity on the model. + + ### Filter by Value + + Shows events for specific values in a category you select. Select a value key, then choose one or more specific values from the secondary dropdown: + + - **Artifacts**[^12] — Artifacts logged on the model, by type.[^13] Only types of artifacts logged on the model will appear for selection. + - **Model Inventory Fields**[^14] — Inventory fields updated on the model. Only inventory fields populated on the model will appear for selection. + - **Model Documents**[^15] — Updates made to documents on the model, including any logged test results from the {{< var validmind.developer >}}.[^16] Only documents with activity will appear for selection. + - **Workflows**[^17] — Workflow activity on the model. Only workflows applied to the model will appear for selection. + - **User**[^18] — Events performed by user. Only users with activity on the model will appear for selection. + + ::: + + (Optional) Add more rules by repeating steps 5 and 6. Multiple rules combine with AND logic.[^19] + +6. Click **Apply Filters** to apply your filter rules. + +#### Remove active filters After applying filters, active conditions appear as chips below the header. Click {{< fa xmark >}} on any chip to remove that filter, or click **Clear All Filters** to reset. -### Share filtered views +#### Share filtered views Filters persist in the URL, so you can share or bookmark a filtered view. +## View individual model field activity + +To view activity for individual model inventory fields:[^20] + +1. In the left sidebar, click **{{< fa cubes >}} Inventory**. + +1. Select a model or find your model by applying a filter or searching for it.[^21] + +1. Hover over the field whose individual activity you want to view until the **{{< fa circle-info >}}** icon appears. + +1. Hover over **{{< fa circle-info >}}** to view individual activity for that field. + [^1]: [Register models in the inventory](/guide/model-inventory/register-models-in-inventory.qmd) +[^2]: [Working with the model inventory](/guide/model-inventory/working-with-model-inventory.qmd#search-filter-and-sort-models) + +[^3]: [Working with artifacts](/guide/model-validation/working-with-artifacts.qmd) + +[^4]: For example, selecting **Artifacts: Validation Issue** shows all validation issue artifact type events. + +[^5]: [Work with metrics over time](/guide/monitoring/work-with-metrics-over-time.qmd) + +[^6]: [Manage model stakeholder types](/guide/configuration/manage-model-stakeholder-types.qmd#manage-stakeholder-types-on-models) + +[^7]: [Edit model inventory fields](edit-model-inventory-fields.qmd) + +[^8]: [Working with model documentation](/guide/model-documentation/working-with-model-documentation.qmd) + +[^9]: [{{< var validmind.developer >}}](/developer/validmind-library.qmd) + +[^10]: [Manage model stages](/guide/workflows/manage-model-stages.qmd) + +[^11]: [Working with workflows](/guide/workflows/working-with-workflows.qmd) + +[^12]: [Working with artifacts](/guide/model-validation/working-with-artifacts.qmd) + +[^13]: For example, selecting **Artifacts: Validation Issue** shows a secondary selection for the names of validation issue artifact type events. + +[^14]: [Edit model inventory fields](edit-model-inventory-fields.qmd) + +[^15]: [Working with model documentation](/guide/model-documentation/working-with-model-documentation.qmd) + +[^16]: [{{< var validmind.developer >}}](/developer/validmind-library.qmd) + +[^17]: [Working with workflows](/guide/workflows/working-with-workflows.qmd) + +[^18]: [Manage users](/guide/configuration/manage-users.qmd) + +[^19]: For example, filter by an event category AND a specific user to see only that user's activity in that category. + +[^20]: [Edit model inventory fields](edit-model-inventory-fields.qmd) +[^21]: [Working with the model inventory](/guide/model-inventory/working-with-model-inventory.qmd#search-filter-and-sort-models) diff --git a/site/guide/model-validation/_add-artifacts-on-documentation.qmd b/site/guide/model-validation/_add-artifacts-on-documentation.qmd index 09971d4a0c..41fe565374 100644 --- a/site/guide/model-validation/_add-artifacts-on-documentation.qmd +++ b/site/guide/model-validation/_add-artifacts-on-documentation.qmd @@ -9,11 +9,11 @@ SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial --> 1. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select the **Latest** tab.^[[Work with document versions](/guide/model-documentation/work-with-document-versions.qmd)] -1. Click on a Documentation type file.^[[Working with model documentation](/guide/model-documentation/working-with-model-documentation.qmd#document-overview)] +1. Click on a Development type file.^[[Working with model documentation](/guide/model-documentation/working-with-model-documentation.qmd#document-overview)] -1. On your model's Documentation page, click **{{< fa square-plus >}} Add Model Artifact**. +1. On your model's documentation page, click **{{< fa square-plus >}} Add Model Artifact**. 1. Select the artifact type^[[Manage artifact types](/guide/model-validation/manage-artifact-types.qmd)] you want to add: @@ -34,11 +34,11 @@ SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial --> 1. Select a model or [find your model by applying a filter or searching for it](/guide/model-inventory/working-with-model-inventory.qmd#search-filter-and-sort-models){target="_blank"}. -1. In the left sidebar that appears for your model, click **Documentation** under {{< fa file >}} Documents. +1. In the left sidebar that appears for your model, click **Development** under {{< fa file >}} Documents. -1. On your model's Documentation page, click **{{< fa plus >}} Add Model Artifact**. +1. On your model's documentation page, click **{{< fa plus >}} Add Model Artifact**. 1. Select the [artifact type](/guide/model-validation/manage-artifact-types.qmd){target="_blank"} you want to add: diff --git a/site/guide/model-validation/_add-edit-artifact-types.qmd b/site/guide/model-validation/_add-edit-artifact-types.qmd index 0793931379..a0cb3cb627 100644 --- a/site/guide/model-validation/_add-edit-artifact-types.qmd +++ b/site/guide/model-validation/_add-edit-artifact-types.qmd @@ -50,6 +50,7 @@ SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial --> - **[update permissions]{.smallcaps}** — Model stakeholder types or user roles that can update existing artifacts of this type. :::: + [^add-fields-callout]: diff --git a/site/guide/model-validation/_link-artifacts-to-reports.qmd b/site/guide/model-validation/_link-artifacts-to-reports.qmd index aa7989e3a3..542f725eee 100644 --- a/site/guide/model-validation/_link-artifacts-to-reports.qmd +++ b/site/guide/model-validation/_link-artifacts-to-reports.qmd @@ -9,7 +9,7 @@ SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial --> 1. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select the **Latest** tab.^[[Work with document versions](/guide/model-documentation/work-with-document-versions.qmd)] -1. Click on a Validation Report type file.^[[Preparing validation reports](/guide/model-validation/preparing-validation-reports.qmd#validation-report-overview)] +1. Click on a Validation type file.^[[Preparing validation reports](/guide/model-validation/preparing-validation-reports.qmd#validation-report-overview)] 1. Expand any subsection of the validation report you would like to work with. @@ -31,7 +31,7 @@ SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial --> :::: {.content-hidden unless-format="revealjs"} 1. From the **{{< fa cubes >}} Inventory** in the ValidMind Platform, go to the model you connected to earlier. -1. In the left sidebar that appears for your model, click **Validation Report** under {{< fa file >}} Documents. +1. In the left sidebar that appears for your model, click **Validation** under {{< fa file >}} Documents. 1. Locate the Data Preparation section and click on **2.2.2. Model Performance** to expand that section. diff --git a/site/guide/model-validation/_link-validator-evidence.qmd b/site/guide/model-validation/_link-validator-evidence.qmd index 26e092e2d0..d629572d86 100644 --- a/site/guide/model-validation/_link-validator-evidence.qmd +++ b/site/guide/model-validation/_link-validator-evidence.qmd @@ -13,13 +13,13 @@ To link validator evidence to a report, you must first log tests as a validator 1. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select the **Latest** tab.^[[Work with document versions](/guide/model-documentation/work-with-document-versions.qmd)] -1. Click on a Validation Report type file.^[[Preparing validation reports](/guide/model-validation/preparing-validation-reports.qmd#validation-report-overview)] +1. Click on a Validation type file.^[[Preparing validation reports](/guide/model-validation/preparing-validation-reports.qmd#validation-report-overview)] You can now expand any subsection of the validation report you would like to work with. 1. In any section of the report where the button is available, click **{{< fa link >}} Link Evidence to Report**. -1. On the **Link Validator Evidence to Validation Report** page that opens, select the evidence that is related to your assessment. +1. On the **Link Validator Evidence to Validation Document** page that opens, select the evidence that is related to your assessment. If you are not sure if something is relevant, you can expand the section by clicking **{{< fa chevron-down >}}** for more details. @@ -41,7 +41,7 @@ While the example below focuses on a specific test result, you can follow the sa ### 1. Link data quality test results a. From the **{{< fa cubes >}} Inventory** in the {{< var validmind.platform >}}, go to the model you connected to earlier. -a. In the left sidebar that appears for your model, click **Validation Report** under {{< fa file >}} Documents. +a. In the left sidebar that appears for your model, click **Validation** under {{< fa file >}} Documents. a. Locate the Data Preparation section and click on **2.2.1. Data Quality** to expand that section. diff --git a/site/guide/model-validation/_provide-compliance-assessments.qmd b/site/guide/model-validation/_provide-compliance-assessments.qmd index 075e4db216..a31799f460 100644 --- a/site/guide/model-validation/_provide-compliance-assessments.qmd +++ b/site/guide/model-validation/_provide-compliance-assessments.qmd @@ -13,7 +13,7 @@ Your organization can customize these options to match its own assessment framew 1. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select the **Latest** tab.^[[Work with document versions](/guide/model-documentation/work-with-document-versions.qmd)] -1. Click on a Validation Report type file.^[[Preparing validation reports](/guide/model-validation/preparing-validation-reports.qmd#validation-report-overview)] +1. Click on a Validation type file.^[[Preparing validation reports](/guide/model-validation/preparing-validation-reports.qmd#validation-report-overview)] Expand any subsection of the validation report you would like to work with. diff --git a/site/guide/model-validation/add-manage-artifacts.qmd b/site/guide/model-validation/add-manage-artifacts.qmd index 21751ff3ea..859a3beb46 100644 --- a/site/guide/model-validation/add-manage-artifacts.qmd +++ b/site/guide/model-validation/add-manage-artifacts.qmd @@ -1,16 +1,18 @@ ---- + --- # Copyright © 2023-2026 ValidMind Inc. All rights reserved. # Refer to the LICENSE file in the root of this repository for details. # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial title: "Add and manage artifacts" date: last-modified -aliases: +aliases: - add-update-model-findings.html - add-manage-model-findings.html --- Add artifacts at the model or documentation section level, update your artifacts to include proposed remediation plans and supporting documentation, or delete artifacts that no longer need to be tracked. +{{< include /guide/model-inventory/_reference-field-values.qmd >}} + ::: {.attn} ## Prerequisites @@ -37,7 +39,7 @@ When linking artifacts to validation reports,[^5] you can add artifacts as you a 3. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select the **Latest** tab.[^7] -4. Click on a Validation Report type file.[^8] +4. Click on a Validation type file.[^8] 5. In any section of the report where the button is available, click **{{< fa link >}} Link Artifact to Report** and select the artifact type[^9] you want to add: diff --git a/site/guide/model-validation/assess-compliance.qmd b/site/guide/model-validation/assess-compliance.qmd index a0c98a938e..5887d6baab 100644 --- a/site/guide/model-validation/assess-compliance.qmd +++ b/site/guide/model-validation/assess-compliance.qmd @@ -44,13 +44,13 @@ To link evidence logged by developers^[[{{< var validmind.developer >}}](/develo 3. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select the **Latest** tab.[^7] -4. Click on a Validation Report type file.[^8] +4. Click on a Validation type file.[^8] Expand any subsection of the validation report you would like to work with. 5. In any section of the report where the button is available, click **{{< fa link >}} Link Developer Evidence**. -6. On the **Link Developer Evidence to Validation Report** page that opens, select the evidence that is related to your assessment. +6. On the **Link Developer Evidence to Validation Document** page that opens, select the evidence that is related to your assessment. - Available evidence is associated with the relevant documentation section. - If you are not sure if something is relevant, you can expand the section by clicking **{{< fa chevron-down >}}** for more details. diff --git a/site/guide/model-validation/configure-assessment-options.qmd b/site/guide/model-validation/configure-assessment-options.qmd index 49c16f9bb0..728192fd2f 100644 --- a/site/guide/model-validation/configure-assessment-options.qmd +++ b/site/guide/model-validation/configure-assessment-options.qmd @@ -28,7 +28,7 @@ To configure assessment options on individual validation report templates: 2. Under Documents, select **Templates**.[^2] -3. Select the **Validation Report** tab. +3. Select the **Validation** tab. 4. Click the validation report template you want to configure assessment options for. diff --git a/site/guide/model-validation/manage-artifact-fields.qmd b/site/guide/model-validation/manage-artifact-fields.qmd index d9f5874395..1f2d6d15d4 100644 --- a/site/guide/model-validation/manage-artifact-fields.qmd +++ b/site/guide/model-validation/manage-artifact-fields.qmd @@ -21,6 +21,10 @@ includes: ::: +## Artifact field types + +{{< include /guide/model-inventory/_field-types.qmd >}} + ## View, search, and filter artifacts fields 1. In the left sidebar, click **{{< fa gear >}} Settings**. @@ -86,10 +90,6 @@ To rename the field keys for artifact fields: {{< include /guide/model-inventory/_rename-field-keys.qmd >}} -#### Artifact field types - -{{< include /guide/model-inventory/_field-types.qmd >}} - ## Add artifact field groups To group artifact fields, first create an artifact field group: diff --git a/site/guide/model-validation/manage-artifact-types.qmd b/site/guide/model-validation/manage-artifact-types.qmd index 6e72b18b6f..0f3b7d35a6 100644 --- a/site/guide/model-validation/manage-artifact-types.qmd +++ b/site/guide/model-validation/manage-artifact-types.qmd @@ -11,9 +11,9 @@ aliases: {{< include /guide/model-validation/_artifact-types.qmd >}} -#### Available artifact types +#### Artifact types -By default, the {{< var validmind.platform >}} provides three artifact types: +By default, the {{< var validmind.platform >}} provides three native artifact types: Validation Issue : Problems discovered during model validation that reflect errors, inconsistencies, or gaps in data, methodology, implementation, or documentation that need to be addressed before the model is considered reliable. @@ -24,6 +24,28 @@ Policy Exception Model Limitation : Inherent weaknesses or constraints of the model itself that cannot be fully resolved; definitions under which the model's results should be interpreted with caution. +
+ +::: {.callout-button .pl4 .nt4} +::: {.callout collapse="true" appearance="minimal"} +### Can I create custom artifact types? + +Yes, you can create custom artifact types to track categories of observations beyond the defaults. Common examples include: + +Change management +: Track requested changes to models, their approval status, and implementation timeline. Useful when model updates require formal change control processes. + +Audit findings +: Capture observations from internal or external audits that require follow-up. Separating these from validation issues helps distinguish regulatory audit trails from routine validation work. + +Enhancement requests +: Log suggested improvements that don't rise to the level of a validation issue. Helps capture ideas for future model iterations without cluttering the validation issue backlog. + +Custom artifact types support the same features as default types, including statuses, fields, permissions, workflow triggers, and analytics exports. + +::: +::: + ::: {.attn} ## Prerequisites @@ -50,6 +72,45 @@ Model Limitation {{< include /guide/model-validation/_add-edit-artifact-statuses.qmd >}} +## Archive and delete artifact types + +::: {.callout-important title="Deletion of artifact types is permanent."} +- Native (default) artifact types cannot be deleted, only archived.[^4] When archiving a native artifact type, existing artifacts of that type will be hidden and no new artifacts of that type can be created. +- When archiving a custom artifact type, existing artifacts of that type will remain viewable, but no new artifacts of that type can be created. + +::: + +1. In the left sidebar, click **{{< fa gear >}} Settings**. + +2. Under Artifacts, select **Artifact Types**. + +3. Hover over the artifact type you want to archive or delete. + +5. When the **{{< fa ellipsis-vertical >}}** appears, click [**{{< fa box-archive >}} Archive Artifact Type**]{.red} under the Actions column. + +6. After you confirm, the artifact type will be archived. Once archived, custom artifact types can be deleted permanently or restored for use: + +::: {.panel-tabset} + +#### Delete artifact types + +::: {.callout-important} +Only artifact types with no active artifact records can be deleted. + +::: + +a. Hover over the artifact type you want to delete. +b. When the **{{< fa ellipsis-vertical >}}** appears, click [**{{< fa trash-can >}} Delete**]{.red} under the Actions column. +c. After you confirm, the artifact type will be deleted permanently. + +#### Restore artifact types + +a. Hover over the artifact type you want to restore. +b. When the **{{< fa ellipsis-vertical >}}** appears, click **{{< fa undo >}} Restore** under the Actions column. +c. After you confirm, the artifact type will be restored. + +::: + @@ -59,5 +120,7 @@ Model Limitation [^3]: [Add artifact types](#add-artifact-types) +[^4]: [Artifact types](#artifact-types) + diff --git a/site/guide/model-validation/manage-validation-guidelines.qmd b/site/guide/model-validation/manage-validation-guidelines.qmd index 5149d05cf8..d33bc0a22e 100644 --- a/site/guide/model-validation/manage-validation-guidelines.qmd +++ b/site/guide/model-validation/manage-validation-guidelines.qmd @@ -126,7 +126,7 @@ By default, the [{{< fa hand >}} Customer Admin]{.bubble} role has all these per 2. Under Workspace, select **Templates**.[^5] -3. Select the **Validation Report** tab, then click on the template you want to edit. +3. Select the **Validation** tab, then click on the template you want to edit. 4. On the template details page, click **{{< fa pencil >}} Edit Outline**. diff --git a/site/guide/model-validation/preparing-validation-reports.qmd b/site/guide/model-validation/preparing-validation-reports.qmd index 99c1ebc398..1f39722977 100644 --- a/site/guide/model-validation/preparing-validation-reports.qmd +++ b/site/guide/model-validation/preparing-validation-reports.qmd @@ -75,13 +75,13 @@ Use the {{< var validmind.platform >}} to prepare detailed validation reports in :::: -## Validation Report Overview +## Validation Overview ::: {.callout title="Static PDF uploads do not include a Document Overview."} -Only PDFs converted to editable documents,[^5] reports generated by the {{< var validmind.developer >}}, or reports created in the {{< var validmind.platform >}} include a Validation Report Overview. +Only PDFs converted to editable documents,[^5] reports generated by the {{< var validmind.developer >}}, or reports created in the {{< var validmind.platform >}} include a Validation Overview. ::: -The **Validation Report Overview** shows a section-by-section outline of your model's validation report, as well as summary of overall compliance.[^6] +The **Validation Overview** shows a section-by-section outline of your model's validation report, as well as summary of overall compliance.[^6] 1. In the left sidebar, click **{{< fa cubes >}} Inventory**. @@ -89,7 +89,7 @@ The **Validation Report Overview** shows a section-by-section outline of your mo 3. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select the **Latest** or **Saved Versions** tab.[^8] -4. Click on a Validation Report type file.[^9] +4. Click on a Validation type file.[^9] ### Add or edit validation reports @@ -99,7 +99,7 @@ Only PDFs converted to editable documents,[^10] reports generated by the {{< var Modify the contents of the **Latest** version of your validation report:[^11] -1. From the **Validation Report Overview**, jump to any section of the validation report by clicking on the heading you would like to view. +1. From the **Validation Overview**, jump to any section of the validation report by clicking on the heading you would like to view. 2. Add **Risk Assessment Notes**, as well as assess compliance for each section of your validation report: diff --git a/site/guide/monitoring/enable-monitoring.qmd b/site/guide/monitoring/enable-monitoring.qmd index 6bfbfa212f..6acbd80f1a 100644 --- a/site/guide/monitoring/enable-monitoring.qmd +++ b/site/guide/monitoring/enable-monitoring.qmd @@ -6,12 +6,7 @@ title: "Enable monitoring" date: last-modified --- -To start uploading ongoing monitoring results for a model to {{< var vm.product >}}, you enable monitoring in your code snippet and then select a monitoring template. You can enable monitoring for both new and existing models. - -Enable monitoring with two steps: - -1. [Add `monitoring=True` to your code snippet](#add-monitoringtrue-to-your-code-snippet) -2. [Select a monitoring template](#select-a-monitoring-template) +To start uploading ongoing monitoring results for a model to {{< var vm.product >}}, enable monitoring in your code snippet within the {{< var validmind.platform >}} and then select a monitoring template. You can enable monitoring for both new and existing models. ::: {.callout title="To try out monitoring, check out the code sample for ongoing monitoring of models."} [Quickstart for ongoing monitoring of models with {{< var vm.product >}} {{< fa hand-point-right >}}](/notebooks/use_cases/ongoing_monitoring/quickstart_customer_churn_ongoing_monitoring.ipynb) @@ -27,87 +22,120 @@ Enable monitoring with two steps: ::: -## Add `monitoring=True` to your code snippet +## Steps + +::: {.panel-tabset} + +### 1. Get monitoring code snippet -{{< var vm.product >}} generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the {{< var validmind.developer >}} with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook. +{{< include /developer/model-documentation/_code-snippet.qmd >}} -To enable ongoing monitoring for a model, add `monitoring=True` to your code snippet: +To enable ongoing monitoring for a model, add `monitoring=True` to your code snippet: -1. In the {{< var validmind.platform >}}, either register a new model[^2] or locate an existing model: +a. In the {{< var validmind.platform >}}, either register a new model[^2] or locate an existing model: - In the left sidebar, click **{{< fa cubes >}} Inventory**. - Select a model or find your model by applying a filter or searching for it.[^3] -2. Copy the code snippet for the model: +b. Copy the code snippet for the model: - In the left sidebar that appears for your model, click **{{< fa rocket >}} Getting Started**. - - Locate the code snippet and set **[enable ongoing monitoring]{.smallcaps}[?]{.smallercaps}** to true. - - Click **{{< fa regular copy >}} Copy snippet to clipboard**. + - Select a Monitoring document type from the **[document]{.smallcaps}** drop-down menu.[^4] + - Set **[enable monitoring]{.smallcaps}[?]{.smallercaps}** to true. + - Click **{{< fa regular copy >}} Copy snippet to clipboard**. + +### 2. Select monitoring template + +Before you can start sending ongoing monitoring data using the {{< var validmind.developer >}} to the {{< var validmind.platform >}}, you must select a monitoring template: + +a. In the {{< var validmind.platform >}}, click **{{< fa cubes >}} Inventory** in the left sidebar. + +b. Select a model or find your model by applying a filter or searching for it.[^6] + +c. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select the **Latest** tab.[^7] + +d. Click on a Monitoring type file.[^8] + +e. From the **[template]{.smallcaps}** drop-down, select one of the available monitoring templates. + +f. Click **Use Template** to apply your desired template. + + - After you select a template, the initial document where your ongoing monitoring results will be stored becomes available. + - Review this documents structure to confirm that it meets your requirements, then start running your code to populate the template with monitoring results. + +### 3. Run code snippet -3. Paste the snippet into your development source code.[^4] +a. Paste the code snippet into your Jupyter Notebook or other development environment. + +b. Confirm that `monitoring=True` is present in the `vm.init` method, similar to this example: - Confirm that `monitoring=True` is present in the `vm.init` method, similar to this example: - ```python import validmind as vm - + vm.init( api_host="https://api.prod.validmind.ai/api/v1/tracking", api_key="...", api_secret="...", model="...", + document="monitoring", # Unique key for your monitoring document monitoring=True ) ``` +
-::: {.column-margin} -To enable monitoring when you are storing credentials in an `.env` file:^[[Store model credentials in `.env` files](/developer/model-documentation/store-credentials-in-env-file.qmd)] + ::: {.callout-button .pl4 .nt4} + ::: {.callout collapse="true" appearance="minimal"} + ### How do I enable monitoring when I am storing credentials in an `.env` file? -```python -%load_ext dotenv -%dotenv .env + :::: {.flex .flex-wrap .justify-around} -import validmind as vm + ::: {.w-60-ns .pr3} -vm.init( - monitoring=True -) -``` + Load your model credentials from your `.env` file, then remove the inline credentials from `vm.init()` after importing the {{< var vm.developer >}}. -::: + You'll still need to include your document key and the monitoring flag. -## Select a monitoring template + [Store model credentials in `.env` files](/developer/model-documentation/store-credentials-in-env-file.qmd){.button .button-green} -Before you can start sending ongoing monitoring data from your developer environment to {{< var vm.product >}}, you must select a monitoring template: + ::: -1. In the {{< var validmind.platform >}}, click **{{< fa cubes >}} Inventory** in the left sidebar. + ::: {.w-40-ns} -2. Select a model or find your model by applying a filter or searching for it.[^5] + ```python + %load_ext dotenv + %dotenv .env -3. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select the **Latest** tab.[^6] + import validmind as vm -4. Click on a Ongoing Monitoring type file.[^7] + vm.init( + document="monitoring", + monitoring=True + ) + ``` + + ::: + + :::: -5. From the **[template]{.smallcaps}** drop-down, select one of the available monitoring templates. -6. Click **Use Template** to apply your desired template. + ::: + ::: - - After you select a template, the initial document where your ongoing monitoring results will be stored becomes available. - - Review this documents structure to confirm that it meets your requirements, then start running your code to populate the template with monitoring results. +c. Run the code snippet to initialize the {{< var validmind.developer >}} and connect to the selected monitoring document for your model in the {{< var validmind.platform >}}.[^5] +::: {.callout} +## A template must already be applied to your selected document to populate monitoring test results in the {{< var validmind.platform >}}. -## Troubleshooting +Attempting to initialize the {{< var vm.developer >}} with a document that does not have a template applied will result in an error. -If you see errors similar to this one when running your code, make sure that you have selected your monitoring template and then re-run your code: +::: -```bash -APIRequestError: Please select an ongoing monitoring template on the {{< var validmind.platform >}} first -``` +::: ## What's next -After you have enabled ongoing monitoring and run your code to generate some output, you can start reviewing the monitoring results.[^8] +After you have enabled ongoing monitoring and run your code to generate some output, you can start reviewing the monitoring results.[^9] @@ -118,12 +146,14 @@ After you have enabled ongoing monitoring and run your code to generate some out [^3]: [Working with the model inventory](/guide/model-inventory/working-with-model-inventory.qmd#search-filter-and-sort-models) -[^4]: [Install and initialize the {{< var validmind.developer >}}](/developer/model-documentation/install-and-initialize-validmind-library.qmd) +[^4]: Selecting a **[document]{.smallcaps}** to connect to requires {{< var validmind.developer >}} version >=2.12.0. + +[^5]: [Install and initialize the {{< var validmind.developer >}}](/developer/model-documentation/install-and-initialize-validmind-library.qmd) -[^5]: [Working with the model inventory](/guide/model-inventory/working-with-model-inventory.qmd#search-filter-and-sort-models) +[^6]: [Working with the model inventory](/guide/model-inventory/working-with-model-inventory.qmd#search-filter-and-sort-models) -[^6]: [Work with document versions](/guide/model-documentation/work-with-document-versions.qmd) +[^7]: [Work with document versions](/guide/model-documentation/work-with-document-versions.qmd) -[^7]: [Working with model documents](/guide/templates/working-with-model-documents.qmd) +[^8]: [Working with model documents](/guide/templates/working-with-model-documents.qmd) -[^8]: [Review monitoring results](review-monitoring-results.qmd) \ No newline at end of file +[^9]: [Review monitoring results](review-monitoring-results.qmd) \ No newline at end of file diff --git a/site/guide/monitoring/review-monitoring-results.qmd b/site/guide/monitoring/review-monitoring-results.qmd index 3ba3018b45..00f8cedd1f 100644 --- a/site/guide/monitoring/review-monitoring-results.qmd +++ b/site/guide/monitoring/review-monitoring-results.qmd @@ -6,9 +6,9 @@ title: "Review monitoring results" date: last-modified --- -To ensure your model continues to perform as expected, it's important to regularly review the monitoring results generated during the ongoing monitoring process. +To ensure your model continues to perform as expected, it's important to regularly review the monitoring results generated during the ongoing monitoring process. -- As your code runs, the monitoring template for your model automatically populates with data, providing a comprehensive view of your model's performance over time. +- As your code runs, the monitoring template for your model automatically populates with data, providing a comprehensive view of your model's performance over time. - You access and examine these results within the {{< var validmind.platform >}}, allowing you to identify any deviations from expected performance and take corrective actions as needed. @@ -38,16 +38,18 @@ For results to be present and available for review, ongoing monitoring must be e 2. Select a model or find your model by applying a filter or searching for it.[^5] -3. In the left sidebar that appears for your model, click **Ongoing Monitoring** under {{< fa file >}} Documents.[^6] - +3. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select the **Latest** tab.[^6] + +3. Select the Monitoring type file you previously sent monitoring test results to.[^7] + - Explore the different sections of the template and review the results. - Any deviations from the expected performance should be identified and acted on if necessary. - For example, click **3.1 Target Drift** under 2. Data Quality & Drift Assessment and scroll down: +::: {.callout-button .pl4 .nt4} +::: {.callout collapse="true" appearance="minimal"} +### Example monitoring test results -## Examples - -#### [{{< fa check >}} Satisfactory]{.bubble .green-bg} +## [{{< fa check >}} Satisfactory]{.bubble .green-bg} :::: {.flex .flex-wrap .justify-around .w-100} @@ -74,7 +76,7 @@ All values are within the threshold of 5% (`drift_pct_threshold = 5`), and so al :::: -#### [{{< fa triangle-exclamation >}} Requires Attention]{.bubble .yellow-bg} +## [{{< fa triangle-exclamation >}} Requires Attention]{.bubble .yellow-bg} :::: {.flex .flex-wrap .justify-around .w-100} @@ -91,7 +93,7 @@ The metric analyzes histogram-based distribution metrics for Class 0 and Class 1 Each metric includes reference and monitoring values, % drift, and pass/fail status based on a 10% drift threshold (`drift_pct_threshold = 10`). -The final assessment is that the model requires attention: +The final assessment is that the model requires attention: - The model is stable for Class 0 - Further investigation is needed for Class 1 due to the drift in kurtosis @@ -104,6 +106,10 @@ The final assessment is that the model requires attention: :::: +::: +::: + + @@ -117,5 +123,7 @@ The final assessment is that the model requires attention: [^5]: [Working with the model inventory](/guide/model-inventory/working-with-model-inventory.qmd#search-filter-and-sort-models) -[^6]: [Working with model documents](/guide/templates/working-with-model-documents.qmd) +[^6]: [Work with document versions](/guide/model-documentation/work-with-document-versions.qmd) + +[^7]: [Working with model documents](/guide/templates/working-with-model-documents.qmd) diff --git a/site/guide/monitoring/work-with-metrics-over-time.qmd b/site/guide/monitoring/work-with-metrics-over-time.qmd index e474ab0643..72aa9c2fef 100644 --- a/site/guide/monitoring/work-with-metrics-over-time.qmd +++ b/site/guide/monitoring/work-with-metrics-over-time.qmd @@ -6,7 +6,7 @@ title: "Work with metrics over time" date: last-modified --- -Once generated via the {{< var validmind.developer >}}, view and add metrics over time to your ongoing monitoring plans in the {{< var validmind.platform >}}. +Once generated via the {{< var validmind.developer >}}, view and add metrics over time to your ongoing monitoring reports in the {{< var validmind.platform >}}. Metrics over time refers to the continued monitoring of a model's performance once it is deployed. Tracking how a model performs as new data is introduced or conditions change ensures that it remains accurate and reliable in real-world environments where data distributions or market conditions shift. @@ -40,19 +40,19 @@ Learn how to log metrics over time, set thresholds, and analyze model performanc 2. Select a model or find your model by applying a filter or searching for it.[^4] -3. In the left sidebar that appears for your model, click **Documentation** or **Ongoing Monitoring** under {{< fa file >}} Documents.[^5] +3. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select the **Latest** tab.[^5] -4. Click on a section header to expand that section and add content. +4. Select a Monitoring type file.[^6] -5. Hover your mouse over the space where you want your new block to go until a horizontal line with a {{< fa square-plus >}} sign appears that indicates you can insert a new block: +5. Click on a section header to expand that section and add content. - ![Adding a content block in the UI](/guide/model-documentation/add-content-block.gif){fig-alt="A gif showing the process of adding a content block in the UI" .screenshot} +6. Hover your mouse over the space where you want your new block to go until a horizontal line with a {{< fa square-plus >}} sign appears that indicates you can insert a new block: -6. Click {{< fa square-plus >}} and then select **Metric Over Time**[^6] under [from library]{.smallcaps}. + ![Adding a content block in the UI](/guide/model-documentation/add-content-block.gif){fig-alt="A gif showing the process of adding a content block in the UI" .screenshot} - By default, only the [{{< fa code >}} Developer]{.bubble} role can add metrics over time within model documentation or ongoing monitoring plans. +7. Click {{< fa square-plus >}} and then select **Metric Over Time**[^7] under [from library]{.smallcaps}. -7. Select metric over time results: +8. Select metric over time results: - Choose from available **VM Library** (out-of-the-box) or **Custom** tests under [metric over time]{.smallcaps} in the left sidebar of the test selection modal. - Use **{{}} Search by name** on the top-left to locate specific metric results. @@ -61,9 +61,9 @@ Learn how to log metrics over time, set thresholds, and analyze model performanc To preview what is included in a metric, click on it. By default, the actively selected metric is previewed. -8. Click **Insert # Metric(s) Over Time to Document** when you are ready. +9. Click **Insert # Metric(s) Over Time to Document** when you are ready. -9. After inserting the metrics into your document, review the data to confirm that it is accurate and relevant. +10. After inserting the metrics into your document, review the data to confirm that it is accurate and relevant. ![Example F1 Score — Metric Over Time visualization](example-f1-score.png){fig-alt="A screenshot showing an example F1 Score — Metric Over Time visualization" .screenshot group="time-metric"} @@ -79,13 +79,15 @@ After you have added metrics over time to your document, you can view the follow 1. In the left sidebar, click **{{< fa cubes >}} Inventory**. -2. Select a model or find your model by applying a filter or searching for it.[^7] +2. Select a model or find your model by applying a filter or searching for it.[^8] + +3. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select the **Latest** tab.[^9] -3. In the left sidebar that appears for your model, click **Documentation** or **Ongoing Monitoring** under {{< fa file >}} Documents.[^8] +4. Select a Monitoring type file.[^10] -4. Locate the metric whose metadata you want to view. +5. Locate the metric whose metadata you want to view. -5. Under the metric's name, click on **Data** tab. +6. Under the metric's name, click on **Data** tab. ![Example Data tab within a Metric Over Time](metric-over-time-data.png){fig-alt="A screenshot showing an example Data tab within a Metric Over Time" .screenshot} @@ -100,10 +102,14 @@ After you have added metrics over time to your document, you can view the follow [^4]: [Working with the model inventory](/guide/model-inventory/working-with-model-inventory.qmd#search-filter-and-sort-models) -[^5]: [Working with model documents](/guide/templates/working-with-model-documents.qmd) +[^5]: [Work with document versions](/guide/model-documentation/work-with-document-versions.qmd) + +[^6]: [Working with model documents](/guide/templates/working-with-model-documents.qmd) + +[^7]: [Work with content blocks](/guide/model-documentation/work-with-content-blocks.qmd) -[^6]: [Work with content blocks](/guide/model-documentation/work-with-content-blocks.qmd) +[^8]: [Working with the model inventory](/guide/model-inventory/working-with-model-inventory.qmd#search-filter-and-sort-models) -[^7]: [Working with the model inventory](/guide/model-inventory/working-with-model-inventory.qmd#search-filter-and-sort-models) +[^9]: [Work with document versions](/guide/model-documentation/work-with-document-versions.qmd) -[^8]: [Working with model documents](/guide/templates/working-with-model-documents.qmd) \ No newline at end of file +[^10]: [Working with model documents](/guide/templates/working-with-model-documents.qmd) \ No newline at end of file diff --git a/site/guide/reporting/export-analytics.qmd b/site/guide/reporting/export-analytics.qmd new file mode 100644 index 0000000000..7e9bb4d7c2 --- /dev/null +++ b/site/guide/reporting/export-analytics.qmd @@ -0,0 +1,40 @@ +--- +# Copyright © 2023-2026 ValidMind Inc. All rights reserved. +# Refer to the LICENSE file in the root of this repository for details. +# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial +title: "Export analytics" +date: last-modified +--- + +Export analytics widgets as comma-delimited tables (`.csv`). + +::: {.attn} + +## Prerequisites + +- [x] {{< var link.login >}} +- [x] To export data on models, there must be models registered in the model inventory.[^1] +- [x] To export data on artifacts, there must be artifacts logged on any model.[^2] + +::: + +## Export analytics from shared dashboards + +1. In the left sidebar, click **{{< fa square-poll-vertical >}} Analytics**. + +2. Click on the tab for the type of analytics widgets you want to export. + +3. Hover over the widget you would like to export. + +4. When the **{{< fa ellipsis-vertical >}}** appears, click on it and select **{{< fa download >}} Export CSV**. + +5. After the `.csv` file downloads automatically, open the file to confirm that the data exported is correct and is as expected. + + Exports include any applied filters, groupings, and metrics displayed by the widget. + + + +[^1]: [Register models in the inventory](/guide/model-inventory/register-models-in-inventory.qmd) + +[^2]: [Add and manage artifacts](/guide/model-validation/add-manage-artifacts.qmd) + diff --git a/site/guide/reporting/export-documents.qmd b/site/guide/reporting/export-documents.qmd index f03741173c..9f89a276d9 100644 --- a/site/guide/reporting/export-documents.qmd +++ b/site/guide/reporting/export-documents.qmd @@ -9,7 +9,7 @@ aliases: - /guide/model-documentation/export-documentation.html --- -Export model documentation, validation reports, and ongoing monitoring reports as XML-based Microsoft Word (`.docx`) or Adobe Portable Document Format `.pdf` files. +Export model documents as XML-based Microsoft Word (`.docx`) or Adobe Portable Document Format `.pdf` files. ::: {.callout title="Microsoft Word compatibility"} Documents exported from {{< var vm.product >}} are compatible with Word 365, Word 2019, Word 2016, and Word 2013. @@ -23,11 +23,11 @@ Documents exported from {{< var vm.product >}} are compatible with Word 365, Wor - [x] {{< var link.login >}} - [x] There are models registered in the model inventory.[^1] - [x] Documents exist and are completed or are in progress for your model.[^2] -- [x] You are a [{{< fa code >}} Developer]{.bubble} or [{{< fa circle-check >}} Validator]{.bubble}, or assigned another role with sufficient permissions to perform the tasks in this guide.[^3] +- [x] You are assigned a role with sufficient permissions to perform the tasks in this guide.[^3] ::: -## Export model documentation +## Export documents as Microsoft Word or PDF files 1. In the left sidebar, click **{{< fa cubes >}} Inventory**. @@ -35,58 +35,29 @@ Documents exported from {{< var vm.product >}} are compatible with Word 365, Wor 3. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select the **Latest** or **Saved Versions** tab.[^5] -4. Click on a Documentation type file.[^6] +4. Select a document to export. What information is available to export depends on the document type: -5. In right sidebar, click **{{< fa download >}} Export Document**. - - - Check off whether or not comments[^7] left on your document should be exported in your document - - Choose your file format — DOCX,[^8] PDF[^9] - -6. Click **{{< fa file-arrow-down >}} Download File** to download the file locally on your machine. - -## Export validation reports - -1. In the left sidebar, click **{{< fa cubes >}} Inventory**. - -2. Select a model or find your model by applying a filter or searching for it.[^10] - -3. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select the **Latest** or **Saved Versions** tab.[^11] - -4. Click on a Validation Report type file.[^12] + - **Development**[^6] — Model documentation + - **Validation**[^7] — Validation reports + - **Monitoring**[^8] — Ongoing monitoring reports + - Other custom document types[^9] 5. In right sidebar, click **{{< fa download >}} Export Document**. 6. Configure what is exported in your document by checking off the relevant boxes: - - Include compliance summary[^13] - - Include validation guidelines information[^14] - - Include validation guideline adherence details - - Include comments left on document[^15] + - [validation reports only]{.smallercaps .pink} Include compliance summary[^10] + - [validation reports only]{.smallercaps .pink} Include validation guidelines information[^11] + - [validation reports only]{.smallercaps .pink} Include validation guideline adherence details + - Include comments left on document[^12] 7. Choose your file format: - - **DOCX** — To update the table of contents, click **Yes** when Word prompts you to update fields. - - **PDF** — Your document will be exported as a PDF with formatting preserved. The table of contents and links are generated automatically. + - **DOCX**[^13] — To update the table of contents, click **Yes** when Word prompts you to update fields. + - **PDF**[^14] — Your document will be exported as a PDF with formatting preserved. The table of contents and links are generated automatically. 8. Click **{{< fa file-arrow-down >}} Download File** to download the file locally on your machine. -## Export ongoing monitoring reports - -1. In the left sidebar, click **{{< fa cubes >}} Inventory**. - -2. Select a model or find your model by applying a filter or searching for it.[^16] - -1. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select the **Latest** tab.[^17] - -1. Click on an Ongoing Monitoring type file.[^18] - -4. In right sidebar, click **{{< fa download >}} Export Document**. - - - Check off whether or not comments[^19] left on your document should be exported in your document - - Choose your file format — DOCX,[^20] PDF[^21] - -5. Click **{{< fa file-arrow-down >}} Download File** to download the file locally on your machine. - @@ -103,33 +74,18 @@ Documents exported from {{< var vm.product >}} are compatible with Word 365, Wor [^6]: [Working with model documentation](/guide/model-documentation/working-with-model-documentation.qmd) -[^7]: [Collaborate with others](/guide/model-documentation/collaborate-with-others.qmd#commenting) - -[^8]: To update the table of contents, click **Yes** when Word prompts you to update fields. - -[^9]: Your document will be exported as a PDF with formatting preserved. The table of contents and links are generated automatically. - -[^10]: [Working with the model inventory](/guide/model-inventory/working-with-model-inventory.qmd#search-filter-and-sort-models) - -[^11]: [Work with document versions](/guide/model-documentation/work-with-document-versions.qmd) - -[^12]: [Preparing validation reports](/guide/model-validation/preparing-validation-reports.qmd) - -[^13]: [Assess compliance](/guide/model-validation/assess-compliance.qmd) - -[^14]: [Manage validation guidelines](/guide/model-validation/manage-validation-guidelines.qmd) - -[^15]: [Collaborate with others](/guide/model-documentation/collaborate-with-others.qmd#commenting) +[^7]: [Preparing validation reports](/guide/model-validation/preparing-validation-reports.qmd) -[^16]: [Working with the model inventory](/guide/model-inventory/working-with-model-inventory.qmd#search-filter-and-sort-models) +[^8]: [Ongoing monitoring](/guide/monitoring/ongoing-monitoring.qmd) -[^17]: [Work with document versions](/guide/model-documentation/work-with-document-versions.qmd) +[^9]: [Manage document types](/guide/templates/manage-document-types.qmd) -[^18]: [Ongoing monitoring](/guide/monitoring/ongoing-monitoring.qmd) +[^10]: [Assess compliance](/guide/model-validation/assess-compliance.qmd) -[^19]: [Collaborate with others](/guide/model-documentation/collaborate-with-others.qmd#commenting) +[^11]: [Manage validation guidelines](/guide/model-validation/manage-validation-guidelines.qmd) -[^20]: To update the table of contents, click **Yes** when Word prompts you to update fields. +[^12]: [Collaborate with others](/guide/model-documentation/collaborate-with-others.qmd#commenting) -[^21]: Your document will be exported as a PDF with formatting preserved. The table of contents and links are generated automatically. +[^13]: To update the table of contents, click **Yes** when Word prompts you to update fields. +[^14]: Your document will be exported as a PDF with formatting preserved. The table of contents and links are generated automatically. diff --git a/site/guide/reporting/generating-exports.qmd b/site/guide/reporting/generating-exports.qmd index 4c1340052f..d265051d9f 100644 --- a/site/guide/reporting/generating-exports.qmd +++ b/site/guide/reporting/generating-exports.qmd @@ -9,12 +9,13 @@ listing: type: grid max-description-length: 250 sort: false - grid-columns: 3 + grid-columns: 2 fields: [title, description] contents: - export-inventory.qmd - export-documents.qmd - export-artifacts.qmd + - export-analytics.qmd --- -Export your documents, model inventory, and artifacts for use outside of the {{< var validmind.platform >}}. \ No newline at end of file +Export your documents, model inventory, artifacts, and analytics for use outside of the {{< var validmind.platform >}}. \ No newline at end of file diff --git a/site/guide/reporting/working-with-analytics.qmd b/site/guide/reporting/working-with-analytics.qmd index bc5463f0f3..37efc01dbe 100644 --- a/site/guide/reporting/working-with-analytics.qmd +++ b/site/guide/reporting/working-with-analytics.qmd @@ -12,11 +12,12 @@ listing: type: grid max-description-length: 250 sort: false - grid-columns: 2 + grid-columns: 3 fields: [title, description] contents: - view-report-data.qmd - manage-custom-reports.qmd + - export-analytics.qmd --- Analytical reports allow you to gain quick insights into your model validation efforts. Reports detail critical artifacts, risk exposure, and compliance status to ensure effective oversight and management of model-related risks. @@ -126,7 +127,7 @@ Rearrange report widgets on your analytics pages, including your custom pages:[^ 6. Click **Done Editing** to exit the editor. -## Manage analytics +## Work with analytics :::{#reports} ::: diff --git a/site/guide/templates/_add-text-blocks.qmd b/site/guide/templates/_add-text-blocks.qmd index 73516f8a51..f063e29499 100644 --- a/site/guide/templates/_add-text-blocks.qmd +++ b/site/guide/templates/_add-text-blocks.qmd @@ -2,6 +2,52 @@ Refer to the LICENSE file in the root of this repository for details. SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial --> + + + +:::: {.content-visible unless-format="revealjs"} +To create a new text block via the block library: + +1. In the left sidebar, click **{{< fa gear >}} Settings**. + +2. Under Documents, select **Block Library**. + +3. Click **Add Block** and enter: + + - The **[name]{.smallcaps}** of your text block + - (Optional) A **[description]{.smallcaps}** for your text block + - The templated **[content]{.smallcaps}** for your text block + +::: {.panel-tabset} + +### Reference field values + +While composing content for a library text block, you can reference values in the form of variables from model inventory fields. When the text block is inserted into a document, values from the associated model's inventory fields will automatically be populated in the content block if not empty. + +{{< include /guide/model-documentation/content_blocks/_reference-field-values.qmd >}} + +### Insert mathematical formulas + +While composing content for a library text block, you can insert math equations using the formula editor: + +{{< include /guide/model-documentation/content_blocks/_insert-mathematical-formulas.qmd >}} + + +::: + +4. Select a **[sharing]{.smallcaps}** option: + + - **Private** — Visible only to you + - **Shared** — Accessible across your organization + +5. Click **Add Block** to save your text block. + +:::: + + + + +:::: {.content-hidden unless-format="revealjs"} To create a new text block via the block library: 1. In the left sidebar, click **{{< fa gear >}} Settings**. @@ -9,7 +55,7 @@ To create a new text block via the block library: 1. Under Documents, select **Block Library**. 1. Click **Add Block** and enter: - + - The **[name]{.smallcaps}** of your text block - (Optional) A **[description]{.smallcaps}** for your text block - The templated **[content]{.smallcaps}** for your text block @@ -20,3 +66,5 @@ To create a new text block via the block library: - **Shared** — Accessible across your organization 1. Click **Add Block** to save your text block. + +:::: diff --git a/site/guide/templates/_customize-document-templates.qmd b/site/guide/templates/_customize-document-templates.qmd index 854e3a2c20..e18b543881 100644 --- a/site/guide/templates/_customize-document-templates.qmd +++ b/site/guide/templates/_customize-document-templates.qmd @@ -3,61 +3,89 @@ Refer to the LICENSE file in the root of this repository for details. SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial --> :::: {.content-visible unless-format="revealjs"} -1. In the left sidebar, click **{{< fa gear >}} Settings**. + + +:::: {.content-visible when-format="html" when-meta="includes.customize"} +Edit outlines for document templates from **{{< fa gear >}} Settings** or directly on model documents: + +::: {.panel-tabset} + +#### From **{{< fa gear >}} Settings** + +Editing outlines for document templates from **{{< fa gear >}} Settings** will create a new version of the template that can be applied to model documents: + +1. In the left sidebar, click **{{< fa gear >}} Settings**. 1. Under Documents, select **Templates**. 1. Select one of the tabs for the type of template you want to edit: - - **Documentation**^[[Working with model documentation](/guide/model-documentation/working-with-model-documentation.qmd)] - - **Validation Report**^[[Preparing validation reports](/guide/model-validation/preparing-validation-reports.qmd)] - - **Ongoing Monitoring**^[[Ongoing monitoring](/guide/monitoring/ongoing-monitoring.qmd)] + - **Development**^[[Working with model documentation](/guide/model-documentation/working-with-model-documentation.qmd)] + - **Validation**^[[Preparing validation reports](/guide/model-validation/preparing-validation-reports.qmd)] + - **Monitoring**^[[Ongoing monitoring](/guide/monitoring/ongoing-monitoring.qmd)] - **Custom**^[[Working with model documents](/guide/templates/working-with-model-documents.qmd)] -1. Click the template to edit and on the template details page, select **{{< fa pencil >}} Edit Outline**. +1. Click the template to edit and on the template details page, select **{{< fa pencil >}} Edit Outline**. -1. In outline editing mode, you can: +{{< include /guide/templates/_outline-editing-mode.qmd >}} - ::: {.panel-tabset} +1. When you're done, click **Create New Version** to view a side-by-side comparison of your changes with the previous version: + - Add a description in **[version notes]{.smallcaps}** to track your changes. + - Click **Save** to save the new version. - #### Work with existing sections + Once saved, your new template version becomes available for use. - Expand sections - : Expand the menu for any sections with subsections by clicking **{{< fa chevron-down >}}**. +#### On documents - Rename sections - : 1. Hover over a section until the **{{< fa pencil >}}** appears, then click on it to change the name. - 1. Save by clicking **{{< fa check >}}**. +Editing outlines for templates on documents will only apply changes to that document's active template: - Remove sections - : 1. Hover over the section you want to delete. - 1. When the **{{< fa trash-can >}}** appears, click on it. - 1. After you confirm, that section will be removed. +1. In the left sidebar, click **{{< fa cubes >}} Inventory**. - Reorder sections - : Click **{{< fa ellipsis-vertical >}}{{< fa ellipsis-vertical >}}** and hold firmly to drag to reorder sections. +1. Select a model or find your model by applying a filter or searching for it.^[[Working with the model inventory](/guide/model-inventory/working-with-model-inventory.qmd#search-filter-and-sort-models)] - #### Add new sections and content +1. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select the **Latest** or **Saved Versions** tab.^[[Work with document versions](/guide/model-documentation/work-with-document-versions.qmd)] - Insert new sections - : Click **{{< fa plus >}}** to add a new section or subsection to the template: - - To add a section above the current section, select **{{< fa arrow-up >}} Section Before**. - - To add a section below the current section, select **{{< fa arrow-down >}} Section After**. - - To add a subsection within the current section, select **{{< fa arrow-right >}} Subsection**. +1. Click on the document you would like to customize the template outline for.^[[Working with model documents](/guide/templates/working-with-model-documents.qmd)] - Add content to sections - : In any section of a template that does not have child sections, click **{{< fa plus >}}** to add additional [content]{.smallcaps}: - - **{{< fa align-justify >}} Summary Block** [validation reports only]{.smallercaps .pink} - - **{{< fa circle-check >}} Guideline Block**^[[Add guidelines to templates](/guide/model-validation/manage-validation-guidelines.qmd#add-guidelines-to-templates)] [validation reports only]{.smallercaps .pink} - - **{{< fa align-left >}} Text Block**^[[Add text blocks via template outlines](/guide/templates/customize-document-templates.qmd#add-text-blocks-via-template-outlines)] +1. On the template details page, click **{{< fa pencil >}} Edit Outline**. - ::: +{{< include /guide/templates/_outline-editing-mode.qmd >}} -1. When you're done, click **Create New Version** to view a side-by-side comparison of your changes with the previous version: - - Add a description in **[version notes]{.smallcaps}** to track your changes. - - Click **Save** to save the new version. +1. When you're done, click **Done Editing** to apply your changes to that specific document's active template. + +::: + +:::: + + + +:::: {.content-visible when-format="html" unless-meta="includes.customize"} +Edit outlines for templates directly on documents to only apply changes to that document's active template: + +1. In the left sidebar, click **{{< fa cubes >}} Inventory**. + +1. Select a model or find your model by applying a filter or searching for it.^[[Working with the model inventory](/guide/model-inventory/working-with-model-inventory.qmd#search-filter-and-sort-models)] + +1. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select the **Latest** or **Saved Versions** tab.^[[Work with document versions](/guide/model-documentation/work-with-document-versions.qmd)] + +1. Click on the document you would like to customize the template outline for.^[[Working with model documents](/guide/templates/working-with-model-documents.qmd)] + +1. On the template details page, click **{{< fa pencil >}} Edit Outline**. + +{{< include /guide/templates/_outline-editing-mode.qmd >}} + +1. When you're done, click **Done Editing** to apply your changes to that specific document's active template. + +:::: - Once saved, your new template version becomes available for use. :::: diff --git a/site/guide/templates/_duplicate-template.qmd b/site/guide/templates/_duplicate-template.qmd index 8da5988fea..5d27e52679 100644 --- a/site/guide/templates/_duplicate-template.qmd +++ b/site/guide/templates/_duplicate-template.qmd @@ -11,9 +11,9 @@ To duplicate an existing template and start with version one of that new templat 1. Select one of the tabs for the type of template you want to duplicate: - - **Documentation**^[[Working with model documentation](/guide/model-documentation/working-with-model-documentation.qmd)] - - **Validation Report**^[[Preparing validation reports](/guide/model-validation/preparing-validation-reports.qmd)] - - **Ongoing Monitoring**^[[Ongoing monitoring](/guide/monitoring/ongoing-monitoring.qmd)] + - **Development**^[[Working with model documentation](/guide/model-documentation/working-with-model-documentation.qmd)] + - **Validation**^[[Preparing validation reports](/guide/model-validation/preparing-validation-reports.qmd)] + - **Monitoring**^[[Ongoing monitoring](/guide/monitoring/ongoing-monitoring.qmd)] - **Custom**^[[Working with model documents](/guide/templates/working-with-model-documents.qmd)] 1. Click on the template to duplicate and on the template details page, select **{{< fa copy >}} Duplicate Template**. @@ -22,7 +22,7 @@ To duplicate an existing template and start with version one of that new templat 1. Click **Duplicate Template** to create a copy of your template. -Once duplicated, customize your new template to your needs, making it available for use with model documentation or validation reports. +Once duplicated, customize your new template to your needs, making it available for use with your model documents. :::: @@ -42,6 +42,6 @@ To duplicate an existing template and start with version one of that new templat 1. Click **Duplicate Template** to create a copy of your template. -Once duplicated, customize your new template to your needs, making it available for use with model documentation, validation reports, or ongoing monitoring reports. +Once duplicated, customize your new template to your needs, making it available for use with your model documents. :::: \ No newline at end of file diff --git a/site/guide/templates/_outline-editing-mode.qmd b/site/guide/templates/_outline-editing-mode.qmd new file mode 100644 index 0000000000..c4d3c4783c --- /dev/null +++ b/site/guide/templates/_outline-editing-mode.qmd @@ -0,0 +1,40 @@ + + +1. In outline editing mode, you can: + + ::: {.panel-tabset} + + #### Work with existing sections + + Expand sections + : Expand the menu for any sections with subsections by clicking **{{< fa chevron-down >}}**. + + Rename sections + : 1. Hover over a section until the **{{< fa pencil >}}** appears, then click on it to change the name. + 1. Save by clicking **{{< fa check >}}**. + + Remove sections + : 1. Hover over the section you want to delete. + 1. When the **{{< fa trash-can >}}** appears, click on it. + 1. After you confirm, that section will be removed. + + Reorder sections + : Click **{{< fa ellipsis-vertical >}}{{< fa ellipsis-vertical >}}** and hold firmly to drag to reorder sections. + + #### Add new sections and content + + Insert new sections + : Click **{{< fa plus >}}** to add a new section or subsection to the template: + - To add a section above the current section, select **{{< fa arrow-up >}} Section Before**. + - To add a section below the current section, select **{{< fa arrow-down >}} Section After**. + - To add a subsection within the current section, select **{{< fa arrow-right >}} Subsection**. + + Add content to sections + : In any section of a template that does not have child sections, click **{{< fa plus >}}** to add additional [content]{.smallcaps}: + - **{{< fa align-justify >}} Summary Block** [validation reports only]{.smallercaps .pink} + - **{{< fa circle-check >}} Guideline Block**^[[Add guidelines to templates](/guide/model-validation/manage-validation-guidelines.qmd#add-guidelines-to-templates)] [validation reports only]{.smallercaps .pink} + - **{{< fa align-left >}} Text Block**^[[Add text blocks via template outlines](/guide/templates/customize-document-templates.qmd#add-text-blocks-via-template-outlines)] + + ::: \ No newline at end of file diff --git a/site/guide/templates/_view-document-templates.qmd b/site/guide/templates/_view-document-templates.qmd index 7993b8d487..d659231431 100644 --- a/site/guide/templates/_view-document-templates.qmd +++ b/site/guide/templates/_view-document-templates.qmd @@ -11,9 +11,9 @@ To review the existing templates available to your organization: 1. Select one of the tabs for the document type with the templates you want to view: - - **Documentation**^[[Working with model documentation](/guide/model-documentation/working-with-model-documentation.qmd)] - - **Validation Report**^[[Preparing validation reports](/guide/model-validation/preparing-validation-reports.qmd)] - - **Ongoing Monitoring**^[[Ongoing monitoring](/guide/monitoring/ongoing-monitoring.qmd)] + - **Development**^[[Working with model documentation](/guide/model-documentation/working-with-model-documentation.qmd)] + - **Validation**^[[Preparing validation reports](/guide/model-validation/preparing-validation-reports.qmd)] + - **Monitoring**^[[Ongoing monitoring](/guide/monitoring/ongoing-monitoring.qmd)] - **Custom**^[[Working with model documents](/guide/templates/working-with-model-documents.qmd)] Displayed in list view is an overview of information for templates under that specific document type. diff --git a/site/guide/templates/customize-document-templates.qmd b/site/guide/templates/customize-document-templates.qmd index 593a4917f5..fcbf2e1876 100644 --- a/site/guide/templates/customize-document-templates.qmd +++ b/site/guide/templates/customize-document-templates.qmd @@ -8,12 +8,14 @@ aliases: - /guide/customize-documentation-templates.html - /guide/model-documentation/customize-documentation-templates.html - /guide/templates/customize-documentation-templates.html +includes: + customize: true --- Customize templates for document types to fit your specific case-by-case needs, such as type or complexity of model or model use case. - - Templates are versioned and saving a template after making changes or reverting to a previous version state always creates a new version, or you can duplicate templates to create new independent templates in your library. - - Documentation templates are stored as YAML files edited directly using the advanced editor, allowing you full control over the minitua such as desired guidelines. + - Templates are versioned and saving a template after making changes or reverting to a previous version state always creates a new version, or you can duplicate templates to create new independent templates in your library. + - Document templates are stored as YAML files edited directly using the advanced editor, allowing you full control over the minutiae such as desired guidelines. - We also provide a simplified editing experience to allow you to easily construct and rearrange the outline of your template. ::: {.attn} @@ -22,7 +24,7 @@ Customize templates for document types to fit your specific case-by-case needs, - [x] {{< var link.login >}} - [x] The template you want to edit must exist in the {{< var validmind.platform >}}.[^1] -- [x] You are a [{{< fa hand >}} Customer Admin]{.bubble} or assigned another role with sufficient permissions to perform the tasks in this guide.[^2] +- [x] You are a [{{< fa hand >}} Customer Admin]{.bubble} or assigned another role with sufficient permissions to perform the tasks in this guide.[^2] ::: {.callout title="Not sure which template or which version of a template your document is using?"} Check the [document template]{.smallcaps} section[^3] in the right sidebar. @@ -50,11 +52,12 @@ Check the [document template]{.smallcaps} section[^3] in the right sidebar. 2. Under Documents, select **Templates**. -3. Select one of the tabs for the type of template you want to edit.[^5] For example: +3. Select one of the tabs for the type of template you want to edit:[^5] - - **Documentation** - - **Validation Report** - - **Ongoing Monitoring** + - **Development** + - **Validation** + - **Monitoring** + - **Custom** 4. Click the template to edit and on the template details page, select **{{< fa pencil >}} Edit Raw**. @@ -70,105 +73,17 @@ Check the [document template]{.smallcaps} section[^3] in the right sidebar. Once saved, your new template version becomes available for use. -#### Template schema +### Template schema -::: {.column-page-inset-right} + -```{=html} - Schema Docs +::: {.column-page-inset-right} - - - - - - -
Type: object

root template_id
Type: string

Unique identifier for the template.

root template_name
Type: string

Name of the template.

root version
Type: string

Version of the template.

root description
Type: string

Description of the template.

root sections
Type: array

Documentation sections of the template.

Each item of this array must be:

root sections section
Type: object

root sections sections items id
Type: string

Unique identifier for the section.

root sections sections items title
Type: string

Title of the section.

root sections sections items description
Type: string

Description of the section.

root sections sections items parent_section
Type: string

ID of the parent section.

root sections sections items order
Type: integer

Order of the section in the navigation menu. By default sections are ordered alphabetically. If order is specified, sections will be ordered by the order value, and then alphabetically.

root sections sections items default_text
Type: string

Default text for the section. If set, a metadata content row will be created with this text when installing the template

root sections sections items index_only
Type: boolean

If true, the section will be displayed in the navigation menu, but it will not be accessible via direct link.

root sections sections items condensed
Type: boolean

If true, the section will condense all of its subsections into a single section.

root sections sections items guidelines
Type: array of string

Documentation or validation guidelines for the section.

Each item of this array must be:

root sections sections items guidelines guidelines items
Type: string

root sections sections items contents
Type: array

Contents to be displayed on the section.

Each item of this array must be:

root sections sections items contents section_contents
Type: object

Single content block of the module.

root sections sections items contents contents items content_type
Type: enum (of string) Default: "metadata_text"

Must be one of:

  • "metadata_text"
  • "dynamic"
  • "metric"
  • "test"

Examples:

"metadata_text"
-
"test"
-

root sections sections items contents contents items content_id
Type: string

ID of the content to be displayed for the given content type (text, tests, etc.).
Examples:

"sample_text"
-
"section_intro"
-

root sections sections items contents contents items options
Type: object

Options for the content block.
Examples:

{
-    "default_text": "This is a sample text block."
-}
-
{
-    "metric_id": "metric_1",
-    "title": "Custom Title for Metric 1"
-}
-
{
-    "test_id": "adf_test"
-}
-

root sections sections items contents contents items options default_text
Type: string

Default text for the content block. Only applicable for metadata_text content blocks.

root sections sections items contents contents items options title
Type: string

Title of the content block. Only applicable for test content blocks.
-``` +{{< include _template-schema-generated.qmd >}} ::: diff --git a/site/guide/templates/manage-document-templates.qmd b/site/guide/templates/manage-document-templates.qmd index c124aaaa3a..beb6943e56 100644 --- a/site/guide/templates/manage-document-templates.qmd +++ b/site/guide/templates/manage-document-templates.qmd @@ -29,6 +29,15 @@ View and swap between different document templates or template versions, or dupl {{< include /guide/templates/_view-document-templates.qmd >}} +## Edit document template outlines + +::: {.callout} +Permissions for directly editing document template outlines are defined by document type and determined by your organization's setup.[^3] + +::: + +{{< include _customize-document-templates.qmd >}} + ## Swap document templates ::: {.callout title="When swapping templates, only the document structure is changed."} @@ -41,14 +50,14 @@ Switch a document to a completely different template, or apply another version o 1. In the left sidebar, click **{{< fa cubes >}} Inventory**. -2. Select a model or find your model by applying a filter or searching for it.[^3] +2. Select a model or find your model by applying a filter or searching for it.[^4] -3. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select the **Latest** tab.[^4] +3. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select the **Latest** tab.[^5] 4. Select the document you want to view the template for: - Click **{{< fa magnifying-glass >}} Search** to search for documents by keywords in the document title. - - Click **{{< fa arrow-up-wide-short >}} Sort** to reorder the list using different criteria in ascending or descending order.[^5] + - Click **{{< fa arrow-up-wide-short >}} Sort** to reorder the list using different criteria in ascending or descending order.[^6] \n", diff --git a/site/notebooks/how_to/metrics/log_metrics_over_time.ipynb b/site/notebooks/how_to/metrics/log_metrics_over_time.ipynb index 31f9f1dcf9..4722eeed69 100644 --- a/site/notebooks/how_to/metrics/log_metrics_over_time.ipynb +++ b/site/notebooks/how_to/metrics/log_metrics_over_time.ipynb @@ -188,7 +188,7 @@ "\n", "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", "\n", "2. Under **TEMPLATE**, select `Credit Risk Scorecard`.\n", "\n", @@ -203,10 +203,11 @@ "\n", "#### Get your code snippet\n", "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -229,7 +230,7 @@ " # api_key=\"...\",\n", " # api_secret=\"...\",\n", " # model=\"...\",\n", - " # document=\"documentation\",\n", + " document=\"documentation\",\n", ")" ] }, @@ -858,11 +859,11 @@ "\n", "### Work with your model documentation\n", "\n", - "1. From the **Model Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", + "1. From the **Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", "\n", - "2. Click and expand the **Model Development** section.\n", + "2. In the left sidebar that appears for your model, click **Development** under Documents.\n", "\n", - "What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation, view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html)\n", + "What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation, view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/working-with-model-documentation.html)\n", "\n", "\n", "\n", diff --git a/site/notebooks/how_to/metrics/run_unit_metrics.ipynb b/site/notebooks/how_to/metrics/run_unit_metrics.ipynb index 3112b8a910..1b8dc8ce69 100644 --- a/site/notebooks/how_to/metrics/run_unit_metrics.ipynb +++ b/site/notebooks/how_to/metrics/run_unit_metrics.ipynb @@ -181,7 +181,7 @@ "\n", "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", "\n", "2. Under **TEMPLATE**, select `Binary classification`.\n", "\n", @@ -196,10 +196,11 @@ "\n", "#### Get your code snippet\n", "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -222,7 +223,7 @@ " # api_key=\"...\",\n", " # api_secret=\"...\",\n", " # model=\"...\",\n", - " # document=\"documentation\",\n", + " document=\"documentation\",\n", ")" ] }, @@ -695,11 +696,11 @@ "\n", "### Work with your model documentation\n", "\n", - "1. From the **Model Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", + "1. From the **Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", "\n", - "2. Click and expand the **Model Development** section.\n", + "2. In the left sidebar that appears for your model, click **Development** under Documents.\n", "\n", - "What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation, view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html)\n", + "What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation, view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/working-with-model-documentation.html)\n", "\n", "\n", "\n", diff --git a/site/notebooks/how_to/scoring/assign_scores_complete_tutorial.ipynb b/site/notebooks/how_to/scoring/assign_scores_complete_tutorial.ipynb index 5dad199c01..586817850d 100644 --- a/site/notebooks/how_to/scoring/assign_scores_complete_tutorial.ipynb +++ b/site/notebooks/how_to/scoring/assign_scores_complete_tutorial.ipynb @@ -183,7 +183,7 @@ "\n", "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", "\n", "2. Under **TEMPLATE**, select `Binary classification`.\n", "\n", @@ -198,10 +198,11 @@ "\n", "#### Get your code snippet\n", "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -224,7 +225,7 @@ " api_key=\"...\",\n", " api_secret=\"...\",\n", " model=\"...\",\n", - " document=\"documentation\",\n", + " # document=\"documentation\",\n", ")\n" ] }, @@ -714,11 +715,11 @@ "\n", "### Work with your model documentation\n", "\n", - "1. From the **Model Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", + "1. From the **Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", "\n", - "2. Click and expand the **Model Development** section.\n", + "2. In the left sidebar that appears for your model, click **Development** under Documents.\n", "\n", - "The scores you've assigned using `assign_scores()` become part of your model's documentation and can be used in ongoing monitoring workflows. You can view these metrics over time, set up alerts for performance drift, and compare models systematically. [Learn more ...](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html)\n", + "What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation, view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/working-with-model-documentation.html)\n", "\n", "\n", "\n", diff --git a/site/notebooks/how_to/tests/custom_tests/implement_custom_tests.ipynb b/site/notebooks/how_to/tests/custom_tests/implement_custom_tests.ipynb index 35b75b4579..8e4be82ac0 100644 --- a/site/notebooks/how_to/tests/custom_tests/implement_custom_tests.ipynb +++ b/site/notebooks/how_to/tests/custom_tests/implement_custom_tests.ipynb @@ -185,7 +185,7 @@ "\n", "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", "\n", "2. Under **TEMPLATE**, select `Binary classification`.\n", "\n", @@ -200,10 +200,11 @@ "\n", "#### Get your code snippet\n", "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -995,11 +996,11 @@ "\n", "### Work with your model documentation\n", "\n", - "1. From the **Model Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", + "1. From the **Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", "\n", - "2. Click and expand the **Model Development** section.\n", + "2. In the left sidebar that appears for your model, click **Development** under Documents.\n", "\n", - "What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation, view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html)\n", + "What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation, view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/working-with-model-documentation.html)\n", "\n", "\n", "\n", diff --git a/site/notebooks/how_to/tests/custom_tests/integrate_external_test_providers.ipynb b/site/notebooks/how_to/tests/custom_tests/integrate_external_test_providers.ipynb index f2411481b7..eaff3a5790 100644 --- a/site/notebooks/how_to/tests/custom_tests/integrate_external_test_providers.ipynb +++ b/site/notebooks/how_to/tests/custom_tests/integrate_external_test_providers.ipynb @@ -198,7 +198,7 @@ "\n", "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", "\n", "2. Under **TEMPLATE**, select `Binary classification`.\n", "\n", @@ -213,10 +213,11 @@ "\n", "#### Get your code snippet\n", "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -239,7 +240,7 @@ " # api_key=\"...\",\n", " # api_secret=\"...\",\n", " # model=\"...\",\n", - " # document=\"documentation\",\n", + " document=\"documentation\",\n", ")" ] }, @@ -895,11 +896,11 @@ "\n", "### Work with your model documentation\n", "\n", - "1. From the **Model Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", + "1. From the **Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", "\n", - "2. Click and expand the **Model Development** section.\n", + "2. In the left sidebar that appears for your model, click **Development** under Documents.\n", "\n", - "What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation, view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html)\n", + "What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation, view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/working-with-model-documentation.html)\n", "\n", "\n", "\n", diff --git a/site/notebooks/how_to/tests/explore_tests/explore_tests.ipynb b/site/notebooks/how_to/tests/explore_tests/explore_tests.ipynb index 629cd09324..56c58b62cf 100644 --- a/site/notebooks/how_to/tests/explore_tests/explore_tests.ipynb +++ b/site/notebooks/how_to/tests/explore_tests/explore_tests.ipynb @@ -4441,9 +4441,9 @@ ], "metadata": { "kernelspec": { - "display_name": ".venv", + "display_name": "ValidMind Library", "language": "python", - "name": "python3" + "name": "validmind" }, "language_info": { "codemirror_mode": { @@ -4455,7 +4455,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.20" + "version": "3.10.13" } }, "nbformat": 4, diff --git a/site/notebooks/how_to/tests/run_tests/1_run_dataset_based_tests.ipynb b/site/notebooks/how_to/tests/run_tests/1_run_dataset_based_tests.ipynb index 3f5ea875ab..9af05b3b3e 100644 --- a/site/notebooks/how_to/tests/run_tests/1_run_dataset_based_tests.ipynb +++ b/site/notebooks/how_to/tests/run_tests/1_run_dataset_based_tests.ipynb @@ -211,7 +211,7 @@ "\n", "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", "\n", "2. Under **TEMPLATE**, select `Binary classification`.\n", "\n", @@ -227,10 +227,11 @@ "\n", "#### Get your code snippet\n", "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -254,7 +255,7 @@ " # api_key=\"...\",\n", " # api_secret=\"...\",\n", " # model=\"...\",\n", - " # document=\"documentation\",\n", + " document=\"documentation\",\n", ")" ] }, @@ -635,7 +636,7 @@ "\n", "1. From the **Inventory** in the ValidMind Platform, go to the model you connected to earlier.\n", "\n", - "2. In the left sidebar that appears for your model, click **Documentation** under Documents.\n", + "2. In the left sidebar that appears for your model, click **Development** under Documents.\n", "\n", "3. Locate the Data Preparation section and click on **2.1. Data Description** to expand that section.\n", "\n", diff --git a/site/notebooks/how_to/tests/run_tests/2_run_comparison_tests.ipynb b/site/notebooks/how_to/tests/run_tests/2_run_comparison_tests.ipynb index 689cfae9b3..1ba4627bc8 100644 --- a/site/notebooks/how_to/tests/run_tests/2_run_comparison_tests.ipynb +++ b/site/notebooks/how_to/tests/run_tests/2_run_comparison_tests.ipynb @@ -1,789 +1,1094 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Run comparison tests\n", - "\n", - "Use the ValidMind Library's `run_test` function to run built-in or custom tests that take any combination of datasets or models as inputs. Comparison tests allow you to run existing test over different groups of inputs and produces a single consolidated list of outputs in the form of text, tables, and images that get populated in model documentation.\n", - "\n", - "You'll learn how to:\n", - "\n", - "- Find tests\n", - "- Initialize a ValidMind dataset\n", - "- Initialize a ValidMind model and assign predictions to a dataset\n", - "- Run a comparison test with `run_test` function\n", - "\n", - "**We recommended that you first complete the [Explore tests](../explore_tests/explore_tests.ipynb) and the [Run dataset based tests](./1_run_dataset_based_tests.ipynb) notebooks** to understand the basics of how to find and describe all the available tests in the ValidMind Library and how to run tests before moving on to this guide.\n", - "\n", - "This interactive notebook provides a step-by-step guide for listing and filtering available tests, building a sample dataset, training a model, initializing the required ValidMind objects, running a comparison test, and then logging the results to ValidMind. " - ] + "cells": [ + { + "cell_type": "markdown", + "id": "1d29276f", + "metadata": {}, + "source": [ + "# Run comparison tests\n", + "\n", + "Learn how to use the ValidMind Library to run comparison tests that take any datasets or models as inputs. Identify comparison tests to run, initialize ValidMind dataset and model objects in preparation for passing them to tests, and then run tests — generating outputs automatically logged to your model's documentation in the ValidMind Platform.\n", + "\n", + "
We recommend that you first complete our introductory notebook on running tests.\n", + "

\n", + "Run dataset-based tests
" + ] + }, + { + "cell_type": "markdown", + "id": "61065444", + "metadata": {}, + "source": [ + "::: {.content-hidden when-format=\"html\"}\n", + "## Contents \n", + "- [About ValidMind](#toc1__) \n", + " - [Before you begin](#toc1_1__) \n", + " - [New to ValidMind?](#toc1_2__) \n", + " - [Key concepts](#toc1_3__) \n", + "- [Setting up](#toc2__) \n", + " - [Install the ValidMind Library](#toc2_1__) \n", + " - [Initialize the ValidMind Library](#toc2_2__) \n", + " - [Register sample model](#toc2_2_1__) \n", + " - [Apply documentation template](#toc2_2_2__) \n", + " - [Get your code snippet](#toc2_2_3__) \n", + " - [Preview the documentation template](#toc2_3__) \n", + " - [Initialize the Python environment](#toc2_4__) \n", + "- [Explore a ValidMind test](#toc3__) \n", + "- [Working with ValidMind datasets](#toc4__) \n", + " - [Import the sample dataset](#toc4_1__) \n", + " - [Split the dataset](#toc4_2__) \n", + " - [Initialize the ValidMind dataset](#toc4_3__) \n", + "- [Working with ValidMind models](#toc5__) \n", + " - [Train a sample model](#toc5_1__) \n", + " - [Initialize the ValidMind model](#toc5_2__) \n", + " - [Assign predictions](#toc5_3__) \n", + "- [Running ValidMind tests](#toc6__) \n", + " - [Run classifier performance test with one model](#toc6_1__) \n", + " - [Run comparison tests](#toc6_2__) \n", + " - [Run classifier performance test with multiple models](#toc6_2_1__) \n", + " - [Run classifier performance test with multiple parameter values](#toc6_2_2__) \n", + " - [Run comparison test with multiple datasets](#toc6_2_3__) \n", + "- [Work with test results](#toc7__) \n", + "- [Next steps](#toc8__) \n", + " - [Discover more learning resources](#toc8_1__) \n", + "- [Upgrade ValidMind](#toc9__) \n", + "\n", + ":::\n", + "\n", + "" + ] + }, + { + "cell_type": "markdown", + "id": "67a4d9dc", + "metadata": {}, + "source": [ + "\n", + "\n", + "## About ValidMind\n", + "\n", + "ValidMind is a suite of tools for managing model risk, including risk associated with AI and statistical models. \n", + "\n", + "You use the ValidMind Library to automate documentation and validation tests, and then use the ValidMind Platform to collaborate on model documentation. Together, these products simplify model risk management, facilitate compliance with regulations and institutional standards, and enhance collaboration between yourself and model validators." + ] + }, + { + "cell_type": "markdown", + "id": "eeb30df8", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Before you begin\n", + "\n", + "This notebook assumes you have basic familiarity with Python, including an understanding of how functions work. If you are new to Python, you can still run the notebook but we recommend further familiarizing yourself with the language. \n", + "\n", + "If you encounter errors due to missing modules in your Python environment, install the modules with `pip install`, and then re-run the notebook. For more help, refer to [Installing Python Modules](https://docs.python.org/3/installing/index.html)." + ] + }, + { + "cell_type": "markdown", + "id": "293c3f98", + "metadata": {}, + "source": [ + "\n", + "\n", + "### New to ValidMind?\n", + "\n", + "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n", + "\n", + "
For access to all features available in this notebook, you'll need access to a ValidMind account.\n", + "

\n", + "Register with ValidMind
" + ] + }, + { + "cell_type": "markdown", + "id": "4fc836d0", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Key concepts\n", + "\n", + "**Model documentation**: A structured and detailed record pertaining to a model, encompassing key components such as its underlying assumptions, methodologies, data sources, inputs, performance metrics, evaluations, limitations, and intended uses. It serves to ensure transparency, adherence to regulatory requirements, and a clear understanding of potential risks associated with the model’s application.\n", + "\n", + "**Documentation template**: Functions as a test suite and lays out the structure of model documentation, segmented into various sections and sub-sections. Documentation templates define the structure of your model documentation, specifying the tests that should be run, and how the results should be displayed.\n", + "\n", + "**Tests**: A function contained in the ValidMind Library, designed to run a specific quantitative test on the dataset or model. Tests are the building blocks of ValidMind, used to evaluate and document models and datasets, and can be run individually or as part of a suite defined by your model documentation template.\n", + "\n", + "**Metrics**: A subset of tests that do not have thresholds. In the context of this notebook, metrics and tests can be thought of as interchangeable concepts.\n", + "\n", + "**Custom metrics**: Custom metrics are functions that you define to evaluate your model or dataset. These functions can be registered with the ValidMind Library to be used in the ValidMind Platform.\n", + "\n", + "**Inputs**: Objects to be evaluated and documented in the ValidMind Library. They can be any of the following:\n", + "\n", + " - **model**: A single model that has been initialized in ValidMind with [`vm.init_model()`](https://docs.validmind.ai/validmind/validmind.html#init_model).\n", + " - **dataset**: Single dataset that has been initialized in ValidMind with [`vm.init_dataset()`](https://docs.validmind.ai/validmind/validmind.html#init_dataset).\n", + " - **models**: A list of ValidMind models - usually this is used when you want to compare multiple models in your custom metric.\n", + " - **datasets**: A list of ValidMind datasets - usually this is used when you want to compare multiple datasets in your custom metric. (Learn more: [Run tests with multiple datasets](https://docs.validmind.ai/notebooks/how_to/tests/run_tests/configure_tests/run_tests_that_require_multiple_datasets.html))\n", + "\n", + "**Parameters**: Additional arguments that can be passed when running a ValidMind test, used to pass additional information to a metric, customize its behavior, or provide additional context.\n", + "\n", + "**Outputs**: Custom metrics can return elements like tables or plots. Tables may be a list of dictionaries (each representing a row) or a pandas DataFrame. Plots may be matplotlib or plotly figures.\n", + "\n", + "**Test suites**: Collections of tests designed to run together to automate and generate model documentation end-to-end for specific use-cases.\n", + "\n", + "Example: the [`classifier_full_suite`](https://docs.validmind.ai/validmind/validmind/test_suites/classifier.html#ClassifierFullSuite) test suite runs tests from the [`tabular_dataset`](https://docs.validmind.ai/validmind/validmind/test_suites/tabular_datasets.html) and [`classifier`](https://docs.validmind.ai/validmind/validmind/test_suites/classifier.html) test suites to fully document the data and model sections for binary classification model use-cases." + ] + }, + { + "cell_type": "markdown", + "id": "8d52b6e0", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Setting up" + ] + }, + { + "cell_type": "markdown", + "id": "e0d2daaf", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Install the ValidMind Library\n", + "\n", + "
Recommended Python versions\n", + "

\n", + "Python 3.8 <= x <= 3.11
\n", + "\n", + "To install the library:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fc97888f", + "metadata": {}, + "outputs": [], + "source": [ + "%pip install -q validmind" + ] + }, + { + "cell_type": "markdown", + "id": "1ff56571", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Initialize the ValidMind Library" + ] + }, + { + "cell_type": "markdown", + "id": "c4d9f164", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Register sample model\n", + "\n", + "Let's first register a sample model for use with this notebook.\n", + "\n", + "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n", + "\n", + "2. In the left sidebar, navigate to **Inventory** and click **+ Register Model**.\n", + "\n", + "3. Enter the model details and click **Next >** to continue to assignment of model stakeholders. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/register-models-in-inventory.html))\n", + "\n", + "4. Select your own name under the **MODEL OWNER** drop-down.\n", + "\n", + "5. Click **Register Model** to add the model to your inventory." + ] + }, + { + "cell_type": "markdown", + "id": "852392e5", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Apply documentation template\n", + "\n", + "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", + "\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", + "\n", + "2. Under **TEMPLATE**, select `Binary classification`.\n", + "\n", + "3. Click **Use Template** to apply the template." + ] + }, + { + "cell_type": "markdown", + "id": "6490e991", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Get your code snippet\n", + "\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", + "\n", + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet::" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c51ae01c", + "metadata": {}, + "outputs": [], + "source": [ + "# Load your model identifier credentials from an `.env` file\n", + "\n", + "%load_ext dotenv\n", + "%dotenv .env\n", + "\n", + "# Or replace with your code snippet\n", + "\n", + "import validmind as vm\n", + "\n", + "vm.init(\n", + " # api_host=\"...\",\n", + " # api_key=\"...\",\n", + " # api_secret=\"...\",\n", + " # model=\"...\",\n", + " document=\"documentation\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "99e9d14f", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Preview the documentation template\n", + "\n", + "Let's verify that you have connected the ValidMind Library to the ValidMind Platform and that the appropriate *template* is selected for your model.\n", + "\n", + "You will upload documentation and test results unique to your model based on this template later on. For now, **take a look at the default structure that the template provides with [the `vm.preview_template()` function](https://docs.validmind.ai/validmind/validmind.html#preview_template)** from the ValidMind library and note the empty sections:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fd332a9d", + "metadata": {}, + "outputs": [], + "source": [ + "vm.preview_template()" + ] + }, + { + "cell_type": "markdown", + "id": "f805ec38", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Initialize the Python environment\n", + "\n", + "Next, let's import the necessary libraries and set up your Python environment for data analysis:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8e2127cd", + "metadata": {}, + "outputs": [], + "source": [ + "import xgboost as xgb\n", + "\n", + "%matplotlib inline" + ] + }, + { + "cell_type": "markdown", + "id": "1783e13c", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Explore a ValidMind test\n", + "\n", + "Before we run a test, use [the `vm.tests.list_tests()` function](https://docs.validmind.ai/validmind/validmind/tests.html#list_tests) to return information on out-of-the-box tests available in the ValidMind Library.\n", + "\n", + "Let's assume you want to evaluate *classifier performance* for a model. Classifier performance measures how well a classification model correctly predicts outcomes, using metrics like [precision, recall, and F1 score](https://en.wikipedia.org/wiki/Precision_and_recall).\n", + "\n", + "We'll pass in a `filter` to the `list_tests` function to find the test ID for classifier performance:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a6a6f715", + "metadata": {}, + "outputs": [], + "source": [ + "vm.tests.list_tests(filter=\"ClassifierPerformance\")" + ] + }, + { + "cell_type": "markdown", + "id": "96a56e4b", + "metadata": {}, + "source": [ + "We've identified from the output that the test ID for the classifier performance test is `validmind.model_validation.ClassifierPerformance`.\n", + "\n", + "Use this ID combined with [the `describe_test()` function](https://docs.validmind.ai/validmind/validmind/tests.html#describe_test) to retrieve more information about the test, including its **Required Inputs**:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f8a46c7d", + "metadata": {}, + "outputs": [], + "source": [ + "test_id = \"validmind.model_validation.sklearn.ClassifierPerformance\"\n", + "vm.tests.describe_test(test_id)" + ] + }, + { + "cell_type": "markdown", + "id": "97053f50", + "metadata": {}, + "source": [ + "Since this test requires a dataset and a model, you can expect it to throw an error when we run it without passing in either as input:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f853c272", + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " vm.tests.run_test(test_id)\n", + "except Exception as e:\n", + " print(e)" + ] + }, + { + "cell_type": "markdown", + "id": "1a3115ed", + "metadata": {}, + "source": [ + "
Learn more about the individual tests available in the ValidMind Library\n", + "

\n", + "Check out our Explore tests notebook for more code examples and usage of key functions.
" + ] + }, + { + "cell_type": "markdown", + "id": "89da851b", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Working with ValidMind datasets" + ] + }, + { + "cell_type": "markdown", + "id": "50bfdb1b", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Import the sample dataset\n", + "\n", + "Since we need a dataset to run tests, let's import the public [Bank Customer Churn Prediction](https://www.kaggle.com/datasets/shantanudhakadd/bank-customer-churn-prediction) dataset from Kaggle so that we have something to work with.\n", + "\n", + "In our below example, note that:\n", + "\n", + "- The target column, `Exited` has a value of `1` when a customer has churned and `0` otherwise.\n", + "- The ValidMind Library provides a wrapper to automatically load the dataset as a [Pandas DataFrame](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html) object. A Pandas Dataframe is a two-dimensional tabular data structure that makes use of rows and columns." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3ef2dfbb", + "metadata": {}, + "outputs": [], + "source": [ + "# Import the sample dataset from the library\n", + "\n", + "from validmind.datasets.classification import customer_churn\n", + "\n", + "print(\n", + " f\"Loaded demo dataset with: \\n\\n\\t• Target column: '{customer_churn.target_column}' \\n\\t• Class labels: {customer_churn.class_labels}\"\n", + ")\n", + "\n", + "raw_df = customer_churn.load_data()\n", + "raw_df.head()" + ] + }, + { + "cell_type": "markdown", + "id": "a5a8212f", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Split the dataset\n", + "\n", + "Let's first split our dataset to help assess how well the model generalizes to unseen data.\n", + "\n", + "Use [`preprocess()`](https://docs.validmind.ai/validmind/validmind/datasets/classification/customer_churn.html#preprocess) to split our dataset into three subsets:\n", + "\n", + "1. **train_df** — Used to train the model.\n", + "2. **validation_df** — Used to evaluate the model's performance during training.\n", + "3. **test_df** — Used later on to asses the model's performance on new, unseen data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "88c87d4a", + "metadata": {}, + "outputs": [], + "source": [ + "train_df, validation_df, test_df = customer_churn.preprocess(raw_df)" + ] + }, + { + "cell_type": "markdown", + "id": "2ae225d7", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Initialize the ValidMind dataset\n", + "\n", + "The next step is to connect your data with a ValidMind `Dataset` object. **This step is always necessary every time you want to connect a dataset to documentation and produce test results through ValidMind,** but you only need to do it once per dataset.\n", + "\n", + "ValidMind dataset objects provide a wrapper to any type of dataset (NumPy, Pandas, Polars, etc.) so that tests can run transparently regardless of the underlying library.\n", + "\n", + "Initialize a ValidMind dataset object using the [`init_dataset` function](https://docs.validmind.ai/validmind/validmind.html#init_dataset) from the ValidMind (`vm`) module. For this example, we'll pass in the following arguments:\n", + "\n", + "- **`dataset`** — The raw dataset that you want to provide as input to tests.\n", + "- **`input_id`** — A unique identifier that allows tracking what inputs are used when running each individual test.\n", + "- **`target_column`** — A required argument if tests require access to true values. This is the name of the target column in the dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bf0ec747", + "metadata": {}, + "outputs": [], + "source": [ + "vm_train_ds = vm.init_dataset(\n", + " dataset=train_df,\n", + " input_id=\"train_dataset\",\n", + " target_column=customer_churn.target_column,\n", + ")\n", + "\n", + "vm_test_ds = vm.init_dataset(\n", + " dataset=test_df,\n", + " input_id=\"test_dataset\",\n", + " target_column=customer_churn.target_column,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "6d26f65b", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Working with ValidMind models" + ] + }, + { + "cell_type": "markdown", + "id": "6d1677f6", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Train a sample model\n", + "\n", + "To train the model, we need to provide it with:\n", + "\n", + "1. **Inputs** — Features such as customer age, usage, etc.\n", + "2. **Outputs (Expected answers/labels)** — in our case, we would like to know whether the customer churned or not.\n", + "\n", + "Here, we'll use `x_train` and `x_val` to hold the input data (features), and `y_train` and `y_val` to hold the answers (the target we want to predict):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "39e8c7ea", + "metadata": {}, + "outputs": [], + "source": [ + "x_train = train_df.drop(customer_churn.target_column, axis=1)\n", + "y_train = train_df[customer_churn.target_column]\n", + "x_val = validation_df.drop(customer_churn.target_column, axis=1)\n", + "y_val = validation_df[customer_churn.target_column]" + ] + }, + { + "cell_type": "markdown", + "id": "4ac628eb", + "metadata": {}, + "source": [ + "Next, let's create an *XGBoost classifier model* that will automatically stop training if it doesn't improve after 10 tries. XGBoost is a gradient-boosted tree ensemble that builds trees sequentially, with each tree correcting the errors of the previous ones — typically known for strong predictive performance and built-in regularization to reduce overfitting.\n", + "\n", + "Setting an explicit threshold avoids wasting time and helps prevent further overfitting by stopping training when further improvement isn't happening. We'll also set three evaluation metrics to get a more complete picture of model performance:\n", + "\n", + "1. **error** — Measures how often the model makes incorrect predictions.\n", + "2. **logloss** — Indicates how confident the predictions are.\n", + "3. **auc** — Evaluates how well the model distinguishes between churn and not churn." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "255e3583", + "metadata": {}, + "outputs": [], + "source": [ + "model = xgb.XGBClassifier(early_stopping_rounds=10)\n", + "model.set_params(\n", + " eval_metric=[\"error\", \"logloss\", \"auc\"],\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "f6430312", + "metadata": {}, + "source": [ + "Finally, our actual training step — where the model learns patterns from the data, so it can make predictions later:\n", + "\n", + "- The model is trained on `x_train` and `y_train`, and evaluates its performance using `x_val` and `y_val` to check if it’s learning well.\n", + "- To turn off printed output while training, we'll set `verbose` to `False`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3aa3657", + "metadata": {}, + "outputs": [], + "source": [ + "model.fit(\n", + " x_train,\n", + " y_train,\n", + " eval_set=[(x_val, y_val)],\n", + " verbose=False,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "c303a046", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Initialize the ValidMind model\n", + "\n", + "You'll also need to initialize a ValidMind model object (`vm_model`) that can be passed to other functions for analysis and tests on the data for our model.\n", + "\n", + "You simply initialize this model object with [`vm.init_model()`](https://docs.validmind.ai/validmind/validmind.html#init_model):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4b2be11f", + "metadata": {}, + "outputs": [], + "source": [ + "vm_model_xgb = vm.init_model(\n", + " model,\n", + " input_id=\"xgboost\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "2fa83857", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Assign predictions\n", + "\n", + "Once the model has been registered, you can assign model predictions to the training and testing datasets.\n", + "\n", + "- The [`assign_predictions()` method](https://docs.validmind.ai/validmind/validmind/vm_models.html#assign_predictions) from the `Dataset` object can link existing predictions to any number of models.\n", + "- This method links the model's class prediction values and probabilities to our `vm_train_ds` and `vm_test_ds` datasets.\n", + "\n", + "If no prediction values are passed, the method will compute predictions automatically:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "229185fd", + "metadata": {}, + "outputs": [], + "source": [ + "vm_train_ds.assign_predictions(model=vm_model_xgb)\n", + "vm_test_ds.assign_predictions(model=vm_model_xgb)" + ] + }, + { + "cell_type": "markdown", + "id": "d0b3312e", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Running ValidMind tests\n", + "\n", + "Now that we know how to initialize ValidMind `dataset` and `model` objects, we're ready to run some tests!\n", + "\n", + "You run individual tests by calling [the `run_test` function](https://docs.validmind.ai/validmind/validmind/tests.html#run_test) provided by the `validmind.tests` module. For the examples below, we'll pass in the following arguments:\n", + "\n", + "- **`test_id`** — The ID of the test to run, as seen in the `ID` column when you run `list_tests`.\n", + "- **`inputs`** — A dictionary of test inputs, such as `dataset`, `model`, `datasets`, or `models`. These are ValidMind objects initialized with [`vm.init_dataset()`](https://docs.validmind.ai/validmind/validmind.html#init_dataset) or [`vm.init_model()`](https://docs.validmind.ai/validmind/validmind.html#init_model)." + ] + }, + { + "cell_type": "markdown", + "id": "96c89f32", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Run classifier performance test with one model\n", + "\n", + "Run `validmind.data_validation.ClassifierPerformance` test with the testing dataset (`vm_test_ds`) and model (`vm_model_xgb`) as inputs:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "85189af9", + "metadata": {}, + "outputs": [], + "source": [ + "result = vm.tests.run_test(\n", + " \"validmind.model_validation.sklearn.ClassifierPerformance\",\n", + " inputs={\n", + " \"dataset\": vm_test_ds,\n", + " \"model\": vm_model_xgb,\n", + " },\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "676dff89", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Run comparison tests\n", + "\n", + "To evaluate which models might be a better fit for a use case based on their performance on selected criteria, we can run the same test with multiple models. We'll train three additional models and run the classifier performance test with for all four models using a single `run_test()` call.\n", + "\n", + "
ValidMind helps streamline your documentation and testing.\n", + "

\n", + "You could call run_test() multiple times passing in different inputs, but you can also pass an input_grid object — a dictionary of test input keys and values that allow you to run a single test for a combination of models and datasets.\n", + "

\n", + "With input_grid, run comparison tests for multiple datasets, or even multiple datasets and models simultaneously — input_grid can be used with run_test() for all possible combinations of inputs, generating a cohesive and comprehensive single output.\n", + "
" + ] + }, + { + "cell_type": "markdown", + "id": "3d9912dc", + "metadata": {}, + "source": [ + "*Random forest classifier* models use an ensemble method that builds multiple decision trees and averages their predictions. Random forest is robust to overfitting and handles non-linear relations well, but is typically less interpretable than simpler models:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1976b7e8", + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.ensemble import RandomForestClassifier\n", + "\n", + "# Train the random forest classifer model\n", + "model_rf = RandomForestClassifier()\n", + "model_rf.fit(x_train, y_train)\n", + "\n", + "# Initialize the ValidMind model object for the random forest classifer model\n", + "vm_model_rf = vm.init_model(\n", + " model_rf,\n", + " input_id=\"random_forest\",\n", + ")\n", + "\n", + "# Assign predictions to the test dataset for the random forest classifer model\n", + "vm_test_ds.assign_predictions(model=vm_model_rf)" + ] + }, + { + "cell_type": "markdown", + "id": "a259927c", + "metadata": {}, + "source": [ + "*Logistic regression* models are linear models that estimate class probabilities via a logistic (sigmoid) function. Logistic regression is highly interpretable with fast training, establishing a strong baseline — however, they struggle when relationships are non-linear as real-world relationships often are:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "90bbf148", + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.linear_model import LogisticRegression\n", + "from sklearn.preprocessing import StandardScaler\n", + "from sklearn.pipeline import Pipeline\n", + "\n", + "# Scaling features ensures the lbfgs solver converges reliably\n", + "model_lr = Pipeline([\n", + " (\"scaler\", StandardScaler()),\n", + " (\"lr\", LogisticRegression()),\n", + "])\n", + "model_lr.fit(x_train, y_train)\n", + "\n", + "# Initialize the ValidMind model object for the logistic regression model\n", + "vm_model_lr = vm.init_model(\n", + " model_lr,\n", + " input_id=\"logistic_regression\",\n", + ")\n", + "\n", + "# Assign predictions to the test dataset for the logistic regression model\n", + "vm_test_ds.assign_predictions(model=vm_model_lr)" + ] + }, + { + "cell_type": "markdown", + "id": "9a666b41", + "metadata": {}, + "source": [ + "*Decision tree classifier* models are a single tree with data split on feature thresholds. Useful as an explanability benchmark, decision trees are easy to visualize and interpret — but are prone to overfitting without pruning or ensemble techniques:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bfa1e17d", + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.tree import DecisionTreeClassifier\n", + "\n", + "# Train the decision tree classifer model\n", + "model_dt = DecisionTreeClassifier()\n", + "model_dt.fit(x_train, y_train)\n", + "\n", + "# Initialize the ValidMind model object for the decision tree classifier model\n", + "vm_model_dt = vm.init_model(\n", + " model_dt,\n", + " input_id=\"decision_tree\",\n", + ")\n", + "\n", + "# Assign predictions to the test dataset for the decision tree classifiermodel\n", + "vm_test_ds.assign_predictions(model=vm_model_dt)" + ] + }, + { + "cell_type": "markdown", + "id": "2c8f3268", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Run classifier performance test with multiple models\n", + "\n", + "Now, we'll use the `input_grid` to run the [`ClassifierPerformance` test](https://docs.validmind.ai/tests/model_validation/sklearn/ClassifierPerformance.html) on all four models using the testing dataset (`vm_test_ds`).\n", + "\n", + "When running individual tests, you can use a custom `result_id` to tag the individual result with a unique identifier by appending this `result_id` to the `test_id` with a `:` separator. We'll append an identifier to signify that this test was run on `all_models` to differentiate this test run from other runs:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2e48ce1e", + "metadata": {}, + "outputs": [], + "source": [ + "perf_comparison_result = vm.tests.run_test(\n", + " \"validmind.model_validation.sklearn.ClassifierPerformance:all_models\",\n", + " input_grid={\n", + " \"dataset\": [vm_test_ds],\n", + " \"model\": [vm_model_xgb, vm_model_rf, vm_model_lr, vm_model_dt],\n", + " },\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "81cbf144", + "metadata": {}, + "source": [ + "Our output indicates that the XGBoost and random forest classification models provide the strongest overall classification performance, so we'll continue our testing with those two models as input only." + ] + }, + { + "cell_type": "markdown", + "id": "3d3fb6ec", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Run classifier performance test with multiple parameter values\n", + "\n", + "Next, let's run the classifier performance test with the `param_grid` object, which runs the same test multiple times with different parameter values. We'll append an identifier to signify that this test was run with our `parameter_grid` configuration:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d0ad94c9", + "metadata": {}, + "outputs": [], + "source": [ + "parameter_comparison_result = vm.tests.run_test(\n", + " \"validmind.model_validation.sklearn.ClassifierPerformance:parameter_grid\",\n", + " input_grid={\n", + " \"dataset\": [vm_test_ds],\n", + " \"model\": [vm_model_xgb,vm_model_rf]\n", + " },\n", + " param_grid={\n", + " \"average\": [\"macro\", \"micro\"]\n", + " },\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "508c7546", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Run comparison test with multiple datasets\n", + "\n", + "Let's also run the [ROCCurve test](https://docs.validmind.ai/tests/model_validation/sklearn/ROCCurve.html) using `input_grid` to iterate through multiple datasets, which plots the ROC curves for the training (`vm_train_ds`) and test (`vm_test_ds`) datasets side by side — a common scenario when you want to compare the performance of a model on the training and test datasets and visually assess how much performance is lost in the test dataset.\n", + "\n", + "We'll also need to assign predictions to the training dataset for the random forest classifier model, since we didn't do that in our earlier setup:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "96c3b426", + "metadata": {}, + "outputs": [], + "source": [ + "vm_train_ds.assign_predictions(model=vm_model_rf)" + ] + }, + { + "cell_type": "markdown", + "id": "2be82bae", + "metadata": {}, + "source": [ + "We'll append an identifier to signify that this test was run with our `train_vs_test` dataset comparison configuration:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4056aa1e", + "metadata": {}, + "outputs": [], + "source": [ + "roc_curve_result = vm.tests.run_test(\n", + " \"validmind.model_validation.sklearn.ROCCurve:train_vs_test\",\n", + " input_grid={\n", + " \"dataset\": [vm_train_ds, vm_test_ds],\n", + " \"model\": [vm_model_xgb,vm_model_rf],\n", + " },\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "a05570d5", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Work with test results\n", + "\n", + "Every test result returned by the `run_test()` function has a [`.log()` method](https://docs.validmind.ai/validmind/validmind/vm_models.html#TestResult.log) that can be used to send the test results to the ValidMind Platform. When logging individual test results to the platform, you'll need to manually add those results to the desired section of the model documentation.\n", + "\n", + "You can do this through the ValidMind Platform interface after logging your test results ([Learn more ...](https://docs.validmind.ai/developer/model-documentation/work-with-test-results.html)), or directly via the ValidMind Library when calling `.log()` by providing an optional `section_id`. The `section_id` should be a string that matches the title of a section in the documentation template in `snake_case`.\n", + "\n", + "Let's log the results of the classifier performance test (`perf_comparison_result`) and the ROCCurve (`roc_curve_result`) test in the `model_evaluation` section of the documentation — present in the template we previewed in the beginning of this notebook:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e119bf1e", + "metadata": {}, + "outputs": [], + "source": [ + "perf_comparison_result.log(section_id=\"model_evaluation\")\n", + "roc_curve_result.log(section_id=\"model_evaluation\")" + ] + }, + { + "cell_type": "markdown", + "id": "ab5205ee", + "metadata": {}, + "source": [ + "Finally, let's head to the model we connected to at the beginning of this notebook and view our inserted test results in the updated documentation ([Need more help?](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html)):\n", + "\n", + "1. From the **Inventory** in the ValidMind Platform, go to the model you connected to earlier.\n", + "\n", + "2. In the left sidebar that appears for your model, click **Development** under Documents.\n", + "\n", + "3. Expand the **3.2. Model Evaluation** section.\n", + "\n", + "4. Confirm that `perf_comparison_result` and `roc_curve_result` display in this section as expected." + ] + }, + { + "cell_type": "markdown", + "id": "eb196aac", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Next steps\n", + "\n", + "Now that you know how to run comparison tests with the ValidMind Library, you’re ready to take the next step. Extend the functionality of `run_test()` with your own custom test functions that can be incorporated into documentation templates just like any default out-of-the-box ValidMind test.\n", + "\n", + "
Learn how to implement custom tests with the ValidMind Library.\n", + "

\n", + "Check out our Implement comparison tests notebook for code examples and usage of key functions.
" + ] + }, + { + "cell_type": "markdown", + "id": "083c1d8d", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Discover more learning resources\n", + "\n", + "We offer many interactive notebooks to help you automate testing, documenting, validating, and more:\n", + "\n", + "- [Run tests & test suites](https://docs.validmind.ai/developer/how-to/testing-overview.html)\n", + "- [Use ValidMind Library features](https://docs.validmind.ai/developer/how-to/feature-overview.html)\n", + "- [Code samples by use case](https://docs.validmind.ai/guide/samples-jupyter-notebooks.html)\n", + "\n", + "Or, visit our [documentation](https://docs.validmind.ai/) to learn more about ValidMind." + ] + }, + { + "cell_type": "markdown", + "id": "efba0f57", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Upgrade ValidMind\n", + "\n", + "
After installing ValidMind, you’ll want to periodically make sure you are on the latest version to access any new features and other enhancements.
\n", + "\n", + "Retrieve the information for the currently installed version of ValidMind:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0d35972c", + "metadata": { + "vscode": { + "languageId": "plaintext" + } + }, + "outputs": [], + "source": [ + "%pip show validmind" + ] + }, + { + "cell_type": "markdown", + "id": "abcd07ef", + "metadata": {}, + "source": [ + "If the version returned is lower than the version indicated in our [production open-source code](https://github.com/validmind/validmind-library/blob/prod/validmind/__version__.py), restart your notebook and run:\n", + "\n", + "```bash\n", + "%pip install --upgrade validmind\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "5fe70b90", + "metadata": {}, + "source": [ + "You may need to restart your kernel after running the upgrade package for changes to be applied." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "\n", + "\n", + "***\n", + "\n", + "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10" + } }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "::: {.content-hidden when-format=\"html\"}\n", - "## Contents \n", - "- [About ValidMind](#toc1__) \n", - " - [Before you begin](#toc1_1__) \n", - " - [New to ValidMind?](#toc1_2__) \n", - " - [Key concepts](#toc1_3__) \n", - "- [Setting up](#toc2__) \n", - " - [Install the ValidMind Library](#toc2_1__) \n", - " - [Initialize the ValidMind Library](#toc2_2__) \n", - " - [Register sample model](#toc2_2_1__) \n", - " - [Apply documentation template](#toc2_2_2__) \n", - " - [Get your code snippet](#toc2_2_3__) \n", - " - [Initialize the Python environment](#toc2_3__) \n", - "- [List and filter available tests](#toc3__) \n", - "- [Load the sample dataset and run a model test](#toc4__) \n", - " - [Initialize a ValidMind dataset](#toc4_1__) \n", - " - [Train a sample XGBoost model](#toc4_2__) \n", - " - [Initialize a ValidMind model](#toc4_3__) \n", - " - [Assign predictions to the datasets](#toc4_4__) \n", - " - [Run the test with one model](#toc4_5__) \n", - "- [Run comparison tests](#toc5__) \n", - " - [Run a comparison test with multiple models](#toc5_1__) \n", - " - [Run a comparison test with multiple parameters' values](#toc5_2__) \n", - " - [Run a comparison test with multiple datasets](#toc5_3__) \n", - "- [Add test results to documentation](#toc6__) \n", - "- [Next steps](#toc7__) \n", - " - [Discover more learning resources](#toc7_1__) \n", - "- [Upgrade ValidMind](#toc8__) \n", - "\n", - ":::\n", - "\n", - "" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## About ValidMind\n", - "ValidMind is a suite of tools for managing model risk, including risk associated with AI and statistical models.\n", - "\n", - "You use the ValidMind Library to automate documentation and validation tests, and then use the ValidMind Platform to collaborate on model documentation. Together, these products simplify model risk management, facilitate compliance with regulations and institutional standards, and enhance collaboration between yourself and model validators.\n", - "\n", - "\n", - "\n", - "### Before you begin\n", - "This notebook assumes you have basic familiarity with Python, including an understanding of how functions work. If you are new to Python, you can still run the notebook but we recommend further familiarizing yourself with the language.\n", - "\n", - "If you encounter errors due to missing modules in your Python environment, install the modules with `pip install`, and then re-run the notebook. For more help, refer to [Installing Python Modules](https://docs.python.org/3/installing/index.html).\n", - "\n", - "\n", - "\n", - "### New to ValidMind?\n", - "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n", - "\n", - "
For access to all features available in this notebook, you'll need access to a ValidMind account.\n", - "

\n", - "Register with ValidMind
" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Key concepts\n", - "\n", - "**Model documentation**: A structured and detailed record pertaining to a model, encompassing key components such as its underlying assumptions, methodologies, data sources, inputs, performance metrics, evaluations, limitations, and intended uses. It serves to ensure transparency, adherence to regulatory requirements, and a clear understanding of potential risks associated with the model’s application.\n", - "\n", - "**Documentation template**: Functions as a test suite and lays out the structure of model documentation, segmented into various sections and sub-sections. Documentation templates define the structure of your model documentation, specifying the tests that should be run, and how the results should be displayed.\n", - "\n", - "**Tests**: A function contained in the ValidMind Library, designed to run a specific quantitative test on the dataset or model. Tests are the building blocks of ValidMind, used to evaluate and document models and datasets, and can be run individually or as part of a suite defined by your model documentation template.\n", - "\n", - "**Custom tests**: Custom tests are functions that you define to evaluate your model or dataset. These functions can be registered via the ValidMind Library to be used with the ValidMind Platform.\n", - "\n", - "**Inputs**: Objects to be evaluated and documented in the ValidMind Library. They can be any of the following:\n", - "\n", - "- **model**: A single model that has been initialized in ValidMind with [`vm.init_model()`](https://docs.validmind.ai/validmind/validmind.html#init_model).\n", - "- **dataset**: Single dataset that has been initialized in ValidMind with [`vm.init_dataset()`](https://docs.validmind.ai/validmind/validmind.html#init_dataset).\n", - "- **models**: A list of ValidMind models - usually this is used when you want to compare multiple models in your custom test.\n", - "- **datasets**: A list of ValidMind datasets - usually this is used when you want to compare multiple datasets in your custom test. See this [example](https://docs.validmind.ai/notebooks/how_to/tests/run_tests/configure_tests/run_tests_that_require_multiple_datasets.html) for more information.\n", - "\n", - "**Parameters**: Additional arguments that can be passed when running a ValidMind test, used to pass additional information to a test, customize its behavior, or provide additional context.\n", - "\n", - "**Outputs**: Custom tests can return elements like tables or plots. Tables may be a list of dictionaries (each representing a row) or a pandas DataFrame. Plots may be matplotlib or plotly figures." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Setting up" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Install the ValidMind Library\n", - "\n", - "To install the library:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%pip install -q validmind" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Initialize the ValidMind Library" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Register sample model\n", - "\n", - "Let's first register a sample model for use with this notebook:\n", - "\n", - "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n", - "\n", - "2. In the left sidebar, navigate to **Inventory** and click **+ Register Model**.\n", - "\n", - "3. Enter the model details and click **Next >** to continue to assignment of model stakeholders. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/register-models-in-inventory.html))\n", - "\n", - "4. Select your own name under the **MODEL OWNER** drop-down.\n", - "\n", - "5. Click **Register Model** to add the model to your inventory." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Apply documentation template\n", - "\n", - "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", - "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", - "\n", - "2. Under **TEMPLATE**, select `Binary classification`.\n", - "\n", - "3. Click **Use Template** to apply the template." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Get your code snippet\n", - "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", - "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Load your model identifier credentials from an `.env` file\n", - "\n", - "%load_ext dotenv\n", - "%dotenv .env\n", - "\n", - "# Or replace with your code snippet\n", - "\n", - "import validmind as vm\n", - "\n", - "vm.init(\n", - " # api_host=\"...\",\n", - " # api_key=\"...\",\n", - " # api_secret=\"...\",\n", - " # model=\"...\",\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Initialize the Python environment\n", - "\n", - "Next, let's import the necessary libraries and set up your Python environment for data analysis:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import xgboost as xgb\n", - "\n", - "%matplotlib inline" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## List and filter available tests\n", - "\n", - "Before we run a comparison test, let's find a suitable test for this demonstration. Let's assume you want to evaluate the performance results for a model.\n", - "\n", - "In the [Explore tests](../explore_tests/explore_tests.ipynb) notebook, we learned how to pass a `filter` to the `list_tests` function. We'll do the same here to find the test ID for the confusion matrix:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "vm.tests.list_tests(filter=\"ClassifierPerformance\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "From the output, you can see that the test ID for the pearson correlation matrix is `validmind.model_validation.sklearn.ClassifierPerformance`. The `describe_test` function gives you more information about the test, including its **Required Inputs**:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "test_id = \"validmind.model_validation.sklearn.ClassifierPerformance\"\n", - "vm.tests.describe_test(test_id)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Since this test requires a **dataset** and a **model** , it should throw an error if you were to run it without passing any of those inputs:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "try:\n", - " vm.tests.run_test(test_id)\n", - "except Exception as e:\n", - " print(e)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Load the sample dataset and run a model test\n", - "\n", - "The sample dataset used here is provided by the ValidMind library. To be able to use it, you need to import the dataset and load it into a pandas [DataFrame](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html), a two-dimensional tabular data structure that makes use of rows and columns:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Import the sample dataset from the library\n", - "\n", - "from validmind.datasets.classification import customer_churn\n", - "\n", - "print(\n", - " f\"Loaded demo dataset with: \\n\\n\\t• Target column: '{customer_churn.target_column}' \\n\\t• Class labels: {customer_churn.class_labels}\"\n", - ")\n", - "\n", - "raw_df = customer_churn.load_data()\n", - "raw_df.head()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Initialize a ValidMind dataset\n", - "\n", - "ValidMind dataset objects provide a wrapper to any type of dataset (NumPy, Pandas, Polars, etc.) so that tests can run transparently regardless of the underlying library. A VM dataset object can be created using the [`init_dataset`](https://docs.validmind.ai/validmind/validmind.html#init_dataset) function from the ValidMind (`vm`) module.\n", - "\n", - "This function takes a number of arguments:\n", - "\n", - "- `dataset`: The raw dataset that you want to provide as input to tests.\n", - "- `input_id`: A unique identifier that allows tracking what inputs are used when running each individual test.\n", - "- `target_column`: A required argument if tests require access to true values. This is the name of the target column in the dataset.\n", - "\n", - "Below you can see how to initialize a VM dataset from a Pandas dataset. Since we are going to train a model and run some tests on it, we will create a dataset split from `raw_df` and initialize a VM dataset for the training and test datasets, which will be used for running comparison tests:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# The test dataset will be used to evaluate the model's performance\n", - "train_df, validation_df, test_df = customer_churn.preprocess(raw_df)\n", - "\n", - "vm_train_ds = vm.init_dataset(\n", - " dataset=train_df,\n", - " input_id=\"train_dataset\",\n", - " target_column=customer_churn.target_column,\n", - ")\n", - "\n", - "vm_test_ds = vm.init_dataset(\n", - " dataset=test_df,\n", - " input_id=\"test_dataset\",\n", - " target_column=customer_churn.target_column,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Train a sample XGBoost model\n", - "\n", - "In the following code snippet you will train an XGBoost model using default parameters. The resulting `model` object will be passed to the `init_model` to initialize a ValidMind model object." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "x_train = train_df.drop(customer_churn.target_column, axis=1)\n", - "y_train = train_df[customer_churn.target_column]\n", - "x_val = validation_df.drop(customer_churn.target_column, axis=1)\n", - "y_val = validation_df[customer_churn.target_column]\n", - "\n", - "model = xgb.XGBClassifier(early_stopping_rounds=10)\n", - "model.set_params(\n", - " eval_metric=[\"error\", \"logloss\", \"auc\"],\n", - ")\n", - "model.fit(\n", - " x_train,\n", - " y_train,\n", - " eval_set=[(x_val, y_val)],\n", - " verbose=False,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Initialize a ValidMind model\n", - "\n", - "Now, you can initialize a ValidMind model object (`vm_model`) that can be passed to other functions for analysis and tests on the data. You simply intialize this model object with [`vm.init_model()`](https://docs.validmind.ai/validmind/validmind.html#init_model):" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "vm_model_xgb = vm.init_model(\n", - " model,\n", - " input_id=\"xgboost\",\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Assign predictions to the datasets\n", - "\n", - "We can now use the assign_predictions() method from the Dataset object to link existing predictions to any model. If no prediction values are passed, the method will compute predictions automatically:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "vm_train_ds.assign_predictions(model=vm_model_xgb)\n", - "vm_test_ds.assign_predictions(model=vm_model_xgb)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Run the test with one model\n", - "\n", - "You can now call `run_test` with the new `vm_test_ds` and `vm_model_xgb` objects as inputs:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "result = vm.tests.run_test(\n", - " \"validmind.model_validation.sklearn.ClassifierPerformance\",\n", - " inputs={\n", - " \"dataset\": vm_test_ds,\n", - " \"model\": vm_model_xgb,\n", - " },\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Run comparison tests\n", - "\n", - "It is possible to run the same `ClassifierPerformance` test with multiple models. You could call the `run_test()` function multiple times passing different inputs, but you can also pass an `input_grid` object that allows you run a single test for a combination of models and datasets.\n", - "\n", - "As an example, you will train 4 models and run the `ClassifierPerformance` test for all of them using a single `run_test()` call. The `input_grid` object is a dictionary where the keys are the test input identifier and the values are a list of inputs to be used in the test. The following code snippet shows how to run the test for our 4 models:\n", - "\n", - "```python\n", - "input_grid = {\n", - " 'model': [vm_model_xgb, vm_model_rf, vm_model_lr, vm_model_dt],\n", - " 'dataset': [vm_test_ds]\n", - "}\n", - "```\n", - "\n", - "This `input_grid` definition will run the `ClassifierPerformance` test for the following 4 groups of inputs:\n", - "\n", - "1. `vm_model_xgb` and `vm_test_ds`\n", - "2. `vm_model_rf` and `vm_test_ds`\n", - "3. `vm_model_lr` and `vm_test_ds`\n", - "4. `vm_model_dt` and `vm_test_ds`\n", - "\n", - "Now, let's train the 3 other models and run the test for all of them:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from sklearn.ensemble import RandomForestClassifier\n", - "from sklearn.linear_model import LogisticRegression\n", - "from sklearn.tree import DecisionTreeClassifier\n", - "\n", - "model_rf = RandomForestClassifier()\n", - "model_rf.fit(x_train, y_train)\n", - "\n", - "vm_model_rf = vm.init_model(\n", - " model_rf,\n", - " input_id=\"random_forest\",\n", - ")\n", - "\n", - "model_lr = LogisticRegression()\n", - "model_lr.fit(x_train, y_train)\n", - "\n", - "vm_model_lr = vm.init_model(\n", - " model_lr,\n", - " input_id=\"logistic_regression\",\n", - ")\n", - "\n", - "model_dt = DecisionTreeClassifier()\n", - "model_dt.fit(x_train, y_train)\n", - "\n", - "vm_model_dt = vm.init_model(\n", - " model_dt,\n", - " input_id=\"decision_tree\",\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You will also need to assign the predictions to the test dataset for the other models:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "vm_test_ds.assign_predictions(model=vm_model_rf)\n", - "vm_test_ds.assign_predictions(model=vm_model_lr)\n", - "vm_test_ds.assign_predictions(model=vm_model_dt)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Run a comparison test with multiple models\n", - "\n", - "Now you can run the test with the `input_grid` object. Note that we're adding a small extra identifier to the test ID to differentiate the results in the documentation. This is optional, but it can be useful to differentiate the results when you have multiple tests with the same ID." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "perf_comparison_result = vm.tests.run_test(\n", - " \"validmind.model_validation.sklearn.ClassifierPerformance:all_models\",\n", - " input_grid={\n", - " \"dataset\": [vm_test_ds],\n", - " \"model\": [vm_model_xgb, vm_model_rf, vm_model_lr, vm_model_dt],\n", - " },\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Run a comparison test with multiple parameters' values\n", - "\n", - "Now you can run the test with the `param_grid` object. Note that we're adding a small extra identifier to the test ID to differentiate the results in the documentation. This is optional, but it can be useful to differentiate the results when you have multiple tests with the same ID." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "parameter_comparison_result = vm.tests.run_test(\n", - " \"validmind.model_validation.sklearn.ClassifierPerformance:parameter_grid\",\n", - " input_grid={\n", - " \"dataset\": [vm_test_ds],\n", - " \"model\": [vm_model_xgb,]\n", - " },\n", - " param_grid={\n", - " \"average\": [\"macro\", \"micro\"]\n", - " },\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Run a comparison test with multiple datasets\n", - "\n", - "Similarly, you can run a comparison test for multiple datasets, or even multiple datasets and models simultaneously. The `input_grid` object can be used to run the test for all possible combinations of inputs and the `run_test()` function will try to combine all results in a single output.\n", - "\n", - "Let's run a test that plots the ROC curves for the training and test datasets side by side. This is a common scenario when you want to compare the performance of a model on the training and test datasets and visually assess how much performance is lost in the test dataset." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "roc_curve_result = vm.tests.run_test(\n", - " \"validmind.model_validation.sklearn.ROCCurve:train_vs_test\",\n", - " input_grid={\n", - " \"dataset\": [vm_train_ds, vm_test_ds],\n", - " \"model\": [vm_model_xgb],\n", - " },\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Add test results to documentation\n", - "\n", - "Using the `.log()` method of a result object, you can log the results of a test to the model documentation. This method takes an optional `section` argument that specifies where the results should be logged in the documentation. The section should be a string that matches the title of a section in the documentation template.\n", - "\n", - "You will log these results in the `model_evaluation` section of the documentation. After logging the results, you can view the updated documentation in the ValidMind Platform following these steps:\n", - "\n", - "1. In the ValidMind Platform, click **Documentation** under Documents for the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html)\n", - "\n", - "2. Expand the **3.2. Model Evaluation** section.\n", - "\n", - "3. View the results of the tests you just ran at the end of the section." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "perf_comparison_result.log(section_id=\"model_evaluation\")\n", - "roc_curve_result.log(section_id=\"model_evaluation\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Next steps\n", - "\n", - "While you can look at the results of this test suite right in the notebook where you ran the code, there is a better way — expand the rest of your documentation in the ValidMind Platform and take a look around. \n", - "\n", - "What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation, view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready.\n", - "\n", - "\n", - "\n", - "### Discover more learning resources\n", - "\n", - "In an upcoming companion notebook, you'll learn how to run tests that require a dataset and model as inputs. This will allow you to generate documentation for model evaluation tests such as ROC-AUC, F1 score, etc.\n", - "\n", - "We also offer many other interactive notebooks to help you document models:\n", - "\n", - "- [Run tests & test suites](https://docs.validmind.ai/developer/how-to/testing-overview.html)\n", - "- [Use ValidMind Library features](https://docs.validmind.ai/developer/how-to/feature-overview.html)\n", - "- [Code samples by use case](https://docs.validmind.ai/guide/samples-jupyter-notebooks.html)\n", - "\n", - "Or, visit our [documentation](https://docs.validmind.ai/) to learn more about ValidMind." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Upgrade ValidMind\n", - "\n", - "
After installing ValidMind, you’ll want to periodically make sure you are on the latest version to access any new features and other enhancements.
\n", - "\n", - "Retrieve the information for the currently installed version of ValidMind:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "%pip show validmind" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If the version returned is lower than the version indicated in our [production open-source code](https://github.com/validmind/validmind-library/blob/prod/validmind/__version__.py), restart your notebook and run:\n", - "\n", - "```bash\n", - "%pip install --upgrade validmind\n", - "```" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You may need to restart your kernel after running the upgrade package for changes to be applied." - ] - }, - { - "cell_type": "markdown", - "id": "copyright-d655a3bbc6e14b15b7466f3808556ef8", - "metadata": {}, - "source": [ - "\n", - "\n", - "\n", - "\n", - "***\n", - "\n", - "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", - "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": ".venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.13" - } - }, - "nbformat": 4, - "nbformat_minor": 2 + "nbformat": 4, + "nbformat_minor": 5 } diff --git a/site/notebooks/how_to/tests/run_tests/configure_tests/customize_test_result_descriptions.ipynb b/site/notebooks/how_to/tests/run_tests/configure_tests/customize_test_result_descriptions.ipynb index a6f4469344..68c659a45d 100644 --- a/site/notebooks/how_to/tests/run_tests/configure_tests/customize_test_result_descriptions.ipynb +++ b/site/notebooks/how_to/tests/run_tests/configure_tests/customize_test_result_descriptions.ipynb @@ -136,7 +136,7 @@ "\n", "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", "\n", "2. Under **TEMPLATE**, select `Binary classification`.\n", "\n", @@ -151,10 +151,11 @@ "\n", "#### Get your code snippet\n", "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -177,7 +178,7 @@ " # api_key=\"...\",\n", " # api_secret=\"...\",\n", " # model=\"...\",\n", - " # document=\"documentation\",\n", + " document=\"documentation\",\n", ")" ] }, diff --git a/site/notebooks/how_to/tests/run_tests/configure_tests/enable_pii_detection.ipynb b/site/notebooks/how_to/tests/run_tests/configure_tests/enable_pii_detection.ipynb index 45de1ac669..e0ae28fff7 100644 --- a/site/notebooks/how_to/tests/run_tests/configure_tests/enable_pii_detection.ipynb +++ b/site/notebooks/how_to/tests/run_tests/configure_tests/enable_pii_detection.ipynb @@ -193,15 +193,11 @@ "\n", "#### Get your code snippet\n", "\n", - "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "2. In the left sidebar, navigate to **Inventory** and click **+ Register Model**.\n", - "\n", - "3. Enter the model details and click **Continue**. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/register-models-in-inventory.html))\n", - "\n", - "4. Go to **Getting Started** and click **Copy snippet to clipboard**.\n", - "\n", - "Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -225,7 +221,7 @@ " # api_key=\"...\",\n", " # api_secret=\"...\",\n", " # model=\"...\",\n", - " # document=\"documentation\",\n", + " document=\"documentation\",\n", ")" ] }, @@ -556,7 +552,7 @@ "\n", "1. From the **Inventory** in the ValidMind Platform, go to the model you registered earlier.\n", "\n", - "2. In the left sidebar that appears for your model, click **Documentation** under Documents.\n", + "2. In the left sidebar that appears for your model, click **Development** under Documents.\n", "\n", "3. Click on any section heading to expand that section to add a new test-driven block ([Need more help?](https://docs.validmind.ai/developer/model-documentation/work-with-test-results.html)).\n", "\n", diff --git a/site/notebooks/how_to/tests/run_tests/configure_tests/run_tests_that_require_multiple_datasets.ipynb b/site/notebooks/how_to/tests/run_tests/configure_tests/run_tests_that_require_multiple_datasets.ipynb index c01a11acea..472d7e59d1 100644 --- a/site/notebooks/how_to/tests/run_tests/configure_tests/run_tests_that_require_multiple_datasets.ipynb +++ b/site/notebooks/how_to/tests/run_tests/configure_tests/run_tests_that_require_multiple_datasets.ipynb @@ -185,7 +185,7 @@ "\n", "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", "\n", "2. Under **TEMPLATE**, select `Binary classification`.\n", "\n", @@ -200,10 +200,11 @@ "\n", "#### Get your code snippet\n", "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -226,7 +227,7 @@ " # api_key=\"...\",\n", " # api_secret=\"...\",\n", " # model=\"...\",\n", - " # document=\"documentation\",\n", + " document=\"documentation\",\n", ")" ] }, @@ -471,11 +472,11 @@ "\n", "### Work with your model documentation\n", "\n", - "1. From the **Model Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", + "1. From the **Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", "\n", - "2. Click and expand the **Model Development** section.\n", + "2. In the left sidebar that appears for your model, click **Development** under Documents.\n", "\n", - "What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation, view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html)\n", + "What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation, view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/working-with-model-documentation.html)\n", "\n", "\n", "\n", diff --git a/site/notebooks/how_to/tests/run_tests/documentation_tests/document_multiple_results_for_the_same_test.ipynb b/site/notebooks/how_to/tests/run_tests/documentation_tests/document_multiple_results_for_the_same_test.ipynb index 395a1a57a2..c7c0aa2cce 100644 --- a/site/notebooks/how_to/tests/run_tests/documentation_tests/document_multiple_results_for_the_same_test.ipynb +++ b/site/notebooks/how_to/tests/run_tests/documentation_tests/document_multiple_results_for_the_same_test.ipynb @@ -190,7 +190,7 @@ "\n", "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", "\n", "2. Under **TEMPLATE**, select `Binary classification`.\n", "\n", @@ -205,10 +205,11 @@ "\n", "#### Get your code snippet\n", "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -231,6 +232,7 @@ " # api_key=\"...\",\n", " # api_secret=\"...\",\n", " # model=\"...\",\n", + " document=\"documentation\",\n", ")" ] }, @@ -517,9 +519,9 @@ "\n", "### Work with your model documentation\n", "\n", - "1. From the **Model Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", + "1. From the **Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", "\n", - "2. Click and expand the **Model Development** section.\n", + "2. In the left sidebar that appears for your model, click **Development** under Documents.\n", "\n", "3. Expand the **2. Data Preparation** section and take a look around.\n", "\n", diff --git a/site/notebooks/how_to/tests/run_tests/documentation_tests/run_documentation_sections.ipynb b/site/notebooks/how_to/tests/run_tests/documentation_tests/run_documentation_sections.ipynb index cfdce6bdb1..63c8d02a62 100644 --- a/site/notebooks/how_to/tests/run_tests/documentation_tests/run_documentation_sections.ipynb +++ b/site/notebooks/how_to/tests/run_tests/documentation_tests/run_documentation_sections.ipynb @@ -183,7 +183,7 @@ "\n", "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", "\n", "2. Under **TEMPLATE**, select `Binary classification`.\n", "\n", @@ -198,10 +198,11 @@ "\n", "#### Get your code snippet\n", "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -224,6 +225,7 @@ " # api_key=\"...\",\n", " # api_secret=\"...\",\n", " # model=\"...\",\n", + " document=\"documentation\",\n", ")" ] }, @@ -493,11 +495,11 @@ "\n", "### Work with your model documentation\n", "\n", - "1. From the **Model Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", + "1. From the **Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", "\n", - "2. Click and expand the **Model Development** section.\n", + "2. In the left sidebar that appears for your model, click **Development** under Documents.\n", "\n", - "What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation, view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html)\n", + "What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation, view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/working-with-model-documentation.html)\n", "\n", "\n", "\n", diff --git a/site/notebooks/how_to/tests/run_tests/documentation_tests/run_documentation_tests_with_config.ipynb b/site/notebooks/how_to/tests/run_tests/documentation_tests/run_documentation_tests_with_config.ipynb index 44b7666c5f..d74a3773f7 100644 --- a/site/notebooks/how_to/tests/run_tests/documentation_tests/run_documentation_tests_with_config.ipynb +++ b/site/notebooks/how_to/tests/run_tests/documentation_tests/run_documentation_tests_with_config.ipynb @@ -187,7 +187,7 @@ "\n", "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", "\n", "2. Under **TEMPLATE**, select `Binary classification`.\n", "\n", @@ -202,10 +202,11 @@ "\n", "#### Get your code snippet\n", "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -228,6 +229,7 @@ " # api_key=\"...\",\n", " # api_secret=\"...\",\n", " # model=\"...\",\n", + " document=\"documentation\",\n", ")" ] }, @@ -618,11 +620,11 @@ "\n", "### Work with your model documentation\n", "\n", - "1. From the **Model Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", + "1. From the **Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", "\n", - "2. Click and expand the **Model Development** section.\n", + "2. In the left sidebar that appears for your model, click **Development** under Documents.\n", "\n", - "What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation, view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html)\n", + "What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation, view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/working-with-model-documentation.html)\n", "\n", "\n", "\n", diff --git a/site/notebooks/quickstart/quickstart_model_documentation.ipynb b/site/notebooks/quickstart/quickstart_model_documentation.ipynb index b2565f20d2..e7c28cf339 100644 --- a/site/notebooks/quickstart/quickstart_model_documentation.ipynb +++ b/site/notebooks/quickstart/quickstart_model_documentation.ipynb @@ -242,7 +242,7 @@ "\n", "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", "\n", "2. Under **TEMPLATE**, select `Binary classification`.\n", "\n", @@ -258,10 +258,11 @@ "\n", "#### Get your code snippet\n", "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -281,10 +282,10 @@ "import validmind as vm\n", "\n", "vm.init(\n", - " api_host=\"https://api.validmind.ai/api/v1/tracking\",\n", - " api_key=\"\",\n", - " api_secret=\"\",\n", - " model=\"\",\n", + " # api_host=\"...\",\n", + " # api_key=\"...\",\n", + " # api_secret=\"...\",\n", + " # model=\"...\",\n", " document=\"documentation\",\n", ")" ] @@ -365,7 +366,7 @@ "\n", "2. In the left sidebar, navigate to **Inventory** and select the model you registered for this notebook.\n", "\n", - "3. Click **Documentation** under Documents for your model and note how the structure of the documentation matches our preview above." + "3. Click **Development** under Documents for your model and note how the structure of the documentation matches our preview above." ] }, { @@ -778,9 +779,9 @@ "\n", "1. From the **Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", "\n", - "2. In the left sidebar that appears for your model, click **Documentation** under Documents.\n", + "2. In the left sidebar that appears for your model, click **Development** under Documents.\n", "\n", - " What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation, view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/working-with-model-documentation.html)" + "What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation, view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/working-with-model-documentation.html)" ] }, { diff --git a/site/notebooks/quickstart/quickstart_model_validation.ipynb b/site/notebooks/quickstart/quickstart_model_validation.ipynb index a3f6b1a25c..640e64015b 100644 --- a/site/notebooks/quickstart/quickstart_model_validation.ipynb +++ b/site/notebooks/quickstart/quickstart_model_validation.ipynb @@ -239,9 +239,9 @@ "\n", "#### Apply validation report template\n", "\n", - "Once you've registered your model, let's select a validation report template. A template predefines sections for your report and provides a general outline to follow, making the validation process much easier.\n", + "Next, let's select a validation report template. A template predefines sections for your report and provides a general outline to follow, making the validation process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Validation**.\n", "\n", "2. Under **TEMPLATE**, select `Generic Validation Report`.\n", "\n", @@ -281,9 +281,7 @@ "source": [ "\n", "\n", - "### Initialize the ValidMind Library\n", - "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your validation environment. You initialize the ValidMind Library with this code snippet, which ensures that your test results are uploaded to the correct model when you run the notebook." + "### Initialize the ValidMind Library" ] }, { @@ -295,8 +293,11 @@ "\n", "#### Get your code snippet\n", "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", + "\n", + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Validation` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -320,7 +321,7 @@ " # api_key=\"...\",\n", " # api_secret=\"...\",\n", " # model=\"...\",\n", - " # document=\"validation-report\",\n", + " document=\"validation-report\",\n", ")" ] }, @@ -368,7 +369,7 @@ "\n", "### Preview the validation report template\n", "\n", - "Let's verify that you have connected the ValidMind Library to the ValidMind Platform and that the appropriate *template* is selected for model validation.\n", + "Let's verify that you have connected the ValidMind Library to the ValidMind Platform and that the appropriate *template* is selected for model validation. A template predefines sections for your validation report and provides a general outline to follow, making the validation process much easier.\n", "\n", "You will attach evidence to this template in the form of risk assessment notes, artifacts, and test results later on. For now, **take a look at the default structure that the template provides with [the `vm.preview_template()` function](https://docs.validmind.ai/validmind/validmind.html#preview_template)** from the ValidMind library:" ] @@ -398,7 +399,7 @@ "\n", "2. In the left sidebar, navigate to **Inventory** and select the model you registered for this notebook.\n", "\n", - "3. Click **Validation Report** under Documents for your model and note:\n", + "3. Click **Validation** under Documents for your model and note:\n", "\n", " - [x] The risk assessment compliance summary at the top of the report (screenshot below)\n", " - [x] How the structure of the validation report reflects the previewed template\n", @@ -1097,11 +1098,13 @@ "\n", "### Work with your validation report\n", "\n", - "1. From the **Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", + "Now that you've logged all your test results and verified the work done by the model development team, head to the ValidMind Platform to wrap up your validation report:\n", + "\n", + "1. From the **Inventory** in the ValidMind Platform, go to the model you connected to earlier.\n", "\n", - "2. In the left sidebar that appears for your model, click **Validation Report** under Documents.\n", + "2. In the left sidebar that appears for your model, click **Validation** under Documents.\n", "\n", - " From here, you can link test results as evidence, add model artifacts, assess compliance, and submit your validation report for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/model-validation/preparing-validation-reports.html)" + "Include your logged test results as evidence, create risk assessment notes, add artifacts, and assess compliance, then submit your report for review when it's ready. **Learn more:** [Preparing validation reports](https://docs.validmind.ai/guide/model-validation/preparing-validation-reports.html)" ] }, { diff --git a/site/notebooks/tutorials/model_development/1-set_up_validmind.ipynb b/site/notebooks/tutorials/model_development/1-set_up_validmind.ipynb index 97a1e9dc15..f82f57eaa1 100644 --- a/site/notebooks/tutorials/model_development/1-set_up_validmind.ipynb +++ b/site/notebooks/tutorials/model_development/1-set_up_validmind.ipynb @@ -231,7 +231,7 @@ "\n", "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", "\n", "2. Under **TEMPLATE**, select `Binary classification`.\n", "\n", @@ -247,10 +247,11 @@ "\n", "#### Get your code snippet\n", "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -274,7 +275,7 @@ " # api_key=\"...\",\n", " # api_secret=\"...\",\n", " # model=\"...\",\n", - " # document=\"documentation\",\n", + " document=\"documentation\",\n", ")" ] }, @@ -327,7 +328,7 @@ "\n", "2. In the left sidebar, navigate to **Inventory** and select the model you registered for this \"ValidMind for model development\" series of notebooks.\n", "\n", - "3. Click **Documentation** under Documents for your model and note how the structure of the documentation matches our preview above." + "3. Click **Development** under Documents for your model and note how the structure of the documentation matches our preview above." ] }, { diff --git a/site/notebooks/tutorials/model_development/2-start_development_process.ipynb b/site/notebooks/tutorials/model_development/2-start_development_process.ipynb index 217df37d1c..b01cb35361 100644 --- a/site/notebooks/tutorials/model_development/2-start_development_process.ipynb +++ b/site/notebooks/tutorials/model_development/2-start_development_process.ipynb @@ -93,13 +93,9 @@ "\n", "First, let's connect up the ValidMind Library to our model we previously registered in the ValidMind Platform:\n", "\n", - "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n", - "\n", - "2. In the left sidebar, navigate to **Inventory** and select the model you registered for this \"ValidMind for model development\" series of notebooks.\n", - "\n", - "3. Go to **Getting Started** and click **Copy snippet to clipboard**.\n", - "\n", - "Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -126,7 +122,7 @@ " # api_key=\"...\",\n", " # api_secret=\"...\",\n", " # model=\"...\",\n", - " # document=\"documentation\",\n", + " document=\"documentation\",\n", ")" ] }, @@ -682,7 +678,7 @@ "\n", "1. From the **Inventory** in the ValidMind Platform, go to the model you connected to earlier.\n", "\n", - "2. In the left sidebar that appears for your model, click **Documentation** under Documents.\n", + "2. In the left sidebar that appears for your model, click **Development** under Documents.\n", "\n", "3. Locate the Data Preparation section and click on **2.3. Correlations and Interactions** to expand that section.\n", "\n", diff --git a/site/notebooks/tutorials/model_development/3-integrate_custom_tests.ipynb b/site/notebooks/tutorials/model_development/3-integrate_custom_tests.ipynb index f5bbf7f533..b4b990346b 100644 --- a/site/notebooks/tutorials/model_development/3-integrate_custom_tests.ipynb +++ b/site/notebooks/tutorials/model_development/3-integrate_custom_tests.ipynb @@ -108,13 +108,9 @@ "\n", "As usual, let's first connect up the ValidMind Library to our model we previously registered in the ValidMind Platform:\n", "\n", - "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n", - "\n", - "2. In the left sidebar, navigate to **Inventory** and select the model you registered for this \"ValidMind for model development\" series of notebooks.\n", - "\n", - "3. Go to **Getting Started** and click **Copy snippet to clipboard**.\n", - "\n", - "Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -141,7 +137,7 @@ " # api_key=\"...\",\n", " # api_secret=\"...\",\n", " # model=\"...\",\n", - " # document=\"documentation\",\n", + " document=\"documentation\",\n", ")" ] }, @@ -909,7 +905,7 @@ "\n", "1. From the **Inventory** in the ValidMind Platform, go to the model you connected to earlier.\n", "\n", - "2. In the left sidebar that appears for your model, click **Documentation** under Documents.\n", + "2. In the left sidebar that appears for your model, click **Development** under Documents.\n", "\n", "3. Locate the Data Preparation section and click on **3.2. Model Evaluation** to expand that section.\n", "\n", diff --git a/site/notebooks/tutorials/model_development/4-finalize_testing_documentation.ipynb b/site/notebooks/tutorials/model_development/4-finalize_testing_documentation.ipynb index b6064a005f..086ccc2029 100644 --- a/site/notebooks/tutorials/model_development/4-finalize_testing_documentation.ipynb +++ b/site/notebooks/tutorials/model_development/4-finalize_testing_documentation.ipynb @@ -105,13 +105,9 @@ "\n", "As usual, let's first connect up the ValidMind Library to our model we previously registered in the ValidMind Platform:\n", "\n", - "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n", - "\n", - "2. In the left sidebar, navigate to **Inventory** and select the model you registered for this \"ValidMind for model development\" series of notebooks.\n", - "\n", - "3. Go to **Getting Started** and click **Copy snippet to clipboard**.\n", - "\n", - "Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -138,7 +134,7 @@ " # api_key=\"...\",\n", " # api_secret=\"...\",\n", " # model=\"...\",\n", - " # document=\"documentation\",\n", + " document=\"documentation\",\n", ")" ] }, diff --git a/site/notebooks/tutorials/model_validation/1-set_up_validmind_for_validation.ipynb b/site/notebooks/tutorials/model_validation/1-set_up_validmind_for_validation.ipynb index 60bf8373d3..c5dc1fb394 100644 --- a/site/notebooks/tutorials/model_validation/1-set_up_validmind_for_validation.ipynb +++ b/site/notebooks/tutorials/model_validation/1-set_up_validmind_for_validation.ipynb @@ -243,7 +243,7 @@ "\n", "Next, let's select a validation report template. A template predefines sections for your report and provides a general outline to follow, making the validation process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Validation**.\n", "\n", "2. Under **TEMPLATE**, select `Generic Validation Report`.\n", "\n", @@ -283,9 +283,7 @@ "source": [ "\n", "\n", - "### Initialize the ValidMind Library\n", - "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your validation environment. You initialize the ValidMind Library with this code snippet, which ensures that your test results are uploaded to the correct model when you run the notebook." + "### Initialize the ValidMind Library" ] }, { @@ -297,13 +295,11 @@ "\n", "#### Get your code snippet\n", "\n", - "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n", - "\n", - "2. In the left sidebar, navigate to **Inventory** and select the model you registered for this \"ValidMind for model validation\" series of notebooks.\n", - "\n", - "3. Go to **Getting Started** and click **Copy snippet to clipboard**.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Validation` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -327,7 +323,7 @@ " # api_key=\"...\",\n", " # api_secret=\"...\",\n", " # model=\"...\",\n", - " # document=\"validation-report\",\n", + " document=\"validation-report\",\n", ")" ] }, @@ -350,7 +346,7 @@ "\n", "### Preview the validation report template\n", "\n", - "Let's verify that you have connected the ValidMind Library to the ValidMind Platform and that the appropriate *template* is selected for model validation. A template predefines sections for your validation report and provides a general outline to follow, making the validation process much easier.\n", + "Let's verify that you have connected the ValidMind Library to the ValidMind Platform and that the appropriate *template* is selected for your model.\n", "\n", "You will attach evidence to this template in the form of risk assessment notes, artifacts, and test results later on. For now, **take a look at the default structure that the template provides with [the `vm.preview_template()` function](https://docs.validmind.ai/validmind/validmind.html#preview_template)** from the ValidMind library:" ] @@ -380,7 +376,7 @@ "\n", "2. In the left sidebar, navigate to **Inventory** and select the model you registered for this \"ValidMind for model validation\" series of notebooks.\n", "\n", - "3. Click **Validation Report** under Documents for your model and note:\n", + "3. Click **Validation** under Documents for your model and note:\n", "\n", " - [x] The risk assessment compliance summary at the top of the report (screenshot below)\n", " - [x] How the structure of the validation report reflects the previewed template\n", diff --git a/site/notebooks/tutorials/model_validation/2-start_validation_process.ipynb b/site/notebooks/tutorials/model_validation/2-start_validation_process.ipynb index e0618c24a9..7fafa5ab50 100644 --- a/site/notebooks/tutorials/model_validation/2-start_validation_process.ipynb +++ b/site/notebooks/tutorials/model_validation/2-start_validation_process.ipynb @@ -96,13 +96,9 @@ "\n", "First, let's connect up the ValidMind Library to our model we previously registered in the ValidMind Platform:\n", "\n", - "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n", - "\n", - "2. In the left sidebar, navigate to **Inventory** and select the model you registered for this \"ValidMind for model validation\" series of notebooks.\n", - "\n", - "3. Go to **Getting Started** and click **Copy snippet to clipboard**.\n", - "\n", - "Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Validation` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -129,7 +125,7 @@ " # api_key=\"...\",\n", " # api_secret=\"...\",\n", " # model=\"...\",\n", - " # document=\"validation-report\",\n", + " document=\"validation-report\",\n", ")" ] }, @@ -693,7 +689,7 @@ "\n", "1. From the **Inventory** in the ValidMind Platform, go to the model you connected to earlier.\n", "\n", - "2. In the left sidebar that appears for your model, click **Validation Report** under Documents.\n", + "2. In the left sidebar that appears for your model, click **Validation** under Documents.\n", "\n", "3. Locate the Data Preparation section and click on **2.2.1. Data Quality** to expand that section.\n", "\n", diff --git a/site/notebooks/tutorials/model_validation/3-developing_challenger_model.ipynb b/site/notebooks/tutorials/model_validation/3-developing_challenger_model.ipynb index cc3f25c4d8..e86fde85de 100644 --- a/site/notebooks/tutorials/model_validation/3-developing_challenger_model.ipynb +++ b/site/notebooks/tutorials/model_validation/3-developing_challenger_model.ipynb @@ -103,13 +103,9 @@ "\n", "As usual, let's first connect up the ValidMind Library to our model we previously registered in the ValidMind Platform:\n", "\n", - "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n", - "\n", - "2. In the left sidebar, navigate to **Inventory** and select the model you registered for this \"ValidMind for model validation\" series of notebooks.\n", - "\n", - "3. Go to **Getting Started** and click **Copy snippet to clipboard**.\n", - "\n", - "Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Validation` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -136,7 +132,7 @@ " # api_key=\"...\",\n", " # api_secret=\"...\",\n", " # model=\"...\",\n", - " # document=\"validation-report\",\n", + " document=\"validation-report\",\n", ")" ] }, @@ -627,7 +623,7 @@ "\n", "1. From the **Inventory** in the ValidMind Platform, go to the model you connected to earlier.\n", "\n", - "2. In the left sidebar that appears for your model, click **Validation Report** under Documents.\n", + "2. In the left sidebar that appears for your model, click **Validation** under Documents.\n", "\n", "3. Locate the Data Preparation section and click on **2.2.2. Model Performance** to expand that section.\n", "\n", diff --git a/site/notebooks/tutorials/model_validation/4-finalize_validation_reporting.ipynb b/site/notebooks/tutorials/model_validation/4-finalize_validation_reporting.ipynb index 3eaee5a3ac..854ccde766 100644 --- a/site/notebooks/tutorials/model_validation/4-finalize_validation_reporting.ipynb +++ b/site/notebooks/tutorials/model_validation/4-finalize_validation_reporting.ipynb @@ -112,13 +112,9 @@ "\n", "As usual, let's first connect up the ValidMind Library to our model we previously registered in the ValidMind Platform:\n", "\n", - "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n", - "\n", - "2. In the left sidebar, navigate to **Inventory** and select the model you registered for this \"ValidMind for model validation\" series of notebooks.\n", - "\n", - "3. Go to **Getting Started** and click **Copy snippet to clipboard**.\n", - "\n", - "Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Validation` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -145,7 +141,7 @@ " # api_key=\"...\",\n", " # api_secret=\"...\",\n", " # model=\"...\",\n", - " # document=\"validation-report\",\n", + " document=\"validation-report\",\n", ")" ] }, diff --git a/site/notebooks/use_cases/agents/document_agentic_ai.ipynb b/site/notebooks/use_cases/agents/document_agentic_ai.ipynb index 54d9ef3a9f..89e8152214 100644 --- a/site/notebooks/use_cases/agents/document_agentic_ai.ipynb +++ b/site/notebooks/use_cases/agents/document_agentic_ai.ipynb @@ -1,2180 +1,2181 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Document an agentic AI system\n", - "\n", - "Build and document an agentic AI system with the ValidMind Library. Construct a LangGraph-based banking agent, assign AI evaluation metric scores to your agent, and run accuracy, RAGAS, and safety tests, then log those test results to the ValidMind Platform.\n", - "\n", - "An _AI agent_ is an autonomous system that interprets inputs, selects from available tools or actions, and executes multi-step behaviors to achieve defined goals. In this notebook, the agent acts as a banking assistant that analyzes user requests and automatically selects and invokes the appropriate specialized banking tool to deliver accurate, compliant, and actionable responses.\n", - "\n", - "- This agent enables financial institutions to automate complex banking workflows where different customer requests require different specialized tools and knowledge bases.\n", - "- Effective validation of agentic AI systems reduces the risks of agents misinterpreting inputs, failing to extract required parameters, or producing incorrect assessments or actions — such as selecting the wrong tool.\n", - "\n", - "
For the LLM components in this notebook to function properly, you'll need access to OpenAI.\n", - "

\n", - "Before you continue, ensure that a valid OPENAI_API_KEY is set in your .env file.
" - ], - "id": "eee6b64c" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "::: {.content-hidden when-format=\"html\"}\n", - "## Contents \n", - "- [About ValidMind](#toc1__) \n", - " - [Before you begin](#toc1_1__) \n", - " - [New to ValidMind?](#toc1_2__) \n", - " - [Key concepts](#toc1_3__) \n", - "- [Setting up](#toc2__) \n", - " - [Install the ValidMind Library](#toc2_1__) \n", - " - [Initialize the ValidMind Library](#toc2_2__) \n", - " - [Register sample model](#toc2_2_1__) \n", - " - [Apply documentation template](#toc2_2_2__) \n", - " - [Get your code snippet](#toc2_2_3__) \n", - " - [Preview the documentation template](#toc2_2_4__) \n", - " - [Verify OpenAI API access](#toc2_3__) \n", - " - [Initialize the Python environment](#toc2_4__) \n", - "- [Building the LangGraph agent](#toc3__) \n", - " - [Test available banking tools](#toc3_1__) \n", - " - [Create LangGraph banking agent](#toc3_2__) \n", - " - [Define system prompt](#toc3_2_1__) \n", - " - [Initialize the LLM](#toc3_2_2__) \n", - " - [Define agent state structure](#toc3_2_3__) \n", - " - [Create agent workflow function](#toc3_2_4__) \n", - " - [Instantiate the banking agent](#toc3_2_5__) \n", - " - [Integrate agent with ValidMind](#toc3_3__) \n", - " - [Import ValidMind components](#toc3_3_1__) \n", - " - [Create agent wrapper function](#toc3_3_2__) \n", - " - [Initialize the ValidMind model object](#toc3_3_3__) \n", - " - [Store the agent reference](#toc3_3_4__) \n", - " - [Verify integration](#toc3_3_5__) \n", - " - [Validate the system prompt](#toc3_4__) \n", - "- [Initializing the ValidMind dataset](#toc4__) \n", - " - [Assign predictions](#toc4_1__) \n", - "- [Running accuracy tests](#toc5__) \n", - " - [Response accuracy test](#toc5_1__) \n", - " - [Tool selection accuracy test](#toc5_2__) \n", - "- [Assigning AI evaluation metric scores](#toc6__) \n", - " - [Identify relevant DeepEval scorers](#toc6_1__) \n", - " - [Assign reasoning scores](#toc6_2__) \n", - " - [Plan quality score](#toc6_2_1__) \n", - " - [Plan adherence score](#toc6_2_2__) \n", - " - [Assign action scores](#toc6_3__) \n", - " - [Tool correctness score](#toc6_3_1__) \n", - " - [Argument correctness score](#toc6_3_2__) \n", - " - [Assign execution score](#toc6_4__) \n", - " - [Task completion score](#toc6_4_1__) \n", - "- [Running RAGAS tests](#toc7__) \n", - " - [Identify relevant RAGAS tests](#toc7_1__) \n", - " - [Faithfulness](#toc7_1_1__) \n", - " - [Response Relevancy](#toc7_1_2__) \n", - " - [Context Recall](#toc7_1_3__) \n", - "- [Running safety tests](#toc8__) \n", - " - [AspectCritic](#toc8_1_1__) \n", - " - [Bias](#toc8_1_2__) \n", - "- [Next steps](#toc9__) \n", - " - [Work with your model documentation](#toc9_1__) \n", - " - [Customize the banking agent for your use case](#toc9_2__) \n", - " - [Discover more learning resources](#toc9_3__) \n", - "- [Upgrade ValidMind](#toc10__) \n", - "\n", - ":::\n", - "\n", - "" - ], - "id": "30927b2b" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## About ValidMind\n", - "\n", - "ValidMind is a suite of tools for managing model risk, including risk associated with AI and statistical models. \n", - "\n", - "You use the ValidMind Library to automate documentation and validation tests, and then use the ValidMind Platform to collaborate on model documentation. Together, these products simplify model risk management, facilitate compliance with regulations and institutional standards, and enhance collaboration between yourself and model validators." - ], - "id": "b58139db" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Before you begin\n", - "\n", - "This notebook assumes you have basic familiarity with Python, including an understanding of how functions work. If you are new to Python, you can still run the notebook but we recommend further familiarizing yourself with the language. \n", - "\n", - "If you encounter errors due to missing modules in your Python environment, install the modules with `pip install`, and then re-run the notebook. For more help, refer to [Installing Python Modules](https://docs.python.org/3/installing/index.html)." - ], - "id": "7e30d36b" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### New to ValidMind?\n", - "\n", - "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n", - "\n", - "
For access to all features available in this notebook, you'll need access to a ValidMind account.\n", - "

\n", - "Register with ValidMind
" - ], - "id": "1cba586e" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Key concepts\n", - "\n", - "**Model documentation**: A structured and detailed record pertaining to a model, encompassing key components such as its underlying assumptions, methodologies, data sources, inputs, performance metrics, evaluations, limitations, and intended uses. It serves to ensure transparency, adherence to regulatory requirements, and a clear understanding of potential risks associated with the model’s application.\n", - "\n", - "**Documentation template**: Functions as a test suite and lays out the structure of model documentation, segmented into various sections and sub-sections. Documentation templates define the structure of your model documentation, specifying the tests that should be run, and how the results should be displayed.\n", - "\n", - "**Tests**: A function contained in the ValidMind Library, designed to run a specific quantitative test on the dataset or model. Tests are the building blocks of ValidMind, used to evaluate and document models and datasets, and can be run individually or as part of a suite defined by your model documentation template.\n", - "\n", - "**Metrics**: A subset of tests that do not have thresholds. In the context of this notebook, metrics and tests can be thought of as interchangeable concepts.\n", - "\n", - "**Custom metrics**: Custom metrics are functions that you define to evaluate your model or dataset. These functions can be registered with the ValidMind Library to be used in the ValidMind Platform.\n", - "\n", - "**Inputs**: Objects to be evaluated and documented in the ValidMind Library. They can be any of the following:\n", - "\n", - " - **model**: A single model that has been initialized in ValidMind with [`vm.init_model()`](https://docs.validmind.ai/validmind/validmind.html#init_model).\n", - " - **dataset**: Single dataset that has been initialized in ValidMind with [`vm.init_dataset()`](https://docs.validmind.ai/validmind/validmind.html#init_dataset).\n", - " - **models**: A list of ValidMind models - usually this is used when you want to compare multiple models in your custom metric.\n", - " - **datasets**: A list of ValidMind datasets - usually this is used when you want to compare multiple datasets in your custom metric. (Learn more: [Run tests with multiple datasets](https://docs.validmind.ai/notebooks/how_to/run_tests_that_require_multiple_datasets.html))\n", - "\n", - "**Parameters**: Additional arguments that can be passed when running a ValidMind test, used to pass additional information to a metric, customize its behavior, or provide additional context.\n", - "\n", - "**Outputs**: Custom metrics can return elements like tables or plots. Tables may be a list of dictionaries (each representing a row) or a pandas DataFrame. Plots may be matplotlib or plotly figures.\n", - "\n", - "**Test suites**: Collections of tests designed to run together to automate and generate model documentation end-to-end for specific use-cases.\n", - "\n", - "Example: the [`classifier_full_suite`](https://docs.validmind.ai/validmind/validmind/test_suites/classifier.html#ClassifierFullSuite) test suite runs tests from the [`tabular_dataset`](https://docs.validmind.ai/validmind/validmind/test_suites/tabular_datasets.html) and [`classifier`](https://docs.validmind.ai/validmind/validmind/test_suites/classifier.html) test suites to fully document the data and model sections for binary classification model use-cases." - ], - "id": "5c46f003" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Setting up" - ], - "id": "11a2d7a5" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Install the ValidMind Library\n", - "\n", - "
Recommended Python versions\n", - "

\n", - "Python 3.9 <= x <= 3.11
\n", - "\n", - "Let's begin by installing the ValidMind Library with large language model (LLM) support:" - ], - "id": "fbab0edf" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "%pip install -q \"validmind[llm]\" \"langgraph==0.3.21\"" - ], - "execution_count": null, - "outputs": [], - "id": "1982a118" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Initialize the ValidMind Library" - ], - "id": "14578e26" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Register sample model\n", - "\n", - "Let's first register a sample model for use with this notebook.\n", - "\n", - "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n", - "\n", - "2. In the left sidebar, navigate to **Inventory** and click **+ Register Model**.\n", - "\n", - "3. Enter the model details and click **Next >** to continue to assignment of model stakeholders. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/register-models-in-inventory.html))\n", - "\n", - "4. Select your own name under the **MODEL OWNER** drop-down.\n", - "\n", - "5. Click **Register Model** to add the model to your inventory." - ], - "id": "83d47d89" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Apply documentation template\n", - "\n", - "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", - "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", - "\n", - "2. Under **TEMPLATE**, select `Agentic AI`.\n", - "\n", - "3. Click **Use Template** to apply the template." - ], - "id": "bb2c5670" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "
Can't select this template?\n", - "

\n", - "Your organization administrators may need to add it to your template library:\n", - "\n", - "
" - ], - "id": "98e475c1" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Get your code snippet\n", - "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", - "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" - ], - "id": "0d1a13ca" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "# Load your model identifier credentials from an `.env` file\n", - "\n", - "%load_ext dotenv\n", - "%dotenv .env\n", - "\n", - "# Or replace with your code snippet\n", - "\n", - "import validmind as vm\n", - "\n", - "vm.init(\n", - " # api_host=\"...\",\n", - " # api_key=\"...\",\n", - " # api_secret=\"...\",\n", - " # model=\"...\",\n", - " # document=\"documentation\",\n", - ")" - ], - "execution_count": null, - "outputs": [], - "id": "d6ccbefc" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Preview the documentation template\n", - "\n", - "Let's verify that you have connected the ValidMind Library to the ValidMind Platform and that the appropriate *template* is selected for your model.\n", - "\n", - "You will upload documentation and test results unique to your model based on this template later on. For now, **take a look at the default structure that the template provides with [the `vm.preview_template()` function](https://docs.validmind.ai/validmind/validmind.html#preview_template)** from the ValidMind library and note the empty sections:" - ], - "id": "3605df4f" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "vm.preview_template()" - ], - "execution_count": null, - "outputs": [], - "id": "dffdaa6f" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Verify OpenAI API access\n", - "\n", - "Verify that a valid `OPENAI_API_KEY` is set in your `.env` file:" - ], - "id": "d467c1d2" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "# Load environment variables if using .env file\n", - "try:\n", - " from dotenv import load_dotenv\n", - " load_dotenv()\n", - "except ImportError:\n", - " print(\"dotenv not installed. Make sure OPENAI_API_KEY is set in your environment.\")" - ], - "execution_count": null, - "outputs": [], - "id": "22cc39cb" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Initialize the Python environment\n", - "\n", - "Let's import all the necessary libraries to prepare for building our banking LangGraph agentic system:\n", - "\n", - "- **Standard libraries** for data handling and environment management.\n", - "- **pandas**, a Python library for data manipulation and analytics, as an alias. We'll also configure pandas to show all columns and all rows at full width for easier debugging and inspection.\n", - "- **LangChain** components for LLM integration and tool management.\n", - "- **LangGraph** for building stateful, multi-step agent workflows.\n", - "- **Banking tools** for specialized financial services as defined in [banking_tools.py](banking_tools.py)." - ], - "id": "b56c3f39" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "from typing import TypedDict, Annotated, Sequence\n", - "\n", - "from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage\n", - "from langchain_openai import ChatOpenAI\n", - "from langgraph.checkpoint.memory import MemorySaver\n", - "from langgraph.graph import StateGraph, END, START\n", - "from langgraph.graph.message import add_messages\n", - "from langgraph.prebuilt import ToolNode\n", - "\n", - "# LOCAL IMPORTS FROM banking_tools.py\n", - "from banking_tools import AVAILABLE_TOOLS\n", - "\n", - "import pandas as pd\n", - "# Configure pandas to show all columns and all rows at full width\n", - "pd.set_option('display.max_columns', None)\n", - "pd.set_option('display.max_colwidth', None)\n", - "pd.set_option('display.width', None)\n", - "pd.set_option('display.max_rows', None)" - ], - "execution_count": null, - "outputs": [], - "id": "2058d1ac" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Building the LangGraph agent" - ], - "id": "cc1d3265" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Test available banking tools\n", - "\n", - "We'll use the demo banking tools defined in `banking_tools.py` that provide use cases of financial services:\n", - "\n", - "- **Credit Risk Analyzer** - Loan applications and credit decisions\n", - "- **Customer Account Manager** - Account services and customer support\n", - "- **Fraud Detection System** - Security and fraud prevention" - ], - "id": "a3c421c4" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "print(f\"Available tools: {len(AVAILABLE_TOOLS)}\")\n", - "print(\"\\nTool Details:\")\n", - "for i, tool in enumerate(AVAILABLE_TOOLS, 1):\n", - " print(f\" - {tool.name}\")" - ], - "execution_count": null, - "outputs": [], - "id": "1e0a120c" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's test each banking tool individually to ensure they're working correctly before integrating them into our agent:" - ], - "id": "53906630" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "# Test 1: Credit Risk Analyzer\n", - "print(\"TEST 1: Credit Risk Analyzer\")\n", - "print(\"-\" * 40)\n", - "try:\n", - " # Access the underlying function using .func\n", - " credit_result = AVAILABLE_TOOLS[0].func(\n", - " customer_income=75000,\n", - " customer_debt=1200,\n", - " credit_score=720,\n", - " loan_amount=50000,\n", - " loan_type=\"personal\"\n", - " )\n", - " print(credit_result)\n", - " print(\"Credit Risk Analyzer test PASSED\")\n", - "except Exception as e:\n", - " print(f\"Credit Risk Analyzer test FAILED: {e}\")\n", - "\n", - "print(\"\" + \"=\" * 60)" - ], - "execution_count": null, - "outputs": [], - "id": "dc0caff2" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "\n", - "# Test 2: Customer Account Manager\n", - "print(\"TEST 2: Customer Account Manager\")\n", - "print(\"-\" * 40)\n", - "try:\n", - " # Test checking balance\n", - " account_result = AVAILABLE_TOOLS[1].func(\n", - " account_type=\"checking\",\n", - " customer_id=\"12345\",\n", - " action=\"check_balance\"\n", - " )\n", - " print(account_result)\n", - "\n", - " # Test getting account info\n", - " info_result = AVAILABLE_TOOLS[1].func(\n", - " account_type=\"all\",\n", - " customer_id=\"12345\", \n", - " action=\"get_info\"\n", - " )\n", - " print(info_result)\n", - " print(\"Customer Account Manager test PASSED\")\n", - "except Exception as e:\n", - " print(f\"Customer Account Manager test FAILED: {e}\")\n", - "\n", - "print(\"\" + \"=\" * 60)" - ], - "execution_count": null, - "outputs": [], - "id": "b6b227db" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "\n", - "# Test 3: Fraud Detection System\n", - "print(\"TEST 3: Fraud Detection System\")\n", - "print(\"-\" * 40)\n", - "try:\n", - " fraud_result = AVAILABLE_TOOLS[2].func(\n", - " transaction_id=\"TX123\",\n", - " customer_id=\"12345\",\n", - " transaction_amount=500.00,\n", - " transaction_type=\"withdrawal\",\n", - " location=\"Miami, FL\",\n", - " device_id=\"DEVICE_001\"\n", - " )\n", - " print(fraud_result)\n", - " print(\"Fraud Detection System test PASSED\")\n", - "except Exception as e:\n", - " print(f\"Fraud Detection System test FAILED: {e}\")\n", - "\n", - "print(\"\" + \"=\" * 60)" - ], - "execution_count": null, - "outputs": [], - "id": "a983b30d" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Create LangGraph banking agent\n", - "\n", - "With our tools ready to go, we'll create our intelligent banking agent with LangGraph that automatically selects and uses the appropriate banking tool based on a user request." - ], - "id": "1424baed" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Define system prompt\n", - "\n", - "We'll begin by defining our system prompt, which provides the LLM with context about its role as a banking assistant and guidance on when to use each available tool:" - ], - "id": "3469d656" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "\n", - "# Enhanced banking system prompt with tool selection guidance\n", - "system_context = \"\"\"You are a professional banking AI assistant with access to specialized banking tools.\n", - " Analyze the user's banking request and directly use the most appropriate tools to help them.\n", - " \n", - " AVAILABLE BANKING TOOLS:\n", - " \n", - " credit_risk_analyzer - Analyze credit risk for loan applications and credit decisions\n", - " - Use for: loan applications, credit assessments, risk analysis, mortgage eligibility\n", - " - Examples: \"Analyze credit risk for $50k personal loan\", \"Assess mortgage eligibility for $300k home purchase\"\n", - " - Parameters: customer_income, customer_debt, credit_score, loan_amount, loan_type\n", - "\n", - " customer_account_manager - Manage customer accounts and provide banking services\n", - " - Use for: account information, transaction processing, product recommendations, customer service\n", - " - Examples: \"Check balance for checking account 12345\", \"Recommend products for customer with high balance\"\n", - " - Parameters: account_type, customer_id, action, amount, account_details\n", - "\n", - " fraud_detection_system - Analyze transactions for potential fraud and security risks\n", - " - Use for: transaction monitoring, fraud prevention, risk assessment, security alerts\n", - " - Examples: \"Analyze fraud risk for $500 ATM withdrawal in Miami\", \"Check security for $2000 online purchase\"\n", - " - Parameters: transaction_id, customer_id, transaction_amount, transaction_type, location, device_id\n", - "\n", - " BANKING INSTRUCTIONS:\n", - " - Analyze the user's banking request carefully and identify the primary need\n", - " - If they need credit analysis → use credit_risk_analyzer\n", - " - If they need financial calculations → use financial_calculator\n", - " - If they need account services → use customer_account_manager\n", - " - If they need security analysis → use fraud_detection_system\n", - " - Extract relevant parameters from the user's request\n", - " - Provide helpful, accurate banking responses based on tool outputs\n", - " - Always consider banking regulations, risk management, and best practices\n", - " - Be professional and thorough in your analysis\n", - "\n", - " Choose and use tools wisely to provide the most helpful banking assistance.\n", - " Describe the response in user friendly manner with details describing the tool output. \n", - " Provide the response in at least 500 words.\n", - " Generate a concise execution plan for the banking request.\n", - " \"\"\"" - ], - "execution_count": null, - "outputs": [], - "id": "7971c427" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Initialize the LLM\n", - "\n", - "Let's initialize the LLM that will power our banking agent:" - ], - "id": "b66c1ac4" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "# Initialize the main LLM for banking responses\n", - "main_llm = ChatOpenAI(\n", - " model=\"gpt-5-mini\",\n", - " reasoning={\n", - " \"effort\": \"low\",\n", - " \"summary\": \"auto\"\n", - " }\n", - ")" - ], - "execution_count": null, - "outputs": [], - "id": "866066e7" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Then bind the available banking tools to the LLM, enabling the model to automatically recognize and invoke each tool when appropriate based on request input and the system prompt we defined above:" - ], - "id": "8220afd6" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "# Bind all banking tools to the main LLM\n", - "llm_with_tools = main_llm.bind_tools(AVAILABLE_TOOLS)" - ], - "execution_count": null, - "outputs": [], - "id": "906d8132" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Define agent state structure\n", - "\n", - "The agent state defines the data structure that flows through the LangGraph workflow. It includes:\n", - "\n", - "- **messages** — The conversation history between the user and agent\n", - "- **user_input** — The current user request\n", - "- **session_id** — A unique identifier for the conversation session\n", - "- **context** — Additional context that can be passed between nodes\n", - "\n", - "Defining this state structure maintains the structure throughout the agent's execution and allows for multi-turn conversations with memory:" - ], - "id": "43f56651" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "# Banking Agent State Definition\n", - "class BankingAgentState(TypedDict):\n", - " messages: Annotated[Sequence[BaseMessage], add_messages]\n", - " user_input: str\n", - " session_id: str\n", - " context: dict" - ], - "execution_count": null, - "outputs": [], - "id": "6b926ddf" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Create agent workflow function\n", - "\n", - "We'll build the LangGraph agent workflow with two main components:\n", - "\n", - "1. **LLM node** — Processes user requests, applies the system prompt, and decides whether to use tools.\n", - "2. **Tools node** — Executes the selected banking tools when the LLM determines they're needed.\n", - "\n", - "The workflow begins with the LLM analyzing the request, then uses tools if needed — or ends if the response is complete, and finally returns to the LLM to generate the final response." - ], - "id": "387ba780" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "def create_banking_langgraph_agent():\n", - " \"\"\"Create a comprehensive LangGraph banking agent with intelligent tool selection.\"\"\"\n", - " def llm_node(state: BankingAgentState) -> BankingAgentState:\n", - " \"\"\"Main LLM node that processes banking requests and selects appropriate tools.\"\"\"\n", - " messages = state[\"messages\"]\n", - " # Add system context to messages\n", - " enhanced_messages = [SystemMessage(content=system_context)] + list(messages)\n", - " # Get LLM response with tool selection\n", - " response = llm_with_tools.invoke(enhanced_messages)\n", - " return {\n", - " **state,\n", - " \"messages\": messages + [response]\n", - " }\n", - " \n", - " def should_continue(state: BankingAgentState) -> str:\n", - " \"\"\"Decide whether to use tools or end the conversation.\"\"\"\n", - " last_message = state[\"messages\"][-1]\n", - " # Check if the LLM wants to use tools\n", - " if hasattr(last_message, 'tool_calls') and last_message.tool_calls:\n", - " return \"tools\"\n", - " return END\n", - " \n", - " # Create the banking state graph\n", - " workflow = StateGraph(BankingAgentState)\n", - " # Add nodes\n", - " workflow.add_node(\"llm\", llm_node)\n", - " workflow.add_node(\"tools\", ToolNode(AVAILABLE_TOOLS))\n", - " # Simplified entry point - go directly to LLM\n", - " workflow.add_edge(START, \"llm\")\n", - " # From LLM, decide whether to use tools or end\n", - " workflow.add_conditional_edges(\n", - " \"llm\",\n", - " should_continue,\n", - " {\"tools\": \"tools\", END: END}\n", - " )\n", - " # Tool execution flows back to LLM for final response\n", - " workflow.add_edge(\"tools\", \"llm\")\n", - " # Set up memory\n", - " memory = MemorySaver()\n", - " # Compile the graph\n", - " agent = workflow.compile(checkpointer=memory)\n", - " return agent" - ], - "execution_count": null, - "outputs": [], - "id": "2c9bf585" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Instantiate the banking agent\n", - "\n", - "Now, we'll create an instance of the banking agent by calling the workflow creation function.\n", - "\n", - "This compiled agent is ready to process banking requests and will automatically select and use the appropriate tools based on user queries:" - ], - "id": "765242e9" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "# Create the banking intelligent agent\n", - "banking_agent = create_banking_langgraph_agent()\n", - "\n", - "print(\"Banking LangGraph Agent Created Successfully!\")\n", - "print(\"\\nFeatures:\")\n", - "print(\" - Intelligent banking tool selection\")\n", - "print(\" - Comprehensive banking system prompt\")\n", - "print(\" - Streamlined workflow: LLM → Tools → Response\")\n", - "print(\" - Automatic tool parameter extraction\")\n", - "print(\" - Professional banking assistance\")" - ], - "execution_count": null, - "outputs": [], - "id": "455b8ee4" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Integrate agent with ValidMind\n", - "\n", - "To integrate our LangGraph banking agent with ValidMind, we need to create a wrapper function that ValidMind can use to invoke the agent and extract the necessary information for testing and documentation, allowing ValidMind to run validation tests on the agent's behavior, tool usage, and responses." - ], - "id": "e00dac77" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Import ValidMind components\n", - "\n", - "We'll start with importing the necessary ValidMind components for integrating our agent:\n", - "\n", - "- `Prompt` from `validmind.models` for handling prompt-based model inputs\n", - "- `extract_tool_calls_from_agent_output` and `_convert_to_tool_call_list` from `validmind.scorers.llm.deepeval` for extracting and converting tool calls from agent outputs" - ], - "id": "a124857e" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "from validmind.models import Prompt\n", - "from validmind.scorers.llm.deepeval import extract_tool_calls_from_agent_output, _convert_to_tool_call_list\n", - "from deepeval.tracing import observe, update_current_span\n", - "from deepeval.test_case import LLMTestCase" - ], - "execution_count": null, - "outputs": [], - "id": "9aeb8969" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Create agent wrapper function\n", - "\n", - "We'll then create a wrapper function that:\n", - "\n", - "- Accepts input in ValidMind's expected format (with `input` and `session_id` fields)\n", - "- Invokes the banking agent with the proper state initialization\n", - "- Captures tool outputs and tool calls for evaluation\n", - "- Returns a standardized response format that includes the prediction, full output, tool messages, and tool call information\n", - "- Handles errors gracefully with fallback responses" - ], - "id": "ed72903f" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "@observe(type=\"agent\")\n", - "def banking_agent_fn(input):\n", - " \"\"\"\n", - " Invoke the banking agent with the given input.\n", - " \"\"\"\n", - " try:\n", - " # Initial state for banking agent\n", - " initial_state = {\n", - " \"user_input\": input[\"input\"],\n", - " \"messages\": [HumanMessage(content=input[\"input\"])],\n", - " \"session_id\": input[\"session_id\"],\n", - " \"context\": {}\n", - " }\n", - " session_config = {\"configurable\": {\"thread_id\": input[\"session_id\"]}}\n", - " result = banking_agent.invoke(initial_state, config=session_config)\n", - "\n", - " from utils import capture_tool_output_messages\n", - "\n", - " # Capture all tool outputs and metadata\n", - " captured_data = capture_tool_output_messages(result)\n", - " \n", - " # Access specific tool outputs, this will be used for RAGAS tests\n", - " tool_message = \"\"\n", - " for output in captured_data[\"tool_outputs\"]:\n", - " tool_message += output['content']\n", - " \n", - " tool_calls_found = []\n", - " messages = result['messages']\n", - " for message in messages:\n", - " if hasattr(message, 'tool_calls') and message.tool_calls:\n", - " for tool_call in message.tool_calls:\n", - " # Handle both dictionary and object formats\n", - " if isinstance(tool_call, dict):\n", - " tool_calls_found.append(tool_call['name'])\n", - " else:\n", - " # ToolCall object - use attribute access\n", - " tool_calls_found.append(tool_call.name)\n", - "\n", - " prediction_text = result['messages'][-1].content[0]['text']\n", - " tools_called_value = _convert_to_tool_call_list(extract_tool_calls_from_agent_output(result))\n", - " expected_tools_value = _convert_to_tool_call_list(input.get(\"expected_tools\", []))\n", - "\n", - " # Feed trace data for DeepEval metrics (e.g. PlanQuality) that require tracing\n", - " update_current_span(\n", - " test_case=LLMTestCase(\n", - " input=input[\"input\"],\n", - " actual_output=prediction_text,\n", - " tools_called=tools_called_value,\n", - " expected_tools=expected_tools_value\n", - " )\n", - " )\n", - "\n", - " return {\n", - " \"prediction\": prediction_text,\n", - " \"output\": result,\n", - " \"tool_messages\": [tool_message],\n", - " # \"tool_calls\": tool_calls_found,\n", - " \"tool_called\": tools_called_value\n", - " }\n", - " except Exception as e:\n", - " # Return a fallback response if the agent fails\n", - " error_message = f\"\"\"I apologize, but I encountered an error while processing your banking request: {str(e)}.\n", - " Please try rephrasing your question or contact support if the issue persists.\"\"\"\n", - " return {\n", - " \"prediction\": error_message, \n", - " \"output\": {\n", - " \"messages\": [HumanMessage(content=input[\"input\"]), SystemMessage(content=error_message)],\n", - " \"error\": str(e)\n", - " }\n", - " }" - ], - "execution_count": null, - "outputs": [], - "id": "0e4d5a82" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Initialize the ValidMind model object\n", - "\n", - "We'll also need to register the banking agent as a ValidMind model object (`vm_model`) that can be passed to other functions for analysis and tests on the data.\n", - "\n", - "You simply initialize this model object with [`vm.init_model()`](https://docs.validmind.ai/validmind/validmind.html#init_model) that:\n", - "\n", - "- Associates the wrapper function with the model for prediction\n", - "- Stores the system prompt template for documentation\n", - "- Provides a unique `input_id` for tracking and identification\n", - "- Enables the agent to be used with ValidMind's testing and documentation features" - ], - "id": "fda87401" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "# Initialize the agent as a model\n", - "vm_banking_model = vm.init_model(\n", - " input_id=\"banking_agent_model\",\n", - " predict_fn=banking_agent_fn,\n", - " prompt=Prompt(template=system_context)\n", - ")" - ], - "execution_count": null, - "outputs": [], - "id": "60a2ce7a" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Store the agent reference\n", - "\n", - "We'll also store a reference to the original banking agent object in the ValidMind model. This allows us to access the full agent functionality directly if needed, while still maintaining the wrapper function interface for ValidMind's testing framework." - ], - "id": "949bcf53" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "# Add the banking agent to the vm model\n", - "vm_banking_model.model = banking_agent" - ], - "execution_count": null, - "outputs": [], - "id": "2c653471" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Verify integration\n", - "\n", - "Let's confirm that the banking agent has been successfully integrated with ValidMind:" - ], - "id": "d8d0c1c1" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "print(\"Banking Agent Successfully Integrated with ValidMind!\")\n", - "print(f\"Model ID: {vm_banking_model.input_id}\")" - ], - "execution_count": null, - "outputs": [], - "id": "8e101b0f" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Validate the system prompt\n", - "\n", - "Let's get an initial sense of how well our defined system prompt meets a few best practices for prompt engineering by running a few tests — we'll run evaluation tests later on our agent's performance.\n", - "\n", - "You run individual tests by calling [the `run_test` function](https://docs.validmind.ai/validmind/validmind/tests.html#run_test) provided by the `validmind.tests` module. Passing in our agentic model as an input, the tests below rate the prompt on a scale of 1-10 against the following criteria:\n", - "\n", - "- **[Clarity](https://docs.validmind.ai/tests/prompt_validation/Clarity.html)** — How clearly the prompt states the task.\n", - "- **[Conciseness](https://docs.validmind.ai/tests/prompt_validation/Conciseness.html)** — How succinctly the prompt states the task.\n", - "- **[Delimitation](https://docs.validmind.ai/tests/prompt_validation/Delimitation.html)** — When using complex prompts containing examples, contextual information, or other elements, is the prompt formatted in such a way that each element is clearly separated?\n", - "- **[NegativeInstruction](https://docs.validmind.ai/tests/prompt_validation/NegativeInstruction.html)** — Whether the prompt contains negative instructions.\n", - "- **[Specificity](https://docs.validmind.ai/tests/prompt_validation/NegativeInstruction.html)** — How specific the prompt defines the task." - ], - "id": "2a5f874e" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "vm.tests.run_test(\n", - " \"validmind.prompt_validation.Clarity\",\n", - " inputs={\n", - " \"model\": vm_banking_model,\n", - " },\n", - ").log()" - ], - "execution_count": null, - "outputs": [], - "id": "f52dceb1" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "vm.tests.run_test(\n", - " \"validmind.prompt_validation.Conciseness\",\n", - " inputs={\n", - " \"model\": vm_banking_model,\n", - " },\n", - ").log()" - ], - "execution_count": null, - "outputs": [], - "id": "70d52333" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "vm.tests.run_test(\n", - " \"validmind.prompt_validation.Delimitation\",\n", - " inputs={\n", - " \"model\": vm_banking_model,\n", - " },\n", - ").log()" - ], - "execution_count": null, - "outputs": [], - "id": "5aa89976" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "vm.tests.run_test(\n", - " \"validmind.prompt_validation.NegativeInstruction\",\n", - " inputs={\n", - " \"model\": vm_banking_model,\n", - " },\n", - ").log()" - ], - "execution_count": null, - "outputs": [], - "id": "8630197e" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "vm.tests.run_test(\n", - " \"validmind.prompt_validation.Specificity\",\n", - " inputs={\n", - " \"model\": vm_banking_model,\n", - " },\n", - ").log()" - ], - "execution_count": null, - "outputs": [], - "id": "bba99915" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Initializing the ValidMind dataset\n", - "\n", - "After validation our system prompt, let's import our sample dataset ([banking_test_dataset.py](banking_test_dataset.py)), which we'll use in the next section to evaluate our agent's performance across different banking scenarios:" - ], - "id": "51d61141" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "from banking_test_dataset import banking_test_dataset" - ], - "execution_count": null, - "outputs": [], - "id": "0c70ca2c" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The next step is to connect your data with a ValidMind `Dataset` object. **This step is always necessary every time you want to connect a dataset to documentation and produce test results through ValidMind,** but you only need to do it once per dataset.\n", - "\n", - "Initialize a ValidMind dataset object using the [`init_dataset` function](https://docs.validmind.ai/validmind/validmind.html#init_dataset) from the ValidMind (`vm`) module. For this example, we'll pass in the following arguments:\n", - "\n", - "- **`input_id`** — A unique identifier that allows tracking what inputs are used when running each individual test.\n", - "- **`dataset`** — The raw dataset that you want to provide as input to tests.\n", - "- **`text_column`** — The name of the column containing the text input data.\n", - "- **`target_column`** — A required argument if tests require access to true values. This is the name of the target column in the dataset." - ], - "id": "442ab66d" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "vm_test_dataset = vm.init_dataset(\n", - " input_id=\"banking_test_dataset\",\n", - " dataset=banking_test_dataset,\n", - " text_column=\"input\",\n", - " target_column=\"possible_outputs\",\n", - ")\n", - "\n", - "print(\"Banking Test Dataset Initialized in ValidMind!\")\n", - "print(f\"Dataset ID: {vm_test_dataset.input_id}\")\n", - "print(f\"Dataset columns: {vm_test_dataset._df.columns}\")\n", - "vm_test_dataset._df" - ], - "execution_count": null, - "outputs": [], - "id": "a7e9d158" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Assign predictions\n", - "\n", - "Now that both the model object and the datasets have been registered, we'll assign predictions to capture the banking agent's responses for evaluation:\n", - "\n", - "- The [`assign_predictions()` method](https://docs.validmind.ai/validmind/validmind/vm_models.html#assign_predictions) from the `Dataset` object can link existing predictions to any number of models.\n", - "- This method links the model's class prediction values and probabilities to our `vm_train_ds` and `vm_test_ds` datasets.\n", - "\n", - "If no prediction values are passed, the method will compute predictions automatically:" - ], - "id": "7b01021c" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "vm_test_dataset.assign_predictions(vm_banking_model)\n", - "\n", - "print(\"Banking Agent Predictions Generated Successfully!\")\n", - "print(f\"Predictions assigned to {len(vm_test_dataset._df)} test cases\")\n", - "vm_test_dataset._df.head()" - ], - "execution_count": null, - "outputs": [], - "id": "1d462663" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Running accuracy tests\n", - "\n", - "Using [`@vm.test`](https://docs.validmind.ai/validmind/validmind.html#test), let's implement some reusable custom *inline tests* to assess the accuracy of our banking agent:\n", - "\n", - "- An inline test refers to a test written and executed within the same environment as the code being tested — in this case, right in this Jupyter Notebook — without requiring a separate test file or framework.\n", - "- You'll note that the custom test functions are just regular Python functions that can include and require any Python library as you see fit." - ], - "id": "4e56f556" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Response accuracy test\n", - "\n", - "We'll create a custom test that evaluates the banking agent's ability to provide accurate responses by:\n", - "\n", - "- Testing against a dataset of predefined banking questions and expected answers.\n", - "- Checking if responses contain expected keywords and banking terminology.\n", - "- Providing detailed test results including pass/fail status.\n", - "- Helping identify any gaps in the agent's banking knowledge or response quality." - ], - "id": "1bce9258" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "\n", - "@vm.test(\"my_custom_tests.banking_accuracy_test\")\n", - "def banking_accuracy_test(model, dataset, list_of_columns):\n", - " \"\"\"\n", - " The Banking Accuracy Test evaluates whether the agent’s responses include \n", - " critical domain-specific keywords and phrases that indicate accurate, compliant,\n", - " and contextually appropriate banking information. This test ensures that the agent\n", - " provides responses containing the expected banking terminology, risk classifications,\n", - " account details, or other domain-relevant information required for regulatory compliance,\n", - " customer safety, and operational accuracy.\n", - " \"\"\"\n", - " df = dataset._df\n", - " \n", - " # Pre-compute responses for all tests\n", - " y_true = dataset.y.tolist()\n", - " y_pred = dataset.y_pred(model).tolist()\n", - "\n", - " # Vectorized test results\n", - " test_results = []\n", - " for response, keywords in zip(y_pred, y_true):\n", - " # Convert keywords to list if not already a list\n", - " if not isinstance(keywords, list):\n", - " keywords = [keywords]\n", - " test_results.append(any(str(keyword).lower() in str(response).lower() for keyword in keywords))\n", - " \n", - " results = pd.DataFrame()\n", - " column_names = [col + \"_details\" for col in list_of_columns]\n", - " results[column_names] = df[list_of_columns]\n", - " results[\"actual\"] = y_pred\n", - " results[\"expected\"] = y_true\n", - " results[\"passed\"] = test_results\n", - " results[\"error\"] = None if test_results else f'Response did not contain any expected keywords: {y_true}'\n", - " \n", - " return results" - ], - "execution_count": null, - "outputs": [], - "id": "90232066" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now that we've defined our custom response accuracy test, we can run the test using the same `run_test()` function we used earlier to validate the system prompt using our sample dataset and agentic model as input, and log the test results to the ValidMind Platform with the [`log()` method](https://docs.validmind.ai/validmind/validmind/vm_models.html#log):" - ], - "id": "2a7f71f8" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "result = vm.tests.run_test(\n", - " \"my_custom_tests.banking_accuracy_test\",\n", - " inputs={\n", - " \"dataset\": vm_test_dataset,\n", - " \"model\": vm_banking_model\n", - " },\n", - " params={\n", - " \"list_of_columns\": [\"input\"]\n", - " }\n", - ")\n", - "result.log()" - ], - "execution_count": null, - "outputs": [], - "id": "e68884d5" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Let's review the first five rows of the test dataset to inspect the results to see how well the banking agent performed. Each column in the output serves a specific purpose in evaluating agent performance:\n", - "\n", - "| Column header | Description | Importance |\n", - "|--------------|-------------|------------|\n", - "| **`input`** | Original user query or request | Essential for understanding the context of each test case and tracing which inputs led to specific agent behaviors. |\n", - "| **`expected_tools`** | Banking tools that should be invoked for this request | Enables validation of correct tool selection, which is critical for agentic AI systems where choosing the right tool is a key success metric. |\n", - "| **`expected_output`** | Expected output or keywords that should appear in the response | Defines the success criteria for each test case, enabling objective evaluation of whether the agent produced the correct result. |\n", - "| **`session_id`** | Unique identifier for each test session | Allows tracking and correlation of related test runs, debugging specific sessions, and maintaining audit trails. |\n", - "| **`category`** | Classification of the request type | Helps organize test results by domain and identify performance patterns across different banking use cases. |\n", - "| **`banking_agent_model_output`** | Complete agent response including all messages and reasoning | Allows you to examine the full output to assess response quality, completeness, and correctness beyond just keyword matching. |\n", - "| **`banking_agent_model_tool_messages`** | Messages exchanged with the banking tools | Critical for understanding how the agent interacted with tools, what parameters were passed, and what tool outputs were received. |\n", - "| **`banking_agent_model_tool_called`** | Specific tool that was invoked | Enables validation that the agent selected the correct tool for each request, which is fundamental to agentic AI validation. |\n", - "| **`possible_outputs`** | Alternative valid outputs or keywords that could appear in the response | Provides flexibility in evaluation by accounting for multiple acceptable response formats or variations. |" - ], - "id": "94a717e7" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "vm_test_dataset.df.head(5)" - ], - "execution_count": null, - "outputs": [], - "id": "78f7edb1" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Tool selection accuracy test\n", - "\n", - "We'll also create a custom test that evaluates the banking agent's ability to select the correct tools for different requests by:\n", - "\n", - "- Testing against a dataset of predefined banking queries with expected tool selections.\n", - "- Comparing the tools actually invoked by the agent against the expected tools for each request.\n", - "- Providing quantitative accuracy scores that measure the proportion of expected tools correctly selected.\n", - "- Helping identify gaps in the agent's understanding of user needs and tool selection logic." - ], - "id": "1cb3e8bd" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First, we'll define a helper function that extracts tool calls from the agent's messages and compares them against the expected tools. This function handles different message formats (dictionary or object) and calculates accuracy scores:" - ], - "id": "69263d62" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "def validate_tool_calls_simple(messages, expected_tools):\n", - " \"\"\"Simple validation of tool calls without RAGAS dependency issues.\"\"\"\n", - " \n", - " tool_calls_found = []\n", - " \n", - " for message in messages:\n", - " if hasattr(message, 'tool_calls') and message.tool_calls:\n", - " for tool_call in message.tool_calls:\n", - " # Handle both dictionary and object formats\n", - " if isinstance(tool_call, dict):\n", - " tool_calls_found.append(tool_call['name'])\n", - " else:\n", - " # ToolCall object - use attribute access\n", - " tool_calls_found.append(tool_call.name)\n", - " \n", - " # Check if expected tools were called\n", - " accuracy = 0.0\n", - " matches = 0\n", - " if expected_tools:\n", - " matches = sum(1 for tool in expected_tools if tool in tool_calls_found)\n", - " accuracy = matches / len(expected_tools)\n", - " \n", - " return {\n", - " 'expected_tools': expected_tools,\n", - " 'found_tools': tool_calls_found,\n", - " 'matches': matches,\n", - " 'total_expected': len(expected_tools) if expected_tools else 0,\n", - " 'accuracy': accuracy,\n", - " }" - ], - "execution_count": null, - "outputs": [], - "id": "e68798be" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Now we'll define the main test function that uses the helper function to evaluate tool selection accuracy across all test cases in the dataset:" - ], - "id": "8f494fd3" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "@vm.test(\"my_custom_tests.BankingToolCallAccuracy\")\n", - "def BankingToolCallAccuracy(dataset, agent_output_column, expected_tools_column):\n", - " \"\"\"\n", - " Evaluates the tool selection accuracy of a LangGraph-powered banking agent.\n", - "\n", - " This test measures whether the agent correctly identifies and invokes the required banking tools\n", - " for each user query scenario.\n", - " For each case, the outputs generated by the agent (including its tool calls) are compared against an\n", - " expected set of tools. The test considers both coverage and exactness: it computes the proportion of\n", - " expected tools correctly called by the agent for each instance.\n", - "\n", - " Parameters:\n", - " dataset (VMDataset): The dataset containing user queries, agent outputs, and ground-truth tool expectations.\n", - " agent_output_column (str): Dataset column name containing agent outputs (should include tool call details in 'messages').\n", - " expected_tools_column (str): Dataset column specifying the true expected tools (as lists).\n", - "\n", - " Returns:\n", - " List[dict]: Per-row dictionaries with details: expected tools, found tools, match count, total expected, and accuracy score.\n", - "\n", - " Purpose:\n", - " Provides diagnostic evidence of the banking agent's core reasoning ability—specifically, its capacity to\n", - " interpret user needs and select the correct banking actions. Useful for diagnosing gaps in tool coverage,\n", - " misclassifications, or breakdowns in agent logic.\n", - "\n", - " Interpretation:\n", - " - An accuracy of 1.0 signals perfect tool selection for that example.\n", - " - Lower scores may indicate partial or complete failures to invoke required tools.\n", - " - Review 'found_tools' vs. 'expected_tools' to understand the source of discrepancies.\n", - "\n", - " Strengths:\n", - " - Directly tests a core capability of compositional tool-use agents.\n", - " - Framework-agnostic; robust to tool call output format (object or dict).\n", - " - Supports batch validation and result logging for systematic documentation.\n", - "\n", - " Limitations:\n", - " - Does not penalize extra, unnecessary tool calls.\n", - " - Does not assess result quality—only correct invocation.\n", - "\n", - " \"\"\"\n", - " df = dataset._df\n", - " \n", - " results = []\n", - " for i, row in df.iterrows():\n", - " result = validate_tool_calls_simple(row[agent_output_column]['messages'], row[expected_tools_column])\n", - " results.append(result)\n", - " \n", - " return results" - ], - "execution_count": null, - "outputs": [], - "id": "604d7313" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Finally, we can call our function with `run_test()` and log the test results to the ValidMind Platform:" - ], - "id": "57ab606b" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "result = vm.tests.run_test(\n", - " \"my_custom_tests.BankingToolCallAccuracy\",\n", - " inputs={\n", - " \"dataset\": vm_test_dataset,\n", - " },\n", - " params={\n", - " \"agent_output_column\": \"banking_agent_model_output\",\n", - " \"expected_tools_column\": \"expected_tools\"\n", - " }\n", - ")\n", - "result.log()" - ], - "execution_count": null, - "outputs": [], - "id": "dd14115e" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Assigning AI evaluation metric scores\n", - "\n", - "*AI agent evaluation metrics* are specialized measurements designed to assess how well autonomous LLM-based agents reason, plan, select and execute tools, and ultimately complete user tasks by analyzing the *full execution trace* — including reasoning steps, tool calls, intermediate decisions, and outcomes, rather than just single input–output pairs. These metrics are essential because agent failures often occur in ways traditional LLM metrics miss — for example, choosing the right tool with wrong arguments, creating a good plan but not following it, or completing a task inefficiently.\n", - "\n", - "In this section, we'll evaluate our banking agent's outputs and add scoring to our sample dataset against metrics defined in [DeepEval’s AI agent evaluation framework](https://deepeval.com/guides/guides-ai-agent-evaluation-metrics) which breaks down AI agent evaluation into three layers with corresponding subcategories: **reasoning**, **action**, and **execution**.\n", - "\n", - "Together, these three metrics enable granular diagnosis of agent behavior, help pinpoint where failures occur (reasoning, action, or execution), and support both development benchmarking and production monitoring." - ], - "id": "be8d5270" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Identify relevant DeepEval scorers\n", - "\n", - "*Scorers* are evaluation metrics that analyze model outputs and store their results in the dataset:\n", - "\n", - "- Each scorer adds a new column to the dataset with format: `{scorer_name}_{metric_name}`\n", - "- The column contains the numeric score (typically `0`-`1`) for each example\n", - "- Multiple scorers can be run on the same dataset, each adding their own column\n", - "- Scores are persisted in the dataset for later analysis and visualization\n", - "- Common scorer patterns include:\n", - " - Model performance metrics (accuracy, F1, etc.)\n", - " - Output quality metrics (relevance, faithfulness)\n", - " - Task-specific metrics (completion, correctness)\n", - "\n", - "Use `list_scorers()` from [`validmind.scorers`](https://docs.validmind.ai/validmind/validmind/tests.html#scorer) to discover all available scoring methods and their IDs that can be used with `assign_scores()`. We'll filter these results to return only DeepEval scorers for our desired three metrics in a formatted table with descriptions:" - ], - "id": "25828bef" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "# Load all DeepEval scorers\n", - "llm_scorers_dict = vm.tests.load._load_tests([s for s in vm.scorer.list_scorers() if \"deepeval\" in s.lower()])\n", - "\n", - "# Categorize scorers by metric layer\n", - "reasoning_scorers = {}\n", - "action_scorers = {}\n", - "execution_scorers = {}\n", - "\n", - "for scorer_id, scorer_func in llm_scorers_dict.items():\n", - " tags = getattr(scorer_func, \"__tags__\", [])\n", - " scorer_name = scorer_id.split(\".\")[-1]\n", - "\n", - " if \"reasoning_layer\" in tags:\n", - " reasoning_scorers[scorer_id] = scorer_func\n", - " elif \"action_layer\" in tags:\n", - " action_scorers[scorer_id] = scorer_func\n", - " elif \"TaskCompletion\" in scorer_name:\n", - " execution_scorers[scorer_id] = scorer_func\n", - "\n", - "# Display scorers by category\n", - "print(\"=\" * 80)\n", - "print(\"REASONING LAYER\")\n", - "print(\"=\" * 80)\n", - "if reasoning_scorers:\n", - " reasoning_df = vm.tests.load._pretty_list_tests(reasoning_scorers, truncate=True)\n", - " display(reasoning_df)\n", - "else:\n", - " print(\"No reasoning layer scorers found.\")\n", - "\n", - "print(\"\\n\" + \"=\" * 80)\n", - "print(\"ACTION LAYER\")\n", - "print(\"=\" * 80)\n", - "if action_scorers:\n", - " action_df = vm.tests.load._pretty_list_tests(action_scorers, truncate=True)\n", - " display(action_df)\n", - "else:\n", - " print(\"No action layer scorers found.\")\n", - "\n", - "print(\"\\n\" + \"=\" * 80)\n", - "print(\"EXECUTION LAYER\")\n", - "print(\"=\" * 80)\n", - "if execution_scorers:\n", - " execution_df = vm.tests.load._pretty_list_tests(execution_scorers, truncate=True)\n", - " display(execution_df)\n", - "else:\n", - " print(\"No execution layer scorers found.\")" - ], - "execution_count": null, - "outputs": [], - "id": "730c70ec" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Assign reasoning scores\n", - "\n", - "*Reasoning* evaluates planning and strategy generation:\n", - "\n", - "- **Plan quality** – How logical, complete, and efficient the agent’s plan is.\n", - "- **Plan adherence** – Whether the agent follows its own plan during execution." - ], - "id": "e5fb739b" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Plan quality score\n", - "\n", - "Let's measure how well our banking agent generates a plan before acting. A high score means the plan is logical, complete, and efficient." - ], - "id": "fde94d01" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "vm_test_dataset.assign_scores(\n", - " metrics = \"validmind.scorers.llm.deepeval.PlanQuality\",\n", - " model = vm_banking_model,\n", - " input_column = \"input\",\n", - ")\n", - "vm_test_dataset._df[['banking_agent_model_PlanQuality_score','banking_agent_model_PlanQuality_reason']]" - ], - "execution_count": null, - "outputs": [], - "id": "52f362ba" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Plan adherence score\n", - "\n", - "Let's check whether our banking agent follows the plan it created. Deviations lower this score and indicate gaps between reasoning and execution." - ], - "id": "d631fd12" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "vm_test_dataset.assign_scores(\n", - " metrics = \"validmind.scorers.llm.deepeval.PlanAdherence\",\n", - " input_column = \"input\",\n", - " model = vm_banking_model,\n", - ")\n", - "vm_test_dataset._df[['banking_agent_model_PlanAdherence_score','banking_agent_model_PlanAdherence_reason']]" - ], - "execution_count": null, - "outputs": [], - "id": "4124a7c2" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Assign action scores\n", - "\n", - "*Action* assesses tool usage and argument generation:\n", - "\n", - "- **Tool correctness** – Whether the agent selects and calls the right tools.\n", - "- **Argument correctness** – Whether the agent generates correct tool arguments." - ], - "id": "82e5e6f1" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Tool correctness score\n", - "\n", - "Let's evaluate if our banking agent selects the appropriate tool for the task. Choosing the wrong tool reduces performance even if reasoning was correct." - ], - "id": "e641c9f2" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "vm_test_dataset.assign_scores(\n", - " metrics = \"validmind.scorers.llm.deepeval.ToolCorrectness\",\n", - " input_column = \"input\",\n", - " model = vm_banking_model,\n", - " expected_tools_called_column = \"expected_tools\",\n", - " actual_tools_called_column = \"banking_agent_model_tool_called\",\n", - ")\n", - "vm_test_dataset._df[['banking_agent_model_ToolCorrectness_score','banking_agent_model_ToolCorrectness_reason']]" - ], - "execution_count": null, - "outputs": [], - "id": "8d2e8a25" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Argument correctness score\n", - "\n", - "Let's assesses whether our banking agent provides correct inputs or arguments to the selected tool. Incorrect arguments can lead to failed or unexpected results." - ], - "id": "dd758ba5" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "vm_test_dataset.assign_scores(\n", - " metrics = \"validmind.scorers.llm.deepeval.ArgumentCorrectness\",\n", - " input_column = \"input\",\n", - " model = vm_banking_model,\n", - " actual_tools_called_column = \"banking_agent_model_tool_called\",\n", - ")\n", - "vm_test_dataset._df[['banking_agent_model_ArgumentCorrectness_score','banking_agent_model_ArgumentCorrectness_reason']]" - ], - "execution_count": null, - "outputs": [], - "id": "04f90489" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Assign execution score\n", - "\n", - "*Execution* measures end-to-end performance:\n", - "\n", - "- **Task completion** – Whether the agent successfully completes the intended task." - ], - "id": "1aeec2f5" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Task completion score\n", - "\n", - "Let's evaluate whether our banking agent successfully completes the requested tasks. Incomplete task execution can lead to user dissatisfaction and failed banking operations." - ], - "id": "eb9ab8de" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "vm_test_dataset.assign_scores(\n", - " metrics = \"validmind.scorers.llm.deepeval.TaskCompletion\",\n", - " input_column = \"input\",\n", - " model = vm_banking_model,\n", - " actual_tools_called_column = \"banking_agent_model_tool_called\",\n", - ")\n", - "vm_test_dataset._df[['banking_agent_model_TaskCompletion_score','banking_agent_model_TaskCompletion_reason']]" - ], - "execution_count": null, - "outputs": [], - "id": "05024f1f" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "As you recall from the beginning of this section, when we run scorers through `assign_scores()`, the return values are automatically processed and added as new columns with the format `{scorer_name}_{metric_name}`. Note that the task completion scorer has added a new column `TaskCompletion_score` to our dataset.\n", - "\n", - "We'll use this column to visualize the distribution of task completion scores across our test cases through the [BoxPlot test](https://docs.validmind.ai/validmind/validmind/tests/plots/BoxPlot.html#boxplot):" - ], - "id": "b577c282" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "vm.tests.run_test(\n", - " \"validmind.plots.BoxPlot\",\n", - " inputs={\"dataset\": vm_test_dataset},\n", - " params={\n", - " \"columns\": \"banking_agent_model_TaskCompletion_score\",\n", - " \"title\": \"Distribution of Task Completion Scores\",\n", - " \"ylabel\": \"Score\",\n", - " \"figsize\": (8, 6)\n", - " }\n", - ").log()" - ], - "execution_count": null, - "outputs": [], - "id": "7f6d08ca" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Running RAGAS tests\n", - "\n", - "Next, let's run some out-of-the-box *Retrieval-Augmented Generation Assessment* (RAGAS) tests available in the ValidMind Library. RAGAS provides specialized metrics for evaluating retrieval-augmented generation systems and conversational AI agents. These metrics analyze different aspects of agent performance by assessing how well systems integrate retrieved information with generated responses.\n", - "\n", - "Our banking agent uses tools to retrieve information and generates responses based on that context, making it similar to a RAG system. RAGAS metrics help evaluate the quality of this integration by analyzing the relationship between retrieved tool outputs, user queries, and generated responses.\n", - "\n", - "These tests provide insights into how well our banking agent integrates tool usage with conversational abilities, ensuring it provides accurate, relevant, and helpful responses to banking users while maintaining fidelity to retrieved information." - ], - "id": "30d9ec62" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Identify relevant RAGAS tests\n", - "\n", - "Let's explore some of ValidMind's available tests. Using ValidMind’s repository of tests streamlines your development testing, and helps you ensure that your models are being documented and evaluated appropriately.\n", - "\n", - "You can pass `tasks` and `tags` as parameters to the [`vm.tests.list_tests()` function](https://docs.validmind.ai/validmind/validmind/tests.html#list_tests) to filter the tests based on the tags and task types:\n", - "\n", - "- **`tasks`** represent the kind of modeling task associated with a test. Here we'll focus on `text_qa` tasks.\n", - "- **`tags`** are free-form descriptions providing more details about the test, for example, what category the test falls into. Here we'll focus on the `ragas` tag.\n", - "\n", - "We'll then run three of these tests returned as examples below." - ], - "id": "8288f6c3" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "vm.tests.list_tests(task=\"text_qa\", tags=[\"ragas\"])" - ], - "execution_count": null, - "outputs": [], - "id": "0701f5a9" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Faithfulness\n", - "\n", - "Let's evaluate whether the banking agent's responses accurately reflect the information retrieved from tools. Unfaithful responses can misreport credit analysis, financial calculations, and compliance results—undermining user trust in the banking agent." - ], - "id": "2ce24ba0" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "vm.tests.run_test(\n", - " \"validmind.model_validation.ragas.Faithfulness\",\n", - " inputs={\"dataset\": vm_test_dataset},\n", - " param_grid={\n", - " \"user_input_column\": [\"input\"],\n", - " \"response_column\": [\"banking_agent_model_prediction\"],\n", - " \"retrieved_contexts_column\": [\"banking_agent_model_tool_messages\"],\n", - " },\n", - ").log()" - ], - "execution_count": null, - "outputs": [], - "id": "92044533" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Response Relevancy\n", - "\n", - "Let's evaluate whether the banking agent's answers address the user's original question or request. Irrelevant or off-topic responses can frustrate users and fail to deliver the banking information they need." - ], - "id": "4d1fcfcd" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "vm.tests.run_test(\n", - " \"validmind.model_validation.ragas.ResponseRelevancy\",\n", - " inputs={\"dataset\": vm_test_dataset},\n", - " params={\n", - " \"user_input_column\": \"input\",\n", - " \"response_column\": \"banking_agent_model_prediction\",\n", - " \"retrieved_contexts_column\": \"banking_agent_model_tool_messages\",\n", - " }\n", - ").log()" - ], - "execution_count": null, - "outputs": [], - "id": "d7483bc3" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Context Recall\n", - "\n", - "Let's evaluate how well the banking agent uses the information retrieved from tools when generating its responses. Poor context recall can lead to incomplete or underinformed answers even when the right tools were selected." - ], - "id": "38c1dfb5" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "vm.tests.run_test(\n", - " \"validmind.model_validation.ragas.ContextRecall\",\n", - " inputs={\"dataset\": vm_test_dataset},\n", - " param_grid={\n", - " \"user_input_column\": [\"input\"],\n", - " \"retrieved_contexts_column\": [\"banking_agent_model_tool_messages\"],\n", - " \"reference_column\": [\"banking_agent_model_prediction\"],\n", - " },\n", - ").log()" - ], - "execution_count": null, - "outputs": [], - "id": "e5dc00ce" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Running safety tests\n", - "\n", - "Finally, let's run some out-of-the-box *safety* tests available in the ValidMind Library. Safety tests provide specialized metrics for evaluating whether AI agents operate reliably and securely. These metrics analyze different aspects of agent behavior by assessing adherence to safety guidelines, consistency of outputs, and resistance to harmful or inappropriate requests.\n", - "\n", - "Our banking agent handles sensitive financial information and user requests, making safety and reliability essential. Safety tests help evaluate whether the agent maintains appropriate boundaries, responds consistently and correctly to inputs, and avoids generating harmful, biased, or unprofessional content.\n", - "\n", - "These tests provide insights into how well our banking agent upholds standards of fairness and professionalism, ensuring it operates reliably and securely for banking users." - ], - "id": "95e1e16a" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### AspectCritic\n", - "\n", - "Let's evaluate our banking agent's responses across multiple quality dimensions — conciseness, coherence, correctness, harmfulness, and maliciousness. Weak performance on these dimensions can degrade user experience, fall short of professional banking standards, or introduce safety risks. \n", - "\n", - "We'll use the `AspectCritic` we identified earlier:" - ], - "id": "e0972afa" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "vm.tests.run_test(\n", - " \"validmind.model_validation.ragas.AspectCritic\",\n", - " inputs={\"dataset\": vm_test_dataset},\n", - " param_grid={\n", - " \"user_input_column\": [\"input\"],\n", - " \"response_column\": [\"banking_agent_model_prediction\"],\n", - " \"retrieved_contexts_column\": [\"banking_agent_model_tool_messages\"],\n", - " },\n", - ").log()" - ], - "execution_count": null, - "outputs": [], - "id": "148daa2b" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "#### Bias\n", - "\n", - "Let's evaluate whether our banking agent's prompts contain unintended biases that could affect banking decisions. Biased prompts can lead to unfair or discriminatory outcomes — undermining customer trust and exposing the institution to compliance risk.\n", - "\n", - "We'll first use `list_tests()` again to filter for tests relating to `prompt_validation`:" - ], - "id": "16f29c8d" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "vm.tests.list_tests(filter=\"prompt_validation\")" - ], - "execution_count": null, - "outputs": [], - "id": "74eba86c" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "And then run the identified `Bias` test:" - ], - "id": "e9413803" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "vm.tests.run_test(\n", - " \"validmind.prompt_validation.Bias\",\n", - " inputs={\n", - " \"model\": vm_banking_model,\n", - " },\n", - ").log()" - ], - "execution_count": null, - "outputs": [], - "id": "062cf8e7" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Next steps\n", - "\n", - "You can look at the output produced by the ValidMind Library right in the notebook where you ran the code, as you would expect. But there is a better way — use the ValidMind Platform to work with your model documentation." - ], - "id": "8f3f2dbe" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Work with your model documentation\n", - "\n", - "1. From the **Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", - "\n", - "2. In the left sidebar that appears for your model, click **Documentation** under Documents.\n", - "\n", - " What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation, view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/working-with-model-documentation.html)\n", - "\n", - "3. Click into any section related to the tests we ran in this notebook, for example: **4.3. Prompt Evaluation** to review the results of the tests we logged." - ], - "id": "8716165d" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Customize the banking agent for your use case\n", - "\n", - "You've now built an agentic AI system designed for banking use cases that supports compliance with supervisory guidance such as SR 11-7 and SS1/23, covering credit and fraud risk assessment for both retail and commercial banking. Extend this example agent to real-world banking scenarios and production deployment by:\n", - "\n", - "- Adapting the banking tools to your organization's specific requirements\n", - "- Adding more banking scenarios and edge cases to your test set\n", - "- Connecting the agent to your banking systems and databases\n", - "- Implementing additional banking-specific tools and workflows" - ], - "id": "7c4a78ce" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "### Discover more learning resources\n", - "\n", - "Learn more about the ValidMind Library tools we used in this notebook:\n", - "\n", - "- [Custom prompts](https://docs.validmind.ai/notebooks/how_to/customize_test_result_descriptions.html)\n", - "- [Custom tests](https://docs.validmind.ai/notebooks/code_samples/custom_tests/implement_custom_tests.html)\n", - "- [ValidMind scorers](https://docs.validmind.ai/notebooks/how_to/assign_scores_complete_tutorial.html)\n", - "\n", - "We also offer many more interactive notebooks to help you document models:\n", - "\n", - "- [Run tests & test suites](https://docs.validmind.ai/developer/how-to/testing-overview.html)\n", - "- [Use ValidMind Library features](https://docs.validmind.ai/developer/how-to/feature-overview.html)\n", - "- [Code samples by use case](https://docs.validmind.ai/guide/samples-jupyter-notebooks.html)\n", - "\n", - "Or, visit our [documentation](https://docs.validmind.ai/) to learn more about ValidMind." - ], - "id": "7f9385d3" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## Upgrade ValidMind\n", - "\n", - "
After installing ValidMind, you’ll want to periodically make sure you are on the latest version to access any new features and other enhancements.
\n", - "\n", - "Retrieve the information for the currently installed version of ValidMind:" - ], - "id": "fdd5c0db" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "%pip show validmind" - ], - "execution_count": null, - "outputs": [], - "id": "9733adff" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "If the version returned is lower than the version indicated in our [production open-source code](https://github.com/validmind/validmind-library/blob/prod/validmind/__version__.py), restart your notebook and run:\n", - "\n", - "```bash\n", - "%pip install --upgrade validmind\n", - "```" - ], - "id": "829429fd" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "You may need to restart your kernel after running the upgrade package for changes to be applied." - ], - "id": "55339760" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "\n", - "\n", - "***\n", - "\n", - "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", - "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", - "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" - ], - "id": "copyright-b9e82bcf4e364c4f8e5ae4bb0e4b2865" - } - ], - "metadata": { - "kernelspec": { - "display_name": "validmind-1QuffXMV-py3.11", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.9" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} \ No newline at end of file + "cells": [ + { + "cell_type": "markdown", + "id": "eee6b64c", + "metadata": {}, + "source": [ + "# Document an agentic AI system\n", + "\n", + "Build and document an agentic AI system with the ValidMind Library. Construct a LangGraph-based banking agent, assign AI evaluation metric scores to your agent, and run accuracy, RAGAS, and safety tests, then log those test results to the ValidMind Platform.\n", + "\n", + "An _AI agent_ is an autonomous system that interprets inputs, selects from available tools or actions, and executes multi-step behaviors to achieve defined goals. In this notebook, the agent acts as a banking assistant that analyzes user requests and automatically selects and invokes the appropriate specialized banking tool to deliver accurate, compliant, and actionable responses.\n", + "\n", + "- This agent enables financial institutions to automate complex banking workflows where different customer requests require different specialized tools and knowledge bases.\n", + "- Effective validation of agentic AI systems reduces the risks of agents misinterpreting inputs, failing to extract required parameters, or producing incorrect assessments or actions — such as selecting the wrong tool.\n", + "\n", + "
For the LLM components in this notebook to function properly, you'll need access to OpenAI.\n", + "

\n", + "Before you continue, ensure that a valid OPENAI_API_KEY is set in your .env file.
" + ] + }, + { + "cell_type": "markdown", + "id": "30927b2b", + "metadata": {}, + "source": [ + "::: {.content-hidden when-format=\"html\"}\n", + "## Contents \n", + "- [About ValidMind](#toc1__) \n", + " - [Before you begin](#toc1_1__) \n", + " - [New to ValidMind?](#toc1_2__) \n", + " - [Key concepts](#toc1_3__) \n", + "- [Setting up](#toc2__) \n", + " - [Install the ValidMind Library](#toc2_1__) \n", + " - [Initialize the ValidMind Library](#toc2_2__) \n", + " - [Register sample model](#toc2_2_1__) \n", + " - [Apply documentation template](#toc2_2_2__) \n", + " - [Get your code snippet](#toc2_2_3__) \n", + " - [Preview the documentation template](#toc2_2_4__) \n", + " - [Verify OpenAI API access](#toc2_3__) \n", + " - [Initialize the Python environment](#toc2_4__) \n", + "- [Building the LangGraph agent](#toc3__) \n", + " - [Test available banking tools](#toc3_1__) \n", + " - [Create LangGraph banking agent](#toc3_2__) \n", + " - [Define system prompt](#toc3_2_1__) \n", + " - [Initialize the LLM](#toc3_2_2__) \n", + " - [Define agent state structure](#toc3_2_3__) \n", + " - [Create agent workflow function](#toc3_2_4__) \n", + " - [Instantiate the banking agent](#toc3_2_5__) \n", + " - [Integrate agent with ValidMind](#toc3_3__) \n", + " - [Import ValidMind components](#toc3_3_1__) \n", + " - [Create agent wrapper function](#toc3_3_2__) \n", + " - [Initialize the ValidMind model object](#toc3_3_3__) \n", + " - [Store the agent reference](#toc3_3_4__) \n", + " - [Verify integration](#toc3_3_5__) \n", + " - [Validate the system prompt](#toc3_4__) \n", + "- [Initializing the ValidMind dataset](#toc4__) \n", + " - [Assign predictions](#toc4_1__) \n", + "- [Running accuracy tests](#toc5__) \n", + " - [Response accuracy test](#toc5_1__) \n", + " - [Tool selection accuracy test](#toc5_2__) \n", + "- [Assigning AI evaluation metric scores](#toc6__) \n", + " - [Identify relevant DeepEval scorers](#toc6_1__) \n", + " - [Assign reasoning scores](#toc6_2__) \n", + " - [Plan quality score](#toc6_2_1__) \n", + " - [Plan adherence score](#toc6_2_2__) \n", + " - [Assign action scores](#toc6_3__) \n", + " - [Tool correctness score](#toc6_3_1__) \n", + " - [Argument correctness score](#toc6_3_2__) \n", + " - [Assign execution score](#toc6_4__) \n", + " - [Task completion score](#toc6_4_1__) \n", + "- [Running RAGAS tests](#toc7__) \n", + " - [Identify relevant RAGAS tests](#toc7_1__) \n", + " - [Faithfulness](#toc7_1_1__) \n", + " - [Response Relevancy](#toc7_1_2__) \n", + " - [Context Recall](#toc7_1_3__) \n", + "- [Running safety tests](#toc8__) \n", + " - [AspectCritic](#toc8_1_1__) \n", + " - [Bias](#toc8_1_2__) \n", + "- [Next steps](#toc9__) \n", + " - [Work with your model documentation](#toc9_1__) \n", + " - [Customize the banking agent for your use case](#toc9_2__) \n", + " - [Discover more learning resources](#toc9_3__) \n", + "- [Upgrade ValidMind](#toc10__) \n", + "\n", + ":::\n", + "\n", + "" + ] + }, + { + "cell_type": "markdown", + "id": "b58139db", + "metadata": {}, + "source": [ + "\n", + "\n", + "## About ValidMind\n", + "\n", + "ValidMind is a suite of tools for managing model risk, including risk associated with AI and statistical models. \n", + "\n", + "You use the ValidMind Library to automate documentation and validation tests, and then use the ValidMind Platform to collaborate on model documentation. Together, these products simplify model risk management, facilitate compliance with regulations and institutional standards, and enhance collaboration between yourself and model validators." + ] + }, + { + "cell_type": "markdown", + "id": "7e30d36b", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Before you begin\n", + "\n", + "This notebook assumes you have basic familiarity with Python, including an understanding of how functions work. If you are new to Python, you can still run the notebook but we recommend further familiarizing yourself with the language. \n", + "\n", + "If you encounter errors due to missing modules in your Python environment, install the modules with `pip install`, and then re-run the notebook. For more help, refer to [Installing Python Modules](https://docs.python.org/3/installing/index.html)." + ] + }, + { + "cell_type": "markdown", + "id": "1cba586e", + "metadata": {}, + "source": [ + "\n", + "\n", + "### New to ValidMind?\n", + "\n", + "If you haven't already seen our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html), we recommend you begin by exploring the available resources in this section. There, you can learn more about documenting models and running tests, as well as find code samples and our Python Library API reference.\n", + "\n", + "
For access to all features available in this notebook, you'll need access to a ValidMind account.\n", + "

\n", + "Register with ValidMind
" + ] + }, + { + "cell_type": "markdown", + "id": "5c46f003", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Key concepts\n", + "\n", + "**Model documentation**: A structured and detailed record pertaining to a model, encompassing key components such as its underlying assumptions, methodologies, data sources, inputs, performance metrics, evaluations, limitations, and intended uses. It serves to ensure transparency, adherence to regulatory requirements, and a clear understanding of potential risks associated with the model’s application.\n", + "\n", + "**Documentation template**: Functions as a test suite and lays out the structure of model documentation, segmented into various sections and sub-sections. Documentation templates define the structure of your model documentation, specifying the tests that should be run, and how the results should be displayed.\n", + "\n", + "**Tests**: A function contained in the ValidMind Library, designed to run a specific quantitative test on the dataset or model. Tests are the building blocks of ValidMind, used to evaluate and document models and datasets, and can be run individually or as part of a suite defined by your model documentation template.\n", + "\n", + "**Metrics**: A subset of tests that do not have thresholds. In the context of this notebook, metrics and tests can be thought of as interchangeable concepts.\n", + "\n", + "**Custom metrics**: Custom metrics are functions that you define to evaluate your model or dataset. These functions can be registered with the ValidMind Library to be used in the ValidMind Platform.\n", + "\n", + "**Inputs**: Objects to be evaluated and documented in the ValidMind Library. They can be any of the following:\n", + "\n", + " - **model**: A single model that has been initialized in ValidMind with [`vm.init_model()`](https://docs.validmind.ai/validmind/validmind.html#init_model).\n", + " - **dataset**: Single dataset that has been initialized in ValidMind with [`vm.init_dataset()`](https://docs.validmind.ai/validmind/validmind.html#init_dataset).\n", + " - **models**: A list of ValidMind models - usually this is used when you want to compare multiple models in your custom metric.\n", + " - **datasets**: A list of ValidMind datasets - usually this is used when you want to compare multiple datasets in your custom metric. (Learn more: [Run tests with multiple datasets](https://docs.validmind.ai/notebooks/how_to/run_tests_that_require_multiple_datasets.html))\n", + "\n", + "**Parameters**: Additional arguments that can be passed when running a ValidMind test, used to pass additional information to a metric, customize its behavior, or provide additional context.\n", + "\n", + "**Outputs**: Custom metrics can return elements like tables or plots. Tables may be a list of dictionaries (each representing a row) or a pandas DataFrame. Plots may be matplotlib or plotly figures.\n", + "\n", + "**Test suites**: Collections of tests designed to run together to automate and generate model documentation end-to-end for specific use-cases.\n", + "\n", + "Example: the [`classifier_full_suite`](https://docs.validmind.ai/validmind/validmind/test_suites/classifier.html#ClassifierFullSuite) test suite runs tests from the [`tabular_dataset`](https://docs.validmind.ai/validmind/validmind/test_suites/tabular_datasets.html) and [`classifier`](https://docs.validmind.ai/validmind/validmind/test_suites/classifier.html) test suites to fully document the data and model sections for binary classification model use-cases." + ] + }, + { + "cell_type": "markdown", + "id": "11a2d7a5", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Setting up" + ] + }, + { + "cell_type": "markdown", + "id": "fbab0edf", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Install the ValidMind Library\n", + "\n", + "
Recommended Python versions\n", + "

\n", + "Python 3.9 <= x <= 3.11
\n", + "\n", + "Let's begin by installing the ValidMind Library with large language model (LLM) support:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1982a118", + "metadata": {}, + "outputs": [], + "source": [ + "%pip install -q \"validmind[llm]\" \"langgraph==0.3.21\"" + ] + }, + { + "cell_type": "markdown", + "id": "14578e26", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Initialize the ValidMind Library" + ] + }, + { + "cell_type": "markdown", + "id": "83d47d89", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Register sample model\n", + "\n", + "Let's first register a sample model for use with this notebook.\n", + "\n", + "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n", + "\n", + "2. In the left sidebar, navigate to **Inventory** and click **+ Register Model**.\n", + "\n", + "3. Enter the model details and click **Next >** to continue to assignment of model stakeholders. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/register-models-in-inventory.html))\n", + "\n", + "4. Select your own name under the **MODEL OWNER** drop-down.\n", + "\n", + "5. Click **Register Model** to add the model to your inventory." + ] + }, + { + "cell_type": "markdown", + "id": "bb2c5670", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Apply documentation template\n", + "\n", + "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", + "\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", + "\n", + "2. Under **TEMPLATE**, select `Agentic AI`.\n", + "\n", + "3. Click **Use Template** to apply the template." + ] + }, + { + "cell_type": "markdown", + "id": "98e475c1", + "metadata": {}, + "source": [ + "
Can't select this template?\n", + "

\n", + "Your organization administrators may need to add it to your template library:\n", + "\n", + "
" + ] + }, + { + "cell_type": "markdown", + "id": "0d1a13ca", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Get your code snippet\n", + "\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", + "\n", + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d6ccbefc", + "metadata": {}, + "outputs": [], + "source": [ + "# Load your model identifier credentials from an `.env` file\n", + "\n", + "%load_ext dotenv\n", + "%dotenv .env\n", + "\n", + "# Or replace with your code snippet\n", + "\n", + "import validmind as vm\n", + "\n", + "vm.init(\n", + " # api_host=\"...\",\n", + " # api_key=\"...\",\n", + " # api_secret=\"...\",\n", + " # model=\"...\",\n", + " document=\"documentation\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "3605df4f", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Preview the documentation template\n", + "\n", + "Let's verify that you have connected the ValidMind Library to the ValidMind Platform and that the appropriate *template* is selected for your model.\n", + "\n", + "You will upload documentation and test results unique to your model based on this template later on. For now, **take a look at the default structure that the template provides with [the `vm.preview_template()` function](https://docs.validmind.ai/validmind/validmind.html#preview_template)** from the ValidMind library and note the empty sections:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dffdaa6f", + "metadata": {}, + "outputs": [], + "source": [ + "vm.preview_template()" + ] + }, + { + "cell_type": "markdown", + "id": "d467c1d2", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Verify OpenAI API access\n", + "\n", + "Verify that a valid `OPENAI_API_KEY` is set in your `.env` file:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "22cc39cb", + "metadata": {}, + "outputs": [], + "source": [ + "# Load environment variables if using .env file\n", + "try:\n", + " from dotenv import load_dotenv\n", + " load_dotenv()\n", + "except ImportError:\n", + " print(\"dotenv not installed. Make sure OPENAI_API_KEY is set in your environment.\")" + ] + }, + { + "cell_type": "markdown", + "id": "b56c3f39", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Initialize the Python environment\n", + "\n", + "Let's import all the necessary libraries to prepare for building our banking LangGraph agentic system:\n", + "\n", + "- **Standard libraries** for data handling and environment management.\n", + "- **pandas**, a Python library for data manipulation and analytics, as an alias. We'll also configure pandas to show all columns and all rows at full width for easier debugging and inspection.\n", + "- **LangChain** components for LLM integration and tool management.\n", + "- **LangGraph** for building stateful, multi-step agent workflows.\n", + "- **Banking tools** for specialized financial services as defined in [banking_tools.py](banking_tools.py)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2058d1ac", + "metadata": {}, + "outputs": [], + "source": [ + "from typing import TypedDict, Annotated, Sequence\n", + "\n", + "from langchain_core.messages import BaseMessage, HumanMessage, SystemMessage\n", + "from langchain_openai import ChatOpenAI\n", + "from langgraph.checkpoint.memory import MemorySaver\n", + "from langgraph.graph import StateGraph, END, START\n", + "from langgraph.graph.message import add_messages\n", + "from langgraph.prebuilt import ToolNode\n", + "\n", + "# LOCAL IMPORTS FROM banking_tools.py\n", + "from banking_tools import AVAILABLE_TOOLS\n", + "\n", + "import pandas as pd\n", + "# Configure pandas to show all columns and all rows at full width\n", + "pd.set_option('display.max_columns', None)\n", + "pd.set_option('display.max_colwidth', None)\n", + "pd.set_option('display.width', None)\n", + "pd.set_option('display.max_rows', None)" + ] + }, + { + "cell_type": "markdown", + "id": "cc1d3265", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Building the LangGraph agent" + ] + }, + { + "cell_type": "markdown", + "id": "a3c421c4", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Test available banking tools\n", + "\n", + "We'll use the demo banking tools defined in `banking_tools.py` that provide use cases of financial services:\n", + "\n", + "- **Credit Risk Analyzer** - Loan applications and credit decisions\n", + "- **Customer Account Manager** - Account services and customer support\n", + "- **Fraud Detection System** - Security and fraud prevention" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1e0a120c", + "metadata": {}, + "outputs": [], + "source": [ + "print(f\"Available tools: {len(AVAILABLE_TOOLS)}\")\n", + "print(\"\\nTool Details:\")\n", + "for i, tool in enumerate(AVAILABLE_TOOLS, 1):\n", + " print(f\" - {tool.name}\")" + ] + }, + { + "cell_type": "markdown", + "id": "53906630", + "metadata": {}, + "source": [ + "Let's test each banking tool individually to ensure they're working correctly before integrating them into our agent:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dc0caff2", + "metadata": {}, + "outputs": [], + "source": [ + "# Test 1: Credit Risk Analyzer\n", + "print(\"TEST 1: Credit Risk Analyzer\")\n", + "print(\"-\" * 40)\n", + "try:\n", + " # Access the underlying function using .func\n", + " credit_result = AVAILABLE_TOOLS[0].func(\n", + " customer_income=75000,\n", + " customer_debt=1200,\n", + " credit_score=720,\n", + " loan_amount=50000,\n", + " loan_type=\"personal\"\n", + " )\n", + " print(credit_result)\n", + " print(\"Credit Risk Analyzer test PASSED\")\n", + "except Exception as e:\n", + " print(f\"Credit Risk Analyzer test FAILED: {e}\")\n", + "\n", + "print(\"\" + \"=\" * 60)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b6b227db", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "# Test 2: Customer Account Manager\n", + "print(\"TEST 2: Customer Account Manager\")\n", + "print(\"-\" * 40)\n", + "try:\n", + " # Test checking balance\n", + " account_result = AVAILABLE_TOOLS[1].func(\n", + " account_type=\"checking\",\n", + " customer_id=\"12345\",\n", + " action=\"check_balance\"\n", + " )\n", + " print(account_result)\n", + "\n", + " # Test getting account info\n", + " info_result = AVAILABLE_TOOLS[1].func(\n", + " account_type=\"all\",\n", + " customer_id=\"12345\", \n", + " action=\"get_info\"\n", + " )\n", + " print(info_result)\n", + " print(\"Customer Account Manager test PASSED\")\n", + "except Exception as e:\n", + " print(f\"Customer Account Manager test FAILED: {e}\")\n", + "\n", + "print(\"\" + \"=\" * 60)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a983b30d", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "# Test 3: Fraud Detection System\n", + "print(\"TEST 3: Fraud Detection System\")\n", + "print(\"-\" * 40)\n", + "try:\n", + " fraud_result = AVAILABLE_TOOLS[2].func(\n", + " transaction_id=\"TX123\",\n", + " customer_id=\"12345\",\n", + " transaction_amount=500.00,\n", + " transaction_type=\"withdrawal\",\n", + " location=\"Miami, FL\",\n", + " device_id=\"DEVICE_001\"\n", + " )\n", + " print(fraud_result)\n", + " print(\"Fraud Detection System test PASSED\")\n", + "except Exception as e:\n", + " print(f\"Fraud Detection System test FAILED: {e}\")\n", + "\n", + "print(\"\" + \"=\" * 60)" + ] + }, + { + "cell_type": "markdown", + "id": "1424baed", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Create LangGraph banking agent\n", + "\n", + "With our tools ready to go, we'll create our intelligent banking agent with LangGraph that automatically selects and uses the appropriate banking tool based on a user request." + ] + }, + { + "cell_type": "markdown", + "id": "3469d656", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Define system prompt\n", + "\n", + "We'll begin by defining our system prompt, which provides the LLM with context about its role as a banking assistant and guidance on when to use each available tool:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7971c427", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "# Enhanced banking system prompt with tool selection guidance\n", + "system_context = \"\"\"You are a professional banking AI assistant with access to specialized banking tools.\n", + " Analyze the user's banking request and directly use the most appropriate tools to help them.\n", + " \n", + " AVAILABLE BANKING TOOLS:\n", + " \n", + " credit_risk_analyzer - Analyze credit risk for loan applications and credit decisions\n", + " - Use for: loan applications, credit assessments, risk analysis, mortgage eligibility\n", + " - Examples: \"Analyze credit risk for $50k personal loan\", \"Assess mortgage eligibility for $300k home purchase\"\n", + " - Parameters: customer_income, customer_debt, credit_score, loan_amount, loan_type\n", + "\n", + " customer_account_manager - Manage customer accounts and provide banking services\n", + " - Use for: account information, transaction processing, product recommendations, customer service\n", + " - Examples: \"Check balance for checking account 12345\", \"Recommend products for customer with high balance\"\n", + " - Parameters: account_type, customer_id, action, amount, account_details\n", + "\n", + " fraud_detection_system - Analyze transactions for potential fraud and security risks\n", + " - Use for: transaction monitoring, fraud prevention, risk assessment, security alerts\n", + " - Examples: \"Analyze fraud risk for $500 ATM withdrawal in Miami\", \"Check security for $2000 online purchase\"\n", + " - Parameters: transaction_id, customer_id, transaction_amount, transaction_type, location, device_id\n", + "\n", + " BANKING INSTRUCTIONS:\n", + " - Analyze the user's banking request carefully and identify the primary need\n", + " - If they need credit analysis → use credit_risk_analyzer\n", + " - If they need financial calculations → use financial_calculator\n", + " - If they need account services → use customer_account_manager\n", + " - If they need security analysis → use fraud_detection_system\n", + " - Extract relevant parameters from the user's request\n", + " - Provide helpful, accurate banking responses based on tool outputs\n", + " - Always consider banking regulations, risk management, and best practices\n", + " - Be professional and thorough in your analysis\n", + "\n", + " Choose and use tools wisely to provide the most helpful banking assistance.\n", + " Describe the response in user friendly manner with details describing the tool output. \n", + " Provide the response in at least 500 words.\n", + " Generate a concise execution plan for the banking request.\n", + " \"\"\"" + ] + }, + { + "cell_type": "markdown", + "id": "b66c1ac4", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Initialize the LLM\n", + "\n", + "Let's initialize the LLM that will power our banking agent:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "866066e7", + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize the main LLM for banking responses\n", + "main_llm = ChatOpenAI(\n", + " model=\"gpt-5-mini\",\n", + " reasoning={\n", + " \"effort\": \"low\",\n", + " \"summary\": \"auto\"\n", + " }\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "8220afd6", + "metadata": {}, + "source": [ + "Then bind the available banking tools to the LLM, enabling the model to automatically recognize and invoke each tool when appropriate based on request input and the system prompt we defined above:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "906d8132", + "metadata": {}, + "outputs": [], + "source": [ + "# Bind all banking tools to the main LLM\n", + "llm_with_tools = main_llm.bind_tools(AVAILABLE_TOOLS)" + ] + }, + { + "cell_type": "markdown", + "id": "43f56651", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Define agent state structure\n", + "\n", + "The agent state defines the data structure that flows through the LangGraph workflow. It includes:\n", + "\n", + "- **messages** — The conversation history between the user and agent\n", + "- **user_input** — The current user request\n", + "- **session_id** — A unique identifier for the conversation session\n", + "- **context** — Additional context that can be passed between nodes\n", + "\n", + "Defining this state structure maintains the structure throughout the agent's execution and allows for multi-turn conversations with memory:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6b926ddf", + "metadata": {}, + "outputs": [], + "source": [ + "# Banking Agent State Definition\n", + "class BankingAgentState(TypedDict):\n", + " messages: Annotated[Sequence[BaseMessage], add_messages]\n", + " user_input: str\n", + " session_id: str\n", + " context: dict" + ] + }, + { + "cell_type": "markdown", + "id": "387ba780", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Create agent workflow function\n", + "\n", + "We'll build the LangGraph agent workflow with two main components:\n", + "\n", + "1. **LLM node** — Processes user requests, applies the system prompt, and decides whether to use tools.\n", + "2. **Tools node** — Executes the selected banking tools when the LLM determines they're needed.\n", + "\n", + "The workflow begins with the LLM analyzing the request, then uses tools if needed — or ends if the response is complete, and finally returns to the LLM to generate the final response." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2c9bf585", + "metadata": {}, + "outputs": [], + "source": [ + "def create_banking_langgraph_agent():\n", + " \"\"\"Create a comprehensive LangGraph banking agent with intelligent tool selection.\"\"\"\n", + " def llm_node(state: BankingAgentState) -> BankingAgentState:\n", + " \"\"\"Main LLM node that processes banking requests and selects appropriate tools.\"\"\"\n", + " messages = state[\"messages\"]\n", + " # Add system context to messages\n", + " enhanced_messages = [SystemMessage(content=system_context)] + list(messages)\n", + " # Get LLM response with tool selection\n", + " response = llm_with_tools.invoke(enhanced_messages)\n", + " return {\n", + " **state,\n", + " \"messages\": messages + [response]\n", + " }\n", + " \n", + " def should_continue(state: BankingAgentState) -> str:\n", + " \"\"\"Decide whether to use tools or end the conversation.\"\"\"\n", + " last_message = state[\"messages\"][-1]\n", + " # Check if the LLM wants to use tools\n", + " if hasattr(last_message, 'tool_calls') and last_message.tool_calls:\n", + " return \"tools\"\n", + " return END\n", + " \n", + " # Create the banking state graph\n", + " workflow = StateGraph(BankingAgentState)\n", + " # Add nodes\n", + " workflow.add_node(\"llm\", llm_node)\n", + " workflow.add_node(\"tools\", ToolNode(AVAILABLE_TOOLS))\n", + " # Simplified entry point - go directly to LLM\n", + " workflow.add_edge(START, \"llm\")\n", + " # From LLM, decide whether to use tools or end\n", + " workflow.add_conditional_edges(\n", + " \"llm\",\n", + " should_continue,\n", + " {\"tools\": \"tools\", END: END}\n", + " )\n", + " # Tool execution flows back to LLM for final response\n", + " workflow.add_edge(\"tools\", \"llm\")\n", + " # Set up memory\n", + " memory = MemorySaver()\n", + " # Compile the graph\n", + " agent = workflow.compile(checkpointer=memory)\n", + " return agent" + ] + }, + { + "cell_type": "markdown", + "id": "765242e9", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Instantiate the banking agent\n", + "\n", + "Now, we'll create an instance of the banking agent by calling the workflow creation function.\n", + "\n", + "This compiled agent is ready to process banking requests and will automatically select and use the appropriate tools based on user queries:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "455b8ee4", + "metadata": {}, + "outputs": [], + "source": [ + "# Create the banking intelligent agent\n", + "banking_agent = create_banking_langgraph_agent()\n", + "\n", + "print(\"Banking LangGraph Agent Created Successfully!\")\n", + "print(\"\\nFeatures:\")\n", + "print(\" - Intelligent banking tool selection\")\n", + "print(\" - Comprehensive banking system prompt\")\n", + "print(\" - Streamlined workflow: LLM → Tools → Response\")\n", + "print(\" - Automatic tool parameter extraction\")\n", + "print(\" - Professional banking assistance\")" + ] + }, + { + "cell_type": "markdown", + "id": "e00dac77", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Integrate agent with ValidMind\n", + "\n", + "To integrate our LangGraph banking agent with ValidMind, we need to create a wrapper function that ValidMind can use to invoke the agent and extract the necessary information for testing and documentation, allowing ValidMind to run validation tests on the agent's behavior, tool usage, and responses." + ] + }, + { + "cell_type": "markdown", + "id": "a124857e", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Import ValidMind components\n", + "\n", + "We'll start with importing the necessary ValidMind components for integrating our agent:\n", + "\n", + "- `Prompt` from `validmind.models` for handling prompt-based model inputs\n", + "- `extract_tool_calls_from_agent_output` and `_convert_to_tool_call_list` from `validmind.scorers.llm.deepeval` for extracting and converting tool calls from agent outputs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9aeb8969", + "metadata": {}, + "outputs": [], + "source": [ + "from validmind.models import Prompt\n", + "from validmind.scorers.llm.deepeval import extract_tool_calls_from_agent_output, _convert_to_tool_call_list\n", + "from deepeval.tracing import observe, update_current_span\n", + "from deepeval.test_case import LLMTestCase" + ] + }, + { + "cell_type": "markdown", + "id": "ed72903f", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Create agent wrapper function\n", + "\n", + "We'll then create a wrapper function that:\n", + "\n", + "- Accepts input in ValidMind's expected format (with `input` and `session_id` fields)\n", + "- Invokes the banking agent with the proper state initialization\n", + "- Captures tool outputs and tool calls for evaluation\n", + "- Returns a standardized response format that includes the prediction, full output, tool messages, and tool call information\n", + "- Handles errors gracefully with fallback responses" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0e4d5a82", + "metadata": {}, + "outputs": [], + "source": [ + "@observe(type=\"agent\")\n", + "def banking_agent_fn(input):\n", + " \"\"\"\n", + " Invoke the banking agent with the given input.\n", + " \"\"\"\n", + " try:\n", + " # Initial state for banking agent\n", + " initial_state = {\n", + " \"user_input\": input[\"input\"],\n", + " \"messages\": [HumanMessage(content=input[\"input\"])],\n", + " \"session_id\": input[\"session_id\"],\n", + " \"context\": {}\n", + " }\n", + " session_config = {\"configurable\": {\"thread_id\": input[\"session_id\"]}}\n", + " result = banking_agent.invoke(initial_state, config=session_config)\n", + "\n", + " from utils import capture_tool_output_messages\n", + "\n", + " # Capture all tool outputs and metadata\n", + " captured_data = capture_tool_output_messages(result)\n", + " \n", + " # Access specific tool outputs, this will be used for RAGAS tests\n", + " tool_message = \"\"\n", + " for output in captured_data[\"tool_outputs\"]:\n", + " tool_message += output['content']\n", + " \n", + " tool_calls_found = []\n", + " messages = result['messages']\n", + " for message in messages:\n", + " if hasattr(message, 'tool_calls') and message.tool_calls:\n", + " for tool_call in message.tool_calls:\n", + " # Handle both dictionary and object formats\n", + " if isinstance(tool_call, dict):\n", + " tool_calls_found.append(tool_call['name'])\n", + " else:\n", + " # ToolCall object - use attribute access\n", + " tool_calls_found.append(tool_call.name)\n", + "\n", + " prediction_text = result['messages'][-1].content[0]['text']\n", + " tools_called_value = _convert_to_tool_call_list(extract_tool_calls_from_agent_output(result))\n", + " expected_tools_value = _convert_to_tool_call_list(input.get(\"expected_tools\", []))\n", + "\n", + " # Feed trace data for DeepEval metrics (e.g. PlanQuality) that require tracing\n", + " update_current_span(\n", + " test_case=LLMTestCase(\n", + " input=input[\"input\"],\n", + " actual_output=prediction_text,\n", + " tools_called=tools_called_value,\n", + " expected_tools=expected_tools_value\n", + " )\n", + " )\n", + "\n", + " return {\n", + " \"prediction\": prediction_text,\n", + " \"output\": result,\n", + " \"tool_messages\": [tool_message],\n", + " # \"tool_calls\": tool_calls_found,\n", + " \"tool_called\": tools_called_value\n", + " }\n", + " except Exception as e:\n", + " # Return a fallback response if the agent fails\n", + " error_message = f\"\"\"I apologize, but I encountered an error while processing your banking request: {str(e)}.\n", + " Please try rephrasing your question or contact support if the issue persists.\"\"\"\n", + " return {\n", + " \"prediction\": error_message, \n", + " \"output\": {\n", + " \"messages\": [HumanMessage(content=input[\"input\"]), SystemMessage(content=error_message)],\n", + " \"error\": str(e)\n", + " }\n", + " }" + ] + }, + { + "cell_type": "markdown", + "id": "fda87401", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Initialize the ValidMind model object\n", + "\n", + "We'll also need to register the banking agent as a ValidMind model object (`vm_model`) that can be passed to other functions for analysis and tests on the data.\n", + "\n", + "You simply initialize this model object with [`vm.init_model()`](https://docs.validmind.ai/validmind/validmind.html#init_model) that:\n", + "\n", + "- Associates the wrapper function with the model for prediction\n", + "- Stores the system prompt template for documentation\n", + "- Provides a unique `input_id` for tracking and identification\n", + "- Enables the agent to be used with ValidMind's testing and documentation features" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "60a2ce7a", + "metadata": {}, + "outputs": [], + "source": [ + "# Initialize the agent as a model\n", + "vm_banking_model = vm.init_model(\n", + " input_id=\"banking_agent_model\",\n", + " predict_fn=banking_agent_fn,\n", + " prompt=Prompt(template=system_context)\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "949bcf53", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Store the agent reference\n", + "\n", + "We'll also store a reference to the original banking agent object in the ValidMind model. This allows us to access the full agent functionality directly if needed, while still maintaining the wrapper function interface for ValidMind's testing framework." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2c653471", + "metadata": {}, + "outputs": [], + "source": [ + "# Add the banking agent to the vm model\n", + "vm_banking_model.model = banking_agent" + ] + }, + { + "cell_type": "markdown", + "id": "d8d0c1c1", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Verify integration\n", + "\n", + "Let's confirm that the banking agent has been successfully integrated with ValidMind:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8e101b0f", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Banking Agent Successfully Integrated with ValidMind!\")\n", + "print(f\"Model ID: {vm_banking_model.input_id}\")" + ] + }, + { + "cell_type": "markdown", + "id": "2a5f874e", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Validate the system prompt\n", + "\n", + "Let's get an initial sense of how well our defined system prompt meets a few best practices for prompt engineering by running a few tests — we'll run evaluation tests later on our agent's performance.\n", + "\n", + "You run individual tests by calling [the `run_test` function](https://docs.validmind.ai/validmind/validmind/tests.html#run_test) provided by the `validmind.tests` module. Passing in our agentic model as an input, the tests below rate the prompt on a scale of 1-10 against the following criteria:\n", + "\n", + "- **[Clarity](https://docs.validmind.ai/tests/prompt_validation/Clarity.html)** — How clearly the prompt states the task.\n", + "- **[Conciseness](https://docs.validmind.ai/tests/prompt_validation/Conciseness.html)** — How succinctly the prompt states the task.\n", + "- **[Delimitation](https://docs.validmind.ai/tests/prompt_validation/Delimitation.html)** — When using complex prompts containing examples, contextual information, or other elements, is the prompt formatted in such a way that each element is clearly separated?\n", + "- **[NegativeInstruction](https://docs.validmind.ai/tests/prompt_validation/NegativeInstruction.html)** — Whether the prompt contains negative instructions.\n", + "- **[Specificity](https://docs.validmind.ai/tests/prompt_validation/NegativeInstruction.html)** — How specific the prompt defines the task." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f52dceb1", + "metadata": {}, + "outputs": [], + "source": [ + "vm.tests.run_test(\n", + " \"validmind.prompt_validation.Clarity\",\n", + " inputs={\n", + " \"model\": vm_banking_model,\n", + " },\n", + ").log()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "70d52333", + "metadata": {}, + "outputs": [], + "source": [ + "vm.tests.run_test(\n", + " \"validmind.prompt_validation.Conciseness\",\n", + " inputs={\n", + " \"model\": vm_banking_model,\n", + " },\n", + ").log()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5aa89976", + "metadata": {}, + "outputs": [], + "source": [ + "vm.tests.run_test(\n", + " \"validmind.prompt_validation.Delimitation\",\n", + " inputs={\n", + " \"model\": vm_banking_model,\n", + " },\n", + ").log()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8630197e", + "metadata": {}, + "outputs": [], + "source": [ + "vm.tests.run_test(\n", + " \"validmind.prompt_validation.NegativeInstruction\",\n", + " inputs={\n", + " \"model\": vm_banking_model,\n", + " },\n", + ").log()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bba99915", + "metadata": {}, + "outputs": [], + "source": [ + "vm.tests.run_test(\n", + " \"validmind.prompt_validation.Specificity\",\n", + " inputs={\n", + " \"model\": vm_banking_model,\n", + " },\n", + ").log()" + ] + }, + { + "cell_type": "markdown", + "id": "51d61141", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Initializing the ValidMind dataset\n", + "\n", + "After validation our system prompt, let's import our sample dataset ([banking_test_dataset.py](banking_test_dataset.py)), which we'll use in the next section to evaluate our agent's performance across different banking scenarios:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0c70ca2c", + "metadata": {}, + "outputs": [], + "source": [ + "from banking_test_dataset import banking_test_dataset" + ] + }, + { + "cell_type": "markdown", + "id": "442ab66d", + "metadata": {}, + "source": [ + "The next step is to connect your data with a ValidMind `Dataset` object. **This step is always necessary every time you want to connect a dataset to documentation and produce test results through ValidMind,** but you only need to do it once per dataset.\n", + "\n", + "Initialize a ValidMind dataset object using the [`init_dataset` function](https://docs.validmind.ai/validmind/validmind.html#init_dataset) from the ValidMind (`vm`) module. For this example, we'll pass in the following arguments:\n", + "\n", + "- **`input_id`** — A unique identifier that allows tracking what inputs are used when running each individual test.\n", + "- **`dataset`** — The raw dataset that you want to provide as input to tests.\n", + "- **`text_column`** — The name of the column containing the text input data.\n", + "- **`target_column`** — A required argument if tests require access to true values. This is the name of the target column in the dataset." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a7e9d158", + "metadata": {}, + "outputs": [], + "source": [ + "vm_test_dataset = vm.init_dataset(\n", + " input_id=\"banking_test_dataset\",\n", + " dataset=banking_test_dataset,\n", + " text_column=\"input\",\n", + " target_column=\"possible_outputs\",\n", + ")\n", + "\n", + "print(\"Banking Test Dataset Initialized in ValidMind!\")\n", + "print(f\"Dataset ID: {vm_test_dataset.input_id}\")\n", + "print(f\"Dataset columns: {vm_test_dataset._df.columns}\")\n", + "vm_test_dataset._df" + ] + }, + { + "cell_type": "markdown", + "id": "7b01021c", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Assign predictions\n", + "\n", + "Now that both the model object and the datasets have been registered, we'll assign predictions to capture the banking agent's responses for evaluation:\n", + "\n", + "- The [`assign_predictions()` method](https://docs.validmind.ai/validmind/validmind/vm_models.html#assign_predictions) from the `Dataset` object can link existing predictions to any number of models.\n", + "- This method links the model's class prediction values and probabilities to our `vm_train_ds` and `vm_test_ds` datasets.\n", + "\n", + "If no prediction values are passed, the method will compute predictions automatically:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1d462663", + "metadata": {}, + "outputs": [], + "source": [ + "vm_test_dataset.assign_predictions(vm_banking_model)\n", + "\n", + "print(\"Banking Agent Predictions Generated Successfully!\")\n", + "print(f\"Predictions assigned to {len(vm_test_dataset._df)} test cases\")\n", + "vm_test_dataset._df.head()" + ] + }, + { + "cell_type": "markdown", + "id": "4e56f556", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Running accuracy tests\n", + "\n", + "Using [`@vm.test`](https://docs.validmind.ai/validmind/validmind.html#test), let's implement some reusable custom *inline tests* to assess the accuracy of our banking agent:\n", + "\n", + "- An inline test refers to a test written and executed within the same environment as the code being tested — in this case, right in this Jupyter Notebook — without requiring a separate test file or framework.\n", + "- You'll note that the custom test functions are just regular Python functions that can include and require any Python library as you see fit." + ] + }, + { + "cell_type": "markdown", + "id": "1bce9258", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Response accuracy test\n", + "\n", + "We'll create a custom test that evaluates the banking agent's ability to provide accurate responses by:\n", + "\n", + "- Testing against a dataset of predefined banking questions and expected answers.\n", + "- Checking if responses contain expected keywords and banking terminology.\n", + "- Providing detailed test results including pass/fail status.\n", + "- Helping identify any gaps in the agent's banking knowledge or response quality." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "90232066", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "@vm.test(\"my_custom_tests.banking_accuracy_test\")\n", + "def banking_accuracy_test(model, dataset, list_of_columns):\n", + " \"\"\"\n", + " The Banking Accuracy Test evaluates whether the agent’s responses include \n", + " critical domain-specific keywords and phrases that indicate accurate, compliant,\n", + " and contextually appropriate banking information. This test ensures that the agent\n", + " provides responses containing the expected banking terminology, risk classifications,\n", + " account details, or other domain-relevant information required for regulatory compliance,\n", + " customer safety, and operational accuracy.\n", + " \"\"\"\n", + " df = dataset._df\n", + " \n", + " # Pre-compute responses for all tests\n", + " y_true = dataset.y.tolist()\n", + " y_pred = dataset.y_pred(model).tolist()\n", + "\n", + " # Vectorized test results\n", + " test_results = []\n", + " for response, keywords in zip(y_pred, y_true):\n", + " # Convert keywords to list if not already a list\n", + " if not isinstance(keywords, list):\n", + " keywords = [keywords]\n", + " test_results.append(any(str(keyword).lower() in str(response).lower() for keyword in keywords))\n", + " \n", + " results = pd.DataFrame()\n", + " column_names = [col + \"_details\" for col in list_of_columns]\n", + " results[column_names] = df[list_of_columns]\n", + " results[\"actual\"] = y_pred\n", + " results[\"expected\"] = y_true\n", + " results[\"passed\"] = test_results\n", + " results[\"error\"] = None if test_results else f'Response did not contain any expected keywords: {y_true}'\n", + " \n", + " return results" + ] + }, + { + "cell_type": "markdown", + "id": "2a7f71f8", + "metadata": {}, + "source": [ + "Now that we've defined our custom response accuracy test, we can run the test using the same `run_test()` function we used earlier to validate the system prompt using our sample dataset and agentic model as input, and log the test results to the ValidMind Platform with the [`log()` method](https://docs.validmind.ai/validmind/validmind/vm_models.html#log):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e68884d5", + "metadata": {}, + "outputs": [], + "source": [ + "result = vm.tests.run_test(\n", + " \"my_custom_tests.banking_accuracy_test\",\n", + " inputs={\n", + " \"dataset\": vm_test_dataset,\n", + " \"model\": vm_banking_model\n", + " },\n", + " params={\n", + " \"list_of_columns\": [\"input\"]\n", + " }\n", + ")\n", + "result.log()" + ] + }, + { + "cell_type": "markdown", + "id": "94a717e7", + "metadata": {}, + "source": [ + "Let's review the first five rows of the test dataset to inspect the results to see how well the banking agent performed. Each column in the output serves a specific purpose in evaluating agent performance:\n", + "\n", + "| Column header | Description | Importance |\n", + "|--------------|-------------|------------|\n", + "| **`input`** | Original user query or request | Essential for understanding the context of each test case and tracing which inputs led to specific agent behaviors. |\n", + "| **`expected_tools`** | Banking tools that should be invoked for this request | Enables validation of correct tool selection, which is critical for agentic AI systems where choosing the right tool is a key success metric. |\n", + "| **`expected_output`** | Expected output or keywords that should appear in the response | Defines the success criteria for each test case, enabling objective evaluation of whether the agent produced the correct result. |\n", + "| **`session_id`** | Unique identifier for each test session | Allows tracking and correlation of related test runs, debugging specific sessions, and maintaining audit trails. |\n", + "| **`category`** | Classification of the request type | Helps organize test results by domain and identify performance patterns across different banking use cases. |\n", + "| **`banking_agent_model_output`** | Complete agent response including all messages and reasoning | Allows you to examine the full output to assess response quality, completeness, and correctness beyond just keyword matching. |\n", + "| **`banking_agent_model_tool_messages`** | Messages exchanged with the banking tools | Critical for understanding how the agent interacted with tools, what parameters were passed, and what tool outputs were received. |\n", + "| **`banking_agent_model_tool_called`** | Specific tool that was invoked | Enables validation that the agent selected the correct tool for each request, which is fundamental to agentic AI validation. |\n", + "| **`possible_outputs`** | Alternative valid outputs or keywords that could appear in the response | Provides flexibility in evaluation by accounting for multiple acceptable response formats or variations. |" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "78f7edb1", + "metadata": {}, + "outputs": [], + "source": [ + "vm_test_dataset.df.head(5)" + ] + }, + { + "cell_type": "markdown", + "id": "1cb3e8bd", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Tool selection accuracy test\n", + "\n", + "We'll also create a custom test that evaluates the banking agent's ability to select the correct tools for different requests by:\n", + "\n", + "- Testing against a dataset of predefined banking queries with expected tool selections.\n", + "- Comparing the tools actually invoked by the agent against the expected tools for each request.\n", + "- Providing quantitative accuracy scores that measure the proportion of expected tools correctly selected.\n", + "- Helping identify gaps in the agent's understanding of user needs and tool selection logic." + ] + }, + { + "cell_type": "markdown", + "id": "69263d62", + "metadata": {}, + "source": [ + "First, we'll define a helper function that extracts tool calls from the agent's messages and compares them against the expected tools. This function handles different message formats (dictionary or object) and calculates accuracy scores:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e68798be", + "metadata": {}, + "outputs": [], + "source": [ + "def validate_tool_calls_simple(messages, expected_tools):\n", + " \"\"\"Simple validation of tool calls without RAGAS dependency issues.\"\"\"\n", + " \n", + " tool_calls_found = []\n", + " \n", + " for message in messages:\n", + " if hasattr(message, 'tool_calls') and message.tool_calls:\n", + " for tool_call in message.tool_calls:\n", + " # Handle both dictionary and object formats\n", + " if isinstance(tool_call, dict):\n", + " tool_calls_found.append(tool_call['name'])\n", + " else:\n", + " # ToolCall object - use attribute access\n", + " tool_calls_found.append(tool_call.name)\n", + " \n", + " # Check if expected tools were called\n", + " accuracy = 0.0\n", + " matches = 0\n", + " if expected_tools:\n", + " matches = sum(1 for tool in expected_tools if tool in tool_calls_found)\n", + " accuracy = matches / len(expected_tools)\n", + " \n", + " return {\n", + " 'expected_tools': expected_tools,\n", + " 'found_tools': tool_calls_found,\n", + " 'matches': matches,\n", + " 'total_expected': len(expected_tools) if expected_tools else 0,\n", + " 'accuracy': accuracy,\n", + " }" + ] + }, + { + "cell_type": "markdown", + "id": "8f494fd3", + "metadata": {}, + "source": [ + "Now we'll define the main test function that uses the helper function to evaluate tool selection accuracy across all test cases in the dataset:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "604d7313", + "metadata": {}, + "outputs": [], + "source": [ + "@vm.test(\"my_custom_tests.BankingToolCallAccuracy\")\n", + "def BankingToolCallAccuracy(dataset, agent_output_column, expected_tools_column):\n", + " \"\"\"\n", + " Evaluates the tool selection accuracy of a LangGraph-powered banking agent.\n", + "\n", + " This test measures whether the agent correctly identifies and invokes the required banking tools\n", + " for each user query scenario.\n", + " For each case, the outputs generated by the agent (including its tool calls) are compared against an\n", + " expected set of tools. The test considers both coverage and exactness: it computes the proportion of\n", + " expected tools correctly called by the agent for each instance.\n", + "\n", + " Parameters:\n", + " dataset (VMDataset): The dataset containing user queries, agent outputs, and ground-truth tool expectations.\n", + " agent_output_column (str): Dataset column name containing agent outputs (should include tool call details in 'messages').\n", + " expected_tools_column (str): Dataset column specifying the true expected tools (as lists).\n", + "\n", + " Returns:\n", + " List[dict]: Per-row dictionaries with details: expected tools, found tools, match count, total expected, and accuracy score.\n", + "\n", + " Purpose:\n", + " Provides diagnostic evidence of the banking agent's core reasoning ability—specifically, its capacity to\n", + " interpret user needs and select the correct banking actions. Useful for diagnosing gaps in tool coverage,\n", + " misclassifications, or breakdowns in agent logic.\n", + "\n", + " Interpretation:\n", + " - An accuracy of 1.0 signals perfect tool selection for that example.\n", + " - Lower scores may indicate partial or complete failures to invoke required tools.\n", + " - Review 'found_tools' vs. 'expected_tools' to understand the source of discrepancies.\n", + "\n", + " Strengths:\n", + " - Directly tests a core capability of compositional tool-use agents.\n", + " - Framework-agnostic; robust to tool call output format (object or dict).\n", + " - Supports batch validation and result logging for systematic documentation.\n", + "\n", + " Limitations:\n", + " - Does not penalize extra, unnecessary tool calls.\n", + " - Does not assess result quality—only correct invocation.\n", + "\n", + " \"\"\"\n", + " df = dataset._df\n", + " \n", + " results = []\n", + " for i, row in df.iterrows():\n", + " result = validate_tool_calls_simple(row[agent_output_column]['messages'], row[expected_tools_column])\n", + " results.append(result)\n", + " \n", + " return results" + ] + }, + { + "cell_type": "markdown", + "id": "57ab606b", + "metadata": {}, + "source": [ + "Finally, we can call our function with `run_test()` and log the test results to the ValidMind Platform:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dd14115e", + "metadata": {}, + "outputs": [], + "source": [ + "result = vm.tests.run_test(\n", + " \"my_custom_tests.BankingToolCallAccuracy\",\n", + " inputs={\n", + " \"dataset\": vm_test_dataset,\n", + " },\n", + " params={\n", + " \"agent_output_column\": \"banking_agent_model_output\",\n", + " \"expected_tools_column\": \"expected_tools\"\n", + " }\n", + ")\n", + "result.log()" + ] + }, + { + "cell_type": "markdown", + "id": "be8d5270", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Assigning AI evaluation metric scores\n", + "\n", + "*AI agent evaluation metrics* are specialized measurements designed to assess how well autonomous LLM-based agents reason, plan, select and execute tools, and ultimately complete user tasks by analyzing the *full execution trace* — including reasoning steps, tool calls, intermediate decisions, and outcomes, rather than just single input–output pairs. These metrics are essential because agent failures often occur in ways traditional LLM metrics miss — for example, choosing the right tool with wrong arguments, creating a good plan but not following it, or completing a task inefficiently.\n", + "\n", + "In this section, we'll evaluate our banking agent's outputs and add scoring to our sample dataset against metrics defined in [DeepEval’s AI agent evaluation framework](https://deepeval.com/guides/guides-ai-agent-evaluation-metrics) which breaks down AI agent evaluation into three layers with corresponding subcategories: **reasoning**, **action**, and **execution**.\n", + "\n", + "Together, these three metrics enable granular diagnosis of agent behavior, help pinpoint where failures occur (reasoning, action, or execution), and support both development benchmarking and production monitoring." + ] + }, + { + "cell_type": "markdown", + "id": "25828bef", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Identify relevant DeepEval scorers\n", + "\n", + "*Scorers* are evaluation metrics that analyze model outputs and store their results in the dataset:\n", + "\n", + "- Each scorer adds a new column to the dataset with format: `{scorer_name}_{metric_name}`\n", + "- The column contains the numeric score (typically `0`-`1`) for each example\n", + "- Multiple scorers can be run on the same dataset, each adding their own column\n", + "- Scores are persisted in the dataset for later analysis and visualization\n", + "- Common scorer patterns include:\n", + " - Model performance metrics (accuracy, F1, etc.)\n", + " - Output quality metrics (relevance, faithfulness)\n", + " - Task-specific metrics (completion, correctness)\n", + "\n", + "Use `list_scorers()` from [`validmind.scorers`](https://docs.validmind.ai/validmind/validmind/tests.html#scorer) to discover all available scoring methods and their IDs that can be used with `assign_scores()`. We'll filter these results to return only DeepEval scorers for our desired three metrics in a formatted table with descriptions:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "730c70ec", + "metadata": {}, + "outputs": [], + "source": [ + "# Load all DeepEval scorers\n", + "llm_scorers_dict = vm.tests.load._load_tests([s for s in vm.scorer.list_scorers() if \"deepeval\" in s.lower()])\n", + "\n", + "# Categorize scorers by metric layer\n", + "reasoning_scorers = {}\n", + "action_scorers = {}\n", + "execution_scorers = {}\n", + "\n", + "for scorer_id, scorer_func in llm_scorers_dict.items():\n", + " tags = getattr(scorer_func, \"__tags__\", [])\n", + " scorer_name = scorer_id.split(\".\")[-1]\n", + "\n", + " if \"reasoning_layer\" in tags:\n", + " reasoning_scorers[scorer_id] = scorer_func\n", + " elif \"action_layer\" in tags:\n", + " action_scorers[scorer_id] = scorer_func\n", + " elif \"TaskCompletion\" in scorer_name:\n", + " execution_scorers[scorer_id] = scorer_func\n", + "\n", + "# Display scorers by category\n", + "print(\"=\" * 80)\n", + "print(\"REASONING LAYER\")\n", + "print(\"=\" * 80)\n", + "if reasoning_scorers:\n", + " reasoning_df = vm.tests.load._pretty_list_tests(reasoning_scorers, truncate=True)\n", + " display(reasoning_df)\n", + "else:\n", + " print(\"No reasoning layer scorers found.\")\n", + "\n", + "print(\"\\n\" + \"=\" * 80)\n", + "print(\"ACTION LAYER\")\n", + "print(\"=\" * 80)\n", + "if action_scorers:\n", + " action_df = vm.tests.load._pretty_list_tests(action_scorers, truncate=True)\n", + " display(action_df)\n", + "else:\n", + " print(\"No action layer scorers found.\")\n", + "\n", + "print(\"\\n\" + \"=\" * 80)\n", + "print(\"EXECUTION LAYER\")\n", + "print(\"=\" * 80)\n", + "if execution_scorers:\n", + " execution_df = vm.tests.load._pretty_list_tests(execution_scorers, truncate=True)\n", + " display(execution_df)\n", + "else:\n", + " print(\"No execution layer scorers found.\")" + ] + }, + { + "cell_type": "markdown", + "id": "e5fb739b", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Assign reasoning scores\n", + "\n", + "*Reasoning* evaluates planning and strategy generation:\n", + "\n", + "- **Plan quality** – How logical, complete, and efficient the agent’s plan is.\n", + "- **Plan adherence** – Whether the agent follows its own plan during execution." + ] + }, + { + "cell_type": "markdown", + "id": "fde94d01", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Plan quality score\n", + "\n", + "Let's measure how well our banking agent generates a plan before acting. A high score means the plan is logical, complete, and efficient." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "52f362ba", + "metadata": {}, + "outputs": [], + "source": [ + "vm_test_dataset.assign_scores(\n", + " metrics = \"validmind.scorers.llm.deepeval.PlanQuality\",\n", + " model = vm_banking_model,\n", + " input_column = \"input\",\n", + ")\n", + "vm_test_dataset._df[['banking_agent_model_PlanQuality_score','banking_agent_model_PlanQuality_reason']]" + ] + }, + { + "cell_type": "markdown", + "id": "d631fd12", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Plan adherence score\n", + "\n", + "Let's check whether our banking agent follows the plan it created. Deviations lower this score and indicate gaps between reasoning and execution." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4124a7c2", + "metadata": {}, + "outputs": [], + "source": [ + "vm_test_dataset.assign_scores(\n", + " metrics = \"validmind.scorers.llm.deepeval.PlanAdherence\",\n", + " input_column = \"input\",\n", + " model = vm_banking_model,\n", + ")\n", + "vm_test_dataset._df[['banking_agent_model_PlanAdherence_score','banking_agent_model_PlanAdherence_reason']]" + ] + }, + { + "cell_type": "markdown", + "id": "82e5e6f1", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Assign action scores\n", + "\n", + "*Action* assesses tool usage and argument generation:\n", + "\n", + "- **Tool correctness** – Whether the agent selects and calls the right tools.\n", + "- **Argument correctness** – Whether the agent generates correct tool arguments." + ] + }, + { + "cell_type": "markdown", + "id": "e641c9f2", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Tool correctness score\n", + "\n", + "Let's evaluate if our banking agent selects the appropriate tool for the task. Choosing the wrong tool reduces performance even if reasoning was correct." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8d2e8a25", + "metadata": {}, + "outputs": [], + "source": [ + "vm_test_dataset.assign_scores(\n", + " metrics = \"validmind.scorers.llm.deepeval.ToolCorrectness\",\n", + " input_column = \"input\",\n", + " model = vm_banking_model,\n", + " expected_tools_called_column = \"expected_tools\",\n", + " actual_tools_called_column = \"banking_agent_model_tool_called\",\n", + ")\n", + "vm_test_dataset._df[['banking_agent_model_ToolCorrectness_score','banking_agent_model_ToolCorrectness_reason']]" + ] + }, + { + "cell_type": "markdown", + "id": "dd758ba5", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Argument correctness score\n", + "\n", + "Let's assesses whether our banking agent provides correct inputs or arguments to the selected tool. Incorrect arguments can lead to failed or unexpected results." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "04f90489", + "metadata": {}, + "outputs": [], + "source": [ + "vm_test_dataset.assign_scores(\n", + " metrics = \"validmind.scorers.llm.deepeval.ArgumentCorrectness\",\n", + " input_column = \"input\",\n", + " model = vm_banking_model,\n", + " actual_tools_called_column = \"banking_agent_model_tool_called\",\n", + ")\n", + "vm_test_dataset._df[['banking_agent_model_ArgumentCorrectness_score','banking_agent_model_ArgumentCorrectness_reason']]" + ] + }, + { + "cell_type": "markdown", + "id": "1aeec2f5", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Assign execution score\n", + "\n", + "*Execution* measures end-to-end performance:\n", + "\n", + "- **Task completion** – Whether the agent successfully completes the intended task." + ] + }, + { + "cell_type": "markdown", + "id": "eb9ab8de", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Task completion score\n", + "\n", + "Let's evaluate whether our banking agent successfully completes the requested tasks. Incomplete task execution can lead to user dissatisfaction and failed banking operations." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "05024f1f", + "metadata": {}, + "outputs": [], + "source": [ + "vm_test_dataset.assign_scores(\n", + " metrics = \"validmind.scorers.llm.deepeval.TaskCompletion\",\n", + " input_column = \"input\",\n", + " model = vm_banking_model,\n", + " actual_tools_called_column = \"banking_agent_model_tool_called\",\n", + ")\n", + "vm_test_dataset._df[['banking_agent_model_TaskCompletion_score','banking_agent_model_TaskCompletion_reason']]" + ] + }, + { + "cell_type": "markdown", + "id": "b577c282", + "metadata": {}, + "source": [ + "As you recall from the beginning of this section, when we run scorers through `assign_scores()`, the return values are automatically processed and added as new columns with the format `{scorer_name}_{metric_name}`. Note that the task completion scorer has added a new column `TaskCompletion_score` to our dataset.\n", + "\n", + "We'll use this column to visualize the distribution of task completion scores across our test cases through the [BoxPlot test](https://docs.validmind.ai/validmind/validmind/tests/plots/BoxPlot.html#boxplot):" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7f6d08ca", + "metadata": {}, + "outputs": [], + "source": [ + "vm.tests.run_test(\n", + " \"validmind.plots.BoxPlot\",\n", + " inputs={\"dataset\": vm_test_dataset},\n", + " params={\n", + " \"columns\": \"banking_agent_model_TaskCompletion_score\",\n", + " \"title\": \"Distribution of Task Completion Scores\",\n", + " \"ylabel\": \"Score\",\n", + " \"figsize\": (8, 6)\n", + " }\n", + ").log()" + ] + }, + { + "cell_type": "markdown", + "id": "30d9ec62", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Running RAGAS tests\n", + "\n", + "Next, let's run some out-of-the-box *Retrieval-Augmented Generation Assessment* (RAGAS) tests available in the ValidMind Library. RAGAS provides specialized metrics for evaluating retrieval-augmented generation systems and conversational AI agents. These metrics analyze different aspects of agent performance by assessing how well systems integrate retrieved information with generated responses.\n", + "\n", + "Our banking agent uses tools to retrieve information and generates responses based on that context, making it similar to a RAG system. RAGAS metrics help evaluate the quality of this integration by analyzing the relationship between retrieved tool outputs, user queries, and generated responses.\n", + "\n", + "These tests provide insights into how well our banking agent integrates tool usage with conversational abilities, ensuring it provides accurate, relevant, and helpful responses to banking users while maintaining fidelity to retrieved information." + ] + }, + { + "cell_type": "markdown", + "id": "8288f6c3", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Identify relevant RAGAS tests\n", + "\n", + "Let's explore some of ValidMind's available tests. Using ValidMind’s repository of tests streamlines your development testing, and helps you ensure that your models are being documented and evaluated appropriately.\n", + "\n", + "You can pass `tasks` and `tags` as parameters to the [`vm.tests.list_tests()` function](https://docs.validmind.ai/validmind/validmind/tests.html#list_tests) to filter the tests based on the tags and task types:\n", + "\n", + "- **`tasks`** represent the kind of modeling task associated with a test. Here we'll focus on `text_qa` tasks.\n", + "- **`tags`** are free-form descriptions providing more details about the test, for example, what category the test falls into. Here we'll focus on the `ragas` tag.\n", + "\n", + "We'll then run three of these tests returned as examples below." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0701f5a9", + "metadata": {}, + "outputs": [], + "source": [ + "vm.tests.list_tests(task=\"text_qa\", tags=[\"ragas\"])" + ] + }, + { + "cell_type": "markdown", + "id": "2ce24ba0", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Faithfulness\n", + "\n", + "Let's evaluate whether the banking agent's responses accurately reflect the information retrieved from tools. Unfaithful responses can misreport credit analysis, financial calculations, and compliance results—undermining user trust in the banking agent." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "92044533", + "metadata": {}, + "outputs": [], + "source": [ + "vm.tests.run_test(\n", + " \"validmind.model_validation.ragas.Faithfulness\",\n", + " inputs={\"dataset\": vm_test_dataset},\n", + " param_grid={\n", + " \"user_input_column\": [\"input\"],\n", + " \"response_column\": [\"banking_agent_model_prediction\"],\n", + " \"retrieved_contexts_column\": [\"banking_agent_model_tool_messages\"],\n", + " },\n", + ").log()" + ] + }, + { + "cell_type": "markdown", + "id": "4d1fcfcd", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Response Relevancy\n", + "\n", + "Let's evaluate whether the banking agent's answers address the user's original question or request. Irrelevant or off-topic responses can frustrate users and fail to deliver the banking information they need." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d7483bc3", + "metadata": {}, + "outputs": [], + "source": [ + "vm.tests.run_test(\n", + " \"validmind.model_validation.ragas.ResponseRelevancy\",\n", + " inputs={\"dataset\": vm_test_dataset},\n", + " params={\n", + " \"user_input_column\": \"input\",\n", + " \"response_column\": \"banking_agent_model_prediction\",\n", + " \"retrieved_contexts_column\": \"banking_agent_model_tool_messages\",\n", + " }\n", + ").log()" + ] + }, + { + "cell_type": "markdown", + "id": "38c1dfb5", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Context Recall\n", + "\n", + "Let's evaluate how well the banking agent uses the information retrieved from tools when generating its responses. Poor context recall can lead to incomplete or underinformed answers even when the right tools were selected." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e5dc00ce", + "metadata": {}, + "outputs": [], + "source": [ + "vm.tests.run_test(\n", + " \"validmind.model_validation.ragas.ContextRecall\",\n", + " inputs={\"dataset\": vm_test_dataset},\n", + " param_grid={\n", + " \"user_input_column\": [\"input\"],\n", + " \"retrieved_contexts_column\": [\"banking_agent_model_tool_messages\"],\n", + " \"reference_column\": [\"banking_agent_model_prediction\"],\n", + " },\n", + ").log()" + ] + }, + { + "cell_type": "markdown", + "id": "95e1e16a", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Running safety tests\n", + "\n", + "Finally, let's run some out-of-the-box *safety* tests available in the ValidMind Library. Safety tests provide specialized metrics for evaluating whether AI agents operate reliably and securely. These metrics analyze different aspects of agent behavior by assessing adherence to safety guidelines, consistency of outputs, and resistance to harmful or inappropriate requests.\n", + "\n", + "Our banking agent handles sensitive financial information and user requests, making safety and reliability essential. Safety tests help evaluate whether the agent maintains appropriate boundaries, responds consistently and correctly to inputs, and avoids generating harmful, biased, or unprofessional content.\n", + "\n", + "These tests provide insights into how well our banking agent upholds standards of fairness and professionalism, ensuring it operates reliably and securely for banking users." + ] + }, + { + "cell_type": "markdown", + "id": "e0972afa", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### AspectCritic\n", + "\n", + "Let's evaluate our banking agent's responses across multiple quality dimensions — conciseness, coherence, correctness, harmfulness, and maliciousness. Weak performance on these dimensions can degrade user experience, fall short of professional banking standards, or introduce safety risks. \n", + "\n", + "We'll use the `AspectCritic` we identified earlier:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "148daa2b", + "metadata": {}, + "outputs": [], + "source": [ + "vm.tests.run_test(\n", + " \"validmind.model_validation.ragas.AspectCritic\",\n", + " inputs={\"dataset\": vm_test_dataset},\n", + " param_grid={\n", + " \"user_input_column\": [\"input\"],\n", + " \"response_column\": [\"banking_agent_model_prediction\"],\n", + " \"retrieved_contexts_column\": [\"banking_agent_model_tool_messages\"],\n", + " },\n", + ").log()" + ] + }, + { + "cell_type": "markdown", + "id": "16f29c8d", + "metadata": {}, + "source": [ + "\n", + "\n", + "#### Bias\n", + "\n", + "Let's evaluate whether our banking agent's prompts contain unintended biases that could affect banking decisions. Biased prompts can lead to unfair or discriminatory outcomes — undermining customer trust and exposing the institution to compliance risk.\n", + "\n", + "We'll first use `list_tests()` again to filter for tests relating to `prompt_validation`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "74eba86c", + "metadata": {}, + "outputs": [], + "source": [ + "vm.tests.list_tests(filter=\"prompt_validation\")" + ] + }, + { + "cell_type": "markdown", + "id": "e9413803", + "metadata": {}, + "source": [ + "And then run the identified `Bias` test:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "062cf8e7", + "metadata": {}, + "outputs": [], + "source": [ + "vm.tests.run_test(\n", + " \"validmind.prompt_validation.Bias\",\n", + " inputs={\n", + " \"model\": vm_banking_model,\n", + " },\n", + ").log()" + ] + }, + { + "cell_type": "markdown", + "id": "8f3f2dbe", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Next steps\n", + "\n", + "You can look at the output produced by the ValidMind Library right in the notebook where you ran the code, as you would expect. But there is a better way — use the ValidMind Platform to work with your model documentation." + ] + }, + { + "cell_type": "markdown", + "id": "8716165d", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Work with your model documentation\n", + "\n", + "1. From the **Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", + "\n", + "2. In the left sidebar that appears for your model, click **Development** under Documents.\n", + "\n", + " What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation, view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/working-with-model-documentation.html)\n", + "\n", + "3. Click into any section related to the tests we ran in this notebook, for example: **4.3. Prompt Evaluation** to review the results of the tests we logged." + ] + }, + { + "cell_type": "markdown", + "id": "7c4a78ce", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Customize the banking agent for your use case\n", + "\n", + "You've now built an agentic AI system designed for banking use cases that supports compliance with supervisory guidance such as SR 11-7 and SS1/23, covering credit and fraud risk assessment for both retail and commercial banking. Extend this example agent to real-world banking scenarios and production deployment by:\n", + "\n", + "- Adapting the banking tools to your organization's specific requirements\n", + "- Adding more banking scenarios and edge cases to your test set\n", + "- Connecting the agent to your banking systems and databases\n", + "- Implementing additional banking-specific tools and workflows" + ] + }, + { + "cell_type": "markdown", + "id": "7f9385d3", + "metadata": {}, + "source": [ + "\n", + "\n", + "### Discover more learning resources\n", + "\n", + "Learn more about the ValidMind Library tools we used in this notebook:\n", + "\n", + "- [Custom prompts](https://docs.validmind.ai/notebooks/how_to/customize_test_result_descriptions.html)\n", + "- [Custom tests](https://docs.validmind.ai/notebooks/code_samples/custom_tests/implement_custom_tests.html)\n", + "- [ValidMind scorers](https://docs.validmind.ai/notebooks/how_to/assign_scores_complete_tutorial.html)\n", + "\n", + "We also offer many more interactive notebooks to help you document models:\n", + "\n", + "- [Run tests & test suites](https://docs.validmind.ai/developer/how-to/testing-overview.html)\n", + "- [Use ValidMind Library features](https://docs.validmind.ai/developer/how-to/feature-overview.html)\n", + "- [Code samples by use case](https://docs.validmind.ai/guide/samples-jupyter-notebooks.html)\n", + "\n", + "Or, visit our [documentation](https://docs.validmind.ai/) to learn more about ValidMind." + ] + }, + { + "cell_type": "markdown", + "id": "fdd5c0db", + "metadata": {}, + "source": [ + "\n", + "\n", + "## Upgrade ValidMind\n", + "\n", + "
After installing ValidMind, you’ll want to periodically make sure you are on the latest version to access any new features and other enhancements.
\n", + "\n", + "Retrieve the information for the currently installed version of ValidMind:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9733adff", + "metadata": {}, + "outputs": [], + "source": [ + "%pip show validmind" + ] + }, + { + "cell_type": "markdown", + "id": "829429fd", + "metadata": {}, + "source": [ + "If the version returned is lower than the version indicated in our [production open-source code](https://github.com/validmind/validmind-library/blob/prod/validmind/__version__.py), restart your notebook and run:\n", + "\n", + "```bash\n", + "%pip install --upgrade validmind\n", + "```" + ] + }, + { + "cell_type": "markdown", + "id": "55339760", + "metadata": {}, + "source": [ + "You may need to restart your kernel after running the upgrade package for changes to be applied." + ] + }, + { + "cell_type": "markdown", + "id": "copyright-b9e82bcf4e364c4f8e5ae4bb0e4b2865", + "metadata": {}, + "source": [ + "\n", + "\n", + "\n", + "\n", + "***\n", + "\n", + "Copyright © 2023-2026 ValidMind Inc. All rights reserved.
\n", + "Refer to [LICENSE](https://github.com/validmind/validmind-library/blob/main/LICENSE) for details.
\n", + "SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "validmind-1QuffXMV-py3.11", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/site/notebooks/use_cases/capital_markets/quickstart_option_pricing_models.ipynb b/site/notebooks/use_cases/capital_markets/quickstart_option_pricing_models.ipynb index 175b02a598..f372b04c64 100644 --- a/site/notebooks/use_cases/capital_markets/quickstart_option_pricing_models.ipynb +++ b/site/notebooks/use_cases/capital_markets/quickstart_option_pricing_models.ipynb @@ -203,7 +203,7 @@ "\n", "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", "\n", "2. Under **TEMPLATE**, select `Capital markets`.\n", "\n", @@ -219,10 +219,11 @@ "\n", "#### Get your code snippet\n", "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -246,7 +247,7 @@ " # api_key=\"...\",\n", " # api_secret=\"...\",\n", " # model=\"...\",\n", - " # document=\"documentation\",\n", + " document=\"documentation\",\n", ")" ] }, @@ -2042,11 +2043,11 @@ "\n", "### Work with your model documentation\n", "\n", - "1. From the **Model Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", + "1. From the **Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", "\n", - "2. Click and expand the **Model Development** section.\n", + "2. In the left sidebar that appears for your model, click **Development** under Documents.\n", "\n", - "What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation, view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html)\n", + "What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation, view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/working-with-model-documentation.html)\n", "\n", "\n", "\n", diff --git a/site/notebooks/use_cases/capital_markets/quickstart_option_pricing_models_quantlib.ipynb b/site/notebooks/use_cases/capital_markets/quickstart_option_pricing_models_quantlib.ipynb index 3dadf2ca14..17471d33c4 100644 --- a/site/notebooks/use_cases/capital_markets/quickstart_option_pricing_models_quantlib.ipynb +++ b/site/notebooks/use_cases/capital_markets/quickstart_option_pricing_models_quantlib.ipynb @@ -260,7 +260,7 @@ "\n", "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", "\n", "2. Under **TEMPLATE**, select `Capital Markets`.\n", "\n", @@ -291,10 +291,11 @@ "\n", "#### Get your code snippet\n", "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -318,7 +319,7 @@ " # api_key=\"...\",\n", " # api_secret=\"...\",\n", " # model=\"...\",\n", - " # document=\"documentation\",\n", + " document=\"documentation\",\n", ")\n" ] }, @@ -1287,11 +1288,11 @@ "\n", "### Work with your model documentation\n", "\n", - "1. From the **Model Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", + "1. From the **Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", "\n", - "2. Click and expand the **Model Development** section.\n", + "2. In the left sidebar that appears for your model, click **Development** under Documents.\n", "\n", - "What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation, view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html)\n", + "What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation, view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/working-with-model-documentation.html)\n", "\n", "\n", "\n", diff --git a/site/notebooks/use_cases/code_explainer/quickstart_code_explainer_demo.ipynb b/site/notebooks/use_cases/code_explainer/quickstart_code_explainer_demo.ipynb index afd5e19f65..b1d91a6572 100644 --- a/site/notebooks/use_cases/code_explainer/quickstart_code_explainer_demo.ipynb +++ b/site/notebooks/use_cases/code_explainer/quickstart_code_explainer_demo.ipynb @@ -199,7 +199,7 @@ "\n", "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", "\n", "2. Under **TEMPLATE**, select `Model Source Code Documentation`.\n", "\n", @@ -228,10 +228,11 @@ "\n", "#### Get your code snippet\n", "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -254,7 +255,7 @@ " # api_key=\"...\",\n", " # api_secret=\"...\",\n", " # model=\"...\",\n", - " # document=\"documentation\",\n", + " document=\"documentation\",\n", ")" ] }, diff --git a/site/notebooks/use_cases/credit_risk/application_scorecard_executive.ipynb b/site/notebooks/use_cases/credit_risk/application_scorecard_executive.ipynb index 40388556ca..cd6e4f1a19 100644 --- a/site/notebooks/use_cases/credit_risk/application_scorecard_executive.ipynb +++ b/site/notebooks/use_cases/credit_risk/application_scorecard_executive.ipynb @@ -178,7 +178,7 @@ "\n", "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", "\n", "2. Under **[template]{.smallcaps}**, select `Credit Risk Scorecard`.\n", "\n", @@ -193,10 +193,11 @@ "\n", "#### Get your code snippet\n", "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -208,10 +209,10 @@ "import validmind as vm\n", "\n", "vm.init(\n", - " api_host = \"https://api.prod.validmind.ai/api/v1/tracking\",\n", - " api_key = \"...\",\n", - " api_secret = \"...\",\n", - " model = \"...\",\n", + " # api_host = \"...\",\n", + " # api_key = \"...\",\n", + " # api_secret = \"...\",\n", + " # model = \"...\",\n", " document=\"documentation\",\n", ")" ] @@ -279,14 +280,16 @@ "\n", "### Work with your model documentation\n", "\n", - "1. In the ValidMind Platform, click **Documentation** under Documents for the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html))\n", + "1. From the **Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", "\n", - "2. Expand the following sections and take a look around:\n", + "2. In the left sidebar that appears for your model, click **Development** under Documents.\n", + "\n", + "3. Expand the following sections and take a look around:\n", "\n", " - **2. Data Preparation**\n", " - **3. Model Development**\n", "\n", - "What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation (hint: some of the tests in **2.3. Feature Selection and Engineering** look like they need some attention), view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready.\n", + "What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation (hint: some of the tests in **2.3. Feature Selection and Engineering** look like they need some attention), view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/working-with-model-documentation.html)\n", "\n", "\n", "\n", diff --git a/site/notebooks/use_cases/credit_risk/application_scorecard_full_suite.ipynb b/site/notebooks/use_cases/credit_risk/application_scorecard_full_suite.ipynb index e64a206e41..e8e567a5b4 100644 --- a/site/notebooks/use_cases/credit_risk/application_scorecard_full_suite.ipynb +++ b/site/notebooks/use_cases/credit_risk/application_scorecard_full_suite.ipynb @@ -192,7 +192,7 @@ "\n", "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", "\n", "2. Under **TEMPLATE**, select `Credit Risk Scorecard`.\n", "\n", @@ -207,10 +207,11 @@ "\n", "#### Get your code snippet\n", "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -229,10 +230,10 @@ "import validmind as vm\n", "\n", "vm.init(\n", - " api_host = \"https://api.prod.validmind.ai/api/v1/tracking\",\n", - " api_key = \"...\",\n", - " api_secret = \"...\",\n", - " model = \"...\",\n", + " # api_host = \"...\",\n", + " # api_key = \"...\",\n", + " # api_secret = \"...\",\n", + " # model = \"...\",\n", " document=\"documentation\",\n", ")" ] @@ -799,14 +800,16 @@ "\n", "### Work with your model documentation\n", "\n", - "1. In the ValidMind Platform, click **Documentation** under Documents for the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html))\n", + "1. From the **Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", "\n", - "2. Expand the following sections and take a look around:\n", + "2. In the left sidebar that appears for your model, click **Development** under Documents.\n", + "\n", + "3. Expand the following sections and take a look around:\n", "\n", " - **2. Data Preparation**\n", " - **3. Model Development**\n", "\n", - "What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation (hint: some of the tests in **2.3. Feature Selection and Engineering** look like they need some attention), view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready.\n", + "What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation (hint: some of the tests in **2.3. Feature Selection and Engineering** look like they need some attention), view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/working-with-model-documentation.html)\n", "\n", "\n", "\n", diff --git a/site/notebooks/use_cases/credit_risk/application_scorecard_with_bias.ipynb b/site/notebooks/use_cases/credit_risk/application_scorecard_with_bias.ipynb index 41769972a6..e58e831ffc 100644 --- a/site/notebooks/use_cases/credit_risk/application_scorecard_with_bias.ipynb +++ b/site/notebooks/use_cases/credit_risk/application_scorecard_with_bias.ipynb @@ -193,7 +193,7 @@ "\n", "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", "\n", "2. Under **TEMPLATE**, select `Credit Risk Scorecard`.\n", "\n", @@ -208,10 +208,11 @@ "\n", "#### Get your code snippet\n", "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -1444,14 +1445,16 @@ "\n", "### Work with your model documentation\n", "\n", - "1. In the ValidMind Platform, click **Documentation** under Documents for the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html))\n", + "1. From the **Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", "\n", - "2. Expand the following sections and take a look around:\n", + "2. In the left sidebar that appears for your model, click **Development** under Documents.\n", + "\n", + "3. Expand the following sections and take a look around:\n", "\n", " - **2. Data Preparation**\n", " - **3. Model Development**\n", "\n", - "What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation (hint: some of the tests in **2.3. Feature Selection and Engineering** look like they need some attention), view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready.\n", + "What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation (hint: some of the tests in **2.3. Feature Selection and Engineering** look like they need some attention), view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/working-with-model-documentation.html)\n", "\n", "\n", "\n", diff --git a/site/notebooks/use_cases/credit_risk/application_scorecard_with_ml.ipynb b/site/notebooks/use_cases/credit_risk/application_scorecard_with_ml.ipynb index cdbb02b0eb..dd04e2b6a2 100644 --- a/site/notebooks/use_cases/credit_risk/application_scorecard_with_ml.ipynb +++ b/site/notebooks/use_cases/credit_risk/application_scorecard_with_ml.ipynb @@ -205,7 +205,7 @@ "\n", "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", "\n", "2. Under **TEMPLATE**, select `Credit Risk Scorecard`.\n", "\n", @@ -220,10 +220,11 @@ "\n", "#### Get your code snippet\n", "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -242,10 +243,10 @@ "import validmind as vm\n", "\n", "vm.init(\n", - " api_host = \"https://api.prod.validmind.ai/api/v1/tracking\",\n", - " api_key = \"...\",\n", - " api_secret = \"...\",\n", - " model = \"...\",\n", + " # api_host = \"...\",\n", + " # api_key = \"...\",\n", + " # api_secret = \"...\",\n", + " # model = \"...\",\n", " document=\"documentation\",\n", ")" ] @@ -1894,14 +1895,16 @@ "\n", "### Work with your model documentation\n", "\n", - "1. In the ValidMind Platform, click **Documentation** under Documents for the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html))\n", + "1. From the **Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", "\n", - "2. Expand the following sections and take a look around:\n", + "2. In the left sidebar that appears for your model, click **Development** under Documents.\n", + "\n", + "3. Expand the following sections and take a look around:\n", "\n", " - **2. Data Preparation**\n", " - **3. Model Development**\n", "\n", - "What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation (hint: some of the tests in **2.3. Feature Selection and Engineering** look like they need some attention), view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready.\n", + "What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation (hint: some of the tests in **2.3. Feature Selection and Engineering** look like they need some attention), view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/working-with-model-documentation.html)\n", "\n", "\n", "\n", diff --git a/site/notebooks/use_cases/credit_risk/document_excel_application_scorecard.ipynb b/site/notebooks/use_cases/credit_risk/document_excel_application_scorecard.ipynb index 252fb1a60e..be351497c2 100644 --- a/site/notebooks/use_cases/credit_risk/document_excel_application_scorecard.ipynb +++ b/site/notebooks/use_cases/credit_risk/document_excel_application_scorecard.ipynb @@ -194,7 +194,7 @@ "\n", "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", "\n", "2. Under **TEMPLATE**, select `Credit Risk Scorecard`.\n", "\n", @@ -209,10 +209,11 @@ "\n", "#### Get your code snippet\n", "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -235,7 +236,7 @@ " # api_key=\"...\",\n", " # api_secret=\"...\",\n", " # model=\"...\",\n", - " # document=\"documentation\",\n", + " document=\"documentation\",\n", ")" ] }, @@ -883,7 +884,7 @@ "\n", "1. From the **Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", "\n", - "2. In the left sidebar that appears for your model, click **Documentation** under Documents.\n", + "2. In the left sidebar that appears for your model, click **Development** under Documents.\n", "\n", " What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation, view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready.\n", "\n", diff --git a/site/notebooks/use_cases/model_validation/validate_application_scorecard.ipynb b/site/notebooks/use_cases/model_validation/validate_application_scorecard.ipynb index 3e20fa1eb0..f3df8a617e 100644 --- a/site/notebooks/use_cases/model_validation/validate_application_scorecard.ipynb +++ b/site/notebooks/use_cases/model_validation/validate_application_scorecard.ipynb @@ -228,9 +228,9 @@ "\n", "#### Apply validation report template\n", "\n", - "Once you've registered your model, let's select a validation report template. A template predefines sections for your report and provides a general outline to follow, making the validation process much easier.\n", + "Next, let's select a validation report template. A template predefines sections for your report and provides a general outline to follow, making the validation process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Validation**.\n", "\n", "2. Under **TEMPLATE**, select `Generic Validation Report`.\n", "\n", @@ -267,9 +267,7 @@ "source": [ "\n", "\n", - "### Initialize the ValidMind Library\n", - "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your validation environment. You initialize the ValidMind Library with this code snippet, which ensures that your test results are uploaded to the correct model when you run the notebook." + "### Initialize the ValidMind Library" ] }, { @@ -280,13 +278,11 @@ "\n", "#### Get your code snippet\n", "\n", - "1. In a browser, [log in to ValidMind](https://docs.validmind.ai/guide/configuration/log-in-to-validmind.html).\n", - "\n", - "2. In the left sidebar, navigate to **Inventory** and select the model you registered for this notebook.\n", - "\n", - "3. Go to **Getting Started** and click **Copy snippet to clipboard**.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Validation` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -309,7 +305,7 @@ " # api_key=\"...\",\n", " # api_secret=\"...\",\n", " # model=\"...\",\n", - " # document=\"validation-report\",\n", + " document=\"validation-report\",\n", ")" ] }, @@ -1783,7 +1779,7 @@ "\n", "1. From the **Inventory** in the ValidMind Platform, go to the model you connected to earlier.\n", "\n", - "2. In the left sidebar that appears for your model, click **Validation Report** under Documents.\n", + "2. In the left sidebar that appears for your model, click **Validation** under Documents.\n", "\n", "Include your logged test results as evidence, create risk assessment notes, add artifacts, and assess compliance, then submit your report for review when it's ready. **Learn more:** [Preparing validation reports](https://docs.validmind.ai/guide/model-validation/preparing-validation-reports.html)" ] diff --git a/site/notebooks/use_cases/nlp_and_llm/foundation_models_integration_demo.ipynb b/site/notebooks/use_cases/nlp_and_llm/foundation_models_integration_demo.ipynb index 813acd5d9b..a9a82857df 100644 --- a/site/notebooks/use_cases/nlp_and_llm/foundation_models_integration_demo.ipynb +++ b/site/notebooks/use_cases/nlp_and_llm/foundation_models_integration_demo.ipynb @@ -11,6 +11,11 @@ "This interactive notebook shows you how to set up the ValidMind Library, initializes the library, and uses a specific prompt template for analyzing the sentiment of sentences in a dataset. The notebook also includes example data to test the model's ability to correctly identify sentiment as positive, negative, or neutral." ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, { "cell_type": "markdown", "metadata": {}, @@ -145,7 +150,7 @@ "\n", "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", "\n", "2. Under **TEMPLATE**, select `LLM-based Text Classification`.\n", "\n", @@ -160,10 +165,11 @@ "\n", "#### Get your code snippet\n", "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -186,7 +192,7 @@ " # api_key=\"...\",\n", " # api_secret=\"...\",\n", " # model=\"...\",\n", - " # document=\"documentation\",\n", + " document=\"documentation\",\n", ")" ] }, @@ -407,7 +413,7 @@ "\n", "You can look at the results of this test suite right in the notebook where you ran the code, as you would expect. But there is a better way: view the prompt validation test results as part of your model documentation in the ValidMind Platform:\n", "\n", - "1. In the ValidMind Platform, click **Documentation** under Documents for the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html))\n", + "1. In the ValidMind Platform, click **Development** under Documents for the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html))\n", "\n", "2. Expand **2. Data Preparation** or **3. Model Development** to review all test results.\n", "\n", diff --git a/site/notebooks/use_cases/nlp_and_llm/foundation_models_summarization_demo.ipynb b/site/notebooks/use_cases/nlp_and_llm/foundation_models_summarization_demo.ipynb index cfb50ce82c..e4c7e067b7 100644 --- a/site/notebooks/use_cases/nlp_and_llm/foundation_models_summarization_demo.ipynb +++ b/site/notebooks/use_cases/nlp_and_llm/foundation_models_summarization_demo.ipynb @@ -146,7 +146,7 @@ "\n", "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", "\n", "2. Under **TEMPLATE**, select `LLM-based Text Summarization`.\n", "\n", @@ -161,10 +161,11 @@ "\n", "#### Get your code snippet\n", "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -187,7 +188,7 @@ " # api_key=\"...\",\n", " # api_secret=\"...\",\n", " # model=\"...\",\n", - " # document=\"documentation\",\n", + " document=\"documentation\",\n", ")" ] }, @@ -456,7 +457,7 @@ "\n", "You can look at the results of this test suite right in the notebook where you ran the code, as you would expect. But there is a better way: view the prompt validation test results as part of your model documentation in the ValidMind Platform:\n", "\n", - "1. In the ValidMind Platform, click **Documentation** under Documents for the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html)\n", + "1. In the ValidMind Platform, click **Development** under Documents for the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html)\n", "\n", "2. Expand **2. Data Preparation** or **3. Model Development** to review all test results.\n", "\n", diff --git a/site/notebooks/use_cases/nlp_and_llm/hugging_face_integration_demo.ipynb b/site/notebooks/use_cases/nlp_and_llm/hugging_face_integration_demo.ipynb index 082b4fb6eb..a83eeed947 100644 --- a/site/notebooks/use_cases/nlp_and_llm/hugging_face_integration_demo.ipynb +++ b/site/notebooks/use_cases/nlp_and_llm/hugging_face_integration_demo.ipynb @@ -147,7 +147,7 @@ "\n", "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", "\n", "2. Under **TEMPLATE**, select `NLP-based Text Classification`.\n", "\n", @@ -162,10 +162,11 @@ "\n", "#### Get your code snippet\n", "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -188,7 +189,7 @@ " # api_key=\"...\",\n", " # api_secret=\"...\",\n", " # model=\"...\",\n", - " # document=\"documentation\",\n", + " document=\"documentation\",\n", ")" ] }, @@ -391,7 +392,7 @@ "\n", "You can look at the results of this test suite right in the notebook where you ran the code, as you would expect. But there is a better way: view the prompt validation test results as part of your model documentation in the ValidMind Platform:\n", "\n", - "1. In the ValidMind Platform, click **Documentation** under Documents for the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html)\n", + "1. In the ValidMind Platform, click **Development** under Documents for the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html)\n", "\n", "2. Expand **2. Data Preparation** or **3. Model Development** to review all test results.\n", "\n", diff --git a/site/notebooks/use_cases/nlp_and_llm/hugging_face_summarization_demo.ipynb b/site/notebooks/use_cases/nlp_and_llm/hugging_face_summarization_demo.ipynb index fc4eefd751..290bf7c545 100644 --- a/site/notebooks/use_cases/nlp_and_llm/hugging_face_summarization_demo.ipynb +++ b/site/notebooks/use_cases/nlp_and_llm/hugging_face_summarization_demo.ipynb @@ -144,7 +144,7 @@ "\n", "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", "\n", "2. Under **TEMPLATE**, select `NLP-based Text Classification`.\n", "\n", @@ -159,10 +159,11 @@ "\n", "#### Get your code snippet\n", "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -185,7 +186,7 @@ " # api_key=\"...\",\n", " # api_secret=\"...\",\n", " # model=\"...\",\n", - " # document=\"documentation\",\n", + " document=\"documentation\",\n", ")" ] }, @@ -412,7 +413,7 @@ "\n", "You can look at the results of this test suite right in the notebook where you ran the code, as you would expect. But there is a better way: view the prompt validation test results as part of your model documentation in the ValidMind Platform:\n", "\n", - "1. In the ValidMind Platform, click **Documentation** under Documents for the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html)\n", + "1. In the ValidMind Platform, click **Development** under Documents for the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html)\n", "\n", "2. Expand **2. Data Preparation** or **3. Model Development** to review all test results.\n", "\n", diff --git a/site/notebooks/use_cases/nlp_and_llm/llm_summarization_demo.ipynb b/site/notebooks/use_cases/nlp_and_llm/llm_summarization_demo.ipynb index 69f5dcd4cb..a79afa7792 100644 --- a/site/notebooks/use_cases/nlp_and_llm/llm_summarization_demo.ipynb +++ b/site/notebooks/use_cases/nlp_and_llm/llm_summarization_demo.ipynb @@ -178,7 +178,7 @@ "\n", "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", "\n", "2. Under **TEMPLATE**, select `LLM-based Text Summarization`.\n", "\n", @@ -193,10 +193,11 @@ "\n", "#### Get your code snippet\n", "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -219,7 +220,7 @@ " # api_key=\"...\",\n", " # api_secret=\"...\",\n", " # model=\"...\",\n", - " # document=\"documentation\",\n", + " document=\"documentation\",\n", ")" ] }, @@ -929,7 +930,7 @@ "\n", "You can look at the results of this test suite right in the notebook where you ran the code, as you would expect. But there is a better way: view the prompt validation test results as part of your model documentation right in the ValidMind Platform:\n", "\n", - "1. In the ValidMind Platform, click **Documentation** under Documents for the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html)\n", + "1. In the ValidMind Platform, click **Development** under Documents for the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html)\n", "\n", "2. Expand **2. Data Preparation** or **3. Model Development** to review all test results.\n", "\n", diff --git a/site/notebooks/use_cases/nlp_and_llm/prompt_validation_demo.ipynb b/site/notebooks/use_cases/nlp_and_llm/prompt_validation_demo.ipynb index 1f0114a3c1..16b5f7c42c 100644 --- a/site/notebooks/use_cases/nlp_and_llm/prompt_validation_demo.ipynb +++ b/site/notebooks/use_cases/nlp_and_llm/prompt_validation_demo.ipynb @@ -174,7 +174,7 @@ "\n", "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", "\n", "2. Under **TEMPLATE**, select `LLM-based Text Classification`.\n", "\n", @@ -189,10 +189,11 @@ "\n", "#### Get your code snippet\n", "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -215,7 +216,7 @@ " # api_key=\"...\",\n", " # api_secret=\"...\",\n", " # model=\"...\",\n", - " # document=\"documentation\",\n", + " document=\"documentation\",\n", ")" ] }, diff --git a/site/notebooks/use_cases/nlp_and_llm/rag_benchmark_demo.ipynb b/site/notebooks/use_cases/nlp_and_llm/rag_benchmark_demo.ipynb index bb0a89b5b1..1e3eb07b67 100644 --- a/site/notebooks/use_cases/nlp_and_llm/rag_benchmark_demo.ipynb +++ b/site/notebooks/use_cases/nlp_and_llm/rag_benchmark_demo.ipynb @@ -213,7 +213,7 @@ "\n", "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", "\n", "2. Under **TEMPLATE**, select `Gen AI RAG`.\n", "\n", @@ -242,10 +242,11 @@ "\n", "#### Get your code snippet\n", "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -264,10 +265,10 @@ "import validmind as vm\n", "\n", "vm.init(\n", - " api_host = \"https://api.prod.validmind.ai/api/v1/tracking\",\n", - " api_key = \"...\",\n", - " api_secret = \"...\",\n", - " model = \"...\",\n", + " # api_host = \"...\",\n", + " # api_key = \"...\",\n", + " # api_secret = \"...\",\n", + " # model = \"...\",\n", " document=\"documentation\",\n", ")" ] diff --git a/site/notebooks/use_cases/nlp_and_llm/rag_documentation_demo.ipynb b/site/notebooks/use_cases/nlp_and_llm/rag_documentation_demo.ipynb index 7e606f0316..a57ebed3be 100644 --- a/site/notebooks/use_cases/nlp_and_llm/rag_documentation_demo.ipynb +++ b/site/notebooks/use_cases/nlp_and_llm/rag_documentation_demo.ipynb @@ -187,7 +187,7 @@ "\n", "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", "\n", "2. Under **TEMPLATE**, select `Gen AI RAG`.\n", "\n", @@ -216,10 +216,11 @@ "\n", "#### Get your code snippet\n", "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -242,7 +243,7 @@ " # api_key=\"...\",\n", " # api_secret=\"...\",\n", " # model=\"...\",\n", - " # document=\"documentation\",\n", + " document=\"documentation\",\n", ")" ] }, diff --git a/site/notebooks/use_cases/ongoing_monitoring/application_scorecard_ongoing_monitoring.ipynb b/site/notebooks/use_cases/ongoing_monitoring/application_scorecard_ongoing_monitoring.ipynb index b822bbcdeb..df159619f9 100644 --- a/site/notebooks/use_cases/ongoing_monitoring/application_scorecard_ongoing_monitoring.ipynb +++ b/site/notebooks/use_cases/ongoing_monitoring/application_scorecard_ongoing_monitoring.ipynb @@ -25,10 +25,11 @@ " - [Install the ValidMind Library](#toc2_1__) \n", " - [Initialize the ValidMind Library](#toc2_2__) \n", " - [Register sample model](#toc2_2_1__) \n", - " - [Apply documentation template](#toc2_2_2__) \n", + " - [Apply monitoring report template](#toc2_2_2__) \n", " - [Get your code snippet](#toc2_2_3__) \n", - " - [Initialize the Python environment](#toc2_3__) \n", - " - [Preview the monitoring template](#toc2_4__) \n", + " - [Preview the monitoring report template](#toc2_3__) \n", + " - [Initialize the Python environment](#toc2_4__) \n", + " - [Preview the monitoring template](#toc2_5__) \n", "- [Load the reference and monitoring datasets](#toc3__) \n", "- [Train the model](#toc4__) \n", " - [Initialize the ValidMind datasets](#toc4_1__) \n", @@ -95,9 +96,9 @@ "\n", "**Documentation template**: Functions as a test suite and lays out the structure of model documentation, segmented into various sections and sub-sections. Documentation templates define the structure of your model documentation, specifying the tests that should be run, and how the results should be displayed.\n", "\n", - "**Model monitoring documentation**: A comprehensive and structured record of a production model, including key elements such as data sources, inputs, performance metrics, and periodic evaluations. This documentation ensures transparency and visibility of the model's performance in the production environment.\n", + "**Model monitoring report**: A comprehensive and structured record of a production model, including key elements such as data sources, inputs, performance metrics, and periodic evaluations. This documentation ensures transparency and visibility of the model's performance in the production environment.\n", "\n", - "**Monitoring documentation template**: Similar to documentation template, The monitoring documentation template functions as a test suite and lays out the structure of model monitoring documentation, segmented into various sections and sub-sections. Monitoring documentation templates define the structure of your model monitoring documentation, specifying the tests that should be run, and how the results should be displayed.\n", + "**Monitoring report template**: Similar to documentation template, The monitoring report template functions as a test suite and lays out the structure of model monitoring, segmented into various sections and sub-sections. Monitoring report templates define the structure of your model monitoring report, specifying the tests that should be run, and how the results should be displayed.\n", "\n", "**Tests**: A function contained in the ValidMind Library, designed to run a specific quantitative test on the dataset or model. Tests are the building blocks of ValidMind, used to evaluate and document models and datasets, and can be run individually or as part of a suite defined by your model documentation template.\n", "\n", @@ -180,13 +181,13 @@ "source": [ "\n", "\n", - "#### Apply documentation template\n", + "#### Apply monitoring report template\n", "\n", - "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", + "Once you've registered your model, let's select a monitoring report template. A template predefines sections for your monitoring report and provides a general outline to follow, making the monitoring process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Monitoring**.\n", "\n", - "2. Under **TEMPLATE**, select `Binary classification`.\n", + "2. Under **TEMPLATE**, select `Ongoing Monitoring for Classification Models`.\n", "\n", "3. Click **Use Template** to apply the template." ] @@ -199,10 +200,11 @@ "\n", "#### Get your code snippet\n", "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Monitoring` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -221,12 +223,12 @@ "import validmind as vm\n", "\n", "vm.init(\n", - " api_host = \"https://api.prod.validmind.ai/api/v1/tracking\",\n", - " api_key = \"...\",\n", - " api_secret = \"...\",\n", - " model = \"...\",\n", - " monitoring = True,\n", - " document=\"monitoring\",\n", + " # api_host=\"...\",\n", + " # api_key=\"...\",\n", + " # api_secret=\"...\",\n", + " # model=\"...\",\n", + " document=\"monitoring\",\n", + " monitoring = True,\n", ")" ] }, @@ -236,6 +238,28 @@ "source": [ "\n", "\n", + "### Preview the monitoring report template\n", + "\n", + "Let's verify that you have connected the ValidMind Library to the ValidMind Platform and that the appropriate *template* is selected for your model.\n", + "\n", + "You will upload documentation and test results unique to your model based on this template later on. For now, **take a look at the default structure that the template provides with [the `vm.preview_template()` function](https://docs.validmind.ai/validmind/validmind.html#preview_template)** from the ValidMind library and note the empty sections:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "vm.preview_template()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", "### Initialize the Python environment\n", "\n", "Next, let's import the necessary libraries and set up your Python environment for data analysis:" @@ -266,7 +290,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "\n", + "\n", "\n", "### Preview the monitoring template\n", "\n", @@ -1319,7 +1343,7 @@ }, { "cell_type": "markdown", - "id": "copyright-9b204f07d6ef4508a249d0c7d5e7d44b", + "id": "copyright-7d1a3b9bebaf43cdb331352abb7456af", "metadata": {}, "source": [ "\n", diff --git a/site/notebooks/use_cases/ongoing_monitoring/quickstart_customer_churn_ongoing_monitoring.ipynb b/site/notebooks/use_cases/ongoing_monitoring/quickstart_customer_churn_ongoing_monitoring.ipynb index 8ca30216e9..765be36075 100644 --- a/site/notebooks/use_cases/ongoing_monitoring/quickstart_customer_churn_ongoing_monitoring.ipynb +++ b/site/notebooks/use_cases/ongoing_monitoring/quickstart_customer_churn_ongoing_monitoring.ipynb @@ -27,10 +27,10 @@ " - [Install the ValidMind Library](#toc2_1__) \n", " - [Initialize the ValidMind Library](#toc2_2__) \n", " - [Register sample model](#toc2_2_1__) \n", - " - [Apply documentation template](#toc2_2_2__) \n", + " - [Apply monitoring report template](#toc2_2_2__) \n", " - [Get your code snippet](#toc2_2_3__) \n", " - [Initialize the Python environment](#toc2_3__) \n", - " - [Preview the monitoring template](#toc2_4__) \n", + " - [Preview the monitoring report template](#toc2_4__) \n", "- [Load the reference and monitoring datasets](#toc3__) \n", " - [Load the production model](#toc3_1__) \n", " - [Initialize the ValidMind datasets](#toc3_2__) \n", @@ -41,7 +41,8 @@ " - [Feature drift tests](#toc3_6_1__) \n", " - [Model performance monitoring tests](#toc3_7__) \n", "- [Next steps](#toc4__) \n", - " - [Discover more learning resources](#toc4_1__) \n", + " - [Work with your monitoring report](#toc4_1__) \n", + " - [Discover more learning resources](#toc4_2__) \n", "- [Upgrade ValidMind](#toc5__) \n", "\n", ":::\n", @@ -93,9 +94,9 @@ "\n", "**Documentation template**: Functions as a test suite and lays out the structure of model documentation, segmented into various sections and sub-sections. Documentation templates define the structure of your model documentation, specifying the tests that should be run, and how the results should be displayed.\n", "\n", - "**Model monitoring documentation**: A comprehensive and structured record of a production model, including key elements such as data sources, inputs, performance metrics, and periodic evaluations. This documentation ensures transparency and visibility of the model's performance in the production environment.\n", + "**Model monitoring report**: A comprehensive and structured record of a production model, including key elements such as data sources, inputs, performance metrics, and periodic evaluations. This documentation ensures transparency and visibility of the model's performance in the production environment.\n", "\n", - "**Monitoring documentation template**: Similar to documentation template, The monitoring documentation template functions as a test suite and lays out the structure of model monitoring documentation, segmented into various sections and sub-sections. Monitoring documentation templates define the structure of your model monitoring documentation, specifying the tests that should be run, and how the results should be displayed.\n", + "**Monitoring report template**: Similar to documentation template, The monitoring report template functions as a test suite and lays out the structure of model monitoring, segmented into various sections and sub-sections. Monitoring report templates define the structure of your model monitoring report, specifying the tests that should be run, and how the results should be displayed.\n", "\n", "**Tests**: A function contained in the ValidMind Library, designed to run a specific quantitative test on the dataset or model. Tests are the building blocks of ValidMind, used to evaluate and document models and datasets, and can be run individually or as part of a suite defined by your model documentation template.\n", "\n", @@ -178,13 +179,13 @@ "source": [ "\n", "\n", - "#### Apply documentation template\n", + "#### Apply monitoring report template\n", "\n", - "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", + "Once you've registered your model, let's select a monitoring report template. A template predefines sections for your monitoring report and provides a general outline to follow, making the monitoring process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Monitoring**.\n", "\n", - "2. Under **TEMPLATE**, select `Binary classification`.\n", + "2. Under **TEMPLATE**, select `Ongoing Monitoring for Classification Models`.\n", "\n", "3. Click **Use Template** to apply the template." ] @@ -197,10 +198,11 @@ "\n", "#### Get your code snippet\n", "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Monitoring` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -223,8 +225,8 @@ " # api_key=\"...\",\n", " # api_secret=\"...\",\n", " # model=\"...\",\n", - " monitoring = True,\n", " document=\"monitoring\",\n", + " monitoring = True,\n", ")" ] }, @@ -262,11 +264,11 @@ "source": [ "\n", "\n", - "### Preview the monitoring template\n", + "### Preview the monitoring report template\n", "\n", - "A template predefines sections for your model monitoring documentation and provides a general outline to follow, making the documentation process much easier.\n", + "Let's verify that you have connected the ValidMind Library to the ValidMind Platform and that the appropriate *template* is selected for your model.\n", "\n", - "You will upload documentation and test results into this template later on. For now, take a look at the structure that the template provides with the `vm.preview_template()` function from the ValidMind library and note the empty sections:" + "You will upload documentation and test results unique to your model based on this template later on. For now, **take a look at the default structure that the template provides with [the `vm.preview_template()` function](https://docs.validmind.ai/validmind/validmind.html#preview_template)** from the ValidMind library and note the empty sections:" ] }, { @@ -512,7 +514,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "To view the results of the model metadata and data quality tests, select **Ongoing Monitoring** under Documents in the left sidebar of the model in the ValidMind Platform and click on the following sections:\n", + "To view the results of the model metadata and data quality tests, select **Monitoring** under Documents in the left sidebar of the model in the ValidMind Platform and click on the following sections:\n", "\n", "- 1. Model Monitoring Overview > **1.2. Model Details**\n", "- 2. Data Quality & Drift Assessment > **2.1. Prediction Data Description**" @@ -785,14 +787,19 @@ "\n", "## Next steps\n", "\n", - "You can now review all the ongoing monitoring results in the ValidMind Platform.\n", + "You can look at the output produced by the ValidMind Library right in the notebook where you ran the code, as you would expect. But there is a better way — use the ValidMind Platform to work with your monitoring report.\n", "\n", - "1. From the **Model Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", - "2. In the left sidebar that appears for your model, click **Ongoing Monitoring** under Documents.\n", + "\n", "\n", - "What you see is the full draft of your model monitoring documentation in a more easily consumable version. From here, you can make qualitative edits to model monitoring documentation, view guidelines, collaborate with validators, and submit your model monitoring documentation for approval when it's ready.\n", + "### Work with your monitoring report\n", "\n", - "\n", + "1. From the **Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", + "\n", + "2. In the left sidebar that appears for your model, click **Monitoring** under Documents.\n", + "\n", + "What you see is the full draft of your monitoring report in a more easily consumable version. From here, you can make qualitative edits to monitoring reports, view guidelines, review monitoring results, and submit your monitoring report for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/monitoring/ongoing-monitoring.html)\n", + "\n", + "\n", "\n", "### Discover more learning resources\n", "\n", @@ -847,7 +854,7 @@ }, { "cell_type": "markdown", - "id": "copyright-a5f8f86b12d74c72a867a01b3aeb6da2", + "id": "copyright-336984df36024986aa3efd6a4079956f", "metadata": {}, "source": [ "\n", diff --git a/site/notebooks/use_cases/regression/quickstart_regression_full_suite.ipynb b/site/notebooks/use_cases/regression/quickstart_regression_full_suite.ipynb index 078fc73f01..55a3854a1f 100644 --- a/site/notebooks/use_cases/regression/quickstart_regression_full_suite.ipynb +++ b/site/notebooks/use_cases/regression/quickstart_regression_full_suite.ipynb @@ -39,6 +39,8 @@ " - [Assign predictions to the datasets](#toc5_4__) \n", " - [Run the full suite of tests](#toc5_5__) \n", "- [Next steps](#toc6__) \n", + " - [Work with your model documentation](#toc6_1__) \n", + " - [Discover more learning resources](#toc6_2__) \n", "- [Upgrade ValidMind](#toc7__) \n", "\n", ":::\n", @@ -152,7 +154,7 @@ "\n", "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", "\n", "2. Under **TEMPLATE**, select `Binary classification`.\n", "\n", @@ -167,10 +169,11 @@ "\n", "#### Get your code snippet\n", "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -193,7 +196,7 @@ " # api_key=\"...\",\n", " # api_secret=\"...\",\n", " # model=\"...\",\n", - " # document=\"documentation\",\n", + " document=\"documentation\",\n", ")" ] }, @@ -491,18 +494,34 @@ "\n", "## Next steps\n", "\n", - "You can look at the results of this test suite right in the notebook where you ran the code, as you would expect. But there is a better way: view the test results as part of your model documentation right in the ValidMind Platform:\n", + "You can look at the output produced by the ValidMind Library right in the notebook where you ran the code, as you would expect. But there is a better way — use the ValidMind Platform to work with your model documentation.\n", "\n", - "1. In the ValidMind Platform, click **Documentation** under Documents for the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html)\n", + "\n", "\n", - "2. Expand the following sections and take a look around:\n", + "### Work with your model documentation\n", + "\n", + "1. From the **Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", + "\n", + "2. In the left sidebar that appears for your model, click **Development** under Documents.\n", + "\n", + "3. Expand the following sections and take a look around:\n", "\n", " - **2. Data Preparation**\n", " - **3. Model Development**\n", "\n", - "What you can see now is a much more easily consumable version of the documentation, including the results of the tests you just performed, along with other parts of your model documentation that still need to be completed. There is a wealth of information that gets uploaded when you run the full test suite, so take a closer look around, especially at test results that might need attention (hint: some of the tests in **2.1 Data description** look like they need some attention).\n", + "What you can see now is a much more easily consumable version of the documentation, including the results of the tests you just performed, along with other parts of your model documentation that still need to be completed. There is a wealth of information that gets uploaded when you run the full test suite, so take a closer look around, especially at test results that might need attention (hint: some of the tests in **2.1 Data description** look like they need some attention). From here, you can make qualitative edits to model documentation, view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/working-with-model-documentation.html)\n", + "\n", + "\n", + "\n", + "### Discover more learning resources\n", + "\n", + "We offer many interactive notebooks to help you automate testing, documenting, validating, and more:\n", + "\n", + "- [Run tests & test suites](https://docs.validmind.ai/developer/how-to/testing-overview.html)\n", + "- [Use ValidMind Library features](https://docs.validmind.ai/developer/how-to/feature-overview.html)\n", + "- [Code samples by use case](https://docs.validmind.ai/guide/samples-jupyter-notebooks.html)\n", "\n", - "If you want to learn more about where you are in the model documentation process, take a look our documentation on the [ValidMind Library](https://docs.validmind.ai/developer/validmind-library.html)." + "Or, visit our [documentation](https://docs.validmind.ai/) to learn more about ValidMind." ] }, { @@ -547,7 +566,7 @@ }, { "cell_type": "markdown", - "id": "copyright-e2b543078a614a038f27fa63a6c74297", + "id": "copyright-6eb89a7d0bbf4f2a8dd973330f359ed6", "metadata": {}, "source": [ "\n", diff --git a/site/notebooks/use_cases/time_series/quickstart_time_series_full_suite.ipynb b/site/notebooks/use_cases/time_series/quickstart_time_series_full_suite.ipynb index d130403e38..6d712adde6 100644 --- a/site/notebooks/use_cases/time_series/quickstart_time_series_full_suite.ipynb +++ b/site/notebooks/use_cases/time_series/quickstart_time_series_full_suite.ipynb @@ -186,7 +186,7 @@ "\n", "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", "\n", "2. Under **TEMPLATE**, select `Binary classification`.\n", "\n", @@ -201,10 +201,11 @@ "\n", "#### Get your code snippet\n", "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -227,7 +228,7 @@ " # api_key=\"...\",\n", " # api_secret=\"...\",\n", " # model=\"...\",\n", - " # document=\"documentation\",\n", + " document=\"documentation\",\n", ")" ] }, diff --git a/site/notebooks/use_cases/time_series/quickstart_time_series_high_code.ipynb b/site/notebooks/use_cases/time_series/quickstart_time_series_high_code.ipynb index a09cf81d7f..87610c5f9e 100644 --- a/site/notebooks/use_cases/time_series/quickstart_time_series_high_code.ipynb +++ b/site/notebooks/use_cases/time_series/quickstart_time_series_high_code.ipynb @@ -187,7 +187,7 @@ "\n", "Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier.\n", "\n", - "1. In the left sidebar that appears for your model, click **Documents** and select **Documentation**.\n", + "1. In the left sidebar that appears for your model, click **Documents** and select **Development**.\n", "\n", "2. Under **TEMPLATE**, select `Time Series Forecasting with ML`.\n", "\n", @@ -202,10 +202,11 @@ "\n", "#### Get your code snippet\n", "\n", - "ValidMind generates a unique _code snippet_ for each registered model to connect with your developer environment. You initialize the ValidMind Library with this code snippet, which ensures that your documentation and tests are uploaded to the correct model when you run the notebook.\n", + "Initialize the ValidMind Library with the *code snippet* unique to each model per document, ensuring your test results are uploaded to the correct model and automatically populated in the right document in the ValidMind Platform when you run this notebook.\n", "\n", - "1. On the left sidebar that appears for your model, select **Getting Started** and click **Copy snippet to clipboard**.\n", - "2. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" + "1. On the left sidebar that appears for your model, select **Getting Started** and select `Development` from the **DOCUMENT** drop-down menu.\n", + "2. Click **Copy snippet to clipboard**.\n", + "3. Next, [load your model identifier credentials from an `.env` file](https://docs.validmind.ai/developer/model-documentation/store-credentials-in-env-file.html) or replace the placeholder with your own code snippet:" ] }, { @@ -228,7 +229,7 @@ " # api_key=\"...\",\n", " # api_secret=\"...\",\n", " # model=\"...\",\n", - " # document=\"documentation\",\n", + " document=\"documentation\",\n", ")" ] }, @@ -907,11 +908,11 @@ "\n", "### Work with your model documentation\n", "\n", - "1. From the **Model Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", + "1. From the **Inventory** in the ValidMind Platform, go to the model you registered earlier. ([Need more help?](https://docs.validmind.ai/guide/model-inventory/working-with-model-inventory.html))\n", "\n", - "2. Click and expand the **Model Development** section.\n", + "2. In the left sidebar that appears for your model, click **Development** under Documents.\n", "\n", - "What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation, view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/model-documentation/working-with-model-documentation.html)\n", + "What you see is the full draft of your model documentation in a more easily consumable version. From here, you can make qualitative edits to model documentation, view guidelines, collaborate with validators, and submit your model documentation for approval when it's ready. [Learn more ...](https://docs.validmind.ai/guide/working-with-model-documentation.html)\n", "\n", "\n", "\n", diff --git a/site/releases/2024/2024-mar-27/highlights.qmd b/site/releases/2024/2024-mar-27/highlights.qmd index 5d499af572..162ef46335 100644 --- a/site/releases/2024/2024-mar-27/highlights.qmd +++ b/site/releases/2024/2024-mar-27/highlights.qmd @@ -458,7 +458,7 @@ We improved our supported models documentation with additional information about ::: ::: {.w-30-ns .tc} -[Supported models](/developer/supported-models.qmd){.button} +[Supported models and frameworks](/developer/supported-models-and-frameworks.qmd){.button} ::: diff --git a/site/scripts/developer-sidebar/update_how_tos.py b/site/scripts/developer-sidebar/update_how_tos.py index 957e066160..f3a7b729a4 100644 --- a/site/scripts/developer-sidebar/update_how_tos.py +++ b/site/scripts/developer-sidebar/update_how_tos.py @@ -197,6 +197,7 @@ def update_feature_overview(base: Path, subdirs: list) -> None: listing_lines.append(f" - id: {listing_id}") listing_lines.append(f" type: grid") listing_lines.append(f" grid-columns: 2") + listing_lines.append(f" page-size: 4") listing_lines.append(f' image-placeholder: "jupyter-logo-rectangle.svg"') listing_lines.append(f" max-description-length: 350") listing_lines.append(f' image-height: "100%"') diff --git a/site/scripts/developer-sidebar/update_use_cases.py b/site/scripts/developer-sidebar/update_use_cases.py index 0e8713dc8c..b9c39da160 100644 --- a/site/scripts/developer-sidebar/update_use_cases.py +++ b/site/scripts/developer-sidebar/update_use_cases.py @@ -162,6 +162,7 @@ def update_notebooks_page(base: Path, subdirs: list) -> None: listing_lines.append(f" - id: {listing_id}") listing_lines.append(f' type: grid') listing_lines.append(f' grid-columns: 2') + listing_lines.append(f' page-size: 4') listing_lines.append(f' image-placeholder: "jupyter-logo-rectangle.svg"') listing_lines.append(f' max-description-length: 350') listing_lines.append(f' image-height: "100%"') diff --git a/site/training/administrator-fundamentals/customizing-your-model-inventory.qmd b/site/training/administrator-fundamentals/customizing-your-model-inventory.qmd index cd9acb1442..a2daae311d 100644 --- a/site/training/administrator-fundamentals/customizing-your-model-inventory.qmd +++ b/site/training/administrator-fundamentals/customizing-your-model-inventory.qmd @@ -500,7 +500,7 @@ Work with templates **Duplicate a template** ::: -1. Select one of the tabs for the type of template you want to duplicate. For example: **Documentation** +1. Select one of the tabs for the type of template you want to duplicate. For example: **Development** 2. Select the template to duplicate. For example: **Binary classification** 3. On the template details page, click **{{< fa copy >}} Duplicate Template**. 4. Give your copy a **[template name]{.smallcaps}** and a **[template description]{.smallcaps}**. @@ -538,7 +538,7 @@ Customize document templates **Customize a template** ::: -1. Select one of the tabs for the type of template you want to edit. For example: **Documentation** +1. Select one of the tabs for the type of template you want to edit. For example: **Development** 2. Select the template to edit. For example, the duplicate you created earlier: **Copy of Binary classification** 3. Click **{{< fa pencil >}} Edit Outline**. 4. Make a modification to your template. For example: **Add a 1.4 Feature Engineering subsection** @@ -670,7 +670,7 @@ In this second module, you learned how to: - [x] Configure your model inventory's fields - [x] Define granular access to model inventory fields -- [x] Customize and duplicate documentation templates +- [x] Customize and duplicate document templates - [x] Configure reusable blocks for use in templates ::: diff --git a/site/training/administrator-fundamentals/using-validmind-for-risk-management.qmd b/site/training/administrator-fundamentals/using-validmind-for-risk-management.qmd index cdb5f393b6..8c154c56e8 100644 --- a/site/training/administrator-fundamentals/using-validmind-for-risk-management.qmd +++ b/site/training/administrator-fundamentals/using-validmind-for-risk-management.qmd @@ -54,7 +54,7 @@ This third module is part of a four-part series: ## Module 3 — Contents {.center} ::: {.f2} -1. [{{< var vm.product >}} for model risk management](#validmind-for-model-risk-management) +1. [{{< var vm.product >}} for risk management](#validmind-for-model-risk-management) 2. [Set up & manage custom workflows](#set-up-manage-workflows) 3. [Manage regulations & policies](#manage-regulations-policies) 4. [Configure attestations](#configure-attestations) @@ -102,7 +102,7 @@ Already logged in and refreshed this module? Click [{{< fa chevron-right >}}]() -# ValidMind for model risk management {background-color="#083E44" background-image="/training/assets/home-hero.svg"} +# ValidMind for
risk management {background-color="#083E44" background-image="/training/assets/home-hero.svg"} ## {background-iframe="https://app.prod.validmind.ai" background-interactive="true" data-preload="yes"} diff --git a/site/training/ai-governance/compliance-and-oversight.qmd b/site/training/ai-governance/compliance-and-oversight.qmd index 6b58efc5c9..55746c12d3 100644 --- a/site/training/ai-governance/compliance-and-oversight.qmd +++ b/site/training/ai-governance/compliance-and-oversight.qmd @@ -109,7 +109,7 @@ Use our implementation guides for: ::: {.f4} - Inventory setup, custom fields, risk classification -- Documentation templates aligned to frameworks +- Document templates aligned to frameworks - Automated testing for data quality and accuracy - Independent validation, effective challenge, remediation - Ongoing monitoring and drift detection @@ -210,7 +210,7 @@ Generate reports for: To prepare for audits: -1. **Export** documentation and reports +1. **Export** documents and reports 2. **Generate** activity logs 3. **Produce** compliance summaries 4. **Package** evidence for auditors @@ -245,7 +245,7 @@ Continuous compliance requires: ::: ::: {.column width="50%" .pl3 .tr .f3 .nt3 .tilt-left} -![Ongoing monitoring documentation](../assets/vm-ongoing-monitoring.gif){width="80%" fig-alt="An animation of ongoing model monitoring being scrolled through on the ValidMind Platform showing performance metrics and potential issues" .screenshot} +![Ongoing monitoring report](../assets/vm-ongoing-monitoring.gif){width="80%" fig-alt="An animation of ongoing model monitoring being scrolled through on the ValidMind Platform showing performance metrics and potential issues" .screenshot} ![Identify potential issues early](../assets/target-prediction-distribution-plot-zoomed.png){width="80%" fig-alt="A screenshot of a graph that shows potential data drift over time" .screenshot} ::: @@ -281,7 +281,7 @@ AI regulations **evolve**. Stay current by: [{{< fa check >}}]{.dark-pink}   Updating classification criteria -[{{< fa check >}}]{.dark-pink}   Revising documentation templates +[{{< fa check >}}]{.dark-pink}   Revising document templates [{{< fa check >}}]{.dark-pink}   Training staff on changes ::: diff --git a/site/training/ai-governance/managing-ai-use-cases.qmd b/site/training/ai-governance/managing-ai-use-cases.qmd index 17bf589319..34075584b6 100644 --- a/site/training/ai-governance/managing-ai-use-cases.qmd +++ b/site/training/ai-governance/managing-ai-use-cases.qmd @@ -151,7 +151,7 @@ In {{< var vm.product >}}, you can: 1. **Add custom fields** for risk classification 2. **Configure different workflows** per tier -3. **Apply documentation templates** by tier +3. **Apply document templates** by tier 4. **Generate reports** filtered by risk level ::: diff --git a/site/training/common-slides/_get-your-code-snippet.qmd b/site/training/common-slides/_get-your-code-snippet.qmd index 0d4c5cd5cc..cc7ca2d838 100644 --- a/site/training/common-slides/_get-your-code-snippet.qmd +++ b/site/training/common-slides/_get-your-code-snippet.qmd @@ -2,6 +2,14 @@ Refer to the LICENSE file in the root of this repository for details. SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial --> + + + +:::: {.content-visible when-format="revealjs" when-meta="includes.developer"} ## Get your code snippet :::: {.columns} @@ -31,7 +39,8 @@ SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial --> 1. Select the name of your model you registered for this course to open up the model details page. 2. On the left sidebar that appears for your model, click **Getting Started**. -3. Locate the code snippet and click **Copy snippet to clipboard**. +3. Select `Development` from the **[document]{.smallcaps}** drop-down menu. +4. Click **Copy snippet to clipboard**. ::: @@ -46,4 +55,72 @@ When you're done, click [{{< fa chevron-right >}}]() to continue. Make sure you're logged in and have refreshed the page in a Chromium-based web browser. -::: \ No newline at end of file +::: + +:::: + + + +:::: {.content-visible when-format="revealjs" unless-meta="includes.developer"} +## Get your code snippet + +:::: {.columns} + +::: {.column width="80%"} + + + +
+ +
+ +::: + +::: {.column width="20%" .f4} + +::: {.f5} +{{< var vm.product >}} generates a unique *code snippet* for each registered model to connect with your local environment: + +1. Select the name of your model you registered for this course to open up the model details page. +2. On the left sidebar that appears for your model, click **Getting Started**. +3. Select `Validation` from the **[document]{.smallcaps}** drop-down menu. +4. Click **Copy snippet to clipboard**. + +::: + +When you're done, click [{{< fa chevron-right >}}]() to continue. + +::: + +:::: + +:::: {.tc .f6 .embed} +**Can't load the {{< var validmind.platform >}}?** + +Make sure you're logged in and have refreshed the page in a Chromium-based web browser. + +::: + +:::: + + + + + + + + + + diff --git a/site/training/common-slides/_retrieve-your-code-snippet.qmd b/site/training/common-slides/_retrieve-your-code-snippet.qmd index b7d9613d75..8f5bbdd685 100644 --- a/site/training/common-slides/_retrieve-your-code-snippet.qmd +++ b/site/training/common-slides/_retrieve-your-code-snippet.qmd @@ -2,6 +2,62 @@ Refer to the LICENSE file in the root of this repository for details. SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial --> + + +:::: {.content-visible when-format="revealjs" when-meta="includes.developer"} +## Retrieve your code snippet + +:::: {.columns} + +::: {.column width="80%"} + + + +
+ +
+ +::: + +::: {.column width="20%"} + +::: {.f4} +As usual, let's connect back up to your model in the {{< var validmind.platform >}}: + +1. Select the name of your model you registered for this course to open up the model details page. +2. On the left sidebar that appears for your model, click **Getting Started**. +3. Select `Development` from the **[document]{.smallcaps}** drop-down menu. +4. Click **Copy snippet to clipboard**. + +::: + +::: + +:::: + +When you're done, click [{{< fa chevron-right >}}]() to continue. + +:::: + + + +:::: {.content-visible when-format="revealjs" unless-meta="includes.developer"} ## Retrieve your code snippet :::: {.columns} @@ -31,7 +87,8 @@ As usual, let's connect back up to your model in the {{< var validmind.platform 1. Select the name of your model you registered for this course to open up the model details page. 2. On the left sidebar that appears for your model, click **Getting Started**. -3. Locate the code snippet and click **Copy snippet to clipboard**. +3. Select `Validation` from the **[document]{.smallcaps}** drop-down menu. +4. Click **Copy snippet to clipboard**. ::: @@ -39,4 +96,15 @@ As usual, let's connect back up to your model in the {{< var validmind.platform :::: -When you're done, click [{{< fa chevron-right >}}]() to continue. \ No newline at end of file +When you're done, click [{{< fa chevron-right >}}]() to continue. + +:::: + + + + + + + + + diff --git a/site/training/common-slides/_review-model-activity.qmd b/site/training/common-slides/_review-model-activity.qmd index 49c6dbadb1..d66d5b8b85 100644 --- a/site/training/common-slides/_review-model-activity.qmd +++ b/site/training/common-slides/_review-model-activity.qmd @@ -11,7 +11,8 @@ SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial --> 1. Select the name of your model you registered for this course to open up the model details page. 2. In the left sidebar that appears for your model, click **{{< fa wifi >}} Model Activity**. -3. Filter the following activity:
**Comments** | **Status Updates** | **Model Updates** | **Test Results** +3. Confirm activity logged on the model reflects your activities from this training course. +4. (Optional) Filter the activity by category or specific values to narrow down results. When you're done, click [{{< fa chevron-right >}}]() to continue. diff --git a/site/training/developer-fundamentals/finalizing-model-documentation.qmd b/site/training/developer-fundamentals/finalizing-model-documentation.qmd index 0e53cbf143..73fee394f0 100644 --- a/site/training/developer-fundamentals/finalizing-model-documentation.qmd +++ b/site/training/developer-fundamentals/finalizing-model-documentation.qmd @@ -118,7 +118,7 @@ Review test results ::: {.tc} -[Learn more ...](/guide/model-documentation/work-with-test-results.qmd){.button target="_blank"} +[Learn more ...](/guide/model-documentation/work-with-test-results.qmd#view-test-result-metadata){.button target="_blank"} ::: @@ -138,7 +138,7 @@ From the {{< var validmind.platform >}}: 2. Select the name of your model you registered for this course to open up the model details page. -3. In the left sidebar that appears for your model, click **Documentation** under {{< fa file >}} Documents. +3. In the left sidebar that appears for your model, click **Development** under {{< fa file >}} Documents. {{< include /guide/model-documentation/_view-test-result-metadata.qmd >}} @@ -154,7 +154,7 @@ From the {{< var validmind.platform >}}: ::: 1. Select the name of your model you registered for this course to open up the model details page. -2. On the left sidebar that appears for your model, click **Documentation** under {{< fa file >}} Documents. +2. On the left sidebar that appears for your model, click **Development** under {{< fa file >}} Documents. 3. Click into any section of the documentation to review the test results logged via the {{< var validmind.developer >}}.
For example: - **2.3 Correlations and Interactions** - **3.2 Model Evaluation** @@ -216,7 +216,7 @@ For example: ::: 1. Select the name of your model you registered for this course to open up the model details page. -2. On the left sidebar that appears for your model, click **Documentation** under {{< fa file >}} Documents. +2. On the left sidebar that appears for your model, click **Development** under {{< fa file >}} Documents. 3. Click into any section of the documentation to add and edit a content block. When you're done, click [{{< fa chevron-right >}}]() to continue. @@ -258,7 +258,7 @@ Use the {{< var vm.checker >}} ::: 1. Select the name of your model you registered for this course to open up the model details page. -2. On the left sidebar that appears for your model, click **Documentation** under {{< fa file >}} Documents. +2. On the left sidebar that appears for your model, click **Development** under {{< fa file >}} Documents. 3. Locate the {{< var vm.checker >}} box on the right and click **{{< fa check >}} Check Document**. 4. Select a **[regulation]{.smallcaps}** and an associated **[assessment]{.smallcaps}** from the drop-down menus to to check your document against. 5. Scroll to the bottom and click **Check Document**. @@ -313,7 +313,7 @@ Comment threads ::: 1. Select the name of your model you registered for this course to open up the model details page. -2. On the left sidebar that appears for your model, click **Documentation** under {{< fa file >}} Documents. +2. On the left sidebar that appears for your model, click **Development** under {{< fa file >}} Documents. 3. **In the content block you added earlier**: Post a comment, reply to it, and then resolve the thread. When you're done, click [{{< fa chevron-right >}}]() to continue. diff --git a/site/training/developer-fundamentals/implementing-custom-tests.qmd b/site/training/developer-fundamentals/implementing-custom-tests.qmd index bc88dacd23..e4d2750093 100644 --- a/site/training/developer-fundamentals/implementing-custom-tests.qmd +++ b/site/training/developer-fundamentals/implementing-custom-tests.qmd @@ -32,6 +32,8 @@ title-slide-attributes: data-background-color: "#083E44" data-background-image: "../assets/home-hero.svg" skip_preview: true +includes: + developer: true --- # Learning objectives {.center} @@ -344,7 +346,7 @@ With the custom test results logged, let's head to the model we connected to at 1. From the **{{< fa cubes >}} Inventory** in the {{< var validmind.platform >}}, go to the model you connected to earlier. -2. In the left sidebar that appears for your model, click **Documentation** under {{< fa file >}} Documents. +2. In the left sidebar that appears for your model, click **Development** under {{< fa file >}} Documents. 3. Locate the Data Preparation section and click on **3.2 Model Evaluation** to expand that section. @@ -373,7 +375,7 @@ With the custom test results logged, let's head to the model we connected to at ::: 1. Select the name of your model you registered for this course to open up the model details page. -2. In the left sidebar that appears for your model, click **Documentation** under {{< fa file >}} Documents. +2. In the left sidebar that appears for your model, click **Development** under {{< fa file >}} Documents. 3. Locate the Data Preparation section and click on **3.2 Model Evaluation** to expand that section. 4. Hover under the Pearson Correlation Matrix content block until a horizontal line with a **{{< fa plus >}}** button appears, indicating that you can insert a new block. 5. Click **{{< fa plus >}}** and then select **Test-Driven Block** under [from library]{.smallcaps}: diff --git a/site/training/developer-fundamentals/learning-to-run-tests.qmd b/site/training/developer-fundamentals/learning-to-run-tests.qmd index 69ea84d76c..98dff980fb 100644 --- a/site/training/developer-fundamentals/learning-to-run-tests.qmd +++ b/site/training/developer-fundamentals/learning-to-run-tests.qmd @@ -32,6 +32,8 @@ title-slide-attributes: data-background-color: "#083E44" data-background-image: "../assets/home-hero.svg" skip_preview: true +includes: + developer: true --- # Learning objectives {.center} @@ -304,7 +306,7 @@ Work with test results ::: 1. Select the name of your model you registered for this course to open up the model details page. -2. In the left sidebar that appears for your model, click **Documentation** under {{< fa file >}} Documents. +2. In the left sidebar that appears for your model, click **Development** under {{< fa file >}} Documents. 3. Locate the Data Preparation section and click on **2.3 Correlations and Interactions** to expand that section. 4. Hover under the Pearson Correlation Matrix content block until a horizontal line with a **{{< fa plus >}}** button appears, indicating that you can insert a new block. 5. Click **{{< fa plus >}}** and then select **Test-Driven Block** under [from library]{.smallcaps}: diff --git a/site/training/developer-fundamentals/using-validmind-for-model-development.qmd b/site/training/developer-fundamentals/using-validmind-for-model-development.qmd index b05ee89229..c68ea3f71e 100644 --- a/site/training/developer-fundamentals/using-validmind-for-model-development.qmd +++ b/site/training/developer-fundamentals/using-validmind-for-model-development.qmd @@ -183,7 +183,7 @@ To start the documentation process, a model must already be registered in the mo Once you've registered your model, let's select a documentation template. A template predefines sections for your model documentation and provides a general outline to follow, making the documentation process much easier. 1. Click on the name of the model you registered earlier. -2. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select **Documentation**. +2. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select **Development**. 3. Under [create document]{.smallcaps}, select `Binary classification` from the **[template]{.smallcaps}** drop-down menu. 4. Click **Use Template** to apply the template. @@ -241,7 +241,7 @@ When you're done, return to this page and click [{{< fa chevron-right >}}]() to Once you've called `preview_template()`: 1. Select the name of your model you registered for this course to open up the model details page. -2. On the left sidebar that appears for your model, click **Documentation** under {{< fa file >}} Documents. +2. On the left sidebar that appears for your model, click **Development** under {{< fa file >}} Documents. 3. Note how the structure of the Document Overview reflects the previewed template. When you're done, click [{{< fa chevron-right >}}]() to continue. diff --git a/site/training/training-templates/course-slides.qmd b/site/training/training-templates/course-slides.qmd index e61029eb7c..12d8a7e460 100644 --- a/site/training/training-templates/course-slides.qmd +++ b/site/training/training-templates/course-slides.qmd @@ -34,6 +34,8 @@ title-slide-attributes: data-background-color: "#083E44" data-background-image: "../assets/home-hero.svg" skip_preview: true +includes: + developer: true --- # Learning objectives {.center} @@ -84,7 +86,7 @@ This {{#th}} module is part of a {{#}}-part series: Example embedded `iframe` for interactive copy/paste workaround: -- [Get your code snippet](#get-your-code-snippet) +- [Get your code snippet](#get-your-code-snippet) (Developer ver. preview) Topic summary & next steps: diff --git a/site/training/validator-fundamentals/developing-challenger-models.qmd b/site/training/validator-fundamentals/developing-challenger-models.qmd index 2db346d108..a871339984 100644 --- a/site/training/validator-fundamentals/developing-challenger-models.qmd +++ b/site/training/validator-fundamentals/developing-challenger-models.qmd @@ -360,7 +360,7 @@ As we can observe from the output in our notebook, our champion model doesn't pa ::: 1. Select the name of your model you registered for this course to open up the model details page. -2. In the left sidebar that appears for your model, click **Validation Report** under {{< fa file >}} Documents. +2. In the left sidebar that appears for your model, click **Validation** under {{< fa file >}} Documents. 3. Locate the Data Preparation section and click on **2.2.2. Model Performance** to expand that section. 4. Under the Model Performance Metrics section, locate Artifacts then click **Link Artifact to Report** and select **Validation Issue**. 5. Click **{{< fa plus >}} Add Validation Issue** to add a validation issue type artifact. diff --git a/site/training/validator-fundamentals/finalizing-validation-reports.qmd b/site/training/validator-fundamentals/finalizing-validation-reports.qmd index 7d8301e6ac..c03373e4d4 100644 --- a/site/training/validator-fundamentals/finalizing-validation-reports.qmd +++ b/site/training/validator-fundamentals/finalizing-validation-reports.qmd @@ -146,7 +146,7 @@ Make qualitative edits 1. From the **{{< fa cubes >}} Inventory** in the {{< var validmind.platform >}}, go to the model you connected to earlier. -2. In the left sidebar that appears for your model, click **Validation Report** under {{< fa file >}} Documents. +2. In the left sidebar that appears for your model, click **Validation** under {{< fa file >}} Documents. 3. Click on **1. Executive Summary** to expand and add content to that section. @@ -154,7 +154,7 @@ Make qualitative edits ![Adding a content block in the UI](/guide/model-documentation/add-content-block.gif){width=90% fig-alt="A gif showing the process of adding a content block in the UI" .screenshot} -After adding the block to your documentation, generate a content draft with AI using the [content editing toolbar](/guide/model-documentation/work-with-content-blocks.qmd#content-editing-toolbar){target="_blank"}: +After adding the block to your validation report, generate a content draft with AI using the [content editing toolbar](/guide/model-documentation/work-with-content-blocks.qmd#content-editing-toolbar){target="_blank"}: {{< include /guide/model-documentation/content_blocks/_generate-with-ai.qmd >}} @@ -186,7 +186,7 @@ For example: ::: 1. Select the name of your model you registered for this course to open up the model details page. -2. On the left sidebar that appears for your model, click **Validation Report** under {{< fa file >}} Documents. +2. On the left sidebar that appears for your model, click **Validation** under {{< fa file >}} Documents. 3. Click on **1. Executive Summary** to add and edit a content block. 4. Click on **2.2.1. Data Quality** to edit the description for the linked Class Imbalance Assessment test results. @@ -220,7 +220,7 @@ Assess compliance 1. From the **{{< fa cubes >}} Inventory** in the {{< var validmind.platform >}}, go to the model you connected to earlier. -2. In the left sidebar that appears for your model, click **Validation Report** under {{< fa file >}} Documents. +2. In the left sidebar that appears for your model, click **Validation** under {{< fa file >}} Documents. 3. Click on **2.2.1. Data Quality** to expand that section and locate the Class Imbalance Assessment sub-section. @@ -247,7 +247,7 @@ Assess compliance ::: 1. Select the name of your model you registered for this course to open up the model details page. -2. On the left sidebar that appears for your model, click **Validation Report** under {{< fa file >}} Documents. +2. On the left sidebar that appears for your model, click **Validation** under {{< fa file >}} Documents. 3. Click on **2.2.1. Data Quality** to assess compliance for the Class Imbalance Assessment sub-section. When you're done, click [{{< fa chevron-right >}}]() to continue. @@ -279,7 +279,7 @@ Try it **live** on the next pages. {{< fa hand-point-right >}} ::: {.column width="70%" .bl .pl4 .f4} -Along with adding artifacts directly via validation reports, you can also add artifacts during your review of documentation: +Along with adding artifacts directly via validation reports, you can also add artifacts during your review of model documentation: {{< include /guide/model-validation/_add-artifacts-on-documentation.qmd >}} @@ -296,7 +296,7 @@ Along with adding artifacts directly via validation reports, you can also add ar ::: 1. Select the name of your model you registered for this course to open up the model details page. -2. On the left sidebar that appears for your model, click **Documentation** under {{< fa file >}} Documents. +2. On the left sidebar that appears for your model, click **Development** under {{< fa file >}} Documents. 3. Click **{{< fa plus >}} Add Model Artifact** to add an artifact from the overview. When you're done, click [{{< fa chevron-right >}}]() to continue. @@ -311,7 +311,7 @@ When you're done, click [{{< fa chevron-right >}}]() to continue. ::: 1. Select the name of your model you registered for this course to open up the model details page. -2. On the left sidebar that appears for your model, click **Documentation** under {{< fa file >}} Documents. +2. On the left sidebar that appears for your model, click **Development** under {{< fa file >}} Documents. 3. Click on any section heading to expand and add an artifact to that section via the **{{< var vm.product >}} Insights™** panel. When you're done, click [{{< fa chevron-right >}}]() to continue. @@ -433,7 +433,7 @@ Have a question about the model? Collaborate with your developer right in the mo ::: 1. Select the name of your model you registered for this course to open up the model details page. -2. On the left sidebar that appears for your model, click **Validation Report** under {{< fa file >}} Documents. +2. On the left sidebar that appears for your model, click **Validation** under {{< fa file >}} Documents. 3. **In the content block you added earlier**: Post a comment, reply to it, and then resolve the thread. When you're done, click [{{< fa chevron-right >}}]() to continue. diff --git a/site/training/validator-fundamentals/running-data-quality-tests.qmd b/site/training/validator-fundamentals/running-data-quality-tests.qmd index d40da80cff..b0fd4711d2 100644 --- a/site/training/validator-fundamentals/running-data-quality-tests.qmd +++ b/site/training/validator-fundamentals/running-data-quality-tests.qmd @@ -319,7 +319,7 @@ Add test results to reporting ::: 1. Select the name of your model you registered for this course to open up the model details page. -2. In the left sidebar that appears for your model, click **Validation Report** under {{< fa file >}} Documents. +2. In the left sidebar that appears for your model, click **Validation** under {{< fa file >}} Documents. 3. Locate the Data Preparation section and click on **2.2.1. Data Quality** to expand that section. 4. Under the Class Imbalance Assessment section, locate Validator Evidence then click **Link Evidence to Report**. 5. Select the Class Imbalance test results we logged: **ValidMind Data Validation Class Imbalance** diff --git a/site/training/validator-fundamentals/using-validmind-for-model-validation.qmd b/site/training/validator-fundamentals/using-validmind-for-model-validation.qmd index 630d490e20..e0163bcb81 100644 --- a/site/training/validator-fundamentals/using-validmind-for-model-validation.qmd +++ b/site/training/validator-fundamentals/using-validmind-for-model-validation.qmd @@ -207,7 +207,7 @@ Once you've registered your model, let's select a documentation template. A temp We'll need this documentation template later for reference as we draft our validation report: 1. Click on the name of the model you registered earlier. -2. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select **Documentation**. +2. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select **Development**. 3. Under [create document]{.smallcaps}, select `Binary classification` from the **[template]{.smallcaps}** drop-down menu. 4. Click **Use Template** to apply the template. @@ -225,7 +225,7 @@ When you're done, click [{{< fa chevron-right >}}]() to continue. Next, let's select a validation report template. A template predefines sections for your report and provides a general outline to follow, making the validation process much easier. 1. Click on the name of the model you registered earlier. -2. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select **Validation Report**. +2. In the left sidebar that appears for your model, click **{{< fa file >}} Documents** and select **Validation**. 3. Under [create document]{.smallcaps}, select `Generic Validation Report` from the **[template]{.smallcaps}** drop-down menu. 4. Click **Use Template** to apply the template. @@ -279,7 +279,7 @@ When you're done, return to this page and click [{{< fa chevron-right >}}]() to Once you've called `preview_template()`: 1. Select the name of your model you registered for this course to open up the model details page. -2. On the left sidebar that appears for your model, click **Validation Report** under {{< fa file >}} Documents. +2. On the left sidebar that appears for your model, click **Validation** under {{< fa file >}} Documents. 3. Note how the structure of the validation report reflects the previewed template. When you're done exploring, click [{{< fa chevron-right >}}]() to continue. @@ -300,7 +300,7 @@ Typically, a champion model is submitted to you for validation with completed mo Here, review the empty sample model documentation for the model you registered earlier to familiarize you with what the structure of documentation could look like when presented to you for review: 1. Select the name of your model you registered for this course to open up the model details page. -2. In the left sidebar that appears for your model, click **Documentation** under {{< fa file >}} Documents. +2. In the left sidebar that appears for your model, click **Development** under {{< fa file >}} Documents. ::: {.embed} The structure of the model documentation reflects the documentation template applied to the model, just like the validation report template. diff --git a/site/training/what-is-validmind/what-is-validmind.qmd b/site/training/what-is-validmind/what-is-validmind.qmd index 2e06a521bd..a121a7573b 100644 --- a/site/training/what-is-validmind/what-is-validmind.qmd +++ b/site/training/what-is-validmind/what-is-validmind.qmd @@ -907,7 +907,7 @@ Use our implementation guides for: ::: {.f4} - Inventory setup, custom fields, risk classification -- Documentation templates aligned to frameworks +- Document templates aligned to frameworks - Automated testing for data quality and accuracy - Independent validation, effective challenge, remediation - Ongoing monitoring and drift detection