From fd9b1c82b816114b04ab3eb8b0dcf9452cfadf50 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Wed, 4 Mar 2026 17:55:21 +0000 Subject: [PATCH 01/18] feat: tnc connector implementation --- .../src/integrations/index.ts | 25 +++ .../tnc/certificates/buildSourceQuery.ts | 96 ++++++++++ .../tnc/certificates/transformer.ts | 153 +++++++++++++++ .../tnc/course-actions/buildSourceQuery.ts | 50 +++++ .../tnc/course-actions/transformer.ts | 68 +++++++ .../tnc/enrollments/buildSourceQuery.ts | 95 ++++++++++ .../tnc/enrollments/transformer.ts | 174 ++++++++++++++++++ .../src/integrations/types.ts | 3 + 8 files changed, 664 insertions(+) create mode 100644 services/apps/snowflake_connectors/src/integrations/tnc/certificates/buildSourceQuery.ts create mode 100644 services/apps/snowflake_connectors/src/integrations/tnc/certificates/transformer.ts create mode 100644 services/apps/snowflake_connectors/src/integrations/tnc/course-actions/buildSourceQuery.ts create mode 100644 services/apps/snowflake_connectors/src/integrations/tnc/course-actions/transformer.ts create mode 100644 services/apps/snowflake_connectors/src/integrations/tnc/enrollments/buildSourceQuery.ts create mode 100644 services/apps/snowflake_connectors/src/integrations/tnc/enrollments/transformer.ts diff --git a/services/apps/snowflake_connectors/src/integrations/index.ts b/services/apps/snowflake_connectors/src/integrations/index.ts index 84f6f4656f..aced040958 100644 --- a/services/apps/snowflake_connectors/src/integrations/index.ts +++ b/services/apps/snowflake_connectors/src/integrations/index.ts @@ -8,6 +8,12 @@ import { PlatformType } from '@crowd/types' import { buildSourceQuery as cventBuildSourceQuery } from './cvent/event-registrations/buildSourceQuery' import { CventTransformer } from './cvent/event-registrations/transformer' +import { buildSourceQuery as tncCertificatesBuildQuery } from './tnc/certificates/buildSourceQuery' +import { TncCertificatesTransformer } from './tnc/certificates/transformer' +import { buildSourceQuery as tncCourseActionsBuildQuery } from './tnc/course-actions/buildSourceQuery' +import { TncCourseActionsTransformer } from './tnc/course-actions/transformer' +import { buildSourceQuery as tncEnrollmentsBuildQuery } from './tnc/enrollments/buildSourceQuery' +import { TncEnrollmentsTransformer } from './tnc/enrollments/transformer' import { DataSource, DataSourceName, PlatformDefinition } from './types' export type { BuildSourceQuery, DataSource, PlatformDefinition } from './types' @@ -23,6 +29,25 @@ const supported: Partial> = { }, ], }, + [PlatformType.TNC]: { + sources: [ + { + name: DataSourceName.TNC_ENROLLMENTS, + buildSourceQuery: tncEnrollmentsBuildQuery, + transformer: new TncEnrollmentsTransformer(), + }, + { + name: DataSourceName.TNC_CERTIFICATES, + buildSourceQuery: tncCertificatesBuildQuery, + transformer: new TncCertificatesTransformer(), + }, + { + name: DataSourceName.TNC_COURSE_ACTIONS, + buildSourceQuery: tncCourseActionsBuildQuery, + transformer: new TncCourseActionsTransformer(), + }, + ], + }, } const enabled = (process.env.CROWD_SNOWFLAKE_ENABLED_PLATFORMS || '') diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/certificates/buildSourceQuery.ts b/services/apps/snowflake_connectors/src/integrations/tnc/certificates/buildSourceQuery.ts new file mode 100644 index 0000000000..16d2370254 --- /dev/null +++ b/services/apps/snowflake_connectors/src/integrations/tnc/certificates/buildSourceQuery.ts @@ -0,0 +1,96 @@ +import { IS_PROD_ENV } from '@crowd/common' + +const CDP_MATCHED_SEGMENTS = ` + cdp_matched_segments AS ( + SELECT DISTINCT + s.SOURCE_ID AS sourceId, + s.slug + FROM ANALYTICS.SILVER_DIM._CROWD_DEV_SEGMENTS_UNION s + WHERE s.PARENT_SLUG IS NOT NULL + AND s.GRANDPARENTS_SLUG IS NOT NULL + AND s.SOURCE_ID IS NOT NULL + )` + +const ORG_ACCOUNTS = ` + org_accounts AS ( + SELECT account_id, account_name, website, domain_aliases, LOGO_URL, INDUSTRY, N_EMPLOYEES + FROM analytics.bronze_fivetran_salesforce.accounts + WHERE website IS NOT NULL + UNION ALL + SELECT account_id, account_name, website, domain_aliases, NULL AS LOGO_URL, NULL AS INDUSTRY, NULL AS N_EMPLOYEES + FROM analytics.bronze_fivetran_salesforce_b2b.accounts + WHERE website IS NOT NULL + )` + +const LFID_COALESCE = `COALESCE(mu.user_name, u.lf_username)` + +export const buildSourceQuery = (sinceTimestamp?: string): string => { + let select = ` + SELECT + c.*, + cms.slug AS PROJECT_SLUG, + org.account_name AS ORGANIZATION_NAME, + org.website AS ORG_WEBSITE, + org.domain_aliases AS ORG_DOMAIN_ALIASES, + org.logo_url AS LOGO_URL, + org.industry AS ORGANIZATION_INDUSTRY, + org.n_employees AS ORGANIZATION_SIZE, + ${LFID_COALESCE} AS LFID + FROM analytics.silver_fact.certificates c + INNER JOIN cdp_matched_segments cms + ON cms.sourceId = c.project_id + LEFT JOIN analytics.bronze_fivetran_salesforce.bronze_salesforce_merged_user mu + ON c.user_id = mu.user_id + AND mu.user_name IS NOT NULL + LEFT JOIN analytics.silver_dim.users u + ON LOWER(c.user_email) = LOWER(u.email) + AND u.lf_username IS NOT NULL + LEFT JOIN org_accounts org + ON c.account_id = org.account_id + WHERE c.user_email IS NOT NULL + AND c.is_deleted = false` + + if (!IS_PROD_ENV) { + select += ` AND cms.slug = 'pytorch'` + } + + const dedup = ` + QUALIFY ROW_NUMBER() OVER (PARTITION BY c.certificate_id ORDER BY org.website DESC) = 1` + + if (!sinceTimestamp) { + return ` + WITH ${ORG_ACCOUNTS}, + ${CDP_MATCHED_SEGMENTS} + ${select} + ${dedup}`.trim() + } + + return ` + WITH ${ORG_ACCOUNTS}, + ${CDP_MATCHED_SEGMENTS}, + new_cdp_segments AS ( + SELECT DISTINCT + s.SOURCE_ID AS sourceId, + s.slug + FROM ANALYTICS.SILVER_DIM._CROWD_DEV_SEGMENTS_UNION s + WHERE s.CREATED_TS >= '${sinceTimestamp}' + AND s.PARENT_SLUG IS NOT NULL + AND s.GRANDPARENTS_SLUG IS NOT NULL + AND s.SOURCE_ID IS NOT NULL + ) + + -- New certificates since last export + ${select} + AND c.issued_ts >= '${sinceTimestamp}' + ${dedup} + + UNION + + -- All certificates in newly created segments + ${select} + AND EXISTS ( + SELECT 1 FROM new_cdp_segments ncs + WHERE ncs.slug = cms.slug AND ncs.sourceId = cms.sourceId + ) + ${dedup}`.trim() +} diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/certificates/transformer.ts b/services/apps/snowflake_connectors/src/integrations/tnc/certificates/transformer.ts new file mode 100644 index 0000000000..beae20ae30 --- /dev/null +++ b/services/apps/snowflake_connectors/src/integrations/tnc/certificates/transformer.ts @@ -0,0 +1,153 @@ +import { TNC_GRID, TncActivityType } from '@crowd/integrations' +import { getServiceChildLogger } from '@crowd/logging' +import { + IActivityData, + IMemberData, + IOrganizationIdentity, + MemberAttributeName, + MemberIdentityType, + OrganizationIdentityType, + OrganizationSource, + PlatformType, +} from '@crowd/types' + +import { TransformedActivity, TransformerBase } from '../../../core/transformerBase' + +const log = getServiceChildLogger('tncCertificatesTransformer') + +export class TncCertificatesTransformer extends TransformerBase { + readonly platform = PlatformType.TNC + + transformRow(row: Record): TransformedActivity | null { + const email = (row.USER_EMAIL as string | null)?.trim() || null + if (!email) { + log.debug({ certificateId: row.CERTIFICATE_ID }, 'Skipping row: missing email') + return null + } + + const certificateId = (row.CERTIFICATE_ID as string)?.trim() + const learnerName = (row.LEARNER_NAME as string | null)?.trim() || null + const lfUsername = (row.LFID as string | null)?.trim() || null + + const identities: IMemberData['identities'] = [] + const sourceId = (row.USER_ID as string | null) || undefined + + if (lfUsername) { + identities.push( + { + platform: PlatformType.TNC, + value: email, + type: MemberIdentityType.EMAIL, + verified: true, + sourceId, + }, + { + platform: PlatformType.TNC, + value: lfUsername, + type: MemberIdentityType.USERNAME, + verified: true, + sourceId, + }, + { + platform: PlatformType.LFID, + value: lfUsername, + type: MemberIdentityType.USERNAME, + verified: true, + sourceId, + }, + ) + } else { + identities.push({ + platform: PlatformType.TNC, + value: email, + type: MemberIdentityType.USERNAME, + verified: true, + sourceId, + }) + } + + const activity: IActivityData = { + type: TncActivityType.ISSUED_CERTIFICATION, + platform: PlatformType.TNC, + timestamp: (row.ISSUED_TS as string | null) || null, + score: TNC_GRID[TncActivityType.ISSUED_CERTIFICATION].score, + sourceId: certificateId, + sourceParentId: (row.COURSE_ID as string | null) || undefined, + member: { + displayName: learnerName || email, + identities, + organizations: this.buildOrganizations(row), + attributes: { + ...((row.JOB_TITLE as string | null) && { + [MemberAttributeName.JOB_TITLE]: { [PlatformType.TNC]: row.JOB_TITLE as string }, + }), + ...((row.USER_COUNTRY as string | null) && { + [MemberAttributeName.COUNTRY]: { [PlatformType.TNC]: row.USER_COUNTRY as string }, + }), + }, + }, + attributes: { + productName: (row.COURSE_NAME as string | null) || null, + productType: 'Certification', + technology: (row.TECHNOLOGIES_LIST as string | null) || null, + didExpire: row.DID_EXPIRE as boolean | null, + expirationDate: (row.EXPIRATION_DATE as string | null) || null, + }, + } + + const segmentSlug = (row.PROJECT_SLUG as string | null)?.trim() || null + const segmentSourceId = (row.PROJECT_ID as string | null)?.trim() || null + if (!segmentSlug || !segmentSourceId) { + return null + } + + return { activity, segment: { slug: segmentSlug, sourceId: segmentSourceId } } + } + + private buildOrganizations( + row: Record, + ): IActivityData['member']['organizations'] { + const website = (row.ORG_WEBSITE as string | null)?.trim() || null + const domainAliases = (row.ORG_DOMAIN_ALIASES as string | null)?.trim() || null + + if (!website && !domainAliases) { + return undefined + } + + const identities: IOrganizationIdentity[] = [] + + if (website) { + identities.push({ + platform: PlatformType.TNC, + value: website, + type: OrganizationIdentityType.PRIMARY_DOMAIN, + verified: true, + }) + } + + if (domainAliases) { + for (const alias of domainAliases.split(',')) { + const trimmed = alias.trim() + if (trimmed) { + identities.push({ + platform: PlatformType.TNC, + value: trimmed, + type: OrganizationIdentityType.ALTERNATIVE_DOMAIN, + verified: true, + }) + } + } + } + + return [ + { + displayName: (row.ORGANIZATION_NAME as string | null)?.trim() || website, + source: OrganizationSource.TNC, + identities, + logo: (row.LOGO_URL as string | null)?.trim() || undefined, + size: (row.ORGANIZATION_SIZE as string | null)?.trim() || undefined, + industry: (row.ORGANIZATION_INDUSTRY as string | null)?.trim() || undefined, + }, + ] + } +} diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/course-actions/buildSourceQuery.ts b/services/apps/snowflake_connectors/src/integrations/tnc/course-actions/buildSourceQuery.ts new file mode 100644 index 0000000000..2d7e0eb593 --- /dev/null +++ b/services/apps/snowflake_connectors/src/integrations/tnc/course-actions/buildSourceQuery.ts @@ -0,0 +1,50 @@ +import { IS_PROD_ENV } from '@crowd/common' + +// TODO: user resolution — course_actions only has INTERNAL_TI_USER_ID. +// Need to identify the join table that maps INTERNAL_TI_USER_ID to user_email, learner_name, account_id, etc. + +const CDP_MATCHED_SEGMENTS = ` + cdp_matched_segments AS ( + SELECT DISTINCT + s.SOURCE_ID AS sourceId, + s.slug + FROM ANALYTICS.SILVER_DIM._CROWD_DEV_SEGMENTS_UNION s + WHERE s.PARENT_SLUG IS NOT NULL + AND s.GRANDPARENTS_SLUG IS NOT NULL + AND s.SOURCE_ID IS NOT NULL + )` + +export const buildSourceQuery = (sinceTimestamp?: string): string => { + let select = ` + SELECT + ca.course_action_id, + ca.course_id, + ca.timestamp, + ca.type, + ca.source, + ca.internal_ti_user_id, + co.title AS COURSE_NAME, + co.course_group_id, + co.slug AS COURSE_SLUG, + co.instruction_type, + co.product_type, + co.is_training, + co.is_certification + FROM analytics.bronze_census_ti.course_actions ca + INNER JOIN analytics.bronze_census_ti.courses co + ON ca.course_id = co.course_id + WHERE ca.type = 'status_change' + AND ca.source = 'course_started' + AND co.is_test_or_archived = false` + + if (!IS_PROD_ENV) { + select += ` AND 1=1` // TODO: add non-prod project filter once segment join is available + } + + if (!sinceTimestamp) { + return select.trim() + } + + return `${select} + AND ca.timestamp >= '${sinceTimestamp}'`.trim() +} diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/course-actions/transformer.ts b/services/apps/snowflake_connectors/src/integrations/tnc/course-actions/transformer.ts new file mode 100644 index 0000000000..718b279b6f --- /dev/null +++ b/services/apps/snowflake_connectors/src/integrations/tnc/course-actions/transformer.ts @@ -0,0 +1,68 @@ +import { TNC_GRID, TncActivityType } from '@crowd/integrations' +import { getServiceChildLogger } from '@crowd/logging' +import { IActivityData, PlatformType } from '@crowd/types' + +import { TransformedActivity, TransformerBase } from '../../../core/transformerBase' + +const log = getServiceChildLogger('tncCourseActionsTransformer') + +// TODO: user resolution — course_actions only has INTERNAL_TI_USER_ID. +// Once the user join table is identified, add email/name/org resolution here. + +export class TncCourseActionsTransformer extends TransformerBase { + readonly platform = PlatformType.TNC + + transformRow(row: Record): TransformedActivity | null { + const courseActionId = (row.COURSE_ACTION_ID as string | null)?.trim() || null + if (!courseActionId) { + log.debug('Skipping row: missing course_action_id') + return null + } + + const internalUserId = (row.INTERNAL_TI_USER_ID as string | null)?.trim() || null + if (!internalUserId) { + log.debug({ courseActionId }, 'Skipping row: missing internal_ti_user_id') + return null + } + + const isCertification = row.IS_CERTIFICATION === true + const type = isCertification + ? TncActivityType.ATTEMPTED_EXAM + : TncActivityType.ATTEMPTED_COURSE + + // TODO: replace with actual email/name once user resolution join is available + const activity: IActivityData = { + type, + platform: PlatformType.TNC, + timestamp: (row.TIMESTAMP as string | null) || null, + score: TNC_GRID[type].score, + sourceId: courseActionId, + sourceParentId: (row.COURSE_ID as string | null) || undefined, + member: { + displayName: internalUserId, + identities: [ + { + platform: PlatformType.TNC, + value: internalUserId, + type: 'username' as never, + verified: false, + sourceId: internalUserId, + }, + ], + }, + attributes: { + courseName: (row.COURSE_NAME as string | null) || null, + courseSlug: (row.COURSE_SLUG as string | null) || null, + instructionType: (row.INSTRUCTION_TYPE as string | null) || null, + productType: (row.PRODUCT_TYPE as string | null) || null, + isCertification, + isTraining: row.IS_TRAINING === true, + }, + } + + // TODO: segment resolution — course_actions doesn't have PROJECT_SLUG/PROJECT_ID. + // Need to determine how to map courses to projects/segments. + log.debug({ courseActionId }, 'Skipping row: segment resolution not yet implemented') + return null + } +} diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/buildSourceQuery.ts b/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/buildSourceQuery.ts new file mode 100644 index 0000000000..4f0fa2eeca --- /dev/null +++ b/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/buildSourceQuery.ts @@ -0,0 +1,95 @@ +import { IS_PROD_ENV } from '@crowd/common' + +const CDP_MATCHED_SEGMENTS = ` + cdp_matched_segments AS ( + SELECT DISTINCT + s.SOURCE_ID AS sourceId, + s.slug + FROM ANALYTICS.SILVER_DIM._CROWD_DEV_SEGMENTS_UNION s + WHERE s.PARENT_SLUG IS NOT NULL + AND s.GRANDPARENTS_SLUG IS NOT NULL + AND s.SOURCE_ID IS NOT NULL + )` + +const ORG_ACCOUNTS = ` + org_accounts AS ( + SELECT account_id, account_name, website, domain_aliases, LOGO_URL, INDUSTRY, N_EMPLOYEES + FROM analytics.bronze_fivetran_salesforce.accounts + WHERE website IS NOT NULL + UNION ALL + SELECT account_id, account_name, website, domain_aliases, NULL AS LOGO_URL, NULL AS INDUSTRY, NULL AS N_EMPLOYEES + FROM analytics.bronze_fivetran_salesforce_b2b.accounts + WHERE website IS NOT NULL + )` + +const LFID_COALESCE = `COALESCE(mu.user_name, u.lf_username)` + +export const buildSourceQuery = (sinceTimestamp?: string): string => { + let select = ` + SELECT + e.*, + org.account_name AS ORGANIZATION_NAME, + org.website AS ORG_WEBSITE, + org.domain_aliases AS ORG_DOMAIN_ALIASES, + org.logo_url AS LOGO_URL, + org.industry AS ORGANIZATION_INDUSTRY, + org.n_employees AS ORGANIZATION_SIZE, + ${LFID_COALESCE} AS LFID + FROM analytics.silver_fact.enrollments e + INNER JOIN cdp_matched_segments cms + ON cms.slug = e.project_slug + AND cms.sourceId = e.project_id + LEFT JOIN analytics.bronze_fivetran_salesforce.bronze_salesforce_merged_user mu + ON e.user_id = mu.user_id + AND mu.user_name IS NOT NULL + LEFT JOIN analytics.silver_dim.users u + ON LOWER(e.user_email) = LOWER(u.email) + AND u.lf_username IS NOT NULL + LEFT JOIN org_accounts org + ON e.account_id = org.account_id + WHERE e.user_email IS NOT NULL` + + if (!IS_PROD_ENV) { + select += ` AND e.project_slug = 'pytorch'` + } + + const dedup = ` + QUALIFY ROW_NUMBER() OVER (PARTITION BY e.enrollment_id ORDER BY org.website DESC) = 1` + + if (!sinceTimestamp) { + return ` + WITH ${ORG_ACCOUNTS}, + ${CDP_MATCHED_SEGMENTS} + ${select} + ${dedup}`.trim() + } + + return ` + WITH ${ORG_ACCOUNTS}, + ${CDP_MATCHED_SEGMENTS}, + new_cdp_segments AS ( + SELECT DISTINCT + s.SOURCE_ID AS sourceId, + s.slug + FROM ANALYTICS.SILVER_DIM._CROWD_DEV_SEGMENTS_UNION s + WHERE s.CREATED_TS >= '${sinceTimestamp}' + AND s.PARENT_SLUG IS NOT NULL + AND s.GRANDPARENTS_SLUG IS NOT NULL + AND s.SOURCE_ID IS NOT NULL + ) + + -- New enrollments since last export + ${select} + AND e.enrollment_ts >= '${sinceTimestamp}' + ${dedup} + + UNION + + -- All enrollments in newly created segments + ${select} + AND EXISTS ( + SELECT 1 FROM new_cdp_segments ncs + WHERE ncs.slug = cms.slug AND ncs.sourceId = cms.sourceId + ) + ${dedup}`.trim() +} diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/transformer.ts b/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/transformer.ts new file mode 100644 index 0000000000..3d313559b4 --- /dev/null +++ b/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/transformer.ts @@ -0,0 +1,174 @@ +import { TNC_GRID, TncActivityType } from '@crowd/integrations' +import { getServiceChildLogger } from '@crowd/logging' +import { + IActivityData, + IMemberData, + IOrganizationIdentity, + MemberAttributeName, + MemberIdentityType, + OrganizationIdentityType, + OrganizationSource, + PlatformType, +} from '@crowd/types' + +import { TransformedActivity, TransformerBase } from '../../../core/transformerBase' + +const log = getServiceChildLogger('tncEnrollmentsTransformer') + +export class TncEnrollmentsTransformer extends TransformerBase { + readonly platform = PlatformType.TNC + + transformRow(row: Record): TransformedActivity | null { + const email = (row.USER_EMAIL as string | null)?.trim() || null + if (!email) { + log.debug({ enrollmentId: row.ENROLLMENT_ID }, 'Skipping row: missing email') + return null + } + + const enrollmentId = (row.ENROLLMENT_ID as string)?.trim() + const learnerName = (row.LEARNER_NAME as string | null)?.trim() || null + const lfUsername = (row.LFID as string | null)?.trim() || null + + const identities: IMemberData['identities'] = [] + const sourceId = (row.USER_ID as string | null) || undefined + + if (lfUsername) { + identities.push( + { + platform: PlatformType.TNC, + value: email, + type: MemberIdentityType.EMAIL, + verified: true, + sourceId, + }, + { + platform: PlatformType.TNC, + value: lfUsername, + type: MemberIdentityType.USERNAME, + verified: true, + sourceId, + }, + { + platform: PlatformType.LFID, + value: lfUsername, + type: MemberIdentityType.USERNAME, + verified: true, + sourceId, + }, + ) + } else { + identities.push({ + platform: PlatformType.TNC, + value: email, + type: MemberIdentityType.USERNAME, + verified: true, + sourceId, + }) + } + + const productType = (row.PRODUCT_TYPE as string | null)?.trim() || null + const instructionType = (row.INSTRUCTION_TYPE as string | null)?.trim() || null + + let type: TncActivityType + if ( + productType?.toLowerCase() === 'certification' && + instructionType?.toLowerCase() === 'certification exam' + ) { + type = TncActivityType.ENROLLED_CERTIFICATION + } else if (productType?.toLowerCase() === 'training') { + type = TncActivityType.ENROLLED_TRAINING + } else { + log.debug( + { enrollmentId, productType, instructionType }, + 'Skipping row: unrecognized product/instruction type', + ) + return null + } + + const activity: IActivityData = { + type, + platform: PlatformType.TNC, + timestamp: (row.ENROLLMENT_TS as string | null) || null, + score: TNC_GRID[type].score, + sourceId: enrollmentId, + sourceParentId: (row.COURSE_ID as string | null) || undefined, + member: { + displayName: learnerName || email, + identities, + organizations: this.buildOrganizations(row), + attributes: { + ...((row.LEARNER_TITLE as string | null) && { + [MemberAttributeName.JOB_TITLE]: { [PlatformType.TNC]: row.LEARNER_TITLE as string }, + }), + ...((row.USER_COUNTRY as string | null) && { + [MemberAttributeName.COUNTRY]: { [PlatformType.TNC]: row.USER_COUNTRY as string }, + }), + }, + }, + attributes: { + productName: (row.COURSE_NAME as string | null) || null, + productType, + technology: (row.TECHNOLOGIES_LIST as string | null) || null, + courseGroupId: (row.COURSE_GROUP_ID as string | null) || null, + courseCode: (row.COURSE_CODE as string | null) || null, + instructionType, + location: (row.LOCATION as string | null) || null, + }, + } + + const segmentSlug = (row.PROJECT_SLUG as string | null)?.trim() || null + const segmentSourceId = (row.PROJECT_ID as string | null)?.trim() || null + if (!segmentSlug || !segmentSourceId) { + return null + } + + return { activity, segment: { slug: segmentSlug, sourceId: segmentSourceId } } + } + + private buildOrganizations( + row: Record, + ): IActivityData['member']['organizations'] { + const website = (row.ORG_WEBSITE as string | null)?.trim() || null + const domainAliases = (row.ORG_DOMAIN_ALIASES as string | null)?.trim() || null + + if (!website && !domainAliases) { + return undefined + } + + const identities: IOrganizationIdentity[] = [] + + if (website) { + identities.push({ + platform: PlatformType.TNC, + value: website, + type: OrganizationIdentityType.PRIMARY_DOMAIN, + verified: true, + }) + } + + if (domainAliases) { + for (const alias of domainAliases.split(',')) { + const trimmed = alias.trim() + if (trimmed) { + identities.push({ + platform: PlatformType.TNC, + value: trimmed, + type: OrganizationIdentityType.ALTERNATIVE_DOMAIN, + verified: true, + }) + } + } + } + + return [ + { + displayName: (row.ORGANIZATION_NAME as string | null)?.trim() || website, + source: OrganizationSource.TNC, + identities, + logo: (row.LOGO_URL as string | null)?.trim() || undefined, + size: (row.ORGANIZATION_SIZE as string | null)?.trim() || undefined, + industry: (row.ORGANIZATION_INDUSTRY as string | null)?.trim() || undefined, + }, + ] + } +} diff --git a/services/apps/snowflake_connectors/src/integrations/types.ts b/services/apps/snowflake_connectors/src/integrations/types.ts index 01208e1783..53db366c16 100644 --- a/services/apps/snowflake_connectors/src/integrations/types.ts +++ b/services/apps/snowflake_connectors/src/integrations/types.ts @@ -5,6 +5,9 @@ export type BuildSourceQuery = (sinceTimestamp?: string) => string // Each data source maps to a distinct Snowflake table (or joined set of tables) that is exported and transformed independently. export enum DataSourceName { CVENT_EVENT_REGISTRATIONS = 'event-registrations', + TNC_ENROLLMENTS = 'enrollments', + TNC_CERTIFICATES = 'certificates', + TNC_COURSE_ACTIONS = 'course-actions', } export interface DataSource { From 4e3cbe00521bd62261522d09f5af8b89fd6b80ea Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Wed, 4 Mar 2026 17:55:38 +0000 Subject: [PATCH 02/18] feat: tnc connector implementation --- .../V1772556158__addTncActivityTypes.sql | 6 ++++++ .../libs/integrations/src/integrations/index.ts | 2 ++ .../integrations/src/integrations/tnc/types.ts | 17 +++++++++++++++++ services/libs/types/src/enums/organizations.ts | 2 ++ 4 files changed, 27 insertions(+) create mode 100644 backend/src/database/migrations/V1772556158__addTncActivityTypes.sql create mode 100644 services/libs/integrations/src/integrations/tnc/types.ts diff --git a/backend/src/database/migrations/V1772556158__addTncActivityTypes.sql b/backend/src/database/migrations/V1772556158__addTncActivityTypes.sql new file mode 100644 index 0000000000..790fa31fc3 --- /dev/null +++ b/backend/src/database/migrations/V1772556158__addTncActivityTypes.sql @@ -0,0 +1,6 @@ +INSERT INTO "activityTypes" ("activityType", platform, "isCodeContribution", "isCollaboration", description, "label") VALUES +('enrolled-certification', 'tnc', false, false, 'Successful payment purchase of certification enrollment', 'Enrolled in certification'), +('enrolled-training', 'tnc', false, false, 'Successful payment purchase of training enrollment', 'Enrolled in training'), +('issued-certification', 'tnc', false, false, 'User is granted a certification', 'Issued certification'), +('attempted-course', 'tnc', false, false, 'Certification course is completed', 'Attempted course'), +('attempted-exam', 'tnc', false, false, 'Certification exam is completed', 'Attempted exam'); diff --git a/services/libs/integrations/src/integrations/index.ts b/services/libs/integrations/src/integrations/index.ts index f58014ec98..3db691cd2b 100644 --- a/services/libs/integrations/src/integrations/index.ts +++ b/services/libs/integrations/src/integrations/index.ts @@ -50,4 +50,6 @@ export * from './groupsio/memberAttributes' export * from './cvent/types' +export * from './tnc/types' + export * from './activityDisplayService' diff --git a/services/libs/integrations/src/integrations/tnc/types.ts b/services/libs/integrations/src/integrations/tnc/types.ts new file mode 100644 index 0000000000..5c97ad6c02 --- /dev/null +++ b/services/libs/integrations/src/integrations/tnc/types.ts @@ -0,0 +1,17 @@ +import { IActivityScoringGrid } from '@crowd/types' + +export enum TncActivityType { + ENROLLED_CERTIFICATION = 'enrolled-certification', + ENROLLED_TRAINING = 'enrolled-training', + ISSUED_CERTIFICATION = 'issued-certification', + ATTEMPTED_COURSE = 'attempted-course', + ATTEMPTED_EXAM = 'attempted-exam', +} + +export const TNC_GRID: Record = { + [TncActivityType.ENROLLED_CERTIFICATION]: { score: 1 }, + [TncActivityType.ENROLLED_TRAINING]: { score: 1 }, + [TncActivityType.ISSUED_CERTIFICATION]: { score: 1 }, + [TncActivityType.ATTEMPTED_COURSE]: { score: 1 }, + [TncActivityType.ATTEMPTED_EXAM]: { score: 1 }, +} diff --git a/services/libs/types/src/enums/organizations.ts b/services/libs/types/src/enums/organizations.ts index 24dd1cceeb..3dc62f28dc 100644 --- a/services/libs/types/src/enums/organizations.ts +++ b/services/libs/types/src/enums/organizations.ts @@ -13,6 +13,7 @@ export enum OrganizationSource { GITHUB = 'github', UI = 'ui', CVENT = 'cvent', + TNC = 'tnc', } export enum OrganizationMergeSuggestionType { @@ -40,6 +41,7 @@ export enum OrganizationAttributeSource { ENRICHMENT_LFX_INTERNAL_API = 'enrichment-lfx-internal-api', ENRICHMENT_PEOPLEDATALABS = 'enrichment-peopledatalabs', CVENT = 'cvent', + TNC = 'tnc', // legacy - keeping this for backward compatibility ENRICHMENT = 'enrichment', GITHUB = 'github', From 3a96f5260c02cbe7c5f5276b28f3f125c199377f Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Thu, 5 Mar 2026 15:27:38 +0000 Subject: [PATCH 03/18] feat: finalize queries & transformations --- .../src/integrations/index.ts | 10 +- .../tnc/certificates/buildSourceQuery.ts | 10 +- .../tnc/certificates/transformer.ts | 1 - .../tnc/course-actions/buildSourceQuery.ts | 50 ------ .../tnc/course-actions/transformer.ts | 68 -------- .../tnc/courses/buildSourceQuery.ts | 113 ++++++++++++ .../integrations/tnc/courses/transformer.ts | 164 ++++++++++++++++++ .../tnc/enrollments/buildSourceQuery.ts | 36 +++- .../tnc/enrollments/transformer.ts | 5 +- .../src/integrations/types.ts | 2 +- 10 files changed, 329 insertions(+), 130 deletions(-) delete mode 100644 services/apps/snowflake_connectors/src/integrations/tnc/course-actions/buildSourceQuery.ts delete mode 100644 services/apps/snowflake_connectors/src/integrations/tnc/course-actions/transformer.ts create mode 100644 services/apps/snowflake_connectors/src/integrations/tnc/courses/buildSourceQuery.ts create mode 100644 services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts diff --git a/services/apps/snowflake_connectors/src/integrations/index.ts b/services/apps/snowflake_connectors/src/integrations/index.ts index aced040958..7adc90bec0 100644 --- a/services/apps/snowflake_connectors/src/integrations/index.ts +++ b/services/apps/snowflake_connectors/src/integrations/index.ts @@ -10,8 +10,8 @@ import { buildSourceQuery as cventBuildSourceQuery } from './cvent/event-registr import { CventTransformer } from './cvent/event-registrations/transformer' import { buildSourceQuery as tncCertificatesBuildQuery } from './tnc/certificates/buildSourceQuery' import { TncCertificatesTransformer } from './tnc/certificates/transformer' -import { buildSourceQuery as tncCourseActionsBuildQuery } from './tnc/course-actions/buildSourceQuery' -import { TncCourseActionsTransformer } from './tnc/course-actions/transformer' +import { buildSourceQuery as tncCoursesBuildQuery } from './tnc/courses/buildSourceQuery' +import { TncCoursesTransformer } from './tnc/courses/transformer' import { buildSourceQuery as tncEnrollmentsBuildQuery } from './tnc/enrollments/buildSourceQuery' import { TncEnrollmentsTransformer } from './tnc/enrollments/transformer' import { DataSource, DataSourceName, PlatformDefinition } from './types' @@ -42,9 +42,9 @@ const supported: Partial> = { transformer: new TncCertificatesTransformer(), }, { - name: DataSourceName.TNC_COURSE_ACTIONS, - buildSourceQuery: tncCourseActionsBuildQuery, - transformer: new TncCourseActionsTransformer(), + name: DataSourceName.TNC_COURSES, + buildSourceQuery: tncCoursesBuildQuery, + transformer: new TncCoursesTransformer(), }, ], }, diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/certificates/buildSourceQuery.ts b/services/apps/snowflake_connectors/src/integrations/tnc/certificates/buildSourceQuery.ts index 16d2370254..ab4045de49 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/certificates/buildSourceQuery.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/certificates/buildSourceQuery.ts @@ -1,5 +1,12 @@ import { IS_PROD_ENV } from '@crowd/common' +// Main: analytics.silver_fact.certificates (certificate data) +// Joins: +// - analytics.silver_dim._crowd_dev_segments_union (segment resolution) +// - analytics.bronze_fivetran_salesforce.bronze_salesforce_merged_user (LFID) +// - analytics.silver_dim.users (LFID fallback) +// - analytics.bronze_fivetran_salesforce.accounts + analytics.bronze_fivetran_salesforce_b2b.accounts (org data) + const CDP_MATCHED_SEGMENTS = ` cdp_matched_segments AS ( SELECT DISTINCT @@ -47,8 +54,7 @@ export const buildSourceQuery = (sinceTimestamp?: string): string => { AND u.lf_username IS NOT NULL LEFT JOIN org_accounts org ON c.account_id = org.account_id - WHERE c.user_email IS NOT NULL - AND c.is_deleted = false` + WHERE c.user_email IS NOT NULL` if (!IS_PROD_ENV) { select += ` AND cms.slug = 'pytorch'` diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/certificates/transformer.ts b/services/apps/snowflake_connectors/src/integrations/tnc/certificates/transformer.ts index beae20ae30..accd5627c8 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/certificates/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/certificates/transformer.ts @@ -88,7 +88,6 @@ export class TncCertificatesTransformer extends TransformerBase { }, attributes: { productName: (row.COURSE_NAME as string | null) || null, - productType: 'Certification', technology: (row.TECHNOLOGIES_LIST as string | null) || null, didExpire: row.DID_EXPIRE as boolean | null, expirationDate: (row.EXPIRATION_DATE as string | null) || null, diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/course-actions/buildSourceQuery.ts b/services/apps/snowflake_connectors/src/integrations/tnc/course-actions/buildSourceQuery.ts deleted file mode 100644 index 2d7e0eb593..0000000000 --- a/services/apps/snowflake_connectors/src/integrations/tnc/course-actions/buildSourceQuery.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { IS_PROD_ENV } from '@crowd/common' - -// TODO: user resolution — course_actions only has INTERNAL_TI_USER_ID. -// Need to identify the join table that maps INTERNAL_TI_USER_ID to user_email, learner_name, account_id, etc. - -const CDP_MATCHED_SEGMENTS = ` - cdp_matched_segments AS ( - SELECT DISTINCT - s.SOURCE_ID AS sourceId, - s.slug - FROM ANALYTICS.SILVER_DIM._CROWD_DEV_SEGMENTS_UNION s - WHERE s.PARENT_SLUG IS NOT NULL - AND s.GRANDPARENTS_SLUG IS NOT NULL - AND s.SOURCE_ID IS NOT NULL - )` - -export const buildSourceQuery = (sinceTimestamp?: string): string => { - let select = ` - SELECT - ca.course_action_id, - ca.course_id, - ca.timestamp, - ca.type, - ca.source, - ca.internal_ti_user_id, - co.title AS COURSE_NAME, - co.course_group_id, - co.slug AS COURSE_SLUG, - co.instruction_type, - co.product_type, - co.is_training, - co.is_certification - FROM analytics.bronze_census_ti.course_actions ca - INNER JOIN analytics.bronze_census_ti.courses co - ON ca.course_id = co.course_id - WHERE ca.type = 'status_change' - AND ca.source = 'course_started' - AND co.is_test_or_archived = false` - - if (!IS_PROD_ENV) { - select += ` AND 1=1` // TODO: add non-prod project filter once segment join is available - } - - if (!sinceTimestamp) { - return select.trim() - } - - return `${select} - AND ca.timestamp >= '${sinceTimestamp}'`.trim() -} diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/course-actions/transformer.ts b/services/apps/snowflake_connectors/src/integrations/tnc/course-actions/transformer.ts deleted file mode 100644 index 718b279b6f..0000000000 --- a/services/apps/snowflake_connectors/src/integrations/tnc/course-actions/transformer.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { TNC_GRID, TncActivityType } from '@crowd/integrations' -import { getServiceChildLogger } from '@crowd/logging' -import { IActivityData, PlatformType } from '@crowd/types' - -import { TransformedActivity, TransformerBase } from '../../../core/transformerBase' - -const log = getServiceChildLogger('tncCourseActionsTransformer') - -// TODO: user resolution — course_actions only has INTERNAL_TI_USER_ID. -// Once the user join table is identified, add email/name/org resolution here. - -export class TncCourseActionsTransformer extends TransformerBase { - readonly platform = PlatformType.TNC - - transformRow(row: Record): TransformedActivity | null { - const courseActionId = (row.COURSE_ACTION_ID as string | null)?.trim() || null - if (!courseActionId) { - log.debug('Skipping row: missing course_action_id') - return null - } - - const internalUserId = (row.INTERNAL_TI_USER_ID as string | null)?.trim() || null - if (!internalUserId) { - log.debug({ courseActionId }, 'Skipping row: missing internal_ti_user_id') - return null - } - - const isCertification = row.IS_CERTIFICATION === true - const type = isCertification - ? TncActivityType.ATTEMPTED_EXAM - : TncActivityType.ATTEMPTED_COURSE - - // TODO: replace with actual email/name once user resolution join is available - const activity: IActivityData = { - type, - platform: PlatformType.TNC, - timestamp: (row.TIMESTAMP as string | null) || null, - score: TNC_GRID[type].score, - sourceId: courseActionId, - sourceParentId: (row.COURSE_ID as string | null) || undefined, - member: { - displayName: internalUserId, - identities: [ - { - platform: PlatformType.TNC, - value: internalUserId, - type: 'username' as never, - verified: false, - sourceId: internalUserId, - }, - ], - }, - attributes: { - courseName: (row.COURSE_NAME as string | null) || null, - courseSlug: (row.COURSE_SLUG as string | null) || null, - instructionType: (row.INSTRUCTION_TYPE as string | null) || null, - productType: (row.PRODUCT_TYPE as string | null) || null, - isCertification, - isTraining: row.IS_TRAINING === true, - }, - } - - // TODO: segment resolution — course_actions doesn't have PROJECT_SLUG/PROJECT_ID. - // Need to determine how to map courses to projects/segments. - log.debug({ courseActionId }, 'Skipping row: segment resolution not yet implemented') - return null - } -} diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/courses/buildSourceQuery.ts b/services/apps/snowflake_connectors/src/integrations/tnc/courses/buildSourceQuery.ts new file mode 100644 index 0000000000..ff0d6117af --- /dev/null +++ b/services/apps/snowflake_connectors/src/integrations/tnc/courses/buildSourceQuery.ts @@ -0,0 +1,113 @@ +import { IS_PROD_ENV } from '@crowd/common' + +// Main: analytics.bronze_census_ti.course_actions (course action data) +// Joins: +// - analytics.bronze_census_ti.users (user resolution via internal_ti_user_id) +// - analytics.bronze_census_ti.courses (course metadata) +// - analytics.silver_fact.enrollments (segment + org resolution via email + course_id) +// - analytics.silver_dim._crowd_dev_segments_union (segment resolution) +// - analytics.bronze_fivetran_salesforce.accounts + analytics.bronze_fivetran_salesforce_b2b.accounts (org data) + +const CDP_MATCHED_SEGMENTS = ` + cdp_matched_segments AS ( + SELECT DISTINCT + s.SOURCE_ID AS sourceId, + s.slug + FROM ANALYTICS.SILVER_DIM._CROWD_DEV_SEGMENTS_UNION s + WHERE s.PARENT_SLUG IS NOT NULL + AND s.GRANDPARENTS_SLUG IS NOT NULL + AND s.SOURCE_ID IS NOT NULL + )` + +const ORG_ACCOUNTS = ` + org_accounts AS ( + SELECT account_id, account_name, website, domain_aliases, LOGO_URL, INDUSTRY, N_EMPLOYEES + FROM analytics.bronze_fivetran_salesforce.accounts + WHERE website IS NOT NULL + UNION ALL + SELECT account_id, account_name, website, domain_aliases, NULL AS LOGO_URL, NULL AS INDUSTRY, NULL AS N_EMPLOYEES + FROM analytics.bronze_fivetran_salesforce_b2b.accounts + WHERE website IS NOT NULL + )` + +export const buildSourceQuery = (sinceTimestamp?: string): string => { + let select = ` + SELECT + ca.*, + co.*, + tu.user_email, + tu.lfid, + tu.learner_name, + tu.user_country, + tu.job_title, + e.project_slug AS PROJECT_SLUG, + e.project_id AS PROJECT_ID, + e.account_id, + org.account_name AS ORGANIZATION_NAME, + org.website AS ORG_WEBSITE, + org.domain_aliases AS ORG_DOMAIN_ALIASES, + org.logo_url AS LOGO_URL, + org.industry AS ORGANIZATION_INDUSTRY, + org.n_employees AS ORGANIZATION_SIZE + FROM analytics.bronze_census_ti.course_actions ca + INNER JOIN analytics.bronze_census_ti.users tu + ON ca.internal_ti_user_id = tu.internal_ti_user_id + INNER JOIN analytics.bronze_census_ti.courses co + ON ca.course_id = co.course_id + INNER JOIN analytics.silver_fact.enrollments e + ON e.course_id = ca.course_id + AND LOWER(e.user_email) = LOWER(tu.user_email) + INNER JOIN cdp_matched_segments cms + ON cms.slug = e.project_slug + AND cms.sourceId = e.project_id + LEFT JOIN org_accounts org + ON e.account_id = org.account_id + WHERE ca.type = 'status_change' + AND ca.source = 'course_started' + AND co.is_test_or_archived = false + AND tu.user_email IS NOT NULL` + + if (!IS_PROD_ENV) { + select += ` AND e.project_slug = 'pytorch'` + } + + const dedup = ` + QUALIFY ROW_NUMBER() OVER (PARTITION BY ca.course_action_id ORDER BY org.website DESC) = 1` + + if (!sinceTimestamp) { + return ` + WITH ${ORG_ACCOUNTS}, + ${CDP_MATCHED_SEGMENTS} + ${select} + ${dedup}`.trim() + } + + return ` + WITH ${ORG_ACCOUNTS}, + ${CDP_MATCHED_SEGMENTS}, + new_cdp_segments AS ( + SELECT DISTINCT + s.SOURCE_ID AS sourceId, + s.slug + FROM ANALYTICS.SILVER_DIM._CROWD_DEV_SEGMENTS_UNION s + WHERE s.CREATED_TS >= '${sinceTimestamp}' + AND s.PARENT_SLUG IS NOT NULL + AND s.GRANDPARENTS_SLUG IS NOT NULL + AND s.SOURCE_ID IS NOT NULL + ) + + -- New course actions since last export + ${select} + AND ca.timestamp >= '${sinceTimestamp}' + ${dedup} + + UNION + + -- All course actions in newly created segments + ${select} + AND EXISTS ( + SELECT 1 FROM new_cdp_segments ncs + WHERE ncs.slug = cms.slug AND ncs.sourceId = cms.sourceId + ) + ${dedup}`.trim() +} diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts b/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts new file mode 100644 index 0000000000..ab3b2875fd --- /dev/null +++ b/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts @@ -0,0 +1,164 @@ +import { TNC_GRID, TncActivityType } from '@crowd/integrations' +import { getServiceChildLogger } from '@crowd/logging' +import { + IActivityData, + IMemberData, + IOrganizationIdentity, + MemberAttributeName, + MemberIdentityType, + OrganizationIdentityType, + OrganizationSource, + PlatformType, +} from '@crowd/types' + +import { TransformedActivity, TransformerBase } from '../../../core/transformerBase' + +const log = getServiceChildLogger('tncCoursesTransformer') + +export class TncCoursesTransformer extends TransformerBase { + readonly platform = PlatformType.TNC + + transformRow(row: Record): TransformedActivity | null { + const email = (row.USER_EMAIL as string | null)?.trim() || null + if (!email) { + log.debug({ courseActionId: row.COURSE_ACTION_ID }, 'Skipping row: missing email') + return null + } + + const courseActionId = (row.COURSE_ACTION_ID as string | null)?.trim() || null + if (!courseActionId) { + return null + } + + const learnerName = (row.LEARNER_NAME as string | null)?.trim() || null + const lfUsername = (row.LFID as string | null)?.trim() || null + + const identities: IMemberData['identities'] = [] + const sourceId = undefined + + if (lfUsername) { + identities.push( + { + platform: PlatformType.TNC, + value: email, + type: MemberIdentityType.EMAIL, + verified: true, + sourceId, + }, + { + platform: PlatformType.TNC, + value: lfUsername, + type: MemberIdentityType.USERNAME, + verified: true, + sourceId, + }, + { + platform: PlatformType.LFID, + value: lfUsername, + type: MemberIdentityType.USERNAME, + verified: true, + sourceId, + }, + ) + } else { + identities.push({ + platform: PlatformType.TNC, + value: email, + type: MemberIdentityType.USERNAME, + verified: true, + sourceId, + }) + } + + const isCertification = row.IS_CERTIFICATION === true + const type = isCertification + ? TncActivityType.ATTEMPTED_EXAM + : TncActivityType.ATTEMPTED_COURSE + + const activity: IActivityData = { + type, + platform: PlatformType.TNC, + timestamp: (row.ACTION_TIMESTAMP as string | null) || null, + score: TNC_GRID[type].score, + sourceId: courseActionId, + sourceParentId: (row.COURSE_ID as string | null) || undefined, + member: { + displayName: learnerName || email, + identities, + organizations: this.buildOrganizations(row), + attributes: { + ...((row.JOB_TITLE as string | null) && { + [MemberAttributeName.JOB_TITLE]: { [PlatformType.TNC]: row.JOB_TITLE as string }, + }), + ...((row.USER_COUNTRY as string | null) && { + [MemberAttributeName.COUNTRY]: { [PlatformType.TNC]: row.USER_COUNTRY as string }, + }), + }, + }, + attributes: { + productName: (row.COURSE_NAME as string | null) || null, + productType: (row.PRODUCT_TYPE as string | null) || null, + parentProduct: (row.COURSE_GROUP_ID as string | null) || null, + courseSlug: (row.COURSE_SLUG as string | null) || null, + instructionType: (row.INSTRUCTION_TYPE as string | null) || null, + isCertification, + isTraining: row.IS_TRAINING === true, + }, + } + + const segmentSlug = (row.PROJECT_SLUG as string | null)?.trim() || null + const segmentSourceId = (row.PROJECT_ID as string | null)?.trim() || null + if (!segmentSlug || !segmentSourceId) { + return null + } + + return { activity, segment: { slug: segmentSlug, sourceId: segmentSourceId } } + } + + private buildOrganizations( + row: Record, + ): IActivityData['member']['organizations'] { + const website = (row.ORG_WEBSITE as string | null)?.trim() || null + const domainAliases = (row.ORG_DOMAIN_ALIASES as string | null)?.trim() || null + + if (!website && !domainAliases) { + return undefined + } + + const identities: IOrganizationIdentity[] = [] + + if (website) { + identities.push({ + platform: PlatformType.TNC, + value: website, + type: OrganizationIdentityType.PRIMARY_DOMAIN, + verified: true, + }) + } + + if (domainAliases) { + for (const alias of domainAliases.split(',')) { + const trimmed = alias.trim() + if (trimmed) { + identities.push({ + platform: PlatformType.TNC, + value: trimmed, + type: OrganizationIdentityType.ALTERNATIVE_DOMAIN, + verified: true, + }) + } + } + } + + return [ + { + displayName: (row.ORGANIZATION_NAME as string | null)?.trim() || website, + source: OrganizationSource.TNC, + identities, + logo: (row.LOGO_URL as string | null)?.trim() || undefined, + size: (row.ORGANIZATION_SIZE as string | null)?.trim() || undefined, + industry: (row.ORGANIZATION_INDUSTRY as string | null)?.trim() || undefined, + }, + ] + } +} diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/buildSourceQuery.ts b/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/buildSourceQuery.ts index 4f0fa2eeca..682ca8d7e0 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/buildSourceQuery.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/buildSourceQuery.ts @@ -1,5 +1,13 @@ import { IS_PROD_ENV } from '@crowd/common' +// Main: analytics.silver_fact.enrollments (enrollment data) +// Joins: +// - analytics.silver_dim._crowd_dev_segments_union (segment resolution) +// - analytics.bronze_fivetran_salesforce.bronze_salesforce_merged_user (LFID) +// - analytics.silver_dim.users (LFID fallback) +// - analytics.bronze_fivetran_salesforce.accounts + analytics.bronze_fivetran_salesforce_b2b.accounts (org data) +// - analytics.bronze_census_ti.course_actions + analytics.bronze_census_ti.users (course status) + const CDP_MATCHED_SEGMENTS = ` cdp_matched_segments AS ( SELECT DISTINCT @@ -24,6 +32,22 @@ const ORG_ACCOUNTS = ` const LFID_COALESCE = `COALESCE(mu.user_name, u.lf_username)` +const COURSE_STATUS = ` + course_status AS ( + SELECT + tu.user_email, + ca.course_id, + MAX(CASE WHEN ca.source = 'course_started' THEN ca.timestamp END) AS COURSE_STARTED_DATE, + MAX(CASE WHEN ca.source = 'course_completed' THEN ca.timestamp END) AS COURSE_COMPLETED_DATE, + MAX_BY(ca.source, ca.timestamp) AS COURSE_STATUS + FROM analytics.bronze_census_ti.course_actions ca + INNER JOIN analytics.bronze_census_ti.users tu + ON ca.internal_ti_user_id = tu.internal_ti_user_id + WHERE ca.type = 'status_change' + AND ca.source IN ('course_started', 'course_completed') + GROUP BY tu.user_email, ca.course_id + )` + export const buildSourceQuery = (sinceTimestamp?: string): string => { let select = ` SELECT @@ -34,7 +58,10 @@ export const buildSourceQuery = (sinceTimestamp?: string): string => { org.logo_url AS LOGO_URL, org.industry AS ORGANIZATION_INDUSTRY, org.n_employees AS ORGANIZATION_SIZE, - ${LFID_COALESCE} AS LFID + ${LFID_COALESCE} AS LFID, + cs.COURSE_STARTED_DATE, + cs.COURSE_COMPLETED_DATE, + cs.COURSE_STATUS FROM analytics.silver_fact.enrollments e INNER JOIN cdp_matched_segments cms ON cms.slug = e.project_slug @@ -47,6 +74,9 @@ export const buildSourceQuery = (sinceTimestamp?: string): string => { AND u.lf_username IS NOT NULL LEFT JOIN org_accounts org ON e.account_id = org.account_id + LEFT JOIN course_status cs + ON LOWER(e.user_email) = LOWER(cs.user_email) + AND e.course_id = cs.course_id WHERE e.user_email IS NOT NULL` if (!IS_PROD_ENV) { @@ -59,7 +89,8 @@ export const buildSourceQuery = (sinceTimestamp?: string): string => { if (!sinceTimestamp) { return ` WITH ${ORG_ACCOUNTS}, - ${CDP_MATCHED_SEGMENTS} + ${CDP_MATCHED_SEGMENTS}, + ${COURSE_STATUS} ${select} ${dedup}`.trim() } @@ -67,6 +98,7 @@ export const buildSourceQuery = (sinceTimestamp?: string): string => { return ` WITH ${ORG_ACCOUNTS}, ${CDP_MATCHED_SEGMENTS}, + ${COURSE_STATUS}, new_cdp_segments AS ( SELECT DISTINCT s.SOURCE_ID AS sourceId, diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/transformer.ts b/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/transformer.ts index 3d313559b4..46196174be 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/transformer.ts @@ -109,10 +109,13 @@ export class TncEnrollmentsTransformer extends TransformerBase { productName: (row.COURSE_NAME as string | null) || null, productType, technology: (row.TECHNOLOGIES_LIST as string | null) || null, - courseGroupId: (row.COURSE_GROUP_ID as string | null) || null, + parentProduct: (row.COURSE_GROUP_ID as string | null) || null, courseCode: (row.COURSE_CODE as string | null) || null, instructionType, location: (row.LOCATION as string | null) || null, + courseStatus: (row.COURSE_STATUS as string | null) || null, + courseStartedDate: (row.COURSE_STARTED_DATE as string | null) || null, + courseCompletedDate: (row.COURSE_COMPLETED_DATE as string | null) || null, }, } diff --git a/services/apps/snowflake_connectors/src/integrations/types.ts b/services/apps/snowflake_connectors/src/integrations/types.ts index 53db366c16..36377a4f4f 100644 --- a/services/apps/snowflake_connectors/src/integrations/types.ts +++ b/services/apps/snowflake_connectors/src/integrations/types.ts @@ -7,7 +7,7 @@ export enum DataSourceName { CVENT_EVENT_REGISTRATIONS = 'event-registrations', TNC_ENROLLMENTS = 'enrollments', TNC_CERTIFICATES = 'certificates', - TNC_COURSE_ACTIONS = 'course-actions', + TNC_COURSES = 'courses', } export interface DataSource { From 8595dfa8cd7181dd7a3d64eccc7ee12aeea463a0 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Thu, 5 Mar 2026 15:28:12 +0000 Subject: [PATCH 04/18] chore: format --- .../src/integrations/tnc/courses/transformer.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts b/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts index ab3b2875fd..689dfaddde 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts @@ -71,9 +71,7 @@ export class TncCoursesTransformer extends TransformerBase { } const isCertification = row.IS_CERTIFICATION === true - const type = isCertification - ? TncActivityType.ATTEMPTED_EXAM - : TncActivityType.ATTEMPTED_COURSE + const type = isCertification ? TncActivityType.ATTEMPTED_EXAM : TncActivityType.ATTEMPTED_COURSE const activity: IActivityData = { type, From 3a22b7451810213c985dd6d76d834c730e1b0934 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Thu, 5 Mar 2026 16:04:25 +0000 Subject: [PATCH 05/18] fix: timestamp column name --- .../src/integrations/tnc/courses/transformer.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts b/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts index 689dfaddde..0704c6bf5c 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts @@ -76,7 +76,7 @@ export class TncCoursesTransformer extends TransformerBase { const activity: IActivityData = { type, platform: PlatformType.TNC, - timestamp: (row.ACTION_TIMESTAMP as string | null) || null, + timestamp: (row.TIMESTAMP as string | null) || null, score: TNC_GRID[type].score, sourceId: courseActionId, sourceParentId: (row.COURSE_ID as string | null) || undefined, From 678edcc3686a1d3b826b5760c8b89fae6fb526d1 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Thu, 5 Mar 2026 16:26:07 +0000 Subject: [PATCH 06/18] fix: avoid buildOrg duplication --- .../tnc/certificates/transformer.ts | 57 +----------------- .../integrations/tnc/courses/transformer.ts | 57 +----------------- .../tnc/enrollments/transformer.ts | 57 +----------------- .../integrations/tnc/tncTransformerBase.ts | 60 +++++++++++++++++++ 4 files changed, 69 insertions(+), 162 deletions(-) create mode 100644 services/apps/snowflake_connectors/src/integrations/tnc/tncTransformerBase.ts diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/certificates/transformer.ts b/services/apps/snowflake_connectors/src/integrations/tnc/certificates/transformer.ts index accd5627c8..b5846ce507 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/certificates/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/certificates/transformer.ts @@ -3,21 +3,17 @@ import { getServiceChildLogger } from '@crowd/logging' import { IActivityData, IMemberData, - IOrganizationIdentity, MemberAttributeName, MemberIdentityType, - OrganizationIdentityType, - OrganizationSource, PlatformType, } from '@crowd/types' -import { TransformedActivity, TransformerBase } from '../../../core/transformerBase' +import { TransformedActivity } from '../../../core/transformerBase' +import { TncTransformerBase } from '../tncTransformerBase' const log = getServiceChildLogger('tncCertificatesTransformer') -export class TncCertificatesTransformer extends TransformerBase { - readonly platform = PlatformType.TNC - +export class TncCertificatesTransformer extends TncTransformerBase { transformRow(row: Record): TransformedActivity | null { const email = (row.USER_EMAIL as string | null)?.trim() || null if (!email) { @@ -102,51 +98,4 @@ export class TncCertificatesTransformer extends TransformerBase { return { activity, segment: { slug: segmentSlug, sourceId: segmentSourceId } } } - - private buildOrganizations( - row: Record, - ): IActivityData['member']['organizations'] { - const website = (row.ORG_WEBSITE as string | null)?.trim() || null - const domainAliases = (row.ORG_DOMAIN_ALIASES as string | null)?.trim() || null - - if (!website && !domainAliases) { - return undefined - } - - const identities: IOrganizationIdentity[] = [] - - if (website) { - identities.push({ - platform: PlatformType.TNC, - value: website, - type: OrganizationIdentityType.PRIMARY_DOMAIN, - verified: true, - }) - } - - if (domainAliases) { - for (const alias of domainAliases.split(',')) { - const trimmed = alias.trim() - if (trimmed) { - identities.push({ - platform: PlatformType.TNC, - value: trimmed, - type: OrganizationIdentityType.ALTERNATIVE_DOMAIN, - verified: true, - }) - } - } - } - - return [ - { - displayName: (row.ORGANIZATION_NAME as string | null)?.trim() || website, - source: OrganizationSource.TNC, - identities, - logo: (row.LOGO_URL as string | null)?.trim() || undefined, - size: (row.ORGANIZATION_SIZE as string | null)?.trim() || undefined, - industry: (row.ORGANIZATION_INDUSTRY as string | null)?.trim() || undefined, - }, - ] - } } diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts b/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts index 0704c6bf5c..ecb32a4c89 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts @@ -3,21 +3,17 @@ import { getServiceChildLogger } from '@crowd/logging' import { IActivityData, IMemberData, - IOrganizationIdentity, MemberAttributeName, MemberIdentityType, - OrganizationIdentityType, - OrganizationSource, PlatformType, } from '@crowd/types' -import { TransformedActivity, TransformerBase } from '../../../core/transformerBase' +import { TransformedActivity } from '../../../core/transformerBase' +import { TncTransformerBase } from '../tncTransformerBase' const log = getServiceChildLogger('tncCoursesTransformer') -export class TncCoursesTransformer extends TransformerBase { - readonly platform = PlatformType.TNC - +export class TncCoursesTransformer extends TncTransformerBase { transformRow(row: Record): TransformedActivity | null { const email = (row.USER_EMAIL as string | null)?.trim() || null if (!email) { @@ -112,51 +108,4 @@ export class TncCoursesTransformer extends TransformerBase { return { activity, segment: { slug: segmentSlug, sourceId: segmentSourceId } } } - - private buildOrganizations( - row: Record, - ): IActivityData['member']['organizations'] { - const website = (row.ORG_WEBSITE as string | null)?.trim() || null - const domainAliases = (row.ORG_DOMAIN_ALIASES as string | null)?.trim() || null - - if (!website && !domainAliases) { - return undefined - } - - const identities: IOrganizationIdentity[] = [] - - if (website) { - identities.push({ - platform: PlatformType.TNC, - value: website, - type: OrganizationIdentityType.PRIMARY_DOMAIN, - verified: true, - }) - } - - if (domainAliases) { - for (const alias of domainAliases.split(',')) { - const trimmed = alias.trim() - if (trimmed) { - identities.push({ - platform: PlatformType.TNC, - value: trimmed, - type: OrganizationIdentityType.ALTERNATIVE_DOMAIN, - verified: true, - }) - } - } - } - - return [ - { - displayName: (row.ORGANIZATION_NAME as string | null)?.trim() || website, - source: OrganizationSource.TNC, - identities, - logo: (row.LOGO_URL as string | null)?.trim() || undefined, - size: (row.ORGANIZATION_SIZE as string | null)?.trim() || undefined, - industry: (row.ORGANIZATION_INDUSTRY as string | null)?.trim() || undefined, - }, - ] - } } diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/transformer.ts b/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/transformer.ts index 46196174be..291816dcb2 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/transformer.ts @@ -3,21 +3,17 @@ import { getServiceChildLogger } from '@crowd/logging' import { IActivityData, IMemberData, - IOrganizationIdentity, MemberAttributeName, MemberIdentityType, - OrganizationIdentityType, - OrganizationSource, PlatformType, } from '@crowd/types' -import { TransformedActivity, TransformerBase } from '../../../core/transformerBase' +import { TransformedActivity } from '../../../core/transformerBase' +import { TncTransformerBase } from '../tncTransformerBase' const log = getServiceChildLogger('tncEnrollmentsTransformer') -export class TncEnrollmentsTransformer extends TransformerBase { - readonly platform = PlatformType.TNC - +export class TncEnrollmentsTransformer extends TncTransformerBase { transformRow(row: Record): TransformedActivity | null { const email = (row.USER_EMAIL as string | null)?.trim() || null if (!email) { @@ -127,51 +123,4 @@ export class TncEnrollmentsTransformer extends TransformerBase { return { activity, segment: { slug: segmentSlug, sourceId: segmentSourceId } } } - - private buildOrganizations( - row: Record, - ): IActivityData['member']['organizations'] { - const website = (row.ORG_WEBSITE as string | null)?.trim() || null - const domainAliases = (row.ORG_DOMAIN_ALIASES as string | null)?.trim() || null - - if (!website && !domainAliases) { - return undefined - } - - const identities: IOrganizationIdentity[] = [] - - if (website) { - identities.push({ - platform: PlatformType.TNC, - value: website, - type: OrganizationIdentityType.PRIMARY_DOMAIN, - verified: true, - }) - } - - if (domainAliases) { - for (const alias of domainAliases.split(',')) { - const trimmed = alias.trim() - if (trimmed) { - identities.push({ - platform: PlatformType.TNC, - value: trimmed, - type: OrganizationIdentityType.ALTERNATIVE_DOMAIN, - verified: true, - }) - } - } - } - - return [ - { - displayName: (row.ORGANIZATION_NAME as string | null)?.trim() || website, - source: OrganizationSource.TNC, - identities, - logo: (row.LOGO_URL as string | null)?.trim() || undefined, - size: (row.ORGANIZATION_SIZE as string | null)?.trim() || undefined, - industry: (row.ORGANIZATION_INDUSTRY as string | null)?.trim() || undefined, - }, - ] - } } diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/tncTransformerBase.ts b/services/apps/snowflake_connectors/src/integrations/tnc/tncTransformerBase.ts new file mode 100644 index 0000000000..6dc0dfda2b --- /dev/null +++ b/services/apps/snowflake_connectors/src/integrations/tnc/tncTransformerBase.ts @@ -0,0 +1,60 @@ +import { + IActivityData, + IOrganizationIdentity, + OrganizationIdentityType, + OrganizationSource, + PlatformType, +} from '@crowd/types' + +import { TransformerBase } from '../../core/transformerBase' + +export abstract class TncTransformerBase extends TransformerBase { + readonly platform = PlatformType.TNC + + protected buildOrganizations( + row: Record, + ): IActivityData['member']['organizations'] { + const website = (row.ORG_WEBSITE as string | null)?.trim() || null + const domainAliases = (row.ORG_DOMAIN_ALIASES as string | null)?.trim() || null + + if (!website && !domainAliases) { + return undefined + } + + const identities: IOrganizationIdentity[] = [] + + if (website) { + identities.push({ + platform: PlatformType.TNC, + value: website, + type: OrganizationIdentityType.PRIMARY_DOMAIN, + verified: true, + }) + } + + if (domainAliases) { + for (const alias of domainAliases.split(',')) { + const trimmed = alias.trim() + if (trimmed) { + identities.push({ + platform: PlatformType.TNC, + value: trimmed, + type: OrganizationIdentityType.ALTERNATIVE_DOMAIN, + verified: true, + }) + } + } + } + + return [ + { + displayName: (row.ORGANIZATION_NAME as string | null)?.trim() || website, + source: OrganizationSource.TNC, + identities, + logo: (row.LOGO_URL as string | null)?.trim() || undefined, + size: (row.ORGANIZATION_SIZE as string | null)?.trim() || undefined, + industry: (row.ORGANIZATION_INDUSTRY as string | null)?.trim() || undefined, + }, + ] + } +} From 34dcd63572388ef63673ae719dffc9a246b6491c Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Thu, 5 Mar 2026 17:54:31 +0000 Subject: [PATCH 07/18] fix: transformation errors and add logs --- .../src/core/transformerBase.ts | 12 +++++++++++- .../tnc/certificates/buildSourceQuery.ts | 4 ++-- .../tnc/courses/buildSourceQuery.ts | 4 ++-- .../src/integrations/tnc/courses/transformer.ts | 16 +++++++++++++++- .../tnc/enrollments/buildSourceQuery.ts | 4 ++-- .../integrations/tnc/enrollments/transformer.ts | 17 +++++++++++++++-- .../src/integrations/tnc/tncTransformerBase.ts | 5 ++++- 7 files changed, 51 insertions(+), 11 deletions(-) diff --git a/services/apps/snowflake_connectors/src/core/transformerBase.ts b/services/apps/snowflake_connectors/src/core/transformerBase.ts index cdbeeca284..33caeec635 100644 --- a/services/apps/snowflake_connectors/src/core/transformerBase.ts +++ b/services/apps/snowflake_connectors/src/core/transformerBase.ts @@ -36,7 +36,17 @@ export abstract class TransformerBase { try { return this.transformRow(row) } catch (err) { - log.warn({ err, platform: this.platform }, 'Failed to transform row, skipping') + const message = err instanceof Error ? err.message : String(err) + const stack = err instanceof Error ? err.stack : undefined + log.warn( + { + errMessage: message, + errStack: stack, + platform: this.platform, + rowKeys: Object.keys(row), + }, + 'Failed to transform row, skipping', + ) return null } } diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/certificates/buildSourceQuery.ts b/services/apps/snowflake_connectors/src/integrations/tnc/certificates/buildSourceQuery.ts index ab4045de49..8b1230bb3b 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/certificates/buildSourceQuery.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/certificates/buildSourceQuery.ts @@ -41,7 +41,7 @@ export const buildSourceQuery = (sinceTimestamp?: string): string => { org.domain_aliases AS ORG_DOMAIN_ALIASES, org.logo_url AS LOGO_URL, org.industry AS ORGANIZATION_INDUSTRY, - org.n_employees AS ORGANIZATION_SIZE, + CAST(org.n_employees AS VARCHAR) AS ORGANIZATION_SIZE, ${LFID_COALESCE} AS LFID FROM analytics.silver_fact.certificates c INNER JOIN cdp_matched_segments cms @@ -57,7 +57,7 @@ export const buildSourceQuery = (sinceTimestamp?: string): string => { WHERE c.user_email IS NOT NULL` if (!IS_PROD_ENV) { - select += ` AND cms.slug = 'pytorch'` + select += ` AND cms.slug = 'cncf'` } const dedup = ` diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/courses/buildSourceQuery.ts b/services/apps/snowflake_connectors/src/integrations/tnc/courses/buildSourceQuery.ts index ff0d6117af..1fceb011d6 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/courses/buildSourceQuery.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/courses/buildSourceQuery.ts @@ -48,7 +48,7 @@ export const buildSourceQuery = (sinceTimestamp?: string): string => { org.domain_aliases AS ORG_DOMAIN_ALIASES, org.logo_url AS LOGO_URL, org.industry AS ORGANIZATION_INDUSTRY, - org.n_employees AS ORGANIZATION_SIZE + CAST(org.n_employees AS VARCHAR) AS ORGANIZATION_SIZE FROM analytics.bronze_census_ti.course_actions ca INNER JOIN analytics.bronze_census_ti.users tu ON ca.internal_ti_user_id = tu.internal_ti_user_id @@ -68,7 +68,7 @@ export const buildSourceQuery = (sinceTimestamp?: string): string => { AND tu.user_email IS NOT NULL` if (!IS_PROD_ENV) { - select += ` AND e.project_slug = 'pytorch'` + select += ` AND e.project_slug = 'openssf'` } const dedup = ` diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts b/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts index ecb32a4c89..18bcdf5210 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts @@ -17,12 +17,16 @@ export class TncCoursesTransformer extends TncTransformerBase { transformRow(row: Record): TransformedActivity | null { const email = (row.USER_EMAIL as string | null)?.trim() || null if (!email) { - log.debug({ courseActionId: row.COURSE_ACTION_ID }, 'Skipping row: missing email') + log.warn( + { courseActionId: row.COURSE_ACTION_ID, rawUserEmail: row.USER_EMAIL }, + 'Skipping row: missing email', + ) return null } const courseActionId = (row.COURSE_ACTION_ID as string | null)?.trim() || null if (!courseActionId) { + log.warn('Skipping row: missing courseActionId') return null } @@ -103,6 +107,16 @@ export class TncCoursesTransformer extends TncTransformerBase { const segmentSlug = (row.PROJECT_SLUG as string | null)?.trim() || null const segmentSourceId = (row.PROJECT_ID as string | null)?.trim() || null if (!segmentSlug || !segmentSourceId) { + log.warn( + { + courseActionId, + segmentSlug, + segmentSourceId, + rawProjectSlug: row.PROJECT_SLUG, + rawProjectId: row.PROJECT_ID, + }, + 'Skipping row: missing segment slug or sourceId', + ) return null } diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/buildSourceQuery.ts b/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/buildSourceQuery.ts index 682ca8d7e0..711f0f6a36 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/buildSourceQuery.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/buildSourceQuery.ts @@ -57,7 +57,7 @@ export const buildSourceQuery = (sinceTimestamp?: string): string => { org.domain_aliases AS ORG_DOMAIN_ALIASES, org.logo_url AS LOGO_URL, org.industry AS ORGANIZATION_INDUSTRY, - org.n_employees AS ORGANIZATION_SIZE, + CAST(org.n_employees AS VARCHAR) AS ORGANIZATION_SIZE, ${LFID_COALESCE} AS LFID, cs.COURSE_STARTED_DATE, cs.COURSE_COMPLETED_DATE, @@ -80,7 +80,7 @@ export const buildSourceQuery = (sinceTimestamp?: string): string => { WHERE e.user_email IS NOT NULL` if (!IS_PROD_ENV) { - select += ` AND e.project_slug = 'pytorch'` + select += ` AND e.project_slug = 'openssf'` } const dedup = ` diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/transformer.ts b/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/transformer.ts index 291816dcb2..126f721022 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/transformer.ts @@ -17,7 +17,10 @@ export class TncEnrollmentsTransformer extends TncTransformerBase { transformRow(row: Record): TransformedActivity | null { const email = (row.USER_EMAIL as string | null)?.trim() || null if (!email) { - log.debug({ enrollmentId: row.ENROLLMENT_ID }, 'Skipping row: missing email') + log.warn( + { enrollmentId: row.ENROLLMENT_ID, rawUserEmail: row.USER_EMAIL }, + 'Skipping row: missing email', + ) return null } @@ -74,7 +77,7 @@ export class TncEnrollmentsTransformer extends TncTransformerBase { } else if (productType?.toLowerCase() === 'training') { type = TncActivityType.ENROLLED_TRAINING } else { - log.debug( + log.warn( { enrollmentId, productType, instructionType }, 'Skipping row: unrecognized product/instruction type', ) @@ -118,6 +121,16 @@ export class TncEnrollmentsTransformer extends TncTransformerBase { const segmentSlug = (row.PROJECT_SLUG as string | null)?.trim() || null const segmentSourceId = (row.PROJECT_ID as string | null)?.trim() || null if (!segmentSlug || !segmentSourceId) { + log.warn( + { + enrollmentId, + segmentSlug, + segmentSourceId, + rawProjectSlug: row.PROJECT_SLUG, + rawProjectId: row.PROJECT_ID, + }, + 'Skipping row: missing segment slug or sourceId', + ) return null } diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/tncTransformerBase.ts b/services/apps/snowflake_connectors/src/integrations/tnc/tncTransformerBase.ts index 6dc0dfda2b..e8350dc7e1 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/tncTransformerBase.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/tncTransformerBase.ts @@ -52,7 +52,10 @@ export abstract class TncTransformerBase extends TransformerBase { source: OrganizationSource.TNC, identities, logo: (row.LOGO_URL as string | null)?.trim() || undefined, - size: (row.ORGANIZATION_SIZE as string | null)?.trim() || undefined, + size: + typeof row.ORGANIZATION_SIZE === 'string' + ? row.ORGANIZATION_SIZE.trim() || undefined + : undefined, industry: (row.ORGANIZATION_INDUSTRY as string | null)?.trim() || undefined, }, ] From 913d335c41a6bdcaf6bd9137e0121d0a9fab8a6f Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 6 Mar 2026 10:55:51 +0000 Subject: [PATCH 08/18] fix: use slug for segment matching --- .../src/integrations/tnc/certificates/buildSourceQuery.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/certificates/buildSourceQuery.ts b/services/apps/snowflake_connectors/src/integrations/tnc/certificates/buildSourceQuery.ts index 8b1230bb3b..57629dbdfc 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/certificates/buildSourceQuery.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/certificates/buildSourceQuery.ts @@ -45,7 +45,8 @@ export const buildSourceQuery = (sinceTimestamp?: string): string => { ${LFID_COALESCE} AS LFID FROM analytics.silver_fact.certificates c INNER JOIN cdp_matched_segments cms - ON cms.sourceId = c.project_id + ON cms.slug = c.project_slug + AND cms.sourceId = c.project_id LEFT JOIN analytics.bronze_fivetran_salesforce.bronze_salesforce_merged_user mu ON c.user_id = mu.user_id AND mu.user_name IS NOT NULL From fa8cdcfb3d807d91e23f77463cb11ed3bb882ff1 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 6 Mar 2026 10:56:26 +0000 Subject: [PATCH 09/18] chore: use cncf for staging for data completion --- .../src/integrations/tnc/courses/buildSourceQuery.ts | 2 +- .../src/integrations/tnc/enrollments/buildSourceQuery.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/courses/buildSourceQuery.ts b/services/apps/snowflake_connectors/src/integrations/tnc/courses/buildSourceQuery.ts index 1fceb011d6..69c25c0e13 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/courses/buildSourceQuery.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/courses/buildSourceQuery.ts @@ -68,7 +68,7 @@ export const buildSourceQuery = (sinceTimestamp?: string): string => { AND tu.user_email IS NOT NULL` if (!IS_PROD_ENV) { - select += ` AND e.project_slug = 'openssf'` + select += ` AND e.project_slug = 'cncf'` } const dedup = ` diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/buildSourceQuery.ts b/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/buildSourceQuery.ts index 711f0f6a36..024e83944b 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/buildSourceQuery.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/buildSourceQuery.ts @@ -80,7 +80,7 @@ export const buildSourceQuery = (sinceTimestamp?: string): string => { WHERE e.user_email IS NOT NULL` if (!IS_PROD_ENV) { - select += ` AND e.project_slug = 'openssf'` + select += ` AND e.project_slug = 'cncf'` } const dedup = ` From e5e6c942e3e617291e89dc92d2867357fb25f284 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 6 Mar 2026 10:56:51 +0000 Subject: [PATCH 10/18] fix: improve bool handling --- .../src/integrations/tnc/courses/transformer.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts b/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts index 18bcdf5210..e628e626cf 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts @@ -70,7 +70,7 @@ export class TncCoursesTransformer extends TncTransformerBase { }) } - const isCertification = row.IS_CERTIFICATION === true + const isCertification = Boolean(row.IS_CERTIFICATION) const type = isCertification ? TncActivityType.ATTEMPTED_EXAM : TncActivityType.ATTEMPTED_COURSE const activity: IActivityData = { @@ -100,7 +100,7 @@ export class TncCoursesTransformer extends TncTransformerBase { courseSlug: (row.COURSE_SLUG as string | null) || null, instructionType: (row.INSTRUCTION_TYPE as string | null) || null, isCertification, - isTraining: row.IS_TRAINING === true, + isTraining: Boolean(row.IS_TRAINING), }, } From f9cbea72c6ce19d40cf26666bd6903b1e1f0d810 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 6 Mar 2026 10:57:35 +0000 Subject: [PATCH 11/18] feat: add tnc support on ORG_DB_ATTRIBUTE_SOURCE_PRIORITY --- .../libs/data-access-layer/src/organizations/attributesConfig.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/services/libs/data-access-layer/src/organizations/attributesConfig.ts b/services/libs/data-access-layer/src/organizations/attributesConfig.ts index a1f98261f7..c8d0414c3c 100644 --- a/services/libs/data-access-layer/src/organizations/attributesConfig.ts +++ b/services/libs/data-access-layer/src/organizations/attributesConfig.ts @@ -232,6 +232,7 @@ export const ORG_DB_ATTRIBUTE_SOURCE_PRIORITY = [ OrganizationAttributeSource.ENRICHMENT_LFX_INTERNAL_API, OrganizationAttributeSource.ENRICHMENT_PEOPLEDATALABS, OrganizationAttributeSource.CVENT, + OrganizationAttributeSource.TNC, // legacy - keeping this for backward compatibility OrganizationAttributeSource.ENRICHMENT, OrganizationAttributeSource.GITHUB, From b439205c7f3bc5f3910a79c3d11bcbbfe71eb8e9 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 6 Mar 2026 11:26:38 +0000 Subject: [PATCH 12/18] fix: undo slug matching as it's missing from certificates table --- .../src/integrations/tnc/certificates/buildSourceQuery.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/certificates/buildSourceQuery.ts b/services/apps/snowflake_connectors/src/integrations/tnc/certificates/buildSourceQuery.ts index 57629dbdfc..8b1230bb3b 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/certificates/buildSourceQuery.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/certificates/buildSourceQuery.ts @@ -45,8 +45,7 @@ export const buildSourceQuery = (sinceTimestamp?: string): string => { ${LFID_COALESCE} AS LFID FROM analytics.silver_fact.certificates c INNER JOIN cdp_matched_segments cms - ON cms.slug = c.project_slug - AND cms.sourceId = c.project_id + ON cms.sourceId = c.project_id LEFT JOIN analytics.bronze_fivetran_salesforce.bronze_salesforce_merged_user mu ON c.user_id = mu.user_id AND mu.user_name IS NOT NULL From 64bd8dedb8510109479eaf4c7ab8240439fd1a51 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 6 Mar 2026 13:30:24 +0000 Subject: [PATCH 13/18] chore: add comment explaing usage of specific project on non-prod envs --- .../src/integrations/tnc/certificates/buildSourceQuery.ts | 1 + .../src/integrations/tnc/courses/buildSourceQuery.ts | 1 + .../src/integrations/tnc/enrollments/buildSourceQuery.ts | 1 + 3 files changed, 3 insertions(+) diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/certificates/buildSourceQuery.ts b/services/apps/snowflake_connectors/src/integrations/tnc/certificates/buildSourceQuery.ts index 8b1230bb3b..ffb79daf7e 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/certificates/buildSourceQuery.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/certificates/buildSourceQuery.ts @@ -56,6 +56,7 @@ export const buildSourceQuery = (sinceTimestamp?: string): string => { ON c.account_id = org.account_id WHERE c.user_email IS NOT NULL` + // Limit to a single project in non-prod to avoid exporting all projects data if (!IS_PROD_ENV) { select += ` AND cms.slug = 'cncf'` } diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/courses/buildSourceQuery.ts b/services/apps/snowflake_connectors/src/integrations/tnc/courses/buildSourceQuery.ts index 69c25c0e13..2473d49aaa 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/courses/buildSourceQuery.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/courses/buildSourceQuery.ts @@ -67,6 +67,7 @@ export const buildSourceQuery = (sinceTimestamp?: string): string => { AND co.is_test_or_archived = false AND tu.user_email IS NOT NULL` + // Limit to a single project in non-prod to avoid exporting all projects data if (!IS_PROD_ENV) { select += ` AND e.project_slug = 'cncf'` } diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/buildSourceQuery.ts b/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/buildSourceQuery.ts index 024e83944b..82d1be636a 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/buildSourceQuery.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/buildSourceQuery.ts @@ -79,6 +79,7 @@ export const buildSourceQuery = (sinceTimestamp?: string): string => { AND e.course_id = cs.course_id WHERE e.user_email IS NOT NULL` + // Limit to a single project in non-prod to avoid exporting all projects data if (!IS_PROD_ENV) { select += ` AND e.project_slug = 'cncf'` } From 2b1099518da67412553eede0147789fd626dd735 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 6 Mar 2026 13:35:03 +0000 Subject: [PATCH 14/18] fix: displayName fallback to email without domain --- .../src/integrations/tnc/certificates/transformer.ts | 2 +- .../src/integrations/tnc/courses/transformer.ts | 2 +- .../src/integrations/tnc/enrollments/transformer.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/certificates/transformer.ts b/services/apps/snowflake_connectors/src/integrations/tnc/certificates/transformer.ts index b5846ce507..f58ab2b44b 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/certificates/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/certificates/transformer.ts @@ -70,7 +70,7 @@ export class TncCertificatesTransformer extends TncTransformerBase { sourceId: certificateId, sourceParentId: (row.COURSE_ID as string | null) || undefined, member: { - displayName: learnerName || email, + displayName: learnerName || email.split('@')[0], identities, organizations: this.buildOrganizations(row), attributes: { diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts b/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts index e628e626cf..091e59d738 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts @@ -81,7 +81,7 @@ export class TncCoursesTransformer extends TncTransformerBase { sourceId: courseActionId, sourceParentId: (row.COURSE_ID as string | null) || undefined, member: { - displayName: learnerName || email, + displayName: learnerName || email.split('@')[0], identities, organizations: this.buildOrganizations(row), attributes: { diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/transformer.ts b/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/transformer.ts index 126f721022..6a19dcd84b 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/enrollments/transformer.ts @@ -92,7 +92,7 @@ export class TncEnrollmentsTransformer extends TncTransformerBase { sourceId: enrollmentId, sourceParentId: (row.COURSE_ID as string | null) || undefined, member: { - displayName: learnerName || email, + displayName: learnerName || email.split('@')[0], identities, organizations: this.buildOrganizations(row), attributes: { From 947aa7c947d89ceaf3741dad9d7ca5d6ee0e0810 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 6 Mar 2026 14:31:40 +0000 Subject: [PATCH 15/18] chore: excplicit check for is_training and is_certificate --- .../src/integrations/tnc/courses/transformer.ts | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts b/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts index 091e59d738..fb6c029dfe 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts @@ -71,7 +71,20 @@ export class TncCoursesTransformer extends TncTransformerBase { } const isCertification = Boolean(row.IS_CERTIFICATION) - const type = isCertification ? TncActivityType.ATTEMPTED_EXAM : TncActivityType.ATTEMPTED_COURSE + const isTraining = Boolean(row.IS_TRAINING) + + let type: TncActivityType + if (isTraining) { + type = TncActivityType.ATTEMPTED_COURSE + } else if (isCertification) { + type = TncActivityType.ATTEMPTED_EXAM + } else { + log.warn( + { courseActionId, isCertification, isTraining }, + 'Skipping row: neither training nor certification', + ) + return null + } const activity: IActivityData = { type, From 5a1d27a3b3ccd4d8281587b9f1f78b974c0d7f3f Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 6 Mar 2026 14:46:42 +0000 Subject: [PATCH 16/18] fix: use productType to determine activity type --- .../src/integrations/tnc/courses/transformer.ts | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts b/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts index fb6c029dfe..bce745f572 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts @@ -70,18 +70,17 @@ export class TncCoursesTransformer extends TncTransformerBase { }) } - const isCertification = Boolean(row.IS_CERTIFICATION) - const isTraining = Boolean(row.IS_TRAINING) + const productType = (row.PRODUCT_TYPE as string | null)?.trim() || null let type: TncActivityType - if (isTraining) { + if (productType?.toLowerCase() === 'training') { type = TncActivityType.ATTEMPTED_COURSE - } else if (isCertification) { + } else if (productType?.toLowerCase() === 'certification') { type = TncActivityType.ATTEMPTED_EXAM } else { log.warn( - { courseActionId, isCertification, isTraining }, - 'Skipping row: neither training nor certification', + { courseActionId, productType }, + 'Skipping row: unrecognized product type', ) return null } @@ -112,7 +111,7 @@ export class TncCoursesTransformer extends TncTransformerBase { parentProduct: (row.COURSE_GROUP_ID as string | null) || null, courseSlug: (row.COURSE_SLUG as string | null) || null, instructionType: (row.INSTRUCTION_TYPE as string | null) || null, - isCertification, + isCertification: Boolean(row.IS_CERTIFICATION), isTraining: Boolean(row.IS_TRAINING), }, } From 3f412b41a66a6f05f890a52459598715fd97c6cf Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Fri, 6 Mar 2026 15:30:08 +0000 Subject: [PATCH 17/18] fix: lint --- .../src/integrations/tnc/courses/transformer.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts b/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts index bce745f572..12ac53521d 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts @@ -78,10 +78,7 @@ export class TncCoursesTransformer extends TncTransformerBase { } else if (productType?.toLowerCase() === 'certification') { type = TncActivityType.ATTEMPTED_EXAM } else { - log.warn( - { courseActionId, productType }, - 'Skipping row: unrecognized product type', - ) + log.warn({ courseActionId, productType }, 'Skipping row: unrecognized product type') return null } From bdf6faf9937f27463cbb705cb97f6889196c20f3 Mon Sep 17 00:00:00 2001 From: Mouad BANI Date: Mon, 9 Mar 2026 11:41:39 +0000 Subject: [PATCH 18/18] fix: course title and slug mapping --- .../src/integrations/tnc/courses/transformer.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts b/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts index 12ac53521d..df214cace8 100644 --- a/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts +++ b/services/apps/snowflake_connectors/src/integrations/tnc/courses/transformer.ts @@ -103,10 +103,10 @@ export class TncCoursesTransformer extends TncTransformerBase { }, }, attributes: { - productName: (row.COURSE_NAME as string | null) || null, + productName: (row.TITLE as string | null) || null, productType: (row.PRODUCT_TYPE as string | null) || null, parentProduct: (row.COURSE_GROUP_ID as string | null) || null, - courseSlug: (row.COURSE_SLUG as string | null) || null, + courseSlug: (row.SLUG as string | null) || null, instructionType: (row.INSTRUCTION_TYPE as string | null) || null, isCertification: Boolean(row.IS_CERTIFICATION), isTraining: Boolean(row.IS_TRAINING),