diff --git a/schemas/pulse-job.yml b/schemas/pulse-job.yml index f63f208a3fa..55fbe9a7702 100644 --- a/schemas/pulse-job.yml +++ b/schemas/pulse-job.yml @@ -38,57 +38,28 @@ properties: maxLength: 25 origin: - oneOf: - - type: 'object' - description: | - PREFERRED: An HG job that only has a revision. This is for all - jobs going forward. - additionalProperties: false - title: 'HG Push' - properties: - kind: - type: 'string' - enum: ['hg.mozilla.org'] - project: - type: 'string' - pattern: "^[\\w-]+$" - minLength: 1 - maxLength: 50 - revision: - type: 'string' - pattern: '^[0-9a-f]+$' - minLength: 40 - maxLength: 40 - pushLogID: - type: 'integer' - required: [kind, project, revision] - - - type: 'object' - title: 'Github push or pull request' - additionalProperties: false - properties: - kind: - type: 'string' - enum: ['github.com'] - owner: - description: Unused - type: 'string' - pattern: "^[\\w-]+$" - minLength: 1 - maxLength: 50 - project: - description: 'The repository name as known to Treeherder' - type: 'string' - pattern: "^[\\w-]+$" - minLength: 1 - maxLength: 50 - revision: - type: 'string' - minLength: 40 - maxLength: 40 - pullRequestID: - type: 'integer' - required: [kind, project, revision] + type: 'object' + description: Describes a Mercurial or Git event. + additionalProperties: false + title: 'Origin event metadata' + properties: + project: + description: 'The repository name as known to Treeherder' + type: 'string' + pattern: "^[\\w-]+$" + minLength: 1 + maxLength: 50 + revision: + type: 'string' + pattern: '^[0-9a-f]+$' + minLength: 40 + maxLength: 40 + id: + description: | + For Mercurial refers to the `pushLogID`. For Git it refers to the + `pullRequestID`. + type: 'integer' + required: [project, revision] display: type: 'object' diff --git a/tests/conftest.py b/tests/conftest.py index cb624e0d7f2..87b39da8607 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -267,6 +267,7 @@ def test_repository_2(test_repository): dvcs_type=test_repository.dvcs_type, url=test_repository.url + "_2", codebase=test_repository.codebase, + tc_root_url=test_repository.tc_root_url, ) diff --git a/tests/etl/test_job_loader.py b/tests/etl/test_job_loader.py index 04ab0785f7e..390efa664ed 100644 --- a/tests/etl/test_job_loader.py +++ b/tests/etl/test_job_loader.py @@ -1,5 +1,6 @@ import copy import uuid +from datetime import datetime import pytest import responses @@ -8,7 +9,7 @@ from treeherder.etl.job_loader import JobLoader from treeherder.etl.taskcluster_pulse.handler import handle_message -from treeherder.model.models import Job, JobLog, TaskclusterMetadata +from treeherder.model.models import Job, JobLog, Push, TaskclusterMetadata @pytest.fixture @@ -389,6 +390,45 @@ def test_transition_running_unscheduled_stays_running( change_state_result(first_job, jl, "unscheduled", "unknown", "running", "unknown") +@responses.activate +def test_github_repo_origin( + pulse_jobs, push_stored, failure_classifications, mock_log_parser, test_repository_2 +): + test_repository_2.url = "https://github.com/mozilla-mobile/fenix" + test_repository_2.save() + + jl = JobLoader() + job = pulse_jobs[0] + + responses.add( + responses.GET, + "https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/AI3Nrr3gSDSpZ9E9aBA3rg", + json={"extra": {"treeherder": {}}}, + status=200, + ) + + revision = push_stored[0]["revision"] + Push.objects.create( + repository=test_repository_2, + revision=revision, + author=push_stored[0]["author"], + time=datetime.fromtimestamp(push_stored[0]["push_timestamp"]), + ) + + job["origin"]["revision"] = revision + job["origin"]["project"] = test_repository_2.name + job["origin"]["id"] = 42 + + jl.process_job(job, "https://firefox-ci-tc.services.mozilla.com") + + jobs = Job.objects.all() + assert len(jobs) == 1 + + # Verify the job was processed successfully with the correct repository + job_obj = jobs[0] + assert job_obj.repository_id == test_repository_2.id + + def change_state_result(test_job, job_loader, new_state, new_result, exp_state, exp_result): # make a copy so we can modify it and not affect other tests job = copy.deepcopy(test_job) diff --git a/tests/sample_data/pulse_consumer/job_data.json b/tests/sample_data/pulse_consumer/job_data.json index 2452bb9cf17..8cfa2a8c4fa 100644 --- a/tests/sample_data/pulse_consumer/job_data.json +++ b/tests/sample_data/pulse_consumer/job_data.json @@ -2,7 +2,6 @@ { "taskId": "008dcdae-bde0-4834-a967-d13d681037ae/0", "origin": { - "kind": "hg.mozilla.org", "project": "set by test", "revision": "9c6e4c31ad26efdad0b9af47a2b530bc414ce2c3" }, @@ -55,7 +54,6 @@ { "taskId": "0401bb89-f4b5-41b1-820f-088e3fb36495/0", "origin": { - "kind": "hg.mozilla.org", "project": "set by test", "revision": "9c6e4c31ad26efdad0b9af47a2b530bc414ce2c3" }, @@ -108,7 +106,6 @@ { "taskId": "6e58db2d-115d-4f82-8609-6276fc2e91b1/0", "origin": { - "kind": "hg.mozilla.org", "project": "set by test", "revision": "9c6e4c31ad26efdad0b9af47a2b530bc414ce2c3" }, @@ -161,7 +158,6 @@ { "taskId": "6ff402cc-c8d5-4262-8fc8-ee4897425e77/0", "origin": { - "kind": "hg.mozilla.org", "project": "set by test", "revision": "9c6e4c31ad26efdad0b9af47a2b530bc414ce2c3" }, @@ -214,7 +210,6 @@ { "taskId": "73677162-e702-48c5-8f95-391bef4afcf6/0", "origin": { - "kind": "hg.mozilla.org", "project": "set by test", "revision": "9c6e4c31ad26efdad0b9af47a2b530bc414ce2c3" }, @@ -267,7 +262,6 @@ { "taskId": "09a2ba36-57c1-49fe-85f8-d015aeb2890d/0", "origin": { - "kind": "hg.mozilla.org", "project": "set by test", "revision": "9c6e4c31ad26efdad0b9af47a2b530bc414ce2c3" }, @@ -320,7 +314,6 @@ { "taskId": "0a29590a-79a2-4ca9-9a80-97bf87a1eae4/0", "origin": { - "kind": "hg.mozilla.org", "project": "set by test", "revision": "9c6e4c31ad26efdad0b9af47a2b530bc414ce2c3" }, @@ -373,7 +366,6 @@ { "taskId": "70f7bccb-4ef5-4a96-add3-d765033086ba/0", "origin": { - "kind": "hg.mozilla.org", "project": "set by test", "revision": "9c6e4c31ad26efdad0b9af47a2b530bc414ce2c3" }, @@ -426,7 +418,6 @@ { "taskId": "719ee073-d258-41ad-943c-40bf108c48ba/0", "origin": { - "kind": "hg.mozilla.org", "project": "set by test", "revision": "9c6e4c31ad26efdad0b9af47a2b530bc414ce2c3" }, @@ -479,7 +470,6 @@ { "taskId": "741f11e4-05ce-4597-82a4-37d0958525a3/0", "origin": { - "kind": "hg.mozilla.org", "project": "set by test", "revision": "9c6e4c31ad26efdad0b9af47a2b530bc414ce2c3" }, @@ -532,7 +522,6 @@ { "taskId": "7b560f96-5cfa-4cc6-b021-2a6e824471d6/0", "origin": { - "kind": "hg.mozilla.org", "project": "set by test", "revision": "9c6e4c31ad26efdad0b9af47a2b530bc414ce2c3" }, @@ -585,7 +574,6 @@ { "taskId": "79dce0cc-2a61-4c04-be40-dfd30277fb78/0", "origin": { - "kind": "hg.mozilla.org", "project": "set by test", "revision": "9c6e4c31ad26efdad0b9af47a2b530bc414ce2c3" }, @@ -638,7 +626,6 @@ { "taskId": "789f4f1b-8d6d-49a5-b335-4d6e63bfae49/0", "origin": { - "kind": "hg.mozilla.org", "project": "set by test", "revision": "9c6e4c31ad26efdad0b9af47a2b530bc414ce2c3" }, @@ -691,7 +678,6 @@ { "taskId": "7a441a78-2ff2-4742-bc8c-f2b1dbc13b10/0", "origin": { - "kind": "hg.mozilla.org", "project": "set by test", "revision": "9c6e4c31ad26efdad0b9af47a2b530bc414ce2c3" }, @@ -744,7 +730,6 @@ { "taskId": "15c6c852-855b-4b8b-adc4-5112f38c2b3f/0", "origin": { - "kind": "hg.mozilla.org", "project": "set by test", "revision": "9c6e4c31ad26efdad0b9af47a2b530bc414ce2c3" }, @@ -797,7 +782,6 @@ { "taskId": "15c943ea-003e-4d31-a0be-afebf7e2d20e/0", "origin": { - "kind": "hg.mozilla.org", "project": "set by test", "revision": "9c6e4c31ad26efdad0b9af47a2b530bc414ce2c3" }, @@ -850,7 +834,6 @@ { "taskId": "14d4f704-b890-4473-b5e0-d360a278d042/0", "origin": { - "kind": "hg.mozilla.org", "project": "set by test", "revision": "9c6e4c31ad26efdad0b9af47a2b530bc414ce2c3" }, @@ -903,7 +886,6 @@ { "taskId": "7ed5ec47-23b0-4607-a289-81f22135d438/0", "origin": { - "kind": "hg.mozilla.org", "project": "set by test", "revision": "9c6e4c31ad26efdad0b9af47a2b530bc414ce2c3" }, @@ -956,7 +938,6 @@ { "taskId": "19cbc73f-a1cb-49e3-bfac-a603376cbf4e/0", "origin": { - "kind": "hg.mozilla.org", "project": "set by test", "revision": "9c6e4c31ad26efdad0b9af47a2b530bc414ce2c3" }, @@ -1009,7 +990,6 @@ { "taskId": "18f2e4ef-c9ba-4b3e-934c-b24555c6b85f/0", "origin": { - "kind": "hg.mozilla.org", "project": "set by test", "revision": "9c6e4c31ad26efdad0b9af47a2b530bc414ce2c3" }, @@ -1062,7 +1042,6 @@ { "taskId": "23663e4c-135c-40f4-89ce-c2a507de6b7d/0", "origin": { - "kind": "hg.mozilla.org", "project": "set by test", "revision": "9c6e4c31ad26efdad0b9af47a2b530bc414ce2c3" }, @@ -1115,7 +1094,6 @@ { "taskId": "23e1e0ed-b338-4d43-aae0-9aa75f4a6dd2/0", "origin": { - "kind": "hg.mozilla.org", "project": "set by test", "revision": "9c6e4c31ad26efdad0b9af47a2b530bc414ce2c3" }, @@ -1168,7 +1146,6 @@ { "taskId": "3351028c-f241-4e59-b027-166ae695b2be/0", "origin": { - "kind": "hg.mozilla.org", "project": "set by test", "revision": "9c6e4c31ad26efdad0b9af47a2b530bc414ce2c3" }, @@ -1221,7 +1198,6 @@ { "taskId": "30aabc98-c33e-4483-b1ce-d5cee6783efc/0", "origin": { - "kind": "hg.mozilla.org", "project": "set by test", "revision": "9c6e4c31ad26efdad0b9af47a2b530bc414ce2c3" }, @@ -1269,7 +1245,6 @@ { "taskId": "32badb89-fcc1-4097-9f51-b4b6ca6e106a/0", "origin": { - "kind": "hg.mozilla.org", "project": "set by test", "revision": "9c6e4c31ad26efdad0b9af47a2b530bc414ce2c3" }, @@ -1322,7 +1297,6 @@ { "taskId": "39161834-c844-4f4c-b118-cbde97826e8e/0", "origin": { - "kind": "hg.mozilla.org", "project": "set by test", "revision": "9c6e4c31ad26efdad0b9af47a2b530bc414ce2c3" }, @@ -1375,7 +1349,6 @@ { "taskId": "4ea5830c-6a16-49b0-87f7-74534f13c05a/0", "origin": { - "kind": "hg.mozilla.org", "project": "set by test", "revision": "9c6e4c31ad26efdad0b9af47a2b530bc414ce2c3" }, @@ -1428,7 +1401,6 @@ { "taskId": "57caed20-3ae8-455f-86f5-bce3260952d0/0", "origin": { - "kind": "hg.mozilla.org", "project": "set by test", "revision": "9c6e4c31ad26efdad0b9af47a2b530bc414ce2c3" }, @@ -1470,7 +1442,6 @@ { "taskId": "5556b66a-6ccc-4bed-9c48-36e283d447b3/0", "origin": { - "kind": "hg.mozilla.org", "project": "set by test", "revision": "9c6e4c31ad26efdad0b9af47a2b530bc414ce2c3" }, @@ -1523,7 +1494,6 @@ { "taskId": "60838ae3-4d72-4761-afca-020571f3d506/0", "origin": { - "kind": "hg.mozilla.org", "project": "set by test", "revision": "9c6e4c31ad26efdad0b9af47a2b530bc414ce2c3" }, diff --git a/tests/sample_data/transform.py b/tests/sample_data/transform.py index 375e448d0ba..ac7d4cd162b 100644 --- a/tests/sample_data/transform.py +++ b/tests/sample_data/transform.py @@ -245,7 +245,6 @@ def generate_pulse_job_data(tasks, revision, output_file): blob = { "taskId": task["job_guid"], "origin": { - "kind": "hg.mozilla.org", "project": "set by test", "revision": revision, }, diff --git a/treeherder/etl/job_loader.py b/treeherder/etl/job_loader.py index 1e346041740..fd6a8d7da10 100644 --- a/treeherder/etl/job_loader.py +++ b/treeherder/etl/job_loader.py @@ -1,5 +1,6 @@ import logging import uuid +from urllib.parse import urlparse import jsonschema import newrelic.agent @@ -77,6 +78,20 @@ def process_job(self, pulse_job, root_url): ) return + # Set origin data based on the repo url + parsed_url = urlparse(repository.url) + is_github = parsed_url.netloc == "github.com" + pulse_job["origin"]["kind"] = parsed_url.netloc + + if origin_id := pulse_job["origin"].pop("id", None): + key = "pullRequestID" if is_github else "pushLogID" + pulse_job["origin"][key] = origin_id + + if is_github and parsed_url.path: + path_parts = parsed_url.path.strip("/").split("/") + if len(path_parts) >= 2: + pulse_job["origin"]["owner"] = path_parts[0] + transformed_job = None try: self.validate_revision(repository, pulse_job) diff --git a/treeherder/etl/taskcluster_pulse/handler.py b/treeherder/etl/taskcluster_pulse/handler.py index c9cb3cfa22e..1bbc13da36c 100644 --- a/treeherder/etl/taskcluster_pulse/handler.py +++ b/treeherder/etl/taskcluster_pulse/handler.py @@ -254,17 +254,11 @@ def build_message(push_info, task, run_id, payload): } job["origin"] = { - "kind": push_info["origin"], "project": push_info["project"], "revision": push_info["revision"], + "id": push_info["id"], } - if push_info["origin"] == "hg.mozilla.org": - job["origin"]["pushLogID"] = push_info["id"] - else: - job["origin"]["pullRequestID"] = push_info["id"] - job["origin"]["owner"] = push_info["owner"] - # Transform "collection" into an array of labels if task doesn't # define "labels". labels = treeherder_config.get("labels", []) @@ -327,17 +321,11 @@ def handle_task_defined(push_info, task, message): } job["origin"] = { - "kind": push_info["origin"], "project": push_info["project"], "revision": push_info["revision"], + "id": push_info["id"], } - if push_info["origin"] == "hg.mozilla.org": - job["origin"]["pushLogID"] = push_info["id"] - else: - job["origin"]["pullRequestID"] = push_info["id"] - job["origin"]["owner"] = push_info["owner"] - labels = treeherder_config.get("labels", []) if not labels: if not treeherder_config.get("collection"): diff --git a/treeherder/etl/taskcluster_pulse/parse_route.py b/treeherder/etl/taskcluster_pulse/parse_route.py index a4f50b3331b..57cfdfba4fd 100644 --- a/treeherder/etl/taskcluster_pulse/parse_route.py +++ b/treeherder/etl/taskcluster_pulse/parse_route.py @@ -13,12 +13,12 @@ # https://github.com/taskcluster/taskcluster/blob/32629c562f8d6f5a6b608a3141a8ee2e0984619f/services/treeherder/src/util/route_parser.js def parse_route(route): id = None - owner = None parsed_project = None parsed_route = route.split(".") project = parsed_route[2] + if len(project.split("/")) == 2: - [owner, parsed_project] = project.split("/") + _, parsed_project = project.split("/") else: parsed_project = project @@ -32,10 +32,4 @@ def parse_route(route): "revision": parsed_route[3], } - if owner and parsed_project: - push_info["owner"] = owner - push_info["origin"] = "github.com" - else: - push_info["origin"] = "hg.mozilla.org" - return push_info