diff --git a/backend/helpers/pluginhelper/api/graphql_async_client.go b/backend/helpers/pluginhelper/api/graphql_async_client.go index 32dac66a9eb..61a1c9fa3ca 100644 --- a/backend/helpers/pluginhelper/api/graphql_async_client.go +++ b/backend/helpers/pluginhelper/api/graphql_async_client.go @@ -20,12 +20,14 @@ package api import ( "context" "fmt" + "strconv" + "sync" + "time" + "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/log" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/core/utils" - "sync" - "time" "github.com/merico-ai/graphql" ) @@ -47,30 +49,52 @@ type GraphqlAsyncClient struct { getRateCost func(q interface{}) int } +// defaultRateLimitConst is the generic fallback rate limit for GraphQL requests. +// It is used as the initial remaining quota when dynamic rate limit +// information is unavailable from the provider. +const defaultRateLimitConst = 1000 + // CreateAsyncGraphqlClient creates a new GraphqlAsyncClient func CreateAsyncGraphqlClient( taskCtx plugin.TaskContext, graphqlClient *graphql.Client, logger log.Logger, getRateRemaining func(context.Context, *graphql.Client, log.Logger) (rateRemaining int, resetAt *time.Time, err errors.Error), + opts ...func(*GraphqlAsyncClient), ) (*GraphqlAsyncClient, errors.Error) { ctxWithCancel, cancel := context.WithCancel(taskCtx.GetContext()) + graphqlAsyncClient := &GraphqlAsyncClient{ ctx: ctxWithCancel, cancel: cancel, client: graphqlClient, logger: logger, rateExhaustCond: sync.NewCond(&sync.Mutex{}), - rateRemaining: 0, + rateRemaining: defaultRateLimitConst, getRateRemaining: getRateRemaining, } + // apply options + for _, opt := range opts { + opt(graphqlAsyncClient) + } + + // Env config wins over everything, only if explicitly set + if rateLimit := resolveRateLimit(taskCtx, logger); rateLimit != -1 { + logger.Info("GRAPHQL_RATE_LIMIT env override applied: %d (was %d)", rateLimit, graphqlAsyncClient.rateRemaining) + graphqlAsyncClient.rateRemaining = rateLimit + } + if getRateRemaining != nil { rateRemaining, resetAt, err := getRateRemaining(taskCtx.GetContext(), graphqlClient, logger) if err != nil { - panic(err) + graphqlAsyncClient.logger.Info("failed to fetch initial graphql rate limit, fallback to default: %v", err) + graphqlAsyncClient.updateRateRemaining(graphqlAsyncClient.rateRemaining, nil) + } else { + graphqlAsyncClient.updateRateRemaining(rateRemaining, resetAt) } - graphqlAsyncClient.updateRateRemaining(rateRemaining, resetAt) + } else { + graphqlAsyncClient.updateRateRemaining(graphqlAsyncClient.rateRemaining, nil) } // load retry/timeout from configuration @@ -115,6 +139,10 @@ func (apiClient *GraphqlAsyncClient) updateRateRemaining(rateRemaining int, rese apiClient.rateExhaustCond.Signal() } go func() { + if apiClient.getRateRemaining == nil { + return + } + nextDuring := 3 * time.Minute if resetAt != nil && resetAt.After(time.Now()) { nextDuring = time.Until(*resetAt) @@ -126,7 +154,15 @@ func (apiClient *GraphqlAsyncClient) updateRateRemaining(rateRemaining int, rese case <-time.After(nextDuring): newRateRemaining, newResetAt, err := apiClient.getRateRemaining(apiClient.ctx, apiClient.client, apiClient.logger) if err != nil { - panic(err) + apiClient.logger.Info("failed to update graphql rate limit, will retry next cycle: %v", err) + // Floor the reused value so Signal() always fires; prevents deadlock when + // rateRemaining is 0 and the rate-limit endpoint keeps erroring (e.g. GHE). + fallback := apiClient.rateRemaining + if fallback < defaultRateLimitConst { + fallback = defaultRateLimitConst + } + apiClient.updateRateRemaining(fallback, nil) + return } apiClient.updateRateRemaining(newRateRemaining, newResetAt) } @@ -218,3 +254,25 @@ func (apiClient *GraphqlAsyncClient) Wait() { func (apiClient *GraphqlAsyncClient) Release() { apiClient.cancel() } + +// WithFallbackRateLimit sets the initial/fallback rate limit used when +// rate limit information cannot be fetched dynamically. +// This value may be overridden later by getRateRemaining. +func WithFallbackRateLimit(limit int) func(*GraphqlAsyncClient) { + return func(c *GraphqlAsyncClient) { + if limit > 0 { + c.rateRemaining = limit + } + } +} + +// resolveRateLimit returns -1 if GRAPHQL_RATE_LIMIT is not set or invalid +func resolveRateLimit(taskCtx plugin.TaskContext, logger log.Logger) int { + if v := taskCtx.GetConfig("GRAPHQL_RATE_LIMIT"); v != "" { + if parsed, err := strconv.Atoi(v); err == nil { + return parsed + } + logger.Warn(nil, "invalid GRAPHQL_RATE_LIMIT, using default") + } + return -1 +} diff --git a/backend/plugins/circleci/e2e/job_collector_test.go b/backend/plugins/circleci/e2e/job_collector_test.go new file mode 100644 index 00000000000..ed2f014217f --- /dev/null +++ b/backend/plugins/circleci/e2e/job_collector_test.go @@ -0,0 +1,82 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "reflect" + "sort" + "testing" + + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/circleci/impl" + "github.com/apache/incubator-devlake/plugins/circleci/models" + "github.com/apache/incubator-devlake/plugins/circleci/tasks" + "github.com/stretchr/testify/assert" +) + +// TestCircleciUnfinishedJobsInputIterator is a regression test for +// https://github.com/apache/devlake/issues/8907. The "collect unfinished job +// details" collector builds its URL from "/v2/workflow/{{ .Input.Id }}/job" while +// scanning rows into a models.CircleciJob. Its input query must therefore expose the +// workflow id in the row's Id field; a bare "DISTINCT workflow_id" left Id empty and +// produced "/v2/workflow//job" (HTTP 500). This test runs the production query +// (tasks.UnfinishedJobsInputClauses) through the real iterator and asserts each +// yielded row's Id is the workflow id, that results are DISTINCT, and that the +// status/connection filters hold. +func TestCircleciUnfinishedJobsInputIterator(t *testing.T) { + var circleci impl.Circleci + dataflowTester := e2ehelper.NewDataFlowTester(t, "circleci", circleci) + + const projectSlug = "github/test/repo" + dataflowTester.FlushTabler(&models.CircleciJob{}) + + seed := []models.CircleciJob{ + {ConnectionId: 1, WorkflowId: "wf-onhold", Id: "job-1", ProjectSlug: projectSlug, Status: "on_hold"}, + {ConnectionId: 1, WorkflowId: "wf-onhold", Id: "job-2", ProjectSlug: projectSlug, Status: "running"}, // same workflow -> DISTINCT + {ConnectionId: 1, WorkflowId: "wf-queued", Id: "job-3", ProjectSlug: projectSlug, Status: "queued"}, + {ConnectionId: 1, WorkflowId: "wf-success", Id: "job-4", ProjectSlug: projectSlug, Status: "success"}, // terminal -> excluded + {ConnectionId: 2, WorkflowId: "wf-otherconn", Id: "job-5", ProjectSlug: projectSlug, Status: "on_hold"}, // other connection -> excluded + } + for i := range seed { + assert.Nil(t, dataflowTester.Dal.Create(&seed[i])) + } + + cursor, err := dataflowTester.Dal.Cursor(tasks.UnfinishedJobsInputClauses(1, projectSlug)...) + assert.Nil(t, err) + iter, err := api.NewDalCursorIterator(dataflowTester.Dal, cursor, reflect.TypeOf(models.CircleciJob{})) + assert.Nil(t, err) + defer iter.Close() + + var ids []string + for iter.HasNext() { + item, err := iter.Fetch() + assert.Nil(t, err) + job := item.(*models.CircleciJob) + ids = append(ids, job.Id) + } + sort.Strings(ids) + + // Distinct workflow ids for connection 1's non-terminal jobs, with Id populated + // (the URL template reads .Input.Id). wf-success (terminal) and wf-otherconn + // (connection 2) are excluded. + assert.Equal(t, []string{"wf-onhold", "wf-queued"}, ids) + for _, id := range ids { + assert.NotEmpty(t, id, "Input.Id must be the workflow id, not empty (#8907)") + } +} diff --git a/backend/plugins/circleci/tasks/job_collector.go b/backend/plugins/circleci/tasks/job_collector.go index fd1d78286bf..00fd234524f 100644 --- a/backend/plugins/circleci/tasks/job_collector.go +++ b/backend/plugins/circleci/tasks/job_collector.go @@ -41,6 +41,20 @@ var CollectJobsMeta = plugin.SubTaskMeta{ DomainTypes: []string{plugin.DOMAIN_TYPE_CICD}, } +// UnfinishedJobsInputClauses returns the DAL clauses that select the workflows whose +// jobs are still in a non-terminal status and therefore need their job details +// recollected by the CollectJobs "unfinished details" collector. +func UnfinishedJobsInputClauses(connectionId uint64, projectSlug string) []dal.Clause { + return []dal.Clause{ + dal.Select("DISTINCT workflow_id AS id"), // #8907: alias to id so {{ .Input.Id }} resolves when scanned into CircleciJob + dal.From(&models.CircleciJob{}), + dal.Where( + "connection_id = ? AND project_slug = ? AND status IN ('running', 'not_running', 'queued', 'on_hold')", + connectionId, projectSlug, + ), + } +} + func CollectJobs(taskCtx plugin.SubTaskContext) errors.Error { rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RAW_JOB_TABLE) logger := taskCtx.GetLogger() @@ -94,14 +108,8 @@ func CollectJobs(taskCtx plugin.SubTaskContext) errors.Error { AfterResponse: ignoreDeletedBuilds, }, BuildInputIterator: func() (api.Iterator, errors.Error) { - clauses := []dal.Clause{ - dal.Select("DISTINCT workflow_id"), // Only need to recollect jobs for a workflow once - dal.From(&models.CircleciJob{}), - dal.Where("connection_id = ? AND project_slug = ? AND status IN ('running', 'not_running', 'queued', 'on_hold')", data.Options.ConnectionId, data.Options.ProjectSlug), - } - db := taskCtx.GetDal() - cursor, err := db.Cursor(clauses...) + cursor, err := db.Cursor(UnfinishedJobsInputClauses(data.Options.ConnectionId, data.Options.ProjectSlug)...) if err != nil { return nil, err } diff --git a/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_enterprise_daily_metrics.csv b/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_enterprise_daily_metrics.csv index 57e3f362712..7a74a5cc81d 100644 --- a/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_enterprise_daily_metrics.csv +++ b/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_enterprise_daily_metrics.csv @@ -1,3 +1,3 @@ -connection_id,scope_id,day,enterprise_id,daily_active_users,weekly_active_users,monthly_active_users,monthly_active_chat_users,monthly_active_agent_users,pr_total_reviewed,pr_total_created,pr_total_created_by_copilot,pr_total_reviewed_by_copilot,user_initiated_interaction_count,code_generation_activity_count,code_acceptance_activity_count,loc_suggested_to_add_sum,loc_suggested_to_delete_sum,loc_added_sum,loc_deleted_sum -1,octodemo,2025-09-01T00:00:00.000+00:00,,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 -1,octodemo,2025-09-02T00:00:00.000+00:00,,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +connection_id,scope_id,day,enterprise_id,daily_active_users,weekly_active_users,monthly_active_users,monthly_active_chat_users,monthly_active_agent_users,daily_active_cli_users,daily_active_copilot_code_review_users,daily_passive_copilot_code_review_users,weekly_active_copilot_code_review_users,weekly_passive_copilot_code_review_users,monthly_active_copilot_code_review_users,monthly_passive_copilot_code_review_users,chat_panel_agent_mode,chat_panel_ask_mode,chat_panel_custom_mode,chat_panel_edit_mode,chat_panel_plan_mode,chat_panel_unknown_mode,pr_total_reviewed,pr_total_created,pr_total_merged,pr_median_minutes_to_merge,pr_total_suggestions,pr_total_applied_suggestions,pr_total_created_by_copilot,pr_total_reviewed_by_copilot,pr_total_merged_created_by_copilot,pr_total_merged_reviewed_by_copilot,pr_median_min_to_merge_copilot_authored,pr_median_min_to_merge_copilot_reviewed,pr_total_copilot_suggestions,pr_total_copilot_applied_suggestions,user_initiated_interaction_count,code_generation_activity_count,code_acceptance_activity_count,loc_suggested_to_add_sum,loc_suggested_to_delete_sum,loc_added_sum,loc_deleted_sum,cli_session_count,cli_request_count,cli_prompt_count,cli_output_token_sum,cli_prompt_token_sum +1,octodemo,2025-09-01T00:00:00.000+00:00,,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 +1,octodemo,2025-09-02T00:00:00.000+00:00,,12,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 diff --git a/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_seats.csv b/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_seats.csv index 87b28f6b8fb..291a98ca607 100644 --- a/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_seats.csv +++ b/backend/plugins/gh-copilot/e2e/metrics/snapshot_tables/_tool_copilot_seats.csv @@ -1,3 +1,3 @@ -connection_id,organization,user_login,user_id,plan_type,created_at,last_activity_at,last_activity_editor,last_authenticated_at,pending_cancellation_date,updated_at -1,octodemo,nathos,4215,enterprise,2023-08-28T23:50:42.000+00:00,2025-11-06T16:12:15.000+00:00,copilot_pr_review,2025-12-04T15:53:22.000+00:00,,2024-02-01T00:00:00.000+00:00 -1,octodemo,octocat,1,enterprise,2024-01-10T10:11:12.000+00:00,,vscode/1.0.0/copilot-chat/0.1.0,,,2024-02-02T00:00:00.000+00:00 +connection_id,organization,user_login,user_id,user_name,user_email,plan_type,assigning_team_id,assigning_team_name,assigning_team_slug,created_at,last_activity_at,last_activity_editor,last_authenticated_at,pending_cancellation_date,updated_at +1,octodemo,nathos,4215,,,enterprise,0,,,2023-08-28T23:50:42.000+00:00,2025-11-06T16:12:15.000+00:00,copilot_pr_review,2025-12-04T15:53:22.000+00:00,,2024-02-01T00:00:00.000+00:00 +1,octodemo,octocat,1,,,enterprise,0,,,2024-01-10T10:11:12.000+00:00,,vscode/1.0.0/copilot-chat/0.1.0,,,2024-02-02T00:00:00.000+00:00 diff --git a/backend/plugins/gh-copilot/models/enterprise_metrics.go b/backend/plugins/gh-copilot/models/enterprise_metrics.go index 07663aa6dd5..967e3ecd326 100644 --- a/backend/plugins/gh-copilot/models/enterprise_metrics.go +++ b/backend/plugins/gh-copilot/models/enterprise_metrics.go @@ -44,6 +44,15 @@ type CopilotCodeMetrics struct { LocDeletedSum int `json:"locDeletedSum"` } +// CopilotCliMetrics contains CLI usage breakdown metrics. +type CopilotCliMetrics struct { + CliSessionCount int `json:"cliSessionCount" gorm:"comment:Number of CLI sessions"` + CliRequestCount int `json:"cliRequestCount" gorm:"comment:Number of CLI requests"` + CliPromptCount int `json:"cliPromptCount" gorm:"comment:Number of CLI prompts"` + CliOutputTokenSum int `json:"cliOutputTokenSum" gorm:"comment:Total output tokens from CLI"` + CliPromptTokenSum int `json:"cliPromptTokenSum" gorm:"comment:Total prompt tokens from CLI"` +} + // GhCopilotEnterpriseDailyMetrics captures daily enterprise-level aggregate Copilot metrics. type GhCopilotEnterpriseDailyMetrics struct { ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` @@ -57,12 +66,43 @@ type GhCopilotEnterpriseDailyMetrics struct { MonthlyActiveChatUsers int `json:"monthlyActiveChatUsers"` MonthlyActiveAgentUsers int `json:"monthlyActiveAgentUsers"` - PRTotalReviewed int `json:"prTotalReviewed" gorm:"comment:Total PRs reviewed"` - PRTotalCreated int `json:"prTotalCreated" gorm:"comment:Total PRs created"` - PRTotalCreatedByCopilot int `json:"prTotalCreatedByCopilot" gorm:"comment:PRs created by Copilot"` - PRTotalReviewedByCopilot int `json:"prTotalReviewedByCopilot" gorm:"comment:PRs reviewed by Copilot"` + // CLI active users + DailyActiveCliUsers int `json:"dailyActiveCliUsers" gorm:"comment:Daily active CLI users"` + + // Code review user counts + DailyActiveCopilotCodeReviewUsers int `json:"dailyActiveCopilotCodeReviewUsers"` + DailyPassiveCopilotCodeReviewUsers int `json:"dailyPassiveCopilotCodeReviewUsers"` + WeeklyActiveCopilotCodeReviewUsers int `json:"weeklyActiveCopilotCodeReviewUsers"` + WeeklyPassiveCopilotCodeReviewUsers int `json:"weeklyPassiveCopilotCodeReviewUsers"` + MonthlyActiveCopilotCodeReviewUsers int `json:"monthlyActiveCopilotCodeReviewUsers"` + MonthlyPassiveCopilotCodeReviewUsers int `json:"monthlyPassiveCopilotCodeReviewUsers"` + + // Chat panel mode breakdown + ChatPanelAgentMode int `json:"chatPanelAgentMode" gorm:"comment:Chat panel agent mode interactions"` + ChatPanelAskMode int `json:"chatPanelAskMode" gorm:"comment:Chat panel ask mode interactions"` + ChatPanelCustomMode int `json:"chatPanelCustomMode" gorm:"comment:Chat panel custom mode interactions"` + ChatPanelEditMode int `json:"chatPanelEditMode" gorm:"comment:Chat panel edit mode interactions"` + ChatPanelPlanMode int `json:"chatPanelPlanMode" gorm:"comment:Chat panel plan mode interactions"` + ChatPanelUnknownMode int `json:"chatPanelUnknownMode" gorm:"comment:Chat panel unknown mode interactions"` + + // Pull request metrics (expanded) + PRTotalReviewed int `json:"prTotalReviewed" gorm:"comment:Total PRs reviewed"` + PRTotalCreated int `json:"prTotalCreated" gorm:"comment:Total PRs created"` + PRTotalMerged int `json:"prTotalMerged" gorm:"comment:Total PRs merged"` + PRMedianMinutesToMerge float64 `json:"prMedianMinutesToMerge" gorm:"comment:Median minutes to merge PRs"` + PRTotalSuggestions int `json:"prTotalSuggestions" gorm:"comment:Total PR review suggestions"` + PRTotalAppliedSuggestions int `json:"prTotalAppliedSuggestions" gorm:"comment:Total applied PR suggestions"` + PRTotalCreatedByCopilot int `json:"prTotalCreatedByCopilot" gorm:"comment:PRs created by Copilot"` + PRTotalReviewedByCopilot int `json:"prTotalReviewedByCopilot" gorm:"comment:PRs reviewed by Copilot"` + PRTotalMergedCreatedByCopilot int `json:"prTotalMergedCreatedByCopilot" gorm:"comment:Merged PRs created by Copilot"` + PRTotalMergedReviewedByCopilot int `json:"prTotalMergedReviewedByCopilot" gorm:"comment:Merged PRs reviewed by Copilot"` + PRMedianMinToMergeCopilotAuthored float64 `json:"prMedianMinToMergeCopilotAuthored" gorm:"comment:Median min to merge Copilot-authored PRs"` + PRMedianMinToMergeCopilotReviewed float64 `json:"prMedianMinToMergeCopilotReviewed" gorm:"comment:Median min to merge Copilot-reviewed PRs"` + PRTotalCopilotSuggestions int `json:"prTotalCopilotSuggestions" gorm:"comment:Total Copilot review suggestions"` + PRTotalCopilotAppliedSuggestions int `json:"prTotalCopilotAppliedSuggestions" gorm:"comment:Total Copilot applied suggestions"` CopilotActivityMetrics `mapstructure:",squash"` + CopilotCliMetrics `mapstructure:",squash"` common.NoPKModel } diff --git a/backend/plugins/gh-copilot/models/migrationscripts/20260527_add_copilot_metrics_gaps.go b/backend/plugins/gh-copilot/models/migrationscripts/20260527_add_copilot_metrics_gaps.go new file mode 100644 index 00000000000..f676b0e4bee --- /dev/null +++ b/backend/plugins/gh-copilot/models/migrationscripts/20260527_add_copilot_metrics_gaps.go @@ -0,0 +1,153 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/helpers/migrationhelper" +) + +type addCopilotMetricsGaps struct{} + +// --- Enterprise daily metrics: new columns --- + +type enterpriseDailyMetrics20260527 struct { + // CLI + DailyActiveCliUsers int + + // Code review user counts + DailyActiveCopilotCodeReviewUsers int + DailyPassiveCopilotCodeReviewUsers int + WeeklyActiveCopilotCodeReviewUsers int + WeeklyPassiveCopilotCodeReviewUsers int + MonthlyActiveCopilotCodeReviewUsers int + MonthlyPassiveCopilotCodeReviewUsers int + + // Chat panel mode breakdown + ChatPanelAgentMode int + ChatPanelAskMode int + ChatPanelCustomMode int + ChatPanelEditMode int + ChatPanelPlanMode int + ChatPanelUnknownMode int + + // Expanded PR metrics + PRTotalMerged int + PRMedianMinutesToMerge float64 + PRTotalSuggestions int + PRTotalAppliedSuggestions int + PRTotalMergedCreatedByCopilot int + PRTotalMergedReviewedByCopilot int + PRMedianMinToMergeCopilotAuthored float64 + PRMedianMinToMergeCopilotReviewed float64 + PRTotalCopilotSuggestions int + PRTotalCopilotAppliedSuggestions int + + // CLI breakdown + CliSessionCount int + CliRequestCount int + CliPromptCount int + CliOutputTokenSum int + CliPromptTokenSum int +} + +func (enterpriseDailyMetrics20260527) TableName() string { + return "_tool_copilot_enterprise_daily_metrics" +} + +// --- User daily metrics: new columns --- + +type userDailyMetrics20260527 struct { + UsedCli bool + UsedCopilotCodeReviewActive bool + UsedCopilotCodeReviewPassive bool + + // CLI breakdown + CliSessionCount int + CliRequestCount int + CliPromptCount int + CliOutputTokenSum int + CliPromptTokenSum int +} + +func (userDailyMetrics20260527) TableName() string { + return "_tool_copilot_user_daily_metrics" +} + +// --- Seat: new columns --- + +type seat20260527 struct { + UserName string `gorm:"type:varchar(255)"` + UserEmail string `gorm:"type:varchar(255)"` + AssigningTeamId int64 + AssigningTeamName string `gorm:"type:varchar(255)"` + AssigningTeamSlug string `gorm:"type:varchar(255)"` +} + +func (seat20260527) TableName() string { + return "_tool_copilot_seats" +} + +// --- User-teams: new table --- + +type userTeam20260527 struct { + ConnectionId uint64 `gorm:"primaryKey"` + ScopeId string `gorm:"primaryKey;type:varchar(255)"` + Day time.Time `gorm:"primaryKey;type:date"` + UserId int64 `gorm:"primaryKey"` + TeamId int64 `gorm:"primaryKey"` + + UserLogin string `gorm:"type:varchar(255);index"` + OrganizationId string `gorm:"type:varchar(100)"` + EnterpriseId string `gorm:"type:varchar(100)"` + TeamSlug string `gorm:"type:varchar(255)"` + + archived.NoPKModel +} + +func (userTeam20260527) TableName() string { + return "_tool_copilot_user_teams" +} + +func (script *addCopilotMetricsGaps) Up(basicRes context.BasicRes) errors.Error { + // Add new columns to existing tables + if err := migrationhelper.AutoMigrateTables(basicRes, + &enterpriseDailyMetrics20260527{}, + &userDailyMetrics20260527{}, + &seat20260527{}, + ); err != nil { + return err + } + + // Create new user-teams table + return migrationhelper.AutoMigrateTables(basicRes, + &userTeam20260527{}, + ) +} + +func (*addCopilotMetricsGaps) Version() uint64 { + return 20260527000000 +} + +func (*addCopilotMetricsGaps) Name() string { + return "Add Copilot metrics gaps: CLI, code review, chat modes, PR expansion, user-teams" +} diff --git a/backend/plugins/gh-copilot/models/migrationscripts/register.go b/backend/plugins/gh-copilot/models/migrationscripts/register.go index a9c1a770bfa..399735695e0 100644 --- a/backend/plugins/gh-copilot/models/migrationscripts/register.go +++ b/backend/plugins/gh-copilot/models/migrationscripts/register.go @@ -30,5 +30,6 @@ func All() []plugin.MigrationScript { new(migrateToUsageMetricsV2), new(addPRFieldsToEnterpriseMetrics), new(addOrganizationIdToUserMetrics), + new(addCopilotMetricsGaps), } } diff --git a/backend/plugins/gh-copilot/models/models.go b/backend/plugins/gh-copilot/models/models.go index f223c821827..5143ce5f8b7 100644 --- a/backend/plugins/gh-copilot/models/models.go +++ b/backend/plugins/gh-copilot/models/models.go @@ -45,5 +45,7 @@ func GetTablesInfo() []dal.Tabler { &GhCopilotUserMetricsByModelFeature{}, // Seat assignments &GhCopilotSeat{}, + // User-team mappings + &GhCopilotUserTeam{}, } } diff --git a/backend/plugins/gh-copilot/models/models_test.go b/backend/plugins/gh-copilot/models/models_test.go index 8c61d222079..ef5b3eff6f5 100644 --- a/backend/plugins/gh-copilot/models/models_test.go +++ b/backend/plugins/gh-copilot/models/models_test.go @@ -40,6 +40,7 @@ func TestGetTablesInfo(t *testing.T) { (&GhCopilotUserMetricsByLanguageModel{}).TableName(): false, (&GhCopilotUserMetricsByModelFeature{}).TableName(): false, (&GhCopilotSeat{}).TableName(): false, + (&GhCopilotUserTeam{}).TableName(): false, } if len(tables) != len(expected) { diff --git a/backend/plugins/gh-copilot/models/seat.go b/backend/plugins/gh-copilot/models/seat.go index 85ebf177ae4..d65c80f2e30 100644 --- a/backend/plugins/gh-copilot/models/seat.go +++ b/backend/plugins/gh-copilot/models/seat.go @@ -29,7 +29,12 @@ type GhCopilotSeat struct { Organization string `gorm:"primaryKey;type:varchar(255)"` UserLogin string `gorm:"primaryKey;type:varchar(255)"` UserId int64 `gorm:"index"` + UserName string `gorm:"type:varchar(255)" json:"userName"` + UserEmail string `gorm:"type:varchar(255)" json:"userEmail"` PlanType string `gorm:"type:varchar(32)"` + AssigningTeamId int64 `json:"assigningTeamId" gorm:"comment:Team that assigned the seat"` + AssigningTeamName string `json:"assigningTeamName" gorm:"type:varchar(255)"` + AssigningTeamSlug string `json:"assigningTeamSlug" gorm:"type:varchar(255)"` CreatedAt time.Time LastActivityAt *time.Time LastActivityEditor string diff --git a/backend/plugins/gh-copilot/models/user_metrics.go b/backend/plugins/gh-copilot/models/user_metrics.go index 1f17acad80a..18e9134c226 100644 --- a/backend/plugins/gh-copilot/models/user_metrics.go +++ b/backend/plugins/gh-copilot/models/user_metrics.go @@ -30,13 +30,17 @@ type GhCopilotUserDailyMetrics struct { Day time.Time `gorm:"primaryKey;type:date" json:"day"` UserId int64 `gorm:"primaryKey" json:"userId"` - OrganizationId string `json:"organizationId" gorm:"type:varchar(100)"` - EnterpriseId string `json:"enterpriseId" gorm:"type:varchar(100)"` - UserLogin string `json:"userLogin" gorm:"type:varchar(255);index"` - UsedAgent bool `json:"usedAgent"` - UsedChat bool `json:"usedChat"` + OrganizationId string `json:"organizationId" gorm:"type:varchar(100)"` + EnterpriseId string `json:"enterpriseId" gorm:"type:varchar(100)"` + UserLogin string `json:"userLogin" gorm:"type:varchar(255);index"` + UsedAgent bool `json:"usedAgent"` + UsedChat bool `json:"usedChat"` + UsedCli bool `json:"usedCli" gorm:"comment:Whether user used Copilot CLI"` + UsedCopilotCodeReviewActive bool `json:"usedCopilotCodeReviewActive" gorm:"comment:Whether user actively used code review"` + UsedCopilotCodeReviewPassive bool `json:"usedCopilotCodeReviewPassive" gorm:"comment:Whether user passively used code review"` CopilotActivityMetrics `mapstructure:",squash"` + CopilotCliMetrics `mapstructure:",squash"` common.NoPKModel } diff --git a/backend/plugins/gh-copilot/models/user_team.go b/backend/plugins/gh-copilot/models/user_team.go new file mode 100644 index 00000000000..d04d55ac56f --- /dev/null +++ b/backend/plugins/gh-copilot/models/user_team.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// GhCopilotUserTeam maps users to teams per day from the user-teams-1-day report. +// This enables team-level metrics aggregation by joining with per-user daily metrics. +type GhCopilotUserTeam struct { + ConnectionId uint64 `gorm:"primaryKey" json:"connectionId"` + ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"` + Day time.Time `gorm:"primaryKey;type:date" json:"day"` + UserId int64 `gorm:"primaryKey" json:"userId"` + TeamId int64 `gorm:"primaryKey" json:"teamId"` + + UserLogin string `json:"userLogin" gorm:"type:varchar(255);index"` + OrganizationId string `json:"organizationId" gorm:"type:varchar(100)"` + EnterpriseId string `json:"enterpriseId" gorm:"type:varchar(100)"` + TeamSlug string `json:"teamSlug" gorm:"type:varchar(255)"` + + common.NoPKModel +} + +func (GhCopilotUserTeam) TableName() string { + return "_tool_copilot_user_teams" +} diff --git a/backend/plugins/gh-copilot/tasks/enterprise_metrics_extractor.go b/backend/plugins/gh-copilot/tasks/enterprise_metrics_extractor.go index 8686b8cc415..e98a3c4f0e5 100644 --- a/backend/plugins/gh-copilot/tasks/enterprise_metrics_extractor.go +++ b/backend/plugins/gh-copilot/tasks/enterprise_metrics_extractor.go @@ -30,13 +30,31 @@ import ( // --- Enterprise report JSON structures --- type enterpriseDayTotal struct { - Day string `json:"day"` - EnterpriseId string `json:"enterprise_id"` - DailyActiveUsers int `json:"daily_active_users"` - WeeklyActiveUsers int `json:"weekly_active_users"` - MonthlyActiveUsers int `json:"monthly_active_users"` - MonthlyActiveChatUsers int `json:"monthly_active_chat_users"` - MonthlyActiveAgentUsers int `json:"monthly_active_agent_users"` + Day string `json:"day"` + EnterpriseId string `json:"enterprise_id"` + DailyActiveUsers int `json:"daily_active_users"` + WeeklyActiveUsers int `json:"weekly_active_users"` + MonthlyActiveUsers int `json:"monthly_active_users"` + MonthlyActiveChatUsers int `json:"monthly_active_chat_users"` + MonthlyActiveAgentUsers int `json:"monthly_active_agent_users"` + DailyActiveCliUsers int `json:"daily_active_cli_users"` + + // Code review user counts + DailyActiveCopilotCodeReviewUsers int `json:"daily_active_copilot_code_review_users"` + DailyPassiveCopilotCodeReviewUsers int `json:"daily_passive_copilot_code_review_users"` + WeeklyActiveCopilotCodeReviewUsers int `json:"weekly_active_copilot_code_review_users"` + WeeklyPassiveCopilotCodeReviewUsers int `json:"weekly_passive_copilot_code_review_users"` + MonthlyActiveCopilotCodeReviewUsers int `json:"monthly_active_copilot_code_review_users"` + MonthlyPassiveCopilotCodeReviewUsers int `json:"monthly_passive_copilot_code_review_users"` + + // Chat panel mode breakdown + ChatPanelAgentMode int `json:"chat_panel_agent_mode"` + ChatPanelAskMode int `json:"chat_panel_ask_mode"` + ChatPanelCustomMode int `json:"chat_panel_custom_mode"` + ChatPanelEditMode int `json:"chat_panel_edit_mode"` + ChatPanelPlanMode int `json:"chat_panel_plan_mode"` + ChatPanelUnknownMode int `json:"chat_panel_unknown_mode"` + UserInitiatedInteractionCount int `json:"user_initiated_interaction_count"` CodeGenerationActivityCount int `json:"code_generation_activity_count"` CodeAcceptanceActivityCount int `json:"code_acceptance_activity_count"` @@ -49,6 +67,7 @@ type enterpriseDayTotal struct { TotalsByLanguageFeature []totalsByLangFeature `json:"totals_by_language_feature"` TotalsByLanguageModel []totalsByLangModel `json:"totals_by_language_model"` TotalsByModelFeature []totalsByModelFeature `json:"totals_by_model_feature"` + TotalsByCli *totalsByCli `json:"totals_by_cli"` PullRequests *pullRequestStats `json:"pull_requests"` } @@ -97,10 +116,32 @@ type totalsByLangModel struct { } type pullRequestStats struct { - TotalReviewed int `json:"total_reviewed"` - TotalCreated int `json:"total_created"` - TotalCreatedByCopilot int `json:"total_created_by_copilot"` - TotalReviewedByCopilot int `json:"total_reviewed_by_copilot"` + TotalReviewed int `json:"total_reviewed"` + TotalCreated int `json:"total_created"` + TotalMerged int `json:"total_merged"` + MedianMinutesToMerge float64 `json:"median_minutes_to_merge"` + TotalSuggestions int `json:"total_suggestions"` + TotalAppliedSuggestions int `json:"total_applied_suggestions"` + TotalCreatedByCopilot int `json:"total_created_by_copilot"` + TotalReviewedByCopilot int `json:"total_reviewed_by_copilot"` + TotalMergedCreatedByCopilot int `json:"total_merged_created_by_copilot"` + TotalMergedReviewedByCopilot int `json:"total_merged_reviewed_by_copilot"` + MedianMinToMergeCopilotAuthored float64 `json:"median_minutes_to_merge_copilot_authored"` + MedianMinToMergeCopilotReviewed float64 `json:"median_minutes_to_merge_copilot_reviewed"` + TotalCopilotSuggestions int `json:"total_copilot_suggestions"` + TotalCopilotAppliedSuggestions int `json:"total_copilot_applied_suggestions"` +} + +type totalsByCli struct { + SessionCount int `json:"session_count"` + RequestCount int `json:"request_count"` + PromptCount int `json:"prompt_count"` + TokenUsage *cliTokens `json:"token_usage"` +} + +type cliTokens struct { + OutputTokensSum int `json:"output_tokens_sum"` + PromptTokensSum int `json:"prompt_tokens_sum"` } type totalsByModelFeature struct { @@ -167,6 +208,22 @@ func ExtractEnterpriseMetrics(taskCtx plugin.SubTaskContext) errors.Error { MonthlyActiveUsers: dt.MonthlyActiveUsers, MonthlyActiveChatUsers: dt.MonthlyActiveChatUsers, MonthlyActiveAgentUsers: dt.MonthlyActiveAgentUsers, + DailyActiveCliUsers: dt.DailyActiveCliUsers, + + DailyActiveCopilotCodeReviewUsers: dt.DailyActiveCopilotCodeReviewUsers, + DailyPassiveCopilotCodeReviewUsers: dt.DailyPassiveCopilotCodeReviewUsers, + WeeklyActiveCopilotCodeReviewUsers: dt.WeeklyActiveCopilotCodeReviewUsers, + WeeklyPassiveCopilotCodeReviewUsers: dt.WeeklyPassiveCopilotCodeReviewUsers, + MonthlyActiveCopilotCodeReviewUsers: dt.MonthlyActiveCopilotCodeReviewUsers, + MonthlyPassiveCopilotCodeReviewUsers: dt.MonthlyPassiveCopilotCodeReviewUsers, + + ChatPanelAgentMode: dt.ChatPanelAgentMode, + ChatPanelAskMode: dt.ChatPanelAskMode, + ChatPanelCustomMode: dt.ChatPanelCustomMode, + ChatPanelEditMode: dt.ChatPanelEditMode, + ChatPanelPlanMode: dt.ChatPanelPlanMode, + ChatPanelUnknownMode: dt.ChatPanelUnknownMode, + CopilotActivityMetrics: models.CopilotActivityMetrics{ UserInitiatedInteractionCount: dt.UserInitiatedInteractionCount, CodeGenerationActivityCount: dt.CodeGenerationActivityCount, @@ -177,11 +234,32 @@ func ExtractEnterpriseMetrics(taskCtx plugin.SubTaskContext) errors.Error { LocDeletedSum: dt.LocDeletedSum, }, } + if dt.TotalsByCli != nil { + dailyMetrics.CopilotCliMetrics = models.CopilotCliMetrics{ + CliSessionCount: dt.TotalsByCli.SessionCount, + CliRequestCount: dt.TotalsByCli.RequestCount, + CliPromptCount: dt.TotalsByCli.PromptCount, + } + if dt.TotalsByCli.TokenUsage != nil { + dailyMetrics.CopilotCliMetrics.CliOutputTokenSum = dt.TotalsByCli.TokenUsage.OutputTokensSum + dailyMetrics.CopilotCliMetrics.CliPromptTokenSum = dt.TotalsByCli.TokenUsage.PromptTokensSum + } + } if dt.PullRequests != nil { dailyMetrics.PRTotalReviewed = dt.PullRequests.TotalReviewed dailyMetrics.PRTotalCreated = dt.PullRequests.TotalCreated + dailyMetrics.PRTotalMerged = dt.PullRequests.TotalMerged + dailyMetrics.PRMedianMinutesToMerge = dt.PullRequests.MedianMinutesToMerge + dailyMetrics.PRTotalSuggestions = dt.PullRequests.TotalSuggestions + dailyMetrics.PRTotalAppliedSuggestions = dt.PullRequests.TotalAppliedSuggestions dailyMetrics.PRTotalCreatedByCopilot = dt.PullRequests.TotalCreatedByCopilot dailyMetrics.PRTotalReviewedByCopilot = dt.PullRequests.TotalReviewedByCopilot + dailyMetrics.PRTotalMergedCreatedByCopilot = dt.PullRequests.TotalMergedCreatedByCopilot + dailyMetrics.PRTotalMergedReviewedByCopilot = dt.PullRequests.TotalMergedReviewedByCopilot + dailyMetrics.PRMedianMinToMergeCopilotAuthored = dt.PullRequests.MedianMinToMergeCopilotAuthored + dailyMetrics.PRMedianMinToMergeCopilotReviewed = dt.PullRequests.MedianMinToMergeCopilotReviewed + dailyMetrics.PRTotalCopilotSuggestions = dt.PullRequests.TotalCopilotSuggestions + dailyMetrics.PRTotalCopilotAppliedSuggestions = dt.PullRequests.TotalCopilotAppliedSuggestions } results = append(results, dailyMetrics) diff --git a/backend/plugins/gh-copilot/tasks/metrics_extractor.go b/backend/plugins/gh-copilot/tasks/metrics_extractor.go index 4d635c1723e..d89eababde6 100644 --- a/backend/plugins/gh-copilot/tasks/metrics_extractor.go +++ b/backend/plugins/gh-copilot/tasks/metrics_extractor.go @@ -38,12 +38,21 @@ type copilotSeatResponse struct { LastActivityAt *string `json:"last_activity_at"` LastActivityEditor string `json:"last_activity_editor"` Assignee copilotAssignee `json:"assignee"` + AssigningTeam *copilotTeam `json:"assigning_team"` } type copilotAssignee struct { Login string `json:"login"` Id int64 `json:"id"` Type string `json:"type"` + Name string `json:"name"` + Email string `json:"email"` +} + +type copilotTeam struct { + Id int64 `json:"id"` + Name string `json:"name"` + Slug string `json:"slug"` } // ExtractOrgMetrics parses org report data from the new report download API. @@ -100,6 +109,22 @@ func ExtractOrgMetrics(taskCtx plugin.SubTaskContext) errors.Error { MonthlyActiveUsers: dt.MonthlyActiveUsers, MonthlyActiveChatUsers: dt.MonthlyActiveChatUsers, MonthlyActiveAgentUsers: dt.MonthlyActiveAgentUsers, + DailyActiveCliUsers: dt.DailyActiveCliUsers, + + DailyActiveCopilotCodeReviewUsers: dt.DailyActiveCopilotCodeReviewUsers, + DailyPassiveCopilotCodeReviewUsers: dt.DailyPassiveCopilotCodeReviewUsers, + WeeklyActiveCopilotCodeReviewUsers: dt.WeeklyActiveCopilotCodeReviewUsers, + WeeklyPassiveCopilotCodeReviewUsers: dt.WeeklyPassiveCopilotCodeReviewUsers, + MonthlyActiveCopilotCodeReviewUsers: dt.MonthlyActiveCopilotCodeReviewUsers, + MonthlyPassiveCopilotCodeReviewUsers: dt.MonthlyPassiveCopilotCodeReviewUsers, + + ChatPanelAgentMode: dt.ChatPanelAgentMode, + ChatPanelAskMode: dt.ChatPanelAskMode, + ChatPanelCustomMode: dt.ChatPanelCustomMode, + ChatPanelEditMode: dt.ChatPanelEditMode, + ChatPanelPlanMode: dt.ChatPanelPlanMode, + ChatPanelUnknownMode: dt.ChatPanelUnknownMode, + CopilotActivityMetrics: models.CopilotActivityMetrics{ UserInitiatedInteractionCount: dt.UserInitiatedInteractionCount, CodeGenerationActivityCount: dt.CodeGenerationActivityCount, @@ -110,11 +135,32 @@ func ExtractOrgMetrics(taskCtx plugin.SubTaskContext) errors.Error { LocDeletedSum: dt.LocDeletedSum, }, } + if dt.TotalsByCli != nil { + dailyMetrics.CopilotCliMetrics = models.CopilotCliMetrics{ + CliSessionCount: dt.TotalsByCli.SessionCount, + CliRequestCount: dt.TotalsByCli.RequestCount, + CliPromptCount: dt.TotalsByCli.PromptCount, + } + if dt.TotalsByCli.TokenUsage != nil { + dailyMetrics.CopilotCliMetrics.CliOutputTokenSum = dt.TotalsByCli.TokenUsage.OutputTokensSum + dailyMetrics.CopilotCliMetrics.CliPromptTokenSum = dt.TotalsByCli.TokenUsage.PromptTokensSum + } + } if dt.PullRequests != nil { dailyMetrics.PRTotalReviewed = dt.PullRequests.TotalReviewed dailyMetrics.PRTotalCreated = dt.PullRequests.TotalCreated + dailyMetrics.PRTotalMerged = dt.PullRequests.TotalMerged + dailyMetrics.PRMedianMinutesToMerge = dt.PullRequests.MedianMinutesToMerge + dailyMetrics.PRTotalSuggestions = dt.PullRequests.TotalSuggestions + dailyMetrics.PRTotalAppliedSuggestions = dt.PullRequests.TotalAppliedSuggestions dailyMetrics.PRTotalCreatedByCopilot = dt.PullRequests.TotalCreatedByCopilot dailyMetrics.PRTotalReviewedByCopilot = dt.PullRequests.TotalReviewedByCopilot + dailyMetrics.PRTotalMergedCreatedByCopilot = dt.PullRequests.TotalMergedCreatedByCopilot + dailyMetrics.PRTotalMergedReviewedByCopilot = dt.PullRequests.TotalMergedReviewedByCopilot + dailyMetrics.PRMedianMinToMergeCopilotAuthored = dt.PullRequests.MedianMinToMergeCopilotAuthored + dailyMetrics.PRMedianMinToMergeCopilotReviewed = dt.PullRequests.MedianMinToMergeCopilotReviewed + dailyMetrics.PRTotalCopilotSuggestions = dt.PullRequests.TotalCopilotSuggestions + dailyMetrics.PRTotalCopilotAppliedSuggestions = dt.PullRequests.TotalCopilotAppliedSuggestions } results = append(results, dailyMetrics) diff --git a/backend/plugins/gh-copilot/tasks/org_metrics_collector.go b/backend/plugins/gh-copilot/tasks/org_metrics_collector.go index 8f651c4821f..c3a8b5e4409 100644 --- a/backend/plugins/gh-copilot/tasks/org_metrics_collector.go +++ b/backend/plugins/gh-copilot/tasks/org_metrics_collector.go @@ -20,6 +20,7 @@ package tasks import ( "encoding/json" "fmt" + "io" "net/http" "net/url" "time" diff --git a/backend/plugins/gh-copilot/tasks/register.go b/backend/plugins/gh-copilot/tasks/register.go index ee1dcc797fc..3c7e5b1eeb9 100644 --- a/backend/plugins/gh-copilot/tasks/register.go +++ b/backend/plugins/gh-copilot/tasks/register.go @@ -27,10 +27,12 @@ func GetSubTaskMetas() []plugin.SubTaskMeta { CollectCopilotSeatAssignmentsMeta, CollectEnterpriseMetricsMeta, CollectUserMetricsMeta, + CollectUserTeamsMeta, // Extractors ExtractSeatsMeta, ExtractOrgMetricsMeta, ExtractEnterpriseMetricsMeta, ExtractUserMetricsMeta, + ExtractUserTeamsMeta, } } diff --git a/backend/plugins/gh-copilot/tasks/seat_extractor.go b/backend/plugins/gh-copilot/tasks/seat_extractor.go index 48abc3c0ce1..1a1b6b13518 100644 --- a/backend/plugins/gh-copilot/tasks/seat_extractor.go +++ b/backend/plugins/gh-copilot/tasks/seat_extractor.go @@ -96,6 +96,8 @@ func ExtractSeats(taskCtx plugin.SubTaskContext) errors.Error { Organization: connection.Organization, UserLogin: seat.Assignee.Login, UserId: seat.Assignee.Id, + UserName: seat.Assignee.Name, + UserEmail: seat.Assignee.Email, PlanType: seat.PlanType, CreatedAt: createdAt, LastActivityAt: lastAct, @@ -104,6 +106,11 @@ func ExtractSeats(taskCtx plugin.SubTaskContext) errors.Error { PendingCancellationDate: pendingCancel, UpdatedAt: updatedAt, } + if seat.AssigningTeam != nil { + toolSeat.AssigningTeamId = seat.AssigningTeam.Id + toolSeat.AssigningTeamName = seat.AssigningTeam.Name + toolSeat.AssigningTeamSlug = seat.AssigningTeam.Slug + } return []interface{}{toolSeat}, nil }, diff --git a/backend/plugins/gh-copilot/tasks/subtasks.go b/backend/plugins/gh-copilot/tasks/subtasks.go index 24a2c95f1c5..61ed5799525 100644 --- a/backend/plugins/gh-copilot/tasks/subtasks.go +++ b/backend/plugins/gh-copilot/tasks/subtasks.go @@ -53,6 +53,14 @@ var CollectUserMetricsMeta = plugin.SubTaskMeta{ Description: "Collect GitHub Copilot enterprise user-level usage metrics reports", } +var CollectUserTeamsMeta = plugin.SubTaskMeta{ + Name: "collectUserTeams", + EntryPoint: CollectUserTeams, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Collect GitHub Copilot user-team mappings from user-teams-1-day report", +} + var ExtractOrgMetricsMeta = plugin.SubTaskMeta{ Name: "extractOrgMetrics", EntryPoint: ExtractOrgMetrics, @@ -88,3 +96,12 @@ var ExtractUserMetricsMeta = plugin.SubTaskMeta{ Description: "Extract Copilot user metrics into tool-layer tables", Dependencies: []*plugin.SubTaskMeta{&CollectUserMetricsMeta}, } + +var ExtractUserTeamsMeta = plugin.SubTaskMeta{ + Name: "extractUserTeams", + EntryPoint: ExtractUserTeams, + EnabledByDefault: true, + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + Description: "Extract Copilot user-team mappings into tool-layer table", + Dependencies: []*plugin.SubTaskMeta{&CollectUserTeamsMeta}, +} diff --git a/backend/plugins/gh-copilot/tasks/user_metrics_extractor.go b/backend/plugins/gh-copilot/tasks/user_metrics_extractor.go index 96f5570f758..72992194063 100644 --- a/backend/plugins/gh-copilot/tasks/user_metrics_extractor.go +++ b/backend/plugins/gh-copilot/tasks/user_metrics_extractor.go @@ -46,11 +46,15 @@ type userDailyReport struct { LocDeletedSum int `json:"loc_deleted_sum"` UsedAgent bool `json:"used_agent"` UsedChat bool `json:"used_chat"` + UsedCli bool `json:"used_cli"` + UsedCopilotCodeReviewActive bool `json:"used_copilot_code_review_active"` + UsedCopilotCodeReviewPassive bool `json:"used_copilot_code_review_passive"` TotalsByIde []userTotalsByIde `json:"totals_by_ide"` TotalsByFeature []totalsByFeature `json:"totals_by_feature"` TotalsByLanguageFeature []totalsByLangFeature `json:"totals_by_language_feature"` TotalsByLanguageModel []totalsByLangModel `json:"totals_by_language_model"` TotalsByModelFeature []totalsByModelFeature `json:"totals_by_model_feature"` + TotalsByCli *totalsByCli `json:"totals_by_cli"` } type userTotalsByIde struct { @@ -106,16 +110,19 @@ func ExtractUserMetrics(taskCtx plugin.SubTaskContext) errors.Error { var results []interface{} // Main user daily metrics - results = append(results, &models.GhCopilotUserDailyMetrics{ - ConnectionId: data.Options.ConnectionId, - ScopeId: data.Options.ScopeId, - Day: day, - UserId: u.UserId, - OrganizationId: u.OrganizationId, - EnterpriseId: u.EnterpriseId, - UserLogin: u.UserLogin, - UsedAgent: u.UsedAgent, - UsedChat: u.UsedChat, + userMetrics := &models.GhCopilotUserDailyMetrics{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Day: day, + UserId: u.UserId, + OrganizationId: u.OrganizationId, + EnterpriseId: u.EnterpriseId, + UserLogin: u.UserLogin, + UsedAgent: u.UsedAgent, + UsedChat: u.UsedChat, + UsedCli: u.UsedCli, + UsedCopilotCodeReviewActive: u.UsedCopilotCodeReviewActive, + UsedCopilotCodeReviewPassive: u.UsedCopilotCodeReviewPassive, CopilotActivityMetrics: models.CopilotActivityMetrics{ UserInitiatedInteractionCount: u.UserInitiatedInteractionCount, CodeGenerationActivityCount: u.CodeGenerationActivityCount, @@ -125,7 +132,19 @@ func ExtractUserMetrics(taskCtx plugin.SubTaskContext) errors.Error { LocAddedSum: u.LocAddedSum, LocDeletedSum: u.LocDeletedSum, }, - }) + } + if u.TotalsByCli != nil { + userMetrics.CopilotCliMetrics = models.CopilotCliMetrics{ + CliSessionCount: u.TotalsByCli.SessionCount, + CliRequestCount: u.TotalsByCli.RequestCount, + CliPromptCount: u.TotalsByCli.PromptCount, + } + if u.TotalsByCli.TokenUsage != nil { + userMetrics.CopilotCliMetrics.CliOutputTokenSum = u.TotalsByCli.TokenUsage.OutputTokensSum + userMetrics.CopilotCliMetrics.CliPromptTokenSum = u.TotalsByCli.TokenUsage.PromptTokensSum + } + } + results = append(results, userMetrics) // User by IDE for _, ide := range u.TotalsByIde { diff --git a/backend/plugins/gh-copilot/tasks/user_teams_collector.go b/backend/plugins/gh-copilot/tasks/user_teams_collector.go new file mode 100644 index 00000000000..2ae0200d2ef --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/user_teams_collector.go @@ -0,0 +1,133 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +const rawUserTeamsTable = "copilot_user_teams" + +// CollectUserTeams collects user-team mapping data from the user-teams-1-day report. +// This enables team-level metrics aggregation by joining with per-user daily metrics. +func CollectUserTeams(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*GhCopilotTaskData) + if !ok { + return errors.Default.New("task data is not GhCopilotTaskData") + } + connection := data.Connection + connection.Normalize() + + var urlTemplate string + + if connection.HasEnterprise() { + urlTemplate = fmt.Sprintf("enterprises/%s/copilot/metrics/reports/user-teams-1-day", connection.Enterprise) + } else if connection.Organization != "" { + urlTemplate = fmt.Sprintf("orgs/%s/copilot/metrics/reports/user-teams-1-day", connection.Organization) + } else { + return nil + } + + apiClient, err := CreateApiClient(taskCtx.TaskContext(), connection) + if err != nil { + return err + } + + rawArgs := helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawUserTeamsTable, + Options: copilotRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Organization: connection.Organization, + Endpoint: connection.Endpoint, + }, + } + + collector, err := helper.NewStatefulApiCollector(rawArgs) + if err != nil { + return err + } + + now := time.Now().UTC() + start, until := computeReportDateRange(now, collector.GetSince()) + logger := taskCtx.GetLogger() + + dayIter := newDayIterator(start, until) + + err = collector.InitCollector(helper.ApiCollectorArgs{ + ApiClient: apiClient, + Input: dayIter, + UrlTemplate: urlTemplate, + Query: func(reqData *helper.RequestData) (url.Values, errors.Error) { + input := reqData.Input.(*dayInput) + q := url.Values{} + q.Set("day", input.Day) + return q, nil + }, + Incremental: true, + Concurrency: 1, + AfterResponse: ignoreNoContent, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + body, readErr := io.ReadAll(res.Body) + res.Body.Close() + if readErr != nil { + return nil, errors.Default.Wrap(readErr, "failed to read report metadata") + } + if isEmptyReport(body) { + return nil, nil + } + + var meta reportMetadataResponse + if jsonErr := json.Unmarshal(body, &meta); jsonErr != nil { + return nil, errors.Default.Wrap(jsonErr, "failed to parse report metadata") + } + + var results []json.RawMessage + for _, link := range meta.DownloadLinks { + reportBody, dlErr := downloadReport(link, logger) + if dlErr != nil { + return nil, dlErr + } + if reportBody == nil { + continue + } + // User-teams reports are JSONL format + records, parseErr := parseJSONL(reportBody) + if parseErr != nil { + return nil, parseErr + } + results = append(results, records...) + } + return results, nil + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/gh-copilot/tasks/user_teams_extractor.go b/backend/plugins/gh-copilot/tasks/user_teams_extractor.go new file mode 100644 index 00000000000..72a3de8abe9 --- /dev/null +++ b/backend/plugins/gh-copilot/tasks/user_teams_extractor.go @@ -0,0 +1,93 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/gh-copilot/models" +) + +// userTeamRecord represents a single line from the user-teams-1-day JSONL report. +type userTeamRecord struct { + Day string `json:"day"` + UserId int64 `json:"user_id"` + UserLogin string `json:"user_login"` + OrganizationId string `json:"organization_id"` + EnterpriseId string `json:"enterprise_id"` + TeamId int64 `json:"team_id"` + Slug string `json:"slug"` +} + +// ExtractUserTeams parses user-team JSONL records into the GhCopilotUserTeam model. +func ExtractUserTeams(taskCtx plugin.SubTaskContext) errors.Error { + data, ok := taskCtx.TaskContext().GetData().(*GhCopilotTaskData) + if !ok { + return errors.Default.New("task data is not GhCopilotTaskData") + } + connection := data.Connection + connection.Normalize() + + params := copilotRawParams{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Organization: connection.Organization, + Endpoint: connection.Endpoint, + } + + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Table: rawUserTeamsTable, + Options: params, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + var rec userTeamRecord + if err := errors.Convert(json.Unmarshal(row.Data, &rec)); err != nil { + return nil, err + } + + day, parseErr := time.Parse("2006-01-02", rec.Day) + if parseErr != nil { + return nil, errors.BadInput.Wrap(parseErr, "invalid day in user-teams report") + } + + return []interface{}{ + &models.GhCopilotUserTeam{ + ConnectionId: data.Options.ConnectionId, + ScopeId: data.Options.ScopeId, + Day: day, + UserId: rec.UserId, + TeamId: rec.TeamId, + UserLogin: rec.UserLogin, + OrganizationId: rec.OrganizationId, + EnterpriseId: rec.EnterpriseId, + TeamSlug: rec.Slug, + }, + }, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/github/e2e/account_test.go b/backend/plugins/github/e2e/account_test.go index 817ef6fab22..7b6466aba23 100644 --- a/backend/plugins/github/e2e/account_test.go +++ b/backend/plugins/github/e2e/account_test.go @@ -20,11 +20,15 @@ package e2e import ( "testing" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" "github.com/apache/incubator-devlake/helpers/e2ehelper" "github.com/apache/incubator-devlake/plugins/github/impl" "github.com/apache/incubator-devlake/plugins/github/models" "github.com/apache/incubator-devlake/plugins/github/tasks" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestAccountDataFlow(t *testing.T) { @@ -108,4 +112,33 @@ func TestAccountDataFlow(t *testing.T) { "_raw_data_remark", }, ) + + // Referential-integrity invariant (#8886): every account the repo references in + // _tool_github_repo_accounts must have a domain `accounts` row, so issues.creator_id / + // pull_requests.author_id / merged_by_id never point at a missing account. We generate + // the domain id with the SAME generator the issue/PR convertors use, so this is a + // faithful proxy for the FK join the issue reported as broken. It also fails loudly if a + // future change shrinks ConvertAccounts' coverage or diverges the id generation. + accountIdGen := didgen.NewDomainIdGenerator(&models.GithubAccount{}) + var repoAccounts []models.GithubRepoAccount + require.NoError(t, dataflowTester.Dal.All(&repoAccounts, + dal.Where("repo_github_id = ? AND connection_id = ? AND account_id > 0", + taskData.Options.GithubId, taskData.Options.ConnectionId), + )) + require.NotEmpty(t, repoAccounts, "fixture must reference at least one account") + sawOrphanCase := false + for _, ra := range repoAccounts { + if ra.Login == "milichev" { + sawOrphanCase = true // the non-committer author from the issue repro + } + domainId := accountIdGen.Generate(taskData.Options.ConnectionId, ra.AccountId) + count, err := dataflowTester.Dal.Count( + dal.From(&crossdomain.Account{}), + dal.Where("id = ?", domainId), + ) + require.NoError(t, err) + assert.Equalf(t, int64(1), count, + "orphan FK: repo account %q (id=%d) has no domain accounts row %q", ra.Login, ra.AccountId, domainId) + } + assert.True(t, sawOrphanCase, "fixture should include the non-committer orphan case (milichev)") } diff --git a/backend/plugins/github/e2e/raw_tables/_tool_github_repo_accounts.csv b/backend/plugins/github/e2e/raw_tables/_tool_github_repo_accounts.csv index 8b86664428f..7aa42a4b0a9 100644 --- a/backend/plugins/github/e2e/raw_tables/_tool_github_repo_accounts.csv +++ b/backend/plugins/github/e2e/raw_tables/_tool_github_repo_accounts.csv @@ -1,16 +1,17 @@ -connection_id,account_id,repo_github_id,login -1,21979,134018330,appleboy -1,964542,134018330,sarathsp06 -1,1052632,134018330,runner-mei -1,3794113,134018330,shanhuhai5739 -1,3971390,134018330,ppmoon -1,7496278,134018330,panjf2000 -1,8518239,134018330,gitter-badger -1,11763614,2,Moonlight-Zhao -1,12420699,2,shanghai-Jerry -1,14950473,2,zqkgo -1,22429695,2,codecov[bot] -1,24841832,2,rikewang -1,31087327,2,chensanle -1,32893410,2,zhangyuanxue -1,38849208,2,king526 \ No newline at end of file +connection_id,account_id,repo_github_id,login,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +1,21979,134018330,appleboy,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_accounts,8, +1,964542,134018330,sarathsp06,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_accounts,1, +1,1052632,134018330,runner-mei,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_accounts,13, +1,3794113,134018330,shanhuhai5739,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_accounts,2, +1,3971390,134018330,ppmoon,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_accounts,14, +1,7496278,134018330,panjf2000,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_accounts,5, +1,8518239,134018330,gitter-badger,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_accounts,9, +1,145564,134018330,milichev,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_issues,1, +1,11763614,2,Moonlight-Zhao,,,0, +1,12420699,2,shanghai-Jerry,,,0, +1,14950473,2,zqkgo,,,0, +1,22429695,2,codecov[bot],,,0, +1,24841832,2,rikewang,,,0, +1,31087327,2,chensanle,,,0, +1,32893410,2,zhangyuanxue,,,0, +1,38849208,2,king526,,,0, diff --git a/backend/plugins/github/e2e/snapshot_tables/account.csv b/backend/plugins/github/e2e/snapshot_tables/account.csv index 1092e7a5296..c3022a4a296 100644 --- a/backend/plugins/github/e2e/snapshot_tables/account.csv +++ b/backend/plugins/github/e2e/snapshot_tables/account.csv @@ -1,5 +1,6 @@ id,email,full_name,user_name,avatar_url,organization,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark github:GithubAccount:1:1052632,runner.mei@,runner,runner-mei,https://avatars.githubusercontent.com/u/1052632?v=4,,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_accounts,13, +github:GithubAccount:1:145564,,,milichev,,,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_issues,1, github:GithubAccount:1:21979,appleboy.tw@gmail.com,Bo-Yi Wu,appleboy,https://avatars.githubusercontent.com/u/21979?v=4,"COSCUP,nodejs-tw,moztw,h5bp,CodeIgniter-TW,drone,Getmore,golangtw,laravel-taiwan,go-xorm,gin-gonic,PHPConf-TW,Mediatek-Cloud,SJFinder,go-gitea,laradock,gin-contrib,tagfans,maintainers,go-training,go-ggz,the-benchmarker,golang-queue","{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_accounts,8, github:GithubAccount:1:3794113,shanhu5739@gmail.com,Derek,shanhuhai5739,https://avatars.githubusercontent.com/u/3794113?v=4,,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_accounts,2, github:GithubAccount:1:3971390,cnliuyunpeng@gmail.com,ppmoon,ppmoon,https://avatars.githubusercontent.com/u/3971390?v=4,,"{""ConnectionId"":1,""Name"":""panjf2000/ants""}",_raw_github_api_accounts,14, diff --git a/backend/plugins/github/tasks/account_convertor.go b/backend/plugins/github/tasks/account_convertor.go index e86780d2746..3d4db97d155 100644 --- a/backend/plugins/github/tasks/account_convertor.go +++ b/backend/plugins/github/tasks/account_convertor.go @@ -22,6 +22,7 @@ import ( "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/common" "github.com/apache/incubator-devlake/core/models/domainlayer" "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" @@ -30,6 +31,19 @@ import ( "github.com/apache/incubator-devlake/plugins/github/models" ) +// repoAccountForConvert is the row projected by ConvertAccounts' query: every +// account referenced by the repo (from _tool_github_repo_accounts), enriched +// with profile detail from _tool_github_accounts when it was collected. The +// embedded NoPKModel carries the RawDataOrigin across to the domain row. +type repoAccountForConvert struct { + Id int + Login string + Name string + Email string + AvatarUrl string + common.NoPKModel +} + func init() { RegisterSubtaskMeta(&ConvertAccountsMeta) } @@ -38,12 +52,12 @@ var ConvertAccountsMeta = plugin.SubTaskMeta{ Name: "Convert Users", EntryPoint: ConvertAccounts, EnabledByDefault: true, - Description: "Convert tool layer table github_accounts into domain layer table accounts", + Description: "Convert every account referenced by the repo (tool layer repo_accounts, enriched by github_accounts) into domain layer table accounts", DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, DependencyTables: []string{ - models.GithubAccount{}.TableName(), // cursor - models.GithubRepoAccount{}.TableName(), // cursor - models.GithubAccountOrg{}.TableName()}, // account id gen + models.GithubRepoAccount{}.TableName(), // cursor (every user referenced by the repo) + models.GithubAccount{}.TableName(), // left-join enrichment (profile detail, optional) + models.GithubAccountOrg{}.TableName()}, // org pluck ProductTables: []string{crossdomain.Account{}.TableName()}, } @@ -53,7 +67,7 @@ func ConvertAccounts(taskCtx plugin.SubTaskContext) errors.Error { accountIdGen := didgen.NewDomainIdGenerator(&models.GithubAccount{}) - converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[models.GithubAccount]{ + converter, err := api.NewStatefulDataConverter(&api.StatefulDataConverterArgs[repoAccountForConvert]{ SubtaskCommonArgs: &api.SubtaskCommonArgs{ SubTaskContext: taskCtx, Table: RAW_ACCOUNT_TABLE, @@ -62,29 +76,57 @@ func ConvertAccounts(taskCtx plugin.SubTaskContext) errors.Error { Name: data.Options.Name, }, }, + // Source every account referenced by this repo from _tool_github_repo_accounts + // (which the issue/PR/commit extractors populate for any author, assignee, or + // merged-by user), and LEFT JOIN _tool_github_accounts for profile detail when it + // was collected. This guarantees a domain `accounts` row for every CreatorId / + // AuthorId the other convertors emit, instead of only for users who committed. + // Raw-data provenance follows the same rule as the profile fields: the enriched + // _tool_github_accounts row when we collected one, the repo_accounts row otherwise. + // Note the consequence: fallback-provenance rows carry a _raw_data_table other than + // _raw_github_api_accounts, so the batch-save divider's full-sync delete-then-reinsert + // (keyed on this converter's raw table) never deletes them; they are reconciled by + // upsert only. Scope deletion still covers them via _raw_data_params. + // SQL is kept DB-agnostic (no backtick quoting, COALESCE not IFNULL) so it runs on + // both MySQL and PostgreSQL. Input: func(stateManager *api.SubtaskStateManager) (dal.Rows, errors.Error) { clauses := []dal.Clause{ - dal.Select("_tool_github_accounts.*"), - dal.From(&models.GithubAccount{}), + dal.Select(`_tool_github_repo_accounts.account_id AS id, + _tool_github_repo_accounts.login AS login, + COALESCE(ga.name, '') AS name, + COALESCE(ga.email, '') AS email, + COALESCE(ga.avatar_url, '') AS avatar_url, + COALESCE(ga._raw_data_params, _tool_github_repo_accounts._raw_data_params) AS _raw_data_params, + COALESCE(ga._raw_data_table, _tool_github_repo_accounts._raw_data_table) AS _raw_data_table, + COALESCE(ga._raw_data_id, _tool_github_repo_accounts._raw_data_id) AS _raw_data_id, + COALESCE(ga._raw_data_remark, _tool_github_repo_accounts._raw_data_remark) AS _raw_data_remark`), + dal.From(&models.GithubRepoAccount{}), + dal.Join(`left join _tool_github_accounts ga on ( + ga.connection_id = _tool_github_repo_accounts.connection_id + AND ga.id = _tool_github_repo_accounts.account_id + )`), dal.Where( - "repo_github_id = ? and _tool_github_accounts.connection_id=?", + `_tool_github_repo_accounts.repo_github_id = ? + AND _tool_github_repo_accounts.connection_id = ? + AND _tool_github_repo_accounts.account_id > 0`, data.Options.GithubId, data.Options.ConnectionId, ), - dal.Join(`left join _tool_github_repo_accounts gra on ( - _tool_github_accounts.connection_id = gra.connection_id - AND _tool_github_accounts.id = gra.account_id - )`), } if stateManager.IsIncremental() { since := stateManager.GetSince() if since != nil { - clauses = append(clauses, dal.Where("_tool_github_accounts.updated_at >= ?", since)) + // Incremental cursor intentionally tracks _tool_github_repo_accounts.updated_at + // (repo membership), not _tool_github_accounts.updated_at (profile freshness): + // account-detail re-enrichment is reconciled on the next full sync. Do not switch + // this back to _tool_github_accounts — that is what left issue/PR-only authors + // orphaned (#8886). + clauses = append(clauses, dal.Where("_tool_github_repo_accounts.updated_at >= ?", since)) } } return db.Cursor(clauses...) }, - Convert: func(githubUser *models.GithubAccount) ([]interface{}, errors.Error) { + Convert: func(githubUser *repoAccountForConvert) ([]interface{}, errors.Error) { // query related orgs var orgs []string err := db.Pluck(`org_login`, &orgs, diff --git a/backend/plugins/github/tasks/api_client.go b/backend/plugins/github/tasks/api_client.go index 42181ff139e..c5be8a0f22b 100644 --- a/backend/plugins/github/tasks/api_client.go +++ b/backend/plugins/github/tasks/api_client.go @@ -26,7 +26,6 @@ import ( "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/github/models" - "github.com/apache/incubator-devlake/plugins/github/token" ) func CreateApiClient(taskCtx plugin.TaskContext, connection *models.GithubConnection) (*api.ApiAsyncClient, errors.Error) { @@ -35,40 +34,10 @@ func CreateApiClient(taskCtx plugin.TaskContext, connection *models.GithubConnec return nil, err } - logger := taskCtx.GetLogger() - db := taskCtx.GetDal() - encryptionSecret := taskCtx.GetConfig(plugin.EncodeKeyEnvStr) - - // Inject TokenProvider for OAuth refresh or GitHub App installation tokens. - var tp *token.TokenProvider - if connection.RefreshToken != "" { - tp = token.NewTokenProvider(connection, db, apiClient.GetClient(), logger, encryptionSecret) - } else if connection.AuthMethod == models.AppKey && connection.InstallationID != 0 { - tp = token.NewAppInstallationTokenProvider(connection, db, apiClient.GetClient(), logger, encryptionSecret) - } - if tp != nil { - // Wrap the transport - baseTransport := apiClient.GetClient().Transport - if baseTransport == nil { - baseTransport = http.DefaultTransport - } - - rt := token.NewRefreshRoundTripper(baseTransport, tp) - apiClient.GetClient().Transport = rt - logger.Info("Installed token refresh round tripper for connection %d (authMethod=%s)", - connection.ID, connection.AuthMethod) - } - - // Persist the freshly minted token so the DB has a correctly encrypted value. - // PrepareApiClient (called by NewApiClientFromConnection) mints the token - // in-memory but does not persist it; without this, the DB may contain a stale - // or corrupted token that breaks GET /connections. - if connection.AuthMethod == models.AppKey && connection.Token != "" { - if err := token.PersistEncryptedTokenColumns(db, connection, encryptionSecret, logger, false); err != nil { - logger.Warn(err, "Failed to persist initial token for connection %d", connection.ID) - } else { - logger.Info("Persisted initial token for connection %d", connection.ID) - } + // inject the shared auth layer + _, err = CreateAuthenticatedHttpClient(taskCtx, connection, apiClient.GetClient()) + if err != nil { + return nil, err } // create rate limit calculator diff --git a/backend/plugins/github/tasks/http_client.go b/backend/plugins/github/tasks/http_client.go new file mode 100644 index 00000000000..33ef6a3df6b --- /dev/null +++ b/backend/plugins/github/tasks/http_client.go @@ -0,0 +1,88 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/plugins/github/models" + "github.com/apache/incubator-devlake/plugins/github/token" +) + +func CreateAuthenticatedHttpClient( + taskCtx plugin.TaskContext, + connection *models.GithubConnection, + baseClient *http.Client, +) (*http.Client, errors.Error) { + + logger := taskCtx.GetLogger() + db := taskCtx.GetDal() + encryptionSecret := taskCtx.GetConfig(plugin.EncodeKeyEnvStr) + + if baseClient == nil { + baseClient = &http.Client{} + } + + // Inject TokenProvider for OAuth refresh or GitHub App installation tokens. + var tp *token.TokenProvider + if connection.RefreshToken != "" { + tp = token.NewTokenProvider(connection, db, baseClient, logger, encryptionSecret) + } else if connection.AuthMethod == models.AppKey && connection.InstallationID != 0 { + tp = token.NewAppInstallationTokenProvider(connection, db, baseClient, logger, encryptionSecret) + } + + baseTransport := baseClient.Transport + if baseTransport == nil { + baseTransport = http.DefaultTransport + } + + if tp != nil { + baseClient.Transport = token.NewRefreshRoundTripper(baseTransport, tp) + logger.Info( + "Installed token refresh round tripper for connection %d (authMethod=%s)", + connection.ID, + connection.AuthMethod, + ) + + } else if connection.Token != "" { + baseClient.Transport = token.NewStaticRoundTripper( + baseTransport, + connection.Token, + ) + logger.Info( + "Installed static token round tripper for connection %d", + connection.ID, + ) + } + + // Persist the freshly minted token so the DB has a correctly encrypted value. + // PrepareApiClient (called by NewApiClientFromConnection) mints the token + // in-memory but does not persist it; without this, the DB may contain a stale + // or corrupted token that breaks GET /connections. + if connection.AuthMethod == models.AppKey && connection.Token != "" { + if err := token.PersistEncryptedTokenColumns(db, connection, encryptionSecret, logger, false); err != nil { + logger.Warn(err, "Failed to persist initial token for connection %d", connection.ID) + } else { + logger.Info("Persisted initial token for connection %d", connection.ID) + } + } + + return baseClient, nil +} diff --git a/backend/plugins/github/tasks/pr_convertor.go b/backend/plugins/github/tasks/pr_convertor.go index 48a40cc4e67..c4e27a6b9bb 100644 --- a/backend/plugins/github/tasks/pr_convertor.go +++ b/backend/plugins/github/tasks/pr_convertor.go @@ -87,7 +87,6 @@ func ConvertPullRequests(taskCtx plugin.SubTaskContext) errors.Error { OriginalStatus: pr.State, Title: pr.Title, Url: pr.Url, - AuthorId: accountIdGen.Generate(data.Options.ConnectionId, pr.AuthorId), AuthorName: pr.AuthorName, Description: pr.Body, CreatedDate: pr.GithubCreatedAt, @@ -104,9 +103,17 @@ func ConvertPullRequests(taskCtx plugin.SubTaskContext) errors.Error { Additions: pr.Additions, Deletions: pr.Deletions, MergedByName: pr.MergedByName, - MergedById: accountIdGen.Generate(data.Options.ConnectionId, pr.MergedById), IsDraft: pr.IsDraft, } + // Generate account ids only for real users (#8886): a zero AuthorId (deleted + // user) or zero MergedById (unmerged PR) would otherwise produce an id like + // github:GithubAccount:1:0 that no accounts row can ever match. + if pr.AuthorId != 0 { + domainPr.AuthorId = accountIdGen.Generate(data.Options.ConnectionId, pr.AuthorId) + } + if pr.MergedById != 0 { + domainPr.MergedById = accountIdGen.Generate(data.Options.ConnectionId, pr.MergedById) + } if pr.State == "open" || pr.State == "OPEN" { domainPr.Status = code.OPEN } else if pr.State == "MERGED" || (pr.State == "closed" && (pr.Merged || pr.MergedAt != nil)) { diff --git a/backend/plugins/github/tasks/pr_extractor.go b/backend/plugins/github/tasks/pr_extractor.go index 802846f301c..9c3f1ca8fc9 100644 --- a/backend/plugins/github/tasks/pr_extractor.go +++ b/backend/plugins/github/tasks/pr_extractor.go @@ -151,6 +151,16 @@ func ExtractApiPullRequests(taskCtx plugin.SubTaskContext) errors.Error { githubPr.AuthorName = githubUser.Login githubPr.AuthorId = githubUser.AccountId } + // Emit a repo_account for the merged-by user too, so pull_requests.merged_by_id + // resolves to a domain account instead of an orphan FK (ConvertAccounts sources + // every referenced user from _tool_github_repo_accounts). + if body.MergedBy != nil { + mergedByUser, err := convertAccount(body.MergedBy, data.Options.GithubId, data.Options.ConnectionId) + if err != nil { + return nil, err + } + results = append(results, mergedByUser) + } for _, label := range body.Labels { results = append(results, &models.GithubPrLabel{ ConnectionId: data.Options.ConnectionId, diff --git a/backend/plugins/github/token/round_tripper.go b/backend/plugins/github/token/round_tripper.go index 8868572dae6..8f05f838d0d 100644 --- a/backend/plugins/github/token/round_tripper.go +++ b/backend/plugins/github/token/round_tripper.go @@ -19,6 +19,8 @@ package token import ( "net/http" + "strings" + "sync/atomic" ) // RefreshRoundTripper is an HTTP transport middleware that automatically manages OAuth token refreshes. @@ -93,3 +95,37 @@ func (rt *RefreshRoundTripper) roundTripWithRetry(req *http.Request, refreshAtte return resp, nil } + +// StaticRoundTripper is an HTTP transport that injects a fixed bearer token. +// Unlike RefreshRoundTripper, it does NOT attempt refresh or retries. +type StaticRoundTripper struct { + base http.RoundTripper + tokens []string + idx atomic.Uint64 +} + +func NewStaticRoundTripper(base http.RoundTripper, rawToken string) *StaticRoundTripper { + if base == nil { + base = http.DefaultTransport + } + parts := strings.Split(rawToken, ",") + tokens := make([]string, 0, len(parts)) + for _, t := range parts { + if t = strings.TrimSpace(t); t != "" { + tokens = append(tokens, t) + } + } + if len(tokens) == 0 { + tokens = []string{rawToken} + } + return &StaticRoundTripper{base: base, tokens: tokens} +} + +func (rt *StaticRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + // always overrides headers put by SetupAuthentication, to make sure the token is always injected + // Add(1)-1 yields a 0-based sequence (0, 1, 2, ...) so rotation starts at tokens[0]. + tok := rt.tokens[(rt.idx.Add(1)-1)%uint64(len(rt.tokens))] + reqClone := req.Clone(req.Context()) + reqClone.Header.Set("Authorization", "Bearer "+tok) + return rt.base.RoundTrip(reqClone) +} diff --git a/backend/plugins/github_graphql/impl/impl.go b/backend/plugins/github_graphql/impl/impl.go index 3efbe10a272..f56c77644bf 100644 --- a/backend/plugins/github_graphql/impl/impl.go +++ b/backend/plugins/github_graphql/impl/impl.go @@ -20,10 +20,7 @@ package impl import ( "context" "fmt" - "net/http" - "net/url" "reflect" - "strings" "time" "github.com/apache/incubator-devlake/core/models/domainlayer/devops" @@ -39,7 +36,6 @@ import ( "github.com/apache/incubator-devlake/plugins/github_graphql/model/migrationscripts" "github.com/apache/incubator-devlake/plugins/github_graphql/tasks" "github.com/merico-ai/graphql" - "golang.org/x/oauth2" ) // make sure interface is implemented @@ -180,46 +176,10 @@ func (p GithubGraphql) PrepareTaskData(taskCtx plugin.TaskContext, options map[s return nil, err } - tokens := strings.Split(connection.Token, ",") - src := oauth2.StaticTokenSource( - &oauth2.Token{AccessToken: tokens[0]}, - ) - oauthContext := taskCtx.GetContext() - proxy := connection.GetProxy() - if proxy != "" { - pu, err := url.Parse(proxy) - if err != nil { - return nil, errors.Convert(err) - } - if pu.Scheme == "http" || pu.Scheme == "socks5" { - proxyClient := &http.Client{ - Transport: &http.Transport{Proxy: http.ProxyURL(pu)}, - } - oauthContext = context.WithValue( - taskCtx.GetContext(), - oauth2.HTTPClient, - proxyClient, - ) - logger.Debug("Proxy set in oauthContext to %s", proxy) - } else { - return nil, errors.BadInput.New("Unsupported scheme set in proxy") - } - } - - httpClient := oauth2.NewClient(oauthContext, src) - endpoint, err := errors.Convert01(url.Parse(connection.Endpoint)) - if err != nil { - return nil, errors.BadInput.Wrap(err, fmt.Sprintf("malformed connection endpoint supplied: %s", connection.Endpoint)) - } - - // github.com and github enterprise have different graphql endpoints - endpoint.Path = "/graphql" // see https://docs.github.com/en/graphql/guides/forming-calls-with-graphql - if endpoint.Hostname() != "api.github.com" { - // see https://docs.github.com/en/enterprise-server@3.11/graphql/guides/forming-calls-with-graphql - endpoint.Path = "/api/graphql" - } - client := graphql.NewClient(endpoint.String(), httpClient) - graphqlClient, err := helper.CreateAsyncGraphqlClient(taskCtx, client, taskCtx.GetLogger(), + graphqlClient, err := tasks.CreateGraphqlClient( + taskCtx, + connection, + apiClient.ApiClient.GetClient(), func(ctx context.Context, client *graphql.Client, logger log.Logger) (rateRemaining int, resetAt *time.Time, err errors.Error) { var query GraphQueryRateLimit dataErrors, err := errors.Convert01(client.Query(taskCtx.GetContext(), &query, nil)) @@ -230,8 +190,7 @@ func (p GithubGraphql) PrepareTaskData(taskCtx plugin.TaskContext, options map[s return 0, nil, errors.Default.Wrap(dataErrors[0], `query rate limit fail`) } if query.RateLimit == nil { - logger.Info(`github graphql rate limit are disabled, fallback to 5000req/hour`) - return 5000, nil, nil + return 0, nil, errors.Default.New("rate limit unavailable") } logger.Info(`github graphql init success with remaining %d/%d and will reset at %s`, query.RateLimit.Remaining, query.RateLimit.Limit, query.RateLimit.ResetAt) diff --git a/backend/plugins/github_graphql/tasks/graphql_client.go b/backend/plugins/github_graphql/tasks/graphql_client.go new file mode 100644 index 00000000000..9c248e15cb6 --- /dev/null +++ b/backend/plugins/github_graphql/tasks/graphql_client.go @@ -0,0 +1,66 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "context" + "fmt" + "net/http" + "net/url" + "time" + + "github.com/merico-ai/graphql" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/log" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/github/models" +) + +func CreateGraphqlClient( + taskCtx plugin.TaskContext, + connection *models.GithubConnection, + httpClient *http.Client, + getRateRemaining func(context.Context, *graphql.Client, log.Logger) (rateRemaining int, resetAt *time.Time, err errors.Error), +) (*helper.GraphqlAsyncClient, errors.Error) { + // Build endpoint + endpoint, err := errors.Convert01(url.Parse(connection.Endpoint)) + if err != nil { + return nil, errors.BadInput.Wrap(err, fmt.Sprintf("malformed connection endpoint supplied: %s", connection.Endpoint)) + } + // github.com and github enterprise have different graphql endpoints + if endpoint.Hostname() == "api.github.com" { + // see https://docs.github.com/en/graphql/guides/forming-calls-with-graphql + endpoint.Path = "/graphql" + } else { + // see https://docs.github.com/en/enterprise-server@3.11/graphql/guides/forming-calls-with-graphql + endpoint.Path = "/api/graphql" + } + + gqlClient := graphql.NewClient(endpoint.String(), httpClient) + + return helper.CreateAsyncGraphqlClient( + taskCtx, + gqlClient, + taskCtx.GetLogger(), + getRateRemaining, + // GitHub GraphQL default fallback aligns with GitHub's standard rate limit (~5000) + helper.WithFallbackRateLimit(5000), + ) +} diff --git a/backend/plugins/linear/README.md b/backend/plugins/linear/README.md new file mode 100644 index 00000000000..e52b4527176 --- /dev/null +++ b/backend/plugins/linear/README.md @@ -0,0 +1,115 @@ + + +# Linear + +## Summary + +This plugin collects data from [Linear](https://linear.app) through its +[GraphQL API](https://linear.app/developers/graphql) and maps it into DevLake's +standardized `ticket` domain, so Linear issues appear in DevLake dashboards +(throughput, lead/cycle time, sprint burndown, etc.). + +The selectable **scope** is a Linear **Team**, which maps to a domain `Board`. + +## Supported data + +| Linear entity | Tool-layer table | Domain-layer table | +|-----------------|-----------------------------------|--------------------------------------------| +| Team | `_tool_linear_teams` (scope) | `boards` | +| User | `_tool_linear_accounts` | `accounts` | +| Workflow state | `_tool_linear_workflow_states` | (drives issue status mapping) | +| Issue | `_tool_linear_issues` | `issues`, `board_issues` | +| Label | `_tool_linear_issue_labels` | `issue_labels` | +| Comment | `_tool_linear_comments` | `issue_comments` | +| Cycle | `_tool_linear_cycles` | `sprints`, `board_sprints`, `sprint_issues`| +| Issue history | `_tool_linear_issue_history` | `issue_changelogs` | + +### Field mapping highlights + +- **Status** — derived deterministically from Linear's `WorkflowState.type` + (no manual mapping needed, unlike Jira): + - `backlog`, `unstarted` → `TODO` + - `started` → `IN_PROGRESS` + - `completed`, `canceled` → `DONE` +- **Priority** — Linear's integer priority maps to a label: `0` No priority, + `1` Urgent, `2` High, `3` Medium, `4` Low. +- **Type** — Linear has no native issue type, so issues default to `REQUIREMENT`. +- **Lead time** — `completedAt − createdAt` (Linear provides `startedAt`/`completedAt` + natively; the history changelog captures every status transition). +- **Story points** — Linear's `estimate`. + +## Authentication + +The plugin uses a Linear **personal API key**, passed verbatim in the +`Authorization` header (no `Bearer` prefix). Create one under +**Settings → Security & access → Personal API keys** in Linear. + +## Configuration + +Create a connection: + +``` +curl 'http://localhost:8080/api/plugins/linear/connections' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "name": "linear", + "endpoint": "https://api.linear.app/graphql", + "token": "", + "rateLimitPerHour": 1500 +}' +``` + +Add a team scope (the team id is the Linear team UUID): + +``` +curl 'http://localhost:8080/api/plugins/linear/connections//scopes' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "data": [{ "connectionId": , "teamId": "", "name": "Engineering" }] +}' +``` + +## Collecting data + +``` +curl 'http://localhost:8080/api/pipelines' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "name": "linear pipeline", + "plan": [[{ + "plugin": "linear", + "options": { "connectionId": , "teamId": "" } + }]] +}' +``` + +## Rate limiting + +Linear enforces a per-API-key request budget (1,500 requests/hour) plus a +complexity budget. The collector paces requests against the configured +`rateLimitPerHour` (default 1500). Issues are collected incrementally using +`updatedAt` ordering so re-runs only fetch changes. + +## Limitations / roadmap + +- Authentication is personal API key only; OAuth2 is a planned follow-up. +- Issue type defaults to `REQUIREMENT`; label-based type mapping via the scope + config is a planned follow-up. +- config-ui integration (connection form + team picker) and the website + documentation page are planned follow-ups; for now connections and scopes are + managed via the API calls shown above. diff --git a/backend/plugins/linear/api/blueprint_v200.go b/backend/plugins/linear/api/blueprint_v200.go new file mode 100644 index 00000000000..a26a43ab521 --- /dev/null +++ b/backend/plugins/linear/api/blueprint_v200.go @@ -0,0 +1,99 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/utils" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/srvhelper" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func MakePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + connectionId uint64, + bpScopes []*coreModels.BlueprintScope, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + connection, err := dsHelper.ConnSrv.FindByPk(connectionId) + if err != nil { + return nil, nil, err + } + scopeDetails, err := dsHelper.ScopeSrv.MapScopeDetails(connectionId, bpScopes) + if err != nil { + return nil, nil, err + } + plan, err := makePipelinePlanV200(subtaskMetas, scopeDetails, connection) + if err != nil { + return nil, nil, err + } + scopes, err := makeScopesV200(scopeDetails, connection) + return plan, scopes, err +} + +func makePipelinePlanV200( + subtaskMetas []plugin.SubTaskMeta, + scopeDetails []*srvhelper.ScopeDetail[models.LinearTeam, models.LinearScopeConfig], + connection *models.LinearConnection, +) (coreModels.PipelinePlan, errors.Error) { + plan := make(coreModels.PipelinePlan, len(scopeDetails)) + for i, scopeDetail := range scopeDetails { + stage := plan[i] + if stage == nil { + stage = coreModels.PipelineStage{} + } + scope, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig + task, err := helper.MakePipelinePlanTask( + "linear", + subtaskMetas, + scopeConfig.Entities, + tasks.LinearOptions{ + ConnectionId: connection.ID, + TeamId: scope.TeamId, + ScopeConfigId: scope.ScopeConfigId, + }, + ) + if err != nil { + return nil, err + } + stage = append(stage, task) + plan[i] = stage + } + return plan, nil +} + +func makeScopesV200( + scopeDetails []*srvhelper.ScopeDetail[models.LinearTeam, models.LinearScopeConfig], + connection *models.LinearConnection, +) ([]plugin.Scope, errors.Error) { + scopes := make([]plugin.Scope, 0, len(scopeDetails)) + idgen := didgen.NewDomainIdGenerator(&models.LinearTeam{}) + for _, scopeDetail := range scopeDetails { + scope, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig + id := idgen.Generate(connection.ID, scope.TeamId) + if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_TICKET) { + scopes = append(scopes, ticket.NewBoard(id, scope.Name)) + } + } + return scopes, nil +} diff --git a/backend/plugins/linear/api/blueprint_v200_test.go b/backend/plugins/linear/api/blueprint_v200_test.go new file mode 100644 index 00000000000..fe0d8b05db3 --- /dev/null +++ b/backend/plugins/linear/api/blueprint_v200_test.go @@ -0,0 +1,122 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/srvhelper" + mockplugin "github.com/apache/incubator-devlake/mocks/core/plugin" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/stretchr/testify/assert" +) + +func mockLinearPlugin(t *testing.T) { + mockMeta := mockplugin.NewPluginMeta(t) + mockMeta.On("RootPkgPath").Return("github.com/apache/incubator-devlake/plugins/linear") + mockMeta.On("Name").Return("linear").Maybe() + _ = plugin.RegisterPlugin("linear", mockMeta) +} + +func TestMakeScopesV200(t *testing.T) { + mockLinearPlugin(t) + + const connectionId uint64 = 1 + const teamId = "team-1" + const expectDomainScopeId = "linear:LinearTeam:1:team-1" + + scopes, err := makeScopesV200( + []*srvhelper.ScopeDetail[models.LinearTeam, models.LinearScopeConfig]{ + { + Scope: models.LinearTeam{ + Scope: common.Scope{ConnectionId: connectionId}, + TeamId: teamId, + Name: "Engineering", + }, + ScopeConfig: &models.LinearScopeConfig{ + ScopeConfig: common.ScopeConfig{Entities: []string{plugin.DOMAIN_TYPE_TICKET}}, + }, + }, + }, + &models.LinearConnection{ + BaseConnection: helper.BaseConnection{Model: common.Model{ID: connectionId}}, + }, + ) + assert.Nil(t, err) + assert.Equal(t, 1, len(scopes)) + assert.Equal(t, expectDomainScopeId, scopes[0].ScopeId()) +} + +func TestMakePipelinePlanV200PassesScopeConfigId(t *testing.T) { + const scopeConfigId uint64 = 42 + subtaskMetas := []plugin.SubTaskMeta{ + {Name: "convertIssues", EnabledByDefault: true, DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}}, + } + + plan, err := makePipelinePlanV200( + subtaskMetas, + []*srvhelper.ScopeDetail[models.LinearTeam, models.LinearScopeConfig]{ + { + Scope: models.LinearTeam{ + Scope: common.Scope{ConnectionId: 1, ScopeConfigId: scopeConfigId}, + TeamId: "team-1", + Name: "Engineering", + }, + ScopeConfig: &models.LinearScopeConfig{ + ScopeConfig: common.ScopeConfig{Entities: []string{plugin.DOMAIN_TYPE_TICKET}}, + }, + }, + }, + &models.LinearConnection{ + BaseConnection: helper.BaseConnection{Model: common.Model{ID: 1}}, + }, + ) + assert.Nil(t, err) + assert.Equal(t, 1, len(plan)) + assert.Equal(t, 1, len(plan[0])) + // the scope's scopeConfigId must be threaded into the task options so the + // convertor can resolve label-based issue-type mapping at runtime. + assert.EqualValues(t, scopeConfigId, plan[0][0].Options["scopeConfigId"]) +} + +func TestMakeScopesV200WithoutTicketEntity(t *testing.T) { + mockLinearPlugin(t) + + scopes, err := makeScopesV200( + []*srvhelper.ScopeDetail[models.LinearTeam, models.LinearScopeConfig]{ + { + Scope: models.LinearTeam{ + Scope: common.Scope{ConnectionId: 1}, + TeamId: "team-1", + }, + ScopeConfig: &models.LinearScopeConfig{ + ScopeConfig: common.ScopeConfig{Entities: []string{}}, + }, + }, + }, + &models.LinearConnection{ + BaseConnection: helper.BaseConnection{Model: common.Model{ID: 1}}, + }, + ) + assert.Nil(t, err) + // no ticket entity selected => no domain board scope produced + assert.Equal(t, 0, len(scopes)) +} diff --git a/backend/plugins/linear/api/connection_api.go b/backend/plugins/linear/api/connection_api.go new file mode 100644 index 00000000000..c2c95c54d64 --- /dev/null +++ b/backend/plugins/linear/api/connection_api.go @@ -0,0 +1,177 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "context" + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/server/api/shared" +) + +const defaultEndpoint = "https://api.linear.app/graphql" + +type LinearTestConnResponse struct { + shared.ApiBody + Connection *models.LinearConn +} + +func testConnection(ctx context.Context, connection models.LinearConn) (*LinearTestConnResponse, errors.Error) { + if vld != nil { + if err := vld.Struct(connection); err != nil { + return nil, errors.Default.Wrap(err, "error validating target") + } + } + if connection.Endpoint == "" { + connection.Endpoint = defaultEndpoint + } + apiClient, err := helper.NewApiClientFromConnection(ctx, basicRes, &connection) + if err != nil { + return nil, err + } + // Linear is GraphQL-over-HTTP-POST; a minimal viewer query verifies the key. + reqBody := map[string]interface{}{"query": "{ viewer { id name } }"} + res, err := apiClient.Post("", nil, reqBody, nil) + if err != nil { + return nil, errors.BadInput.Wrap(err, "verify token failed") + } + if res.StatusCode == http.StatusUnauthorized || res.StatusCode == http.StatusForbidden { + return nil, errors.HttpStatus(http.StatusBadRequest).New("authentication failed, please check your API key") + } + if res.StatusCode != http.StatusOK { + return nil, errors.HttpStatus(res.StatusCode).New("unexpected status code while testing connection") + } + connection = connection.Sanitize() + body := LinearTestConnResponse{} + body.Success = true + body.Message = "success" + body.Connection = &connection + return &body, nil +} + +// TestConnection test linear connection +// @Summary test linear connection +// @Description Test linear Connection +// @Tags plugins/linear +// @Param body body models.LinearConn true "json body" +// @Success 200 {object} LinearTestConnResponse "Success" +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/linear/test [POST] +func TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + var connection models.LinearConn + if err := helper.Decode(input.Body, &connection, vld); err != nil { + return nil, err + } + result, err := testConnection(context.TODO(), connection) + if err != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, err) + } + return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil +} + +// TestExistingConnection test linear connection by ID +// @Summary test linear connection +// @Description Test linear Connection +// @Tags plugins/linear +// @Param connectionId path int true "connection ID" +// @Success 200 {object} LinearTestConnResponse "Success" +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/test [POST] +func TestExistingConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection, err := dsHelper.ConnApi.GetMergedConnection(input) + if err != nil { + return nil, errors.BadInput.Wrap(err, "find connection from db") + } + if err := helper.DecodeMapStruct(input.Body, connection, false); err != nil { + return nil, err + } + result, testErr := testConnection(context.TODO(), connection.LinearConn) + if testErr != nil { + return nil, plugin.WrapTestConnectionErrResp(basicRes, testErr) + } + return &plugin.ApiResourceOutput{Body: result, Status: http.StatusOK}, nil +} + +// PostConnections create linear connection +// @Summary create linear connection +// @Description Create linear connection +// @Tags plugins/linear +// @Param body body models.LinearConnection true "json body" +// @Success 200 {object} models.LinearConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/linear/connections [POST] +func PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Post(input) +} + +// PatchConnection patch linear connection +// @Summary patch linear connection +// @Description Patch linear connection +// @Tags plugins/linear +// @Param body body models.LinearConnection true "json body" +// @Success 200 {object} models.LinearConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/linear/connections/{connectionId} [PATCH] +func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Patch(input) +} + +// DeleteConnection delete a linear connection +// @Summary delete a linear connection +// @Description Delete a linear connection +// @Tags plugins/linear +// @Success 200 {object} models.LinearConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 409 {object} services.BlueprintProjectPairs "References exist to this connection" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/linear/connections/{connectionId} [DELETE] +func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.Delete(input) +} + +// ListConnections get all linear connections +// @Summary get all linear connections +// @Description Get all linear connections +// @Tags plugins/linear +// @Success 200 {object} []models.LinearConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/linear/connections [GET] +func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.GetAll(input) +} + +// GetConnection get linear connection detail +// @Summary get linear connection detail +// @Description Get linear connection detail +// @Tags plugins/linear +// @Success 200 {object} models.LinearConnection +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/linear/connections/{connectionId} [GET] +func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ConnApi.GetDetail(input) +} diff --git a/backend/plugins/linear/api/init.go b/backend/plugins/linear/api/init.go new file mode 100644 index 00000000000..850acf54825 --- /dev/null +++ b/backend/plugins/linear/api/init.go @@ -0,0 +1,51 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/go-playground/validator/v10" +) + +var vld *validator.Validate +var basicRes context.BasicRes +var dsHelper *api.DsHelper[models.LinearConnection, models.LinearTeam, models.LinearScopeConfig] +var raProxy *api.DsRemoteApiProxyHelper[models.LinearConnection] +var raScopeList *api.DsRemoteApiScopeListHelper[models.LinearConnection, models.LinearTeam, LinearRemotePagination] + +func Init(br context.BasicRes, p plugin.PluginMeta) { + basicRes = br + vld = validator.New() + dsHelper = api.NewDataSourceHelper[ + models.LinearConnection, models.LinearTeam, models.LinearScopeConfig, + ]( + br, + p.Name(), + []string{"name"}, + func(c models.LinearConnection) models.LinearConnection { + return c.Sanitize() + }, + nil, + nil, + ) + raProxy = api.NewDsRemoteApiProxyHelper[models.LinearConnection](dsHelper.ConnApi.ModelApiHelper) + raScopeList = api.NewDsRemoteApiScopeListHelper[models.LinearConnection, models.LinearTeam, LinearRemotePagination](raProxy, listLinearRemoteScopes) +} diff --git a/backend/plugins/linear/api/remote_api.go b/backend/plugins/linear/api/remote_api.go new file mode 100644 index 00000000000..094c785016d --- /dev/null +++ b/backend/plugins/linear/api/remote_api.go @@ -0,0 +1,137 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "fmt" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + dsmodels "github.com/apache/incubator-devlake/helpers/pluginhelper/api/models" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +// LinearRemotePagination drives cursor-based pagination through the GraphQL +// `teams` connection when listing remote scopes for the config UI. +type LinearRemotePagination struct { + Cursor string `json:"cursor"` +} + +// linearTeamsGraphqlResponse mirrors the shape of the `teams` query response. +type linearTeamsGraphqlResponse struct { + Data struct { + Teams struct { + Nodes []struct { + Id string `json:"id"` + Name string `json:"name"` + Key string `json:"key"` + Description string `json:"description"` + } `json:"nodes"` + PageInfo struct { + HasNextPage bool `json:"hasNextPage"` + EndCursor string `json:"endCursor"` + } `json:"pageInfo"` + } `json:"teams"` + } `json:"data"` + Errors []struct { + Message string `json:"message"` + } `json:"errors"` +} + +const remoteScopesPageSize = 100 + +// listLinearRemoteScopes lists Linear teams as selectable scopes. Linear teams +// are a flat list, so there are no intermediate groups. +func listLinearRemoteScopes( + _ *models.LinearConnection, + apiClient plugin.ApiClient, + _ string, + page LinearRemotePagination, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.LinearTeam], + nextPage *LinearRemotePagination, + err errors.Error, +) { + after := "" + if page.Cursor != "" { + after = fmt.Sprintf(", after: %q", page.Cursor) + } + query := fmt.Sprintf( + "query { teams(first: %d%s) { nodes { id name key description } pageInfo { hasNextPage endCursor } } }", + remoteScopesPageSize, after, + ) + + res, err := apiClient.Post("", nil, map[string]interface{}{"query": query}, nil) + if err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to query Linear teams") + } + var response linearTeamsGraphqlResponse + if err := api.UnmarshalResponse(res, &response); err != nil { + return nil, nil, errors.Default.Wrap(err, "failed to unmarshal Linear teams response") + } + if len(response.Errors) > 0 { + return nil, nil, errors.Default.New("linear graphql teams query failed: " + response.Errors[0].Message) + } + + return mapLinearTeamsToScopeEntries(response), nextPageFrom(response), nil +} + +// mapLinearTeamsToScopeEntries converts a teams response into scope-list +// entries. Each team is a selectable (leaf) scope. +func mapLinearTeamsToScopeEntries(response linearTeamsGraphqlResponse) []dsmodels.DsRemoteApiScopeListEntry[models.LinearTeam] { + children := make([]dsmodels.DsRemoteApiScopeListEntry[models.LinearTeam], 0, len(response.Data.Teams.Nodes)) + for _, team := range response.Data.Teams.Nodes { + team := team + children = append(children, dsmodels.DsRemoteApiScopeListEntry[models.LinearTeam]{ + Type: api.RAS_ENTRY_TYPE_SCOPE, + ParentId: nil, + Id: team.Id, + Name: team.Name, + FullName: team.Name, + Data: &models.LinearTeam{ + TeamId: team.Id, + Name: team.Name, + Key: team.Key, + Description: team.Description, + }, + }) + } + return children +} + +// nextPageFrom returns the cursor for the following page, or nil when the +// teams connection has been fully traversed. +func nextPageFrom(response linearTeamsGraphqlResponse) *LinearRemotePagination { + pageInfo := response.Data.Teams.PageInfo + if pageInfo.HasNextPage && pageInfo.EndCursor != "" { + return &LinearRemotePagination{Cursor: pageInfo.EndCursor} + } + return nil +} + +// RemoteScopes lists the Linear teams available on the connection so the +// config UI can enumerate selectable scopes. +func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raScopeList.Get(input) +} + +// Proxy forwards arbitrary requests to the Linear API through the connection. +func Proxy(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raProxy.Proxy(input) +} diff --git a/backend/plugins/linear/api/remote_api_test.go b/backend/plugins/linear/api/remote_api_test.go new file mode 100644 index 00000000000..801e0d5029f --- /dev/null +++ b/backend/plugins/linear/api/remote_api_test.go @@ -0,0 +1,66 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "encoding/json" + "testing" + + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/stretchr/testify/assert" +) + +func TestMapLinearTeamsToScopeEntries(t *testing.T) { + var response linearTeamsGraphqlResponse + body := `{"data":{"teams":{"nodes":[` + + `{"id":"team-uuid-1","name":"Engineering","key":"ENG","description":"core eng"},` + + `{"id":"team-uuid-2","name":"Design","key":"DSG","description":""}` + + `],"pageInfo":{"hasNextPage":false,"endCursor":""}}}}` + assert.NoError(t, json.Unmarshal([]byte(body), &response)) + + entries := mapLinearTeamsToScopeEntries(response) + assert.Len(t, entries, 2) + + assert.Equal(t, api.RAS_ENTRY_TYPE_SCOPE, entries[0].Type) + assert.Nil(t, entries[0].ParentId) + assert.Equal(t, "team-uuid-1", entries[0].Id) + assert.Equal(t, "Engineering", entries[0].Name) + assert.Equal(t, "Engineering", entries[0].FullName) + // the scope payload must carry the team id used as the scope's primary key + assert.NotNil(t, entries[0].Data) + assert.Equal(t, "team-uuid-1", entries[0].Data.TeamId) + assert.Equal(t, "ENG", entries[0].Data.Key) + + assert.Equal(t, "team-uuid-2", entries[1].Id) + assert.Equal(t, "Design", entries[1].Name) +} + +func TestNextPageFrom(t *testing.T) { + var more linearTeamsGraphqlResponse + more.Data.Teams.PageInfo.HasNextPage = true + more.Data.Teams.PageInfo.EndCursor = "cursor-abc" + next := nextPageFrom(more) + assert.NotNil(t, next) + assert.Equal(t, "cursor-abc", next.Cursor) + + // no further pages -> nil, so the helper stops paginating + var last linearTeamsGraphqlResponse + last.Data.Teams.PageInfo.HasNextPage = false + last.Data.Teams.PageInfo.EndCursor = "cursor-xyz" + assert.Nil(t, nextPageFrom(last)) +} diff --git a/backend/plugins/linear/api/scope_api.go b/backend/plugins/linear/api/scope_api.go new file mode 100644 index 00000000000..9871d1d2bfe --- /dev/null +++ b/backend/plugins/linear/api/scope_api.go @@ -0,0 +1,105 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +type PutScopesReqBody api.PutScopesReqBody[models.LinearTeam] +type ScopeDetail api.ScopeDetail[models.LinearTeam, models.LinearScopeConfig] + +// PutScopes create or update linear teams +// @Summary create or update linear teams +// @Description Create or update linear teams +// @Tags plugins/linear +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param scope body PutScopesReqBody true "json" +// @Success 200 {object} []models.LinearTeam +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scopes [PUT] +func PutScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.PutMultiple(input) +} + +// PatchScope patch to linear team +// @Summary patch to linear team +// @Description patch to linear team +// @Tags plugins/linear +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param scopeId path string false "team ID" +// @Param scope body models.LinearTeam true "json" +// @Success 200 {object} models.LinearTeam +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scopes/{scopeId} [PATCH] +func PatchScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Patch(input) +} + +// GetScopeList get linear teams +// @Summary get linear teams +// @Description get linear teams +// @Tags plugins/linear +// @Param connectionId path int false "connection ID" +// @Param searchTerm query string false "search term for scope name" +// @Param pageSize query int false "page size, default 50" +// @Param page query int false "page size, default 1" +// @Success 200 {object} []ScopeDetail +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scopes/ [GET] +func GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetPage(input) +} + +// GetScope get one linear team +// @Summary get one linear team +// @Description get one linear team +// @Tags plugins/linear +// @Param connectionId path int false "connection ID" +// @Param scopeId path string false "team ID" +// @Success 200 {object} ScopeDetail +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scopes/{scopeId} [GET] +func GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.GetScopeDetail(input) +} + +// DeleteScope delete plugin data associated with the scope and optionally the scope itself +// @Summary delete plugin data associated with the scope and optionally the scope itself +// @Description delete data associated with plugin scope +// @Tags plugins/linear +// @Param connectionId path int true "connection ID" +// @Param scopeId path string true "scope ID" +// @Param delete_data_only query bool false "Only delete the scope data, not the scope itself" +// @Success 200 +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 409 {object} api.ScopeRefDoc "References exist to this scope" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scopes/{scopeId} [DELETE] +func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.Delete(input) +} diff --git a/backend/plugins/linear/api/scope_config_api.go b/backend/plugins/linear/api/scope_config_api.go new file mode 100644 index 00000000000..e70b800fee2 --- /dev/null +++ b/backend/plugins/linear/api/scope_config_api.go @@ -0,0 +1,106 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" +) + +// PostScopeConfig create scope config for Linear +// @Summary create scope config for Linear +// @Description create scope config for Linear +// @Tags plugins/linear +// @Accept application/json +// @Param scopeConfig body models.LinearScopeConfig true "scope config" +// @Success 200 {object} models.LinearScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scope-configs [POST] +func PostScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Post(input) +} + +// PatchScopeConfig update scope config for Linear +// @Summary update scope config for Linear +// @Description update scope config for Linear +// @Tags plugins/linear +// @Accept application/json +// @Param scopeConfigId path int true "scopeConfigId" +// @Param scopeConfig body models.LinearScopeConfig true "scope config" +// @Success 200 {object} models.LinearScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scope-configs/{scopeConfigId} [PATCH] +func PatchScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Patch(input) +} + +// GetScopeConfig return one scope config +// @Summary return one scope config +// @Description return one scope config +// @Tags plugins/linear +// @Param scopeConfigId path int true "scopeConfigId" +// @Success 200 {object} models.LinearScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scope-configs/{scopeConfigId} [GET] +func GetScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetDetail(input) +} + +// GetScopeConfigList return all scope configs +// @Summary return all scope configs +// @Description return all scope configs +// @Tags plugins/linear +// @Param pageSize query int false "page size, default 50" +// @Param page query int false "page size, default 1" +// @Success 200 {object} []models.LinearScopeConfig +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scope-configs [GET] +func GetScopeConfigList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetAll(input) +} + +// DeleteScopeConfig delete a scope config +// @Summary delete a scope config +// @Description delete a scope config +// @Tags plugins/linear +// @Param scopeConfigId path int true "scopeConfigId" +// @Param connectionId path int true "connectionId" +// @Success 200 +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/connections/{connectionId}/scope-configs/{scopeConfigId} [DELETE] +func DeleteScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Delete(input) +} + +// GetProjectsByScopeConfig return projects details related by scope config +// @Summary return all related projects +// @Description return all related projects +// @Tags plugins/linear +// @Param scopeConfigId path int true "scopeConfigId" +// @Success 200 {object} models.ProjectScopeOutput +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/linear/scope-config/{scopeConfigId}/projects [GET] +func GetProjectsByScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.GetProjectsByScopeConfig(input) +} diff --git a/backend/plugins/linear/e2e/account_test.go b/backend/plugins/linear/e2e/account_test.go new file mode 100644 index 00000000000..9cb8f3a8e15 --- /dev/null +++ b/backend/plugins/linear/e2e/account_test.go @@ -0,0 +1,58 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func TestLinearAccountDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // verify extraction: raw -> tool layer + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_accounts.csv", "_raw_linear_accounts") + dataflowTester.FlushTabler(&models.LinearAccount{}) + dataflowTester.Subtask(tasks.ExtractAccountsMeta, taskData) + dataflowTester.VerifyTableWithOptions(models.LinearAccount{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_linear_accounts.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // verify conversion: tool layer -> domain layer + dataflowTester.FlushTabler(&crossdomain.Account{}) + dataflowTester.Subtask(tasks.ConvertAccountsMeta, taskData) + dataflowTester.VerifyTableWithOptions(crossdomain.Account{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/accounts.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/board_test.go b/backend/plugins/linear/e2e/board_test.go new file mode 100644 index 00000000000..eb4e0487323 --- /dev/null +++ b/backend/plugins/linear/e2e/board_test.go @@ -0,0 +1,56 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +// TestLinearBoardDataFlow verifies that a Linear team scope is converted into a +// domain ticket.Board, keyed identically to the board_id that board_issues +// already reference (boardIdGen over LinearTeam). Without this, the boards table +// is empty and board-scoped dashboards return no data. +func TestLinearBoardDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // the team scope lives in _tool_linear_teams (populated via the scope API) + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_linear_teams.csv", &models.LinearTeam{}) + + // convert: team scope -> domain board + dataflowTester.FlushTabler(&ticket.Board{}) + dataflowTester.Subtask(tasks.ConvertTeamsMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.Board{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/boards.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/comment_test.go b/backend/plugins/linear/e2e/comment_test.go new file mode 100644 index 00000000000..67ee95adad5 --- /dev/null +++ b/backend/plugins/linear/e2e/comment_test.go @@ -0,0 +1,64 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func TestLinearCommentDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // the comment convertor joins to issues, so populate _tool_linear_issues first + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // verify extraction: raw -> tool layer + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_comments.csv", "_raw_linear_comments") + dataflowTester.FlushTabler(&models.LinearComment{}) + dataflowTester.Subtask(tasks.ExtractCommentsMeta, taskData) + dataflowTester.VerifyTableWithOptions(models.LinearComment{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_linear_comments.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // verify conversion: tool layer -> domain layer + dataflowTester.FlushTabler(&ticket.IssueComment{}) + dataflowTester.Subtask(tasks.ConvertCommentsMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.IssueComment{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issue_comments.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/cycle_test.go b/backend/plugins/linear/e2e/cycle_test.go new file mode 100644 index 00000000000..207e9c8a3f9 --- /dev/null +++ b/backend/plugins/linear/e2e/cycle_test.go @@ -0,0 +1,77 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func TestLinearCycleDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // issues drive sprint_issues, so populate _tool_linear_issues first + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // verify extraction: raw -> tool layer + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_cycles.csv", "_raw_linear_cycles") + dataflowTester.FlushTabler(&models.LinearCycle{}) + dataflowTester.Subtask(tasks.ExtractCyclesMeta, taskData) + dataflowTester.VerifyTableWithOptions(models.LinearCycle{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_linear_cycles.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // verify conversion: cycles -> sprints + board_sprints + dataflowTester.FlushTabler(&ticket.Sprint{}) + dataflowTester.FlushTabler(&ticket.BoardSprint{}) + dataflowTester.Subtask(tasks.ConvertCyclesMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.Sprint{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/sprints.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + dataflowTester.VerifyTableWithOptions(ticket.BoardSprint{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/board_sprints.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // verify conversion: issues -> sprint_issues + dataflowTester.FlushTabler(&ticket.SprintIssue{}) + dataflowTester.Subtask(tasks.ConvertSprintIssuesMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.SprintIssue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/sprint_issues.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/issue_history_leadtime_test.go b/backend/plugins/linear/e2e/issue_history_leadtime_test.go new file mode 100644 index 00000000000..7d4a4c9798a --- /dev/null +++ b/backend/plugins/linear/e2e/issue_history_leadtime_test.go @@ -0,0 +1,77 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +// TestLinearIssueHistoryLeadTime proves lead time is derived from the recorded +// state transitions. issue-1 was started on 2026-05-02 and completed on +// 2026-05-03 (1440 min of active cycle time), even though its createdAt -> +// resolutionDate span is 2880 min. ConvertIssues sets the coarse 2880 fallback; +// ConvertIssueHistory must then refine it to 1440 from the history. Issues +// without the required transitions keep the fallback (issue-4 = 1440). +func TestLinearIssueHistoryLeadTime(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // flush accounts so assignee/creator names stay empty in this lead-time- + // focused test regardless of other tests sharing the DB. + dataflowTester.FlushTabler(&models.LinearAccount{}) + + // seed issues + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // seed history + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issue_history.csv", "_raw_linear_issue_history") + dataflowTester.FlushTabler(&models.LinearIssueHistory{}) + dataflowTester.Subtask(tasks.ExtractIssueHistoryMeta, taskData) + + // ConvertIssues sets the createdAt -> resolutionDate fallback ... + dataflowTester.FlushTabler(&ticket.Issue{}) + dataflowTester.FlushTabler(&ticket.BoardIssue{}) + dataflowTester.FlushTabler(&ticket.IssueAssignee{}) + dataflowTester.Subtask(tasks.ConvertIssuesMeta, taskData) + + // ... then ConvertIssueHistory refines lead time from the transitions. + dataflowTester.FlushTabler(&ticket.IssueChangelogs{}) + dataflowTester.Subtask(tasks.ConvertIssueHistoryMeta, taskData) + + dataflowTester.VerifyTableWithOptions(ticket.Issue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issues_history_leadtime.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/issue_history_test.go b/backend/plugins/linear/e2e/issue_history_test.go new file mode 100644 index 00000000000..31f559694e5 --- /dev/null +++ b/backend/plugins/linear/e2e/issue_history_test.go @@ -0,0 +1,64 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func TestLinearIssueHistoryDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // the history convertor joins to issues, so populate _tool_linear_issues first + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // verify extraction: raw -> tool layer + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issue_history.csv", "_raw_linear_issue_history") + dataflowTester.FlushTabler(&models.LinearIssueHistory{}) + dataflowTester.Subtask(tasks.ExtractIssueHistoryMeta, taskData) + dataflowTester.VerifyTableWithOptions(models.LinearIssueHistory{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_linear_issue_history.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // verify conversion: tool layer -> domain layer + dataflowTester.FlushTabler(&ticket.IssueChangelogs{}) + dataflowTester.Subtask(tasks.ConvertIssueHistoryMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.IssueChangelogs{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issue_changelogs.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/issue_incident_test.go b/backend/plugins/linear/e2e/issue_incident_test.go new file mode 100644 index 00000000000..fb214fa82e5 --- /dev/null +++ b/backend/plugins/linear/e2e/issue_incident_test.go @@ -0,0 +1,67 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +// TestLinearIssueIncidentMapping verifies label-based issue-type mapping. With a +// scope config whose IssueTypeIncident pattern matches the "Bug" label, ENG-1 +// (labeled "Bug") is converted to an INCIDENT, while issues without a matching +// label stay REQUIREMENT. This is what feeds DORA's change-failure-rate metric. +func TestLinearIssueIncidentMapping(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + ScopeConfig: &models.LinearScopeConfig{ + IssueTypeIncident: "^Bug$", + }, + } + + // extract issues + inline labels from raw + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // accounts needed so the convertor can resolve assignee/creator names + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_linear_accounts.csv", &models.LinearAccount{}) + + // convert -> domain issues; ENG-1 (Bug label) must be INCIDENT, others REQUIREMENT + dataflowTester.FlushTabler(&ticket.Issue{}) + dataflowTester.FlushTabler(&ticket.BoardIssue{}) + dataflowTester.FlushTabler(&ticket.IssueAssignee{}) + dataflowTester.Subtask(tasks.ConvertIssuesMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.Issue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issues_incident.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/issue_leadtime_test.go b/backend/plugins/linear/e2e/issue_leadtime_test.go new file mode 100644 index 00000000000..783820f696d --- /dev/null +++ b/backend/plugins/linear/e2e/issue_leadtime_test.go @@ -0,0 +1,64 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +// TestLinearIssueNegativeLeadTime guards the lead-time fallback against +// resolution timestamps that precede the creation timestamp (clock skew or +// migrated/imported issues). A negative duration must NOT be cast to uint -- +// doing so wraps to a huge bogus value. The expected behaviour is that no lead +// time is derived (lead_time_minutes stays empty). +func TestLinearIssueNegativeLeadTime(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // extraction: raw -> tool layer + // Flush accounts so this lead-time-focused test is independent of any + // account rows left behind by other tests sharing the DB. + dataflowTester.FlushTabler(&models.LinearAccount{}) + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues_negative_leadtime.csv", "_raw_linear_issues") + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // conversion: tool layer -> domain layer + dataflowTester.FlushTabler(&ticket.Issue{}) + dataflowTester.FlushTabler(&ticket.BoardIssue{}) + dataflowTester.Subtask(tasks.ConvertIssuesMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.Issue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issues_negative_leadtime.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/issue_longtitle_test.go b/backend/plugins/linear/e2e/issue_longtitle_test.go new file mode 100644 index 00000000000..f85062cdb8b --- /dev/null +++ b/backend/plugins/linear/e2e/issue_longtitle_test.go @@ -0,0 +1,55 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" + "github.com/stretchr/testify/assert" +) + +// TestLinearIssueLongTitle guards against truncation/insert failure for long +// issue titles and URLs. Linear titles can exceed 255 chars (and the issue URL +// embeds a title slug), which overflowed the old varchar(255) columns and +// failed extraction with "Data too long for column 'title'". The columns are +// now untyped (longtext), matching the domain issues.title and jira's tool +// summary. +func TestLinearIssueLongTitle(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ConnectionId: 1, TeamId: "team-1"}, + } + + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues_long_title.csv", "_raw_linear_issues") + // must not error with "Data too long for column 'title'" + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + var issue models.LinearIssue + err := dataflowTester.Dal.First(&issue, dal.Where("connection_id = ? AND id = ?", 1, "issue-longtitle")) + assert.NoError(t, err) + assert.Len(t, issue.Title, 300, "full 300-char title must be stored untruncated") +} diff --git a/backend/plugins/linear/e2e/issue_test.go b/backend/plugins/linear/e2e/issue_test.go new file mode 100644 index 00000000000..5f98aad8a61 --- /dev/null +++ b/backend/plugins/linear/e2e/issue_test.go @@ -0,0 +1,77 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func TestLinearIssueDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // verify extraction: raw -> tool layer (issues + inline labels) + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + dataflowTester.VerifyTableWithOptions(models.LinearIssue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_linear_issues.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + dataflowTester.VerifyTableWithOptions(models.LinearIssueLabel{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_linear_issue_labels.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // accounts must be present so the convertor can resolve assignee/creator + // display names and emit issue_assignees rows. + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_linear_accounts.csv", &models.LinearAccount{}) + + // verify conversion: tool layer -> domain layer (issues + board_issues + issue_assignees) + dataflowTester.FlushTabler(&ticket.Issue{}) + dataflowTester.FlushTabler(&ticket.BoardIssue{}) + dataflowTester.FlushTabler(&ticket.IssueAssignee{}) + dataflowTester.Subtask(tasks.ConvertIssuesMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.Issue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issues.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + dataflowTester.VerifyTableWithOptions(ticket.BoardIssue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/board_issues.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + dataflowTester.VerifyTableWithOptions(ticket.IssueAssignee{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issue_assignees.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/label_test.go b/backend/plugins/linear/e2e/label_test.go new file mode 100644 index 00000000000..4861f6197da --- /dev/null +++ b/backend/plugins/linear/e2e/label_test.go @@ -0,0 +1,55 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func TestLinearLabelDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // labels are produced inline by the issue extractor + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // verify conversion: tool layer -> domain layer + dataflowTester.FlushTabler(&ticket.IssueLabel{}) + dataflowTester.Subtask(tasks.ConvertIssueLabelsMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.IssueLabel{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issue_labels.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_accounts.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_accounts.csv new file mode 100644 index 00000000000..785c47d21b2 --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_accounts.csv @@ -0,0 +1,4 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""user-1"",""Name"":""alice"",""DisplayName"":""Alice Anderson"",""Email"":""alice@example.com"",""AvatarUrl"":""https://linear.app/avatars/alice.png"",""Active"":true}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 +2,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""user-2"",""Name"":""bob"",""DisplayName"":""Bob Brown"",""Email"":""bob@example.com"",""AvatarUrl"":""https://linear.app/avatars/bob.png"",""Active"":true}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 +3,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""user-3"",""Name"":""carol"",""DisplayName"":""Carol Clark"",""Email"":""carol@example.com"",""AvatarUrl"":""https://linear.app/avatars/carol.png"",""Active"":false}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_comments.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_comments.csv new file mode 100644 index 00000000000..6d6fc21e6a8 --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_comments.csv @@ -0,0 +1,4 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""comment-1"",""Body"":""Looking into this"",""CreatedAt"":""2026-05-02T10:00:00Z"",""UpdatedAt"":""2026-05-02T10:00:00Z"",""User"":{""Id"":""user-2""}}",https://api.linear.app/graphql,"{""issueId"":""issue-1""}",2026-05-02 10:00:00.000 +2,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""comment-2"",""Body"":""Fixed in PR 42"",""CreatedAt"":""2026-05-03T09:00:00Z"",""UpdatedAt"":""2026-05-03T09:30:00Z"",""User"":{""Id"":""user-1""}}",https://api.linear.app/graphql,"{""issueId"":""issue-1""}",2026-05-03 09:00:00.000 +3,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""comment-3"",""Body"":""Any update?"",""CreatedAt"":""2026-05-02T11:00:00Z"",""UpdatedAt"":""2026-05-02T11:00:00Z"",""User"":{""Id"":""user-1""}}",https://api.linear.app/graphql,"{""issueId"":""issue-2""}",2026-05-02 11:00:00.000 diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_cycles.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_cycles.csv new file mode 100644 index 00000000000..8eb59ddb145 --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_cycles.csv @@ -0,0 +1,3 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""cycle-1"",""Number"":1,""Name"":"""",""StartsAt"":""2026-04-20T00:00:00Z"",""EndsAt"":""2026-05-04T00:00:00Z"",""CompletedAt"":""2026-05-04T00:00:00Z""}",https://api.linear.app/graphql,null,2026-05-04 00:00:00.000 +2,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""cycle-2"",""Number"":2,""Name"":""Sprint 2"",""StartsAt"":""2026-05-04T00:00:00Z"",""EndsAt"":""2026-05-18T00:00:00Z"",""CompletedAt"":null}",https://api.linear.app/graphql,null,2026-05-04 00:00:00.000 diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_issue_history.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issue_history.csv new file mode 100644 index 00000000000..d1cef3c37ab --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issue_history.csv @@ -0,0 +1,4 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""hist-1"",""CreatedAt"":""2026-05-01T08:00:00Z"",""Actor"":{""Id"":""user-2""},""FromState"":{""Id"":""state-backlog"",""Name"":""Backlog"",""Type"":""backlog""},""ToState"":{""Id"":""state-todo"",""Name"":""Todo"",""Type"":""unstarted""}}",https://api.linear.app/graphql,"{""issueId"":""issue-1""}",2026-05-01 08:00:00.000 +2,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""hist-2"",""CreatedAt"":""2026-05-02T00:00:00Z"",""Actor"":{""Id"":""user-1""},""FromState"":{""Id"":""state-todo"",""Name"":""Todo"",""Type"":""unstarted""},""ToState"":{""Id"":""state-inprogress"",""Name"":""In Progress"",""Type"":""started""}}",https://api.linear.app/graphql,"{""issueId"":""issue-1""}",2026-05-02 00:00:00.000 +3,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""hist-3"",""CreatedAt"":""2026-05-03T00:00:00Z"",""Actor"":{""Id"":""user-1""},""FromState"":{""Id"":""state-inprogress"",""Name"":""In Progress"",""Type"":""started""},""ToState"":{""Id"":""state-done"",""Name"":""Done"",""Type"":""completed""}}",https://api.linear.app/graphql,"{""issueId"":""issue-1""}",2026-05-03 00:00:00.000 diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues.csv new file mode 100644 index 00000000000..f49f41c73b6 --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues.csv @@ -0,0 +1,6 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""issue-1"",""Identifier"":""ENG-1"",""Number"":1,""Title"":""Fix login bug"",""Description"":""Users cannot log in"",""Url"":""https://linear.app/eng/issue/ENG-1"",""Priority"":1,""Estimate"":3,""CreatedAt"":""2026-05-01T00:00:00Z"",""UpdatedAt"":""2026-05-03T00:00:00Z"",""StartedAt"":""2026-05-02T00:00:00Z"",""CompletedAt"":""2026-05-03T00:00:00Z"",""CanceledAt"":null,""State"":{""Id"":""state-done"",""Name"":""Done"",""Type"":""completed""},""Assignee"":{""Id"":""user-1""},""Creator"":{""Id"":""user-2""},""Cycle"":{""Id"":""cycle-1""},""Parent"":null,""Labels"":{""Nodes"":[{""Id"":""l1"",""Name"":""Bug""},{""Id"":""l2"",""Name"":""P1""}]}}",https://api.linear.app/graphql,null,2026-05-03 00:00:00.000 +2,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""issue-2"",""Identifier"":""ENG-2"",""Number"":2,""Title"":""Add dark mode"",""Description"":""Theme support"",""Url"":""https://linear.app/eng/issue/ENG-2"",""Priority"":2,""Estimate"":5,""CreatedAt"":""2026-05-01T00:00:00Z"",""UpdatedAt"":""2026-05-02T00:00:00Z"",""StartedAt"":""2026-05-02T00:00:00Z"",""CompletedAt"":null,""CanceledAt"":null,""State"":{""Id"":""state-inprogress"",""Name"":""In Progress"",""Type"":""started""},""Assignee"":null,""Creator"":{""Id"":""user-1""},""Cycle"":null,""Parent"":null,""Labels"":{""Nodes"":[{""Id"":""l3"",""Name"":""Feature""}]}}",https://api.linear.app/graphql,null,2026-05-02 00:00:00.000 +3,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""issue-3"",""Identifier"":""ENG-3"",""Number"":3,""Title"":""Investigate flakiness"",""Description"":"""",""Url"":""https://linear.app/eng/issue/ENG-3"",""Priority"":0,""Estimate"":null,""CreatedAt"":""2026-05-01T00:00:00Z"",""UpdatedAt"":""2026-05-01T00:00:00Z"",""StartedAt"":null,""CompletedAt"":null,""CanceledAt"":null,""State"":{""Id"":""state-backlog"",""Name"":""Backlog"",""Type"":""backlog""},""Assignee"":null,""Creator"":{""Id"":""user-1""},""Cycle"":null,""Parent"":null,""Labels"":{""Nodes"":[]}}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 +4,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""issue-4"",""Identifier"":""ENG-4"",""Number"":4,""Title"":""Deprecated feature"",""Description"":""No longer needed"",""Url"":""https://linear.app/eng/issue/ENG-4"",""Priority"":3,""Estimate"":2,""CreatedAt"":""2026-05-01T00:00:00Z"",""UpdatedAt"":""2026-05-02T12:00:00Z"",""StartedAt"":null,""CompletedAt"":null,""CanceledAt"":""2026-05-02T00:00:00Z"",""State"":{""Id"":""state-canceled"",""Name"":""Canceled"",""Type"":""canceled""},""Assignee"":{""Id"":""user-3""},""Creator"":{""Id"":""user-1""},""Cycle"":{""Id"":""cycle-1""},""Parent"":null,""Labels"":{""Nodes"":[]}}",https://api.linear.app/graphql,null,2026-05-02 12:00:00.000 +5,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""issue-5"",""Identifier"":""ENG-5"",""Number"":5,""Title"":""Write docs"",""Description"":""User guide"",""Url"":""https://linear.app/eng/issue/ENG-5"",""Priority"":4,""Estimate"":1,""CreatedAt"":""2026-05-01T00:00:00Z"",""UpdatedAt"":""2026-05-01T06:00:00Z"",""StartedAt"":null,""CompletedAt"":null,""CanceledAt"":null,""State"":{""Id"":""state-todo"",""Name"":""Todo"",""Type"":""unstarted""},""Assignee"":null,""Creator"":{""Id"":""user-2""},""Cycle"":null,""Parent"":null,""Labels"":{""Nodes"":[]}}",https://api.linear.app/graphql,null,2026-05-01 06:00:00.000 diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues_long_title.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues_long_title.csv new file mode 100644 index 00000000000..4842c33a61c --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues_long_title.csv @@ -0,0 +1,2 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""issue-longtitle"",""Identifier"":""ENG-LONG"",""Number"":900,""Title"":""TTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTTT"",""Description"":""long title row"",""Url"":""https://linear.app/eng/issue/ENG-LONG/tttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttttt"",""Priority"":0,""Estimate"":null,""CreatedAt"":""2026-05-01T00:00:00Z"",""UpdatedAt"":""2026-05-01T00:00:00Z"",""StartedAt"":null,""CompletedAt"":null,""CanceledAt"":null,""State"":{""Id"":""state-backlog"",""Name"":""Backlog"",""Type"":""backlog""},""Assignee"":null,""Creator"":null,""Cycle"":null,""Parent"":null,""Labels"":{""Nodes"":[]}}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues_negative_leadtime.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues_negative_leadtime.csv new file mode 100644 index 00000000000..f63750db475 --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_issues_negative_leadtime.csv @@ -0,0 +1,2 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""issue-neg"",""Identifier"":""ENG-NEG"",""Number"":99,""Title"":""Imported issue with skewed timestamps"",""Description"":""canceledAt precedes createdAt"",""Url"":""https://linear.app/eng/issue/ENG-NEG"",""Priority"":0,""Estimate"":null,""CreatedAt"":""2026-05-10T00:00:00Z"",""UpdatedAt"":""2026-05-10T00:00:00Z"",""StartedAt"":null,""CompletedAt"":null,""CanceledAt"":""2026-05-09T00:00:00Z"",""State"":{""Id"":""state-canceled"",""Name"":""Canceled"",""Type"":""canceled""},""Assignee"":null,""Creator"":{""Id"":""user-1""},""Cycle"":null,""Parent"":null,""Labels"":{""Nodes"":[]}}",https://api.linear.app/graphql,null,2026-05-10 00:00:00.000 diff --git a/backend/plugins/linear/e2e/raw_tables/_raw_linear_workflow_states.csv b/backend/plugins/linear/e2e/raw_tables/_raw_linear_workflow_states.csv new file mode 100644 index 00000000000..72c0ddc877b --- /dev/null +++ b/backend/plugins/linear/e2e/raw_tables/_raw_linear_workflow_states.csv @@ -0,0 +1,6 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""state-backlog"",""Name"":""Backlog"",""Type"":""backlog"",""Color"":""#bec2c8"",""Position"":0}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 +2,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""state-todo"",""Name"":""Todo"",""Type"":""unstarted"",""Color"":""#e2e2e2"",""Position"":1}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 +3,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""state-inprogress"",""Name"":""In Progress"",""Type"":""started"",""Color"":""#f2c94c"",""Position"":2}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 +4,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""state-done"",""Name"":""Done"",""Type"":""completed"",""Color"":""#5e6ad2"",""Position"":3}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 +5,"{""ConnectionId"":1,""TeamId"":""team-1""}","{""Id"":""state-canceled"",""Name"":""Canceled"",""Type"":""canceled"",""Color"":""#95a2b3"",""Position"":4}",https://api.linear.app/graphql,null,2026-05-01 00:00:00.000 diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_accounts.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_accounts.csv new file mode 100644 index 00000000000..db07f6b4cf7 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_accounts.csv @@ -0,0 +1,4 @@ +connection_id,id,name,display_name,email,avatar_url,active +1,user-1,alice,Alice Anderson,alice@example.com,https://linear.app/avatars/alice.png,1 +1,user-2,bob,Bob Brown,bob@example.com,https://linear.app/avatars/bob.png,1 +1,user-3,carol,Carol Clark,carol@example.com,https://linear.app/avatars/carol.png,0 diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_comments.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_comments.csv new file mode 100644 index 00000000000..aec4878ecb1 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_comments.csv @@ -0,0 +1,4 @@ +connection_id,id,issue_id,body,author_id +1,comment-1,issue-1,Looking into this,user-2 +1,comment-2,issue-1,Fixed in PR 42,user-1 +1,comment-3,issue-2,Any update?,user-1 diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_cycles.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_cycles.csv new file mode 100644 index 00000000000..9e95c44d986 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_cycles.csv @@ -0,0 +1,3 @@ +connection_id,id,team_id,number,name,starts_at,ends_at,completed_at +1,cycle-1,team-1,1,,2026-04-20T00:00:00.000+00:00,2026-05-04T00:00:00.000+00:00,2026-05-04T00:00:00.000+00:00 +1,cycle-2,team-1,2,Sprint 2,2026-05-04T00:00:00.000+00:00,2026-05-18T00:00:00.000+00:00, diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issue_history.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issue_history.csv new file mode 100644 index 00000000000..bc8fe9ec7ba --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issue_history.csv @@ -0,0 +1,4 @@ +connection_id,id,issue_id,actor_id,from_state_id,from_state_name,from_state_type,to_state_id,to_state_name,to_state_type +1,hist-1,issue-1,user-2,state-backlog,Backlog,backlog,state-todo,Todo,unstarted +1,hist-2,issue-1,user-1,state-todo,Todo,unstarted,state-inprogress,In Progress,started +1,hist-3,issue-1,user-1,state-inprogress,In Progress,started,state-done,Done,completed diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issue_labels.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issue_labels.csv new file mode 100644 index 00000000000..20da8a076b0 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issue_labels.csv @@ -0,0 +1,4 @@ +connection_id,issue_id,label_name +1,issue-1,Bug +1,issue-1,P1 +1,issue-2,Feature diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issues.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issues.csv new file mode 100644 index 00000000000..7c35d53d39e --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_issues.csv @@ -0,0 +1,6 @@ +connection_id,id,team_id,identifier,number,title,description,url,priority,priority_label,estimate,state_id,state_name,state_type,creator_id,assignee_id,cycle_id,parent_id,started_at,completed_at,canceled_at +1,issue-1,team-1,ENG-1,1,Fix login bug,Users cannot log in,https://linear.app/eng/issue/ENG-1,1,Urgent,3,state-done,Done,completed,user-2,user-1,cycle-1,,2026-05-02T00:00:00.000+00:00,2026-05-03T00:00:00.000+00:00, +1,issue-2,team-1,ENG-2,2,Add dark mode,Theme support,https://linear.app/eng/issue/ENG-2,2,High,5,state-inprogress,In Progress,started,user-1,,,,2026-05-02T00:00:00.000+00:00,, +1,issue-3,team-1,ENG-3,3,Investigate flakiness,,https://linear.app/eng/issue/ENG-3,0,No priority,,state-backlog,Backlog,backlog,user-1,,,,,, +1,issue-4,team-1,ENG-4,4,Deprecated feature,No longer needed,https://linear.app/eng/issue/ENG-4,3,Medium,2,state-canceled,Canceled,canceled,user-1,user-3,cycle-1,,,,2026-05-02T00:00:00.000+00:00 +1,issue-5,team-1,ENG-5,5,Write docs,User guide,https://linear.app/eng/issue/ENG-5,4,Low,1,state-todo,Todo,unstarted,user-2,,,,,, diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_teams.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_teams.csv new file mode 100644 index 00000000000..1541f9c95e4 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_teams.csv @@ -0,0 +1,2 @@ +connection_id,scope_config_id,team_id,name,key,description +1,0,team-1,Engineering,ENG,Core engineering team diff --git a/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_workflow_states.csv b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_workflow_states.csv new file mode 100644 index 00000000000..bd1d3443ae9 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/_tool_linear_workflow_states.csv @@ -0,0 +1,6 @@ +connection_id,id,team_id,name,type,color,position +1,state-backlog,team-1,Backlog,backlog,#bec2c8,0 +1,state-canceled,team-1,Canceled,canceled,#95a2b3,4 +1,state-done,team-1,Done,completed,#5e6ad2,3 +1,state-inprogress,team-1,In Progress,started,#f2c94c,2 +1,state-todo,team-1,Todo,unstarted,#e2e2e2,1 diff --git a/backend/plugins/linear/e2e/snapshot_tables/accounts.csv b/backend/plugins/linear/e2e/snapshot_tables/accounts.csv new file mode 100644 index 00000000000..12c2066d191 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/accounts.csv @@ -0,0 +1,4 @@ +id,email,full_name,user_name,avatar_url,organization,created_date,status +linear:LinearAccount:1:user-1,alice@example.com,Alice Anderson,alice,https://linear.app/avatars/alice.png,,,1 +linear:LinearAccount:1:user-2,bob@example.com,Bob Brown,bob,https://linear.app/avatars/bob.png,,,1 +linear:LinearAccount:1:user-3,carol@example.com,Carol Clark,carol,https://linear.app/avatars/carol.png,,,0 diff --git a/backend/plugins/linear/e2e/snapshot_tables/board_issues.csv b/backend/plugins/linear/e2e/snapshot_tables/board_issues.csv new file mode 100644 index 00000000000..9a7d32f9477 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/board_issues.csv @@ -0,0 +1,6 @@ +board_id,issue_id +linear:LinearTeam:1:team-1,linear:LinearIssue:1:issue-1 +linear:LinearTeam:1:team-1,linear:LinearIssue:1:issue-2 +linear:LinearTeam:1:team-1,linear:LinearIssue:1:issue-3 +linear:LinearTeam:1:team-1,linear:LinearIssue:1:issue-4 +linear:LinearTeam:1:team-1,linear:LinearIssue:1:issue-5 diff --git a/backend/plugins/linear/e2e/snapshot_tables/board_sprints.csv b/backend/plugins/linear/e2e/snapshot_tables/board_sprints.csv new file mode 100644 index 00000000000..ff0cfe33dc5 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/board_sprints.csv @@ -0,0 +1,3 @@ +board_id,sprint_id +linear:LinearTeam:1:team-1,linear:LinearCycle:1:cycle-1 +linear:LinearTeam:1:team-1,linear:LinearCycle:1:cycle-2 diff --git a/backend/plugins/linear/e2e/snapshot_tables/boards.csv b/backend/plugins/linear/e2e/snapshot_tables/boards.csv new file mode 100644 index 00000000000..4548d25c385 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/boards.csv @@ -0,0 +1,2 @@ +id,name,description,url,created_date,type +linear:LinearTeam:1:team-1,Engineering,Core engineering team,,,linear diff --git a/backend/plugins/linear/e2e/snapshot_tables/issue_assignees.csv b/backend/plugins/linear/e2e/snapshot_tables/issue_assignees.csv new file mode 100644 index 00000000000..38f569f7094 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issue_assignees.csv @@ -0,0 +1,3 @@ +issue_id,assignee_id,assignee_name +linear:LinearIssue:1:issue-1,linear:LinearAccount:1:user-1,Alice Anderson +linear:LinearIssue:1:issue-4,linear:LinearAccount:1:user-3,Carol Clark diff --git a/backend/plugins/linear/e2e/snapshot_tables/issue_changelogs.csv b/backend/plugins/linear/e2e/snapshot_tables/issue_changelogs.csv new file mode 100644 index 00000000000..9d53280010f --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issue_changelogs.csv @@ -0,0 +1,4 @@ +id,issue_id,author_id,author_name,field_id,field_name,original_from_value,original_to_value,from_value,to_value,created_date +linear:LinearIssueHistory:1:hist-1,linear:LinearIssue:1:issue-1,linear:LinearAccount:1:user-2,,state,status,Backlog,Todo,TODO,TODO,2026-05-01T08:00:00.000+00:00 +linear:LinearIssueHistory:1:hist-2,linear:LinearIssue:1:issue-1,linear:LinearAccount:1:user-1,,state,status,Todo,In Progress,TODO,IN_PROGRESS,2026-05-02T00:00:00.000+00:00 +linear:LinearIssueHistory:1:hist-3,linear:LinearIssue:1:issue-1,linear:LinearAccount:1:user-1,,state,status,In Progress,Done,IN_PROGRESS,DONE,2026-05-03T00:00:00.000+00:00 diff --git a/backend/plugins/linear/e2e/snapshot_tables/issue_comments.csv b/backend/plugins/linear/e2e/snapshot_tables/issue_comments.csv new file mode 100644 index 00000000000..925e84e5648 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issue_comments.csv @@ -0,0 +1,4 @@ +id,issue_id,body,account_id,created_date,updated_date +linear:LinearComment:1:comment-1,linear:LinearIssue:1:issue-1,Looking into this,linear:LinearAccount:1:user-2,2026-05-02T10:00:00.000+00:00,2026-05-02T10:00:00.000+00:00 +linear:LinearComment:1:comment-2,linear:LinearIssue:1:issue-1,Fixed in PR 42,linear:LinearAccount:1:user-1,2026-05-03T09:00:00.000+00:00,2026-05-03T09:30:00.000+00:00 +linear:LinearComment:1:comment-3,linear:LinearIssue:1:issue-2,Any update?,linear:LinearAccount:1:user-1,2026-05-02T11:00:00.000+00:00,2026-05-02T11:00:00.000+00:00 diff --git a/backend/plugins/linear/e2e/snapshot_tables/issue_labels.csv b/backend/plugins/linear/e2e/snapshot_tables/issue_labels.csv new file mode 100644 index 00000000000..e7c911e0005 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issue_labels.csv @@ -0,0 +1,4 @@ +issue_id,label_name +linear:LinearIssue:1:issue-1,Bug +linear:LinearIssue:1:issue-1,P1 +linear:LinearIssue:1:issue-2,Feature diff --git a/backend/plugins/linear/e2e/snapshot_tables/issues.csv b/backend/plugins/linear/e2e/snapshot_tables/issues.csv new file mode 100644 index 00000000000..f55c39e0eb8 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issues.csv @@ -0,0 +1,6 @@ +id,url,icon_url,issue_key,title,description,epic_key,type,original_type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,parent_issue_id,priority,severity,urgency,component,original_project,is_subtask,due_date,fix_versions +linear:LinearIssue:1:issue-1,https://linear.app/eng/issue/ENG-1,,ENG-1,Fix login bug,Users cannot log in,,REQUIREMENT,,DONE,Done,3,2026-05-03T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,2026-05-03T00:00:00.000+00:00,2880,,,,linear:LinearAccount:1:user-2,Bob Brown,linear:LinearAccount:1:user-1,Alice Anderson,,Urgent,,,,,0,, +linear:LinearIssue:1:issue-2,https://linear.app/eng/issue/ENG-2,,ENG-2,Add dark mode,Theme support,,REQUIREMENT,,IN_PROGRESS,In Progress,5,,2026-05-01T00:00:00.000+00:00,2026-05-02T00:00:00.000+00:00,,,,,linear:LinearAccount:1:user-1,Alice Anderson,,,,High,,,,,0,, +linear:LinearIssue:1:issue-3,https://linear.app/eng/issue/ENG-3,,ENG-3,Investigate flakiness,,,REQUIREMENT,,TODO,Backlog,,,2026-05-01T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,,,,,linear:LinearAccount:1:user-1,Alice Anderson,,,,No priority,,,,,0,, +linear:LinearIssue:1:issue-4,https://linear.app/eng/issue/ENG-4,,ENG-4,Deprecated feature,No longer needed,,REQUIREMENT,,DONE,Canceled,2,2026-05-02T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,2026-05-02T12:00:00.000+00:00,1440,,,,linear:LinearAccount:1:user-1,Alice Anderson,linear:LinearAccount:1:user-3,Carol Clark,,Medium,,,,,0,, +linear:LinearIssue:1:issue-5,https://linear.app/eng/issue/ENG-5,,ENG-5,Write docs,User guide,,REQUIREMENT,,TODO,Todo,1,,2026-05-01T00:00:00.000+00:00,2026-05-01T06:00:00.000+00:00,,,,,linear:LinearAccount:1:user-2,Bob Brown,,,,Low,,,,,0,, diff --git a/backend/plugins/linear/e2e/snapshot_tables/issues_history_leadtime.csv b/backend/plugins/linear/e2e/snapshot_tables/issues_history_leadtime.csv new file mode 100644 index 00000000000..d11625e3c3d --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issues_history_leadtime.csv @@ -0,0 +1,6 @@ +id,url,icon_url,issue_key,title,description,epic_key,type,original_type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,parent_issue_id,priority,severity,urgency,component,original_project,is_subtask,due_date,fix_versions +linear:LinearIssue:1:issue-1,https://linear.app/eng/issue/ENG-1,,ENG-1,Fix login bug,Users cannot log in,,REQUIREMENT,,DONE,Done,3,2026-05-03T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,2026-05-03T00:00:00.000+00:00,1440,,,,linear:LinearAccount:1:user-2,,linear:LinearAccount:1:user-1,,,Urgent,,,,,0,, +linear:LinearIssue:1:issue-2,https://linear.app/eng/issue/ENG-2,,ENG-2,Add dark mode,Theme support,,REQUIREMENT,,IN_PROGRESS,In Progress,5,,2026-05-01T00:00:00.000+00:00,2026-05-02T00:00:00.000+00:00,,,,,linear:LinearAccount:1:user-1,,,,,High,,,,,0,, +linear:LinearIssue:1:issue-3,https://linear.app/eng/issue/ENG-3,,ENG-3,Investigate flakiness,,,REQUIREMENT,,TODO,Backlog,,,2026-05-01T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,,,,,linear:LinearAccount:1:user-1,,,,,No priority,,,,,0,, +linear:LinearIssue:1:issue-4,https://linear.app/eng/issue/ENG-4,,ENG-4,Deprecated feature,No longer needed,,REQUIREMENT,,DONE,Canceled,2,2026-05-02T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,2026-05-02T12:00:00.000+00:00,1440,,,,linear:LinearAccount:1:user-1,,linear:LinearAccount:1:user-3,,,Medium,,,,,0,, +linear:LinearIssue:1:issue-5,https://linear.app/eng/issue/ENG-5,,ENG-5,Write docs,User guide,,REQUIREMENT,,TODO,Todo,1,,2026-05-01T00:00:00.000+00:00,2026-05-01T06:00:00.000+00:00,,,,,linear:LinearAccount:1:user-2,,,,,Low,,,,,0,, diff --git a/backend/plugins/linear/e2e/snapshot_tables/issues_incident.csv b/backend/plugins/linear/e2e/snapshot_tables/issues_incident.csv new file mode 100644 index 00000000000..1adc9cab7da --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issues_incident.csv @@ -0,0 +1,6 @@ +id,url,icon_url,issue_key,title,description,epic_key,type,original_type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,parent_issue_id,priority,severity,urgency,component,original_project,is_subtask,due_date,fix_versions +linear:LinearIssue:1:issue-1,https://linear.app/eng/issue/ENG-1,,ENG-1,Fix login bug,Users cannot log in,,INCIDENT,,DONE,Done,3,2026-05-03T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,2026-05-03T00:00:00.000+00:00,2880,,,,linear:LinearAccount:1:user-2,Bob Brown,linear:LinearAccount:1:user-1,Alice Anderson,,Urgent,,,,,0,, +linear:LinearIssue:1:issue-2,https://linear.app/eng/issue/ENG-2,,ENG-2,Add dark mode,Theme support,,REQUIREMENT,,IN_PROGRESS,In Progress,5,,2026-05-01T00:00:00.000+00:00,2026-05-02T00:00:00.000+00:00,,,,,linear:LinearAccount:1:user-1,Alice Anderson,,,,High,,,,,0,, +linear:LinearIssue:1:issue-3,https://linear.app/eng/issue/ENG-3,,ENG-3,Investigate flakiness,,,REQUIREMENT,,TODO,Backlog,,,2026-05-01T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,,,,,linear:LinearAccount:1:user-1,Alice Anderson,,,,No priority,,,,,0,, +linear:LinearIssue:1:issue-4,https://linear.app/eng/issue/ENG-4,,ENG-4,Deprecated feature,No longer needed,,REQUIREMENT,,DONE,Canceled,2,2026-05-02T00:00:00.000+00:00,2026-05-01T00:00:00.000+00:00,2026-05-02T12:00:00.000+00:00,1440,,,,linear:LinearAccount:1:user-1,Alice Anderson,linear:LinearAccount:1:user-3,Carol Clark,,Medium,,,,,0,, +linear:LinearIssue:1:issue-5,https://linear.app/eng/issue/ENG-5,,ENG-5,Write docs,User guide,,REQUIREMENT,,TODO,Todo,1,,2026-05-01T00:00:00.000+00:00,2026-05-01T06:00:00.000+00:00,,,,,linear:LinearAccount:1:user-2,Bob Brown,,,,Low,,,,,0,, diff --git a/backend/plugins/linear/e2e/snapshot_tables/issues_negative_leadtime.csv b/backend/plugins/linear/e2e/snapshot_tables/issues_negative_leadtime.csv new file mode 100644 index 00000000000..4375d47901b --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/issues_negative_leadtime.csv @@ -0,0 +1,2 @@ +id,url,icon_url,issue_key,title,description,epic_key,type,original_type,status,original_status,story_point,resolution_date,created_date,updated_date,lead_time_minutes,original_estimate_minutes,time_spent_minutes,time_remaining_minutes,creator_id,creator_name,assignee_id,assignee_name,parent_issue_id,priority,severity,urgency,component,original_project,is_subtask,due_date,fix_versions +linear:LinearIssue:1:issue-neg,https://linear.app/eng/issue/ENG-NEG,,ENG-NEG,Imported issue with skewed timestamps,canceledAt precedes createdAt,,REQUIREMENT,,DONE,Canceled,,2026-05-09T00:00:00.000+00:00,2026-05-10T00:00:00.000+00:00,2026-05-10T00:00:00.000+00:00,,,,,linear:LinearAccount:1:user-1,,,,,No priority,,,,,0,, diff --git a/backend/plugins/linear/e2e/snapshot_tables/sprint_issues.csv b/backend/plugins/linear/e2e/snapshot_tables/sprint_issues.csv new file mode 100644 index 00000000000..b13b4f85210 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/sprint_issues.csv @@ -0,0 +1,3 @@ +sprint_id,issue_id +linear:LinearCycle:1:cycle-1,linear:LinearIssue:1:issue-1 +linear:LinearCycle:1:cycle-1,linear:LinearIssue:1:issue-4 diff --git a/backend/plugins/linear/e2e/snapshot_tables/sprint_issues_after_leaving_cycle.csv b/backend/plugins/linear/e2e/snapshot_tables/sprint_issues_after_leaving_cycle.csv new file mode 100644 index 00000000000..1074725b86c --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/sprint_issues_after_leaving_cycle.csv @@ -0,0 +1 @@ +sprint_id,issue_id diff --git a/backend/plugins/linear/e2e/snapshot_tables/sprints.csv b/backend/plugins/linear/e2e/snapshot_tables/sprints.csv new file mode 100644 index 00000000000..a09ab2ffe70 --- /dev/null +++ b/backend/plugins/linear/e2e/snapshot_tables/sprints.csv @@ -0,0 +1,3 @@ +id,name,url,status,started_date,ended_date,completed_date,original_board_id +linear:LinearCycle:1:cycle-1,Cycle 1,,CLOSED,2026-04-20T00:00:00.000+00:00,2026-05-04T00:00:00.000+00:00,2026-05-04T00:00:00.000+00:00,linear:LinearTeam:1:team-1 +linear:LinearCycle:1:cycle-2,Sprint 2,,ACTIVE,2026-05-04T00:00:00.000+00:00,2026-05-18T00:00:00.000+00:00,,linear:LinearTeam:1:team-1 diff --git a/backend/plugins/linear/e2e/sprint_issue_test.go b/backend/plugins/linear/e2e/sprint_issue_test.go new file mode 100644 index 00000000000..81e4508e43f --- /dev/null +++ b/backend/plugins/linear/e2e/sprint_issue_test.go @@ -0,0 +1,71 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +// TestLinearSprintIssueStaleCycle guards against stale sprint_issues rows when +// an issue is moved out of a cycle. Sprint membership is derived from the +// issue's cycle_id; once that empties on re-collection, the issue must no +// longer appear in sprint_issues from a previous run. +func TestLinearSprintIssueStaleCycle(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + // seed issues: issue-1 and issue-4 both belong to cycle-1 + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_issues.csv", "_raw_linear_issues") + dataflowTester.FlushTabler(&models.LinearIssue{}) + dataflowTester.FlushTabler(&models.LinearIssueLabel{}) + dataflowTester.Subtask(tasks.ExtractIssuesMeta, taskData) + + // first conversion: both issues land in the sprint + dataflowTester.FlushTabler(&ticket.SprintIssue{}) + dataflowTester.Subtask(tasks.ConvertSprintIssuesMeta, taskData) + + // every issue is moved out of its cycle; on re-collection cycle_id empties. + // The second conversion then produces zero sprint issues, which is the case + // the batch divider's lazy delete fails to cover. + if err := dataflowTester.Dal.Exec( + "UPDATE _tool_linear_issues SET cycle_id = '' WHERE connection_id = ? AND team_id = ?", 1, "team-1", + ); err != nil { + t.Fatal(err) + } + + // second conversion (no flush) must drop ALL stale sprint_issues rows + dataflowTester.Subtask(tasks.ConvertSprintIssuesMeta, taskData) + dataflowTester.VerifyTableWithOptions(ticket.SprintIssue{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/sprint_issues_after_leaving_cycle.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/e2e/workflow_state_test.go b/backend/plugins/linear/e2e/workflow_state_test.go new file mode 100644 index 00000000000..5af5c4a66b9 --- /dev/null +++ b/backend/plugins/linear/e2e/workflow_state_test.go @@ -0,0 +1,48 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +func TestLinearWorkflowStateDataFlow(t *testing.T) { + var linear impl.Linear + dataflowTester := e2ehelper.NewDataFlowTester(t, "linear", linear) + + taskData := &tasks.LinearTaskData{ + Options: &tasks.LinearOptions{ + ConnectionId: 1, + TeamId: "team-1", + }, + } + + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_linear_workflow_states.csv", "_raw_linear_workflow_states") + dataflowTester.FlushTabler(&models.LinearWorkflowState{}) + dataflowTester.Subtask(tasks.ExtractWorkflowStatesMeta, taskData) + dataflowTester.VerifyTableWithOptions(models.LinearWorkflowState{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_linear_workflow_states.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/linear/impl/impl.go b/backend/plugins/linear/impl/impl.go new file mode 100644 index 00000000000..e1bc9849131 --- /dev/null +++ b/backend/plugins/linear/impl/impl.go @@ -0,0 +1,233 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "fmt" + "time" + + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + coreModels "github.com/apache/incubator-devlake/core/models" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/api" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/apache/incubator-devlake/plugins/linear/models/migrationscripts" + "github.com/apache/incubator-devlake/plugins/linear/tasks" +) + +var _ interface { + plugin.PluginMeta + plugin.PluginInit + plugin.PluginTask + plugin.PluginApi + plugin.PluginModel + plugin.PluginSource + plugin.PluginMigration + plugin.CloseablePluginTask + plugin.DataSourcePluginBlueprintV200 +} = (*Linear)(nil) + +type Linear struct{} + +func (p Linear) Init(basicRes context.BasicRes) errors.Error { + api.Init(basicRes, p) + return nil +} + +func (p Linear) Description() string { + return "To collect and enrich data from Linear" +} + +func (p Linear) Name() string { + return "linear" +} + +func (p Linear) RootPkgPath() string { + return "github.com/apache/incubator-devlake/plugins/linear" +} + +func (p Linear) Connection() dal.Tabler { + return &models.LinearConnection{} +} + +func (p Linear) Scope() plugin.ToolLayerScope { + return &models.LinearTeam{} +} + +func (p Linear) ScopeConfig() dal.Tabler { + return &models.LinearScopeConfig{} +} + +func (p Linear) MigrationScripts() []plugin.MigrationScript { + return migrationscripts.All() +} + +func (p Linear) GetTablesInfo() []dal.Tabler { + return []dal.Tabler{ + &models.LinearConnection{}, + &models.LinearTeam{}, + &models.LinearScopeConfig{}, + &models.LinearAccount{}, + &models.LinearIssue{}, + &models.LinearComment{}, + &models.LinearIssueLabel{}, + &models.LinearWorkflowState{}, + &models.LinearCycle{}, + &models.LinearIssueHistory{}, + } +} + +func (p Linear) SubTaskMetas() []plugin.SubTaskMeta { + return []plugin.SubTaskMeta{ + tasks.CollectAccountsMeta, + tasks.ExtractAccountsMeta, + tasks.CollectWorkflowStatesMeta, + tasks.ExtractWorkflowStatesMeta, + tasks.CollectIssuesMeta, + tasks.ExtractIssuesMeta, + tasks.CollectCommentsMeta, + tasks.ExtractCommentsMeta, + tasks.CollectCyclesMeta, + tasks.ExtractCyclesMeta, + tasks.CollectIssueHistoryMeta, + tasks.ExtractIssueHistoryMeta, + tasks.ConvertTeamsMeta, + tasks.ConvertAccountsMeta, + tasks.ConvertIssuesMeta, + tasks.ConvertIssueLabelsMeta, + tasks.ConvertCommentsMeta, + tasks.ConvertCyclesMeta, + tasks.ConvertSprintIssuesMeta, + tasks.ConvertIssueHistoryMeta, + } +} + +func (p Linear) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]interface{}) (interface{}, errors.Error) { + var op tasks.LinearOptions + if err := helper.Decode(options, &op, nil); err != nil { + return nil, errors.Default.Wrap(err, "could not decode Linear options") + } + if op.ConnectionId == 0 { + return nil, errors.BadInput.New("linear connectionId is invalid") + } + if op.TeamId == "" { + return nil, errors.BadInput.New("linear teamId is required") + } + + connection := &models.LinearConnection{} + connectionHelper := helper.NewConnectionHelper(taskCtx, nil, p.Name()) + if err := connectionHelper.FirstById(connection, op.ConnectionId); err != nil { + return nil, errors.Default.Wrap(err, "error getting connection for Linear plugin") + } + + graphqlClient, err := tasks.NewLinearGraphqlClient(taskCtx, connection) + if err != nil { + return nil, errors.Default.Wrap(err, "unable to create Linear GraphQL client") + } + + // Resolve the scope config (label-based issue-type mapping). Default to an + // empty config when none is set so subtasks can rely on it being non-nil. + scopeConfig := &models.LinearScopeConfig{} + if op.ScopeConfigId != 0 { + if err := taskCtx.GetDal().First(scopeConfig, dal.Where("id = ?", op.ScopeConfigId)); err != nil { + return nil, errors.Default.Wrap(err, "error getting scope config for Linear plugin") + } + } + + taskData := &tasks.LinearTaskData{ + Options: &op, + GraphqlClient: graphqlClient, + ScopeConfig: scopeConfig, + } + if op.TimeAfter != "" { + timeAfter, errConv := errors.Convert01(time.Parse(time.RFC3339, op.TimeAfter)) + if errConv != nil { + return nil, errors.BadInput.Wrap(errConv, "invalid timeAfter") + } + taskData.TimeAfter = &timeAfter + } + return taskData, nil +} + +func (p Linear) ApiResources() map[string]map[string]plugin.ApiResourceHandler { + return map[string]map[string]plugin.ApiResourceHandler{ + "test": { + "POST": api.TestConnection, + }, + "connections": { + "POST": api.PostConnections, + "GET": api.ListConnections, + }, + "connections/:connectionId": { + "PATCH": api.PatchConnection, + "DELETE": api.DeleteConnection, + "GET": api.GetConnection, + }, + "connections/:connectionId/test": { + "POST": api.TestExistingConnection, + }, + "connections/:connectionId/remote-scopes": { + "GET": api.RemoteScopes, + }, + "connections/:connectionId/proxy/rest/*path": { + "GET": api.Proxy, + }, + "connections/:connectionId/scope-configs": { + "POST": api.PostScopeConfig, + "GET": api.GetScopeConfigList, + }, + "connections/:connectionId/scope-configs/:scopeConfigId": { + "PATCH": api.PatchScopeConfig, + "GET": api.GetScopeConfig, + "DELETE": api.DeleteScopeConfig, + }, + "connections/:connectionId/scopes/:scopeId": { + "GET": api.GetScope, + "PATCH": api.PatchScope, + "DELETE": api.DeleteScope, + }, + "connections/:connectionId/scopes": { + "GET": api.GetScopeList, + "PUT": api.PutScopes, + }, + "scope-config/:scopeConfigId/projects": { + "GET": api.GetProjectsByScopeConfig, + }, + } +} + +func (p Linear) MakeDataSourcePipelinePlanV200( + connectionId uint64, + scopes []*coreModels.BlueprintScope, +) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { + return api.MakePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes) +} + +func (p Linear) Close(taskCtx plugin.TaskContext) errors.Error { + data, ok := taskCtx.GetData().(*tasks.LinearTaskData) + if !ok { + return errors.Default.New(fmt.Sprintf("GetData failed when try to close %+v", taskCtx)) + } + if data.GraphqlClient != nil { + data.GraphqlClient.Release() + } + return nil +} diff --git a/backend/plugins/linear/impl/impl_test.go b/backend/plugins/linear/impl/impl_test.go new file mode 100644 index 00000000000..d11b3ec5ae1 --- /dev/null +++ b/backend/plugins/linear/impl/impl_test.go @@ -0,0 +1,38 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package impl + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestApiResourcesRegistersRemoteScopes guards the remote-scopes endpoints used +// by the config UI to enumerate Linear teams. +func TestApiResourcesRegistersRemoteScopes(t *testing.T) { + resources := Linear{}.ApiResources() + + remoteScopes, ok := resources["connections/:connectionId/remote-scopes"] + assert.True(t, ok, "remote-scopes route must be registered") + assert.NotNil(t, remoteScopes["GET"], "remote-scopes must handle GET") + + proxy, ok := resources["connections/:connectionId/proxy/rest/*path"] + assert.True(t, ok, "proxy route must be registered") + assert.NotNil(t, proxy["GET"], "proxy must handle GET") +} diff --git a/backend/plugins/linear/linear.go b/backend/plugins/linear/linear.go new file mode 100644 index 00000000000..2cb64a49470 --- /dev/null +++ b/backend/plugins/linear/linear.go @@ -0,0 +1,43 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main // must be main for plugin entry point + +import ( + "github.com/apache/incubator-devlake/core/runner" + "github.com/apache/incubator-devlake/plugins/linear/impl" + "github.com/spf13/cobra" +) + +var PluginEntry impl.Linear //nolint + +// standalone mode for debugging +func main() { + cmd := &cobra.Command{Use: "linear"} + connectionId := cmd.Flags().Uint64P("connection", "c", 0, "linear connection id") + teamId := cmd.Flags().StringP("team", "t", "", "linear team id") + timeAfter := cmd.Flags().StringP("timeAfter", "a", "", "collect data that are created after specified time, ie 2006-01-02T15:04:05Z") + _ = cmd.MarkFlagRequired("connection") + _ = cmd.MarkFlagRequired("team") + cmd.Run = func(c *cobra.Command, args []string) { + runner.DirectRun(c, args, PluginEntry, map[string]interface{}{ + "connectionId": *connectionId, + "teamId": *teamId, + }, *timeAfter) + } + runner.RunCmd(cmd) +} diff --git a/backend/plugins/linear/models/account.go b/backend/plugins/linear/models/account.go new file mode 100644 index 00000000000..7d1c9376f98 --- /dev/null +++ b/backend/plugins/linear/models/account.go @@ -0,0 +1,38 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearAccount is a Linear user (tool layer), converted to crossdomain.Account. +type LinearAccount struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + Name string `gorm:"type:varchar(255)" json:"name"` + DisplayName string `gorm:"type:varchar(255)" json:"displayName"` + Email string `gorm:"type:varchar(255)" json:"email"` + AvatarUrl string `gorm:"type:varchar(255)" json:"avatarUrl"` + Active bool `json:"active"` + common.NoPKModel +} + +func (LinearAccount) TableName() string { + return "_tool_linear_accounts" +} diff --git a/backend/plugins/linear/models/connection.go b/backend/plugins/linear/models/connection.go new file mode 100644 index 00000000000..6f63e431a11 --- /dev/null +++ b/backend/plugins/linear/models/connection.go @@ -0,0 +1,73 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "net/http" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/utils" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +// LinearConn holds the essential information to connect to the Linear API. +// Linear authenticates with a personal API key passed verbatim in the +// `Authorization` header (NO `Bearer` prefix), so we implement our own +// SetupAuthentication instead of reusing helper.AccessToken. +type LinearConn struct { + helper.RestConnection `mapstructure:",squash"` + Token string `mapstructure:"token" validate:"required" json:"token" gorm:"serializer:encdec"` +} + +// SetupAuthentication sets up the HTTP request authentication for the Linear API. +func (lc *LinearConn) SetupAuthentication(req *http.Request) errors.Error { + req.Header.Set("Authorization", lc.Token) + return nil +} + +func (lc *LinearConn) Sanitize() LinearConn { + lc.Token = utils.SanitizeString(lc.Token) + return *lc +} + +// LinearConnection holds LinearConn plus ID/Name for database storage. +type LinearConnection struct { + helper.BaseConnection `mapstructure:",squash"` + LinearConn `mapstructure:",squash"` +} + +func (connection LinearConnection) Sanitize() LinearConnection { + connection.LinearConn = connection.LinearConn.Sanitize() + return connection +} + +func (connection *LinearConnection) MergeFromRequest(target *LinearConnection, body map[string]interface{}) error { + token := target.Token + if err := helper.DecodeMapStruct(body, target, true); err != nil { + return err + } + modifiedToken := target.Token + if modifiedToken == "" || modifiedToken == utils.SanitizeString(token) { + target.Token = token + } + return nil +} + +func (LinearConnection) TableName() string { + return "_tool_linear_connections" +} diff --git a/backend/plugins/linear/models/cycle.go b/backend/plugins/linear/models/cycle.go new file mode 100644 index 00000000000..4f61370dbb2 --- /dev/null +++ b/backend/plugins/linear/models/cycle.go @@ -0,0 +1,41 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearCycle is a Linear cycle (sprint-equivalent), converted to ticket.Sprint. +type LinearCycle struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + TeamId string `gorm:"index;type:varchar(255)" json:"teamId"` + Number int `json:"number"` + Name string `gorm:"type:varchar(255)" json:"name"` + StartsAt *time.Time `json:"startsAt"` + EndsAt *time.Time `json:"endsAt"` + CompletedAt *time.Time `json:"completedAt"` + common.NoPKModel +} + +func (LinearCycle) TableName() string { + return "_tool_linear_cycles" +} diff --git a/backend/plugins/linear/models/issue.go b/backend/plugins/linear/models/issue.go new file mode 100644 index 00000000000..991585de956 --- /dev/null +++ b/backend/plugins/linear/models/issue.go @@ -0,0 +1,56 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearIssue is the tool-layer representation of a Linear issue. +type LinearIssue struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + TeamId string `gorm:"index;type:varchar(255)" json:"teamId"` + Identifier string `gorm:"type:varchar(255)" json:"identifier"` + Number int `json:"number"` + Title string `json:"title"` + Description string `json:"description"` + Url string `json:"url"` + Priority int `json:"priority"` + PriorityLabel string `gorm:"type:varchar(100)" json:"priorityLabel"` + Estimate *float64 `json:"estimate"` + StateId string `gorm:"index;type:varchar(255)" json:"stateId"` + StateName string `gorm:"type:varchar(255)" json:"stateName"` + StateType string `gorm:"type:varchar(100)" json:"stateType"` + CreatorId string `gorm:"type:varchar(255)" json:"creatorId"` + AssigneeId string `gorm:"type:varchar(255)" json:"assigneeId"` + CycleId string `gorm:"index;type:varchar(255)" json:"cycleId"` + ParentId string `gorm:"type:varchar(255)" json:"parentId"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `gorm:"index" json:"updatedAt"` + StartedAt *time.Time `json:"startedAt"` + CompletedAt *time.Time `json:"completedAt"` + CanceledAt *time.Time `json:"canceledAt"` + common.NoPKModel +} + +func (LinearIssue) TableName() string { + return "_tool_linear_issues" +} diff --git a/backend/plugins/linear/models/issue_comment.go b/backend/plugins/linear/models/issue_comment.go new file mode 100644 index 00000000000..8ba18505038 --- /dev/null +++ b/backend/plugins/linear/models/issue_comment.go @@ -0,0 +1,40 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearComment is the tool-layer representation of a comment on a Linear issue. +type LinearComment struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + IssueId string `gorm:"index;type:varchar(255)" json:"issueId"` + Body string `json:"body"` + AuthorId string `gorm:"type:varchar(255)" json:"authorId"` + CreatedAt time.Time `json:"createdAt"` + UpdatedAt time.Time `gorm:"index" json:"updatedAt"` + common.NoPKModel +} + +func (LinearComment) TableName() string { + return "_tool_linear_comments" +} diff --git a/backend/plugins/linear/models/issue_history.go b/backend/plugins/linear/models/issue_history.go new file mode 100644 index 00000000000..027f6cb4a07 --- /dev/null +++ b/backend/plugins/linear/models/issue_history.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearIssueHistory is a single entry in a Linear issue's history, used to +// build domain-layer changelogs and derive lead/cycle time. +type LinearIssueHistory struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + IssueId string `gorm:"index;type:varchar(255)" json:"issueId"` + ActorId string `gorm:"type:varchar(255)" json:"actorId"` + FromStateId string `gorm:"type:varchar(255)" json:"fromStateId"` + FromStateName string `gorm:"type:varchar(255)" json:"fromStateName"` + FromStateType string `gorm:"type:varchar(100)" json:"fromStateType"` + ToStateId string `gorm:"type:varchar(255)" json:"toStateId"` + ToStateName string `gorm:"type:varchar(255)" json:"toStateName"` + ToStateType string `gorm:"type:varchar(100)" json:"toStateType"` + CreatedAt time.Time `gorm:"index" json:"createdAt"` + common.NoPKModel +} + +func (LinearIssueHistory) TableName() string { + return "_tool_linear_issue_history" +} diff --git a/backend/plugins/linear/models/issue_label.go b/backend/plugins/linear/models/issue_label.go new file mode 100644 index 00000000000..76a4517cb7b --- /dev/null +++ b/backend/plugins/linear/models/issue_label.go @@ -0,0 +1,35 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearIssueLabel joins a Linear issue to one of its labels. Labels are +// collected inline with issues, so there is no separate label collector. +type LinearIssueLabel struct { + ConnectionId uint64 `gorm:"primaryKey"` + IssueId string `gorm:"primaryKey;type:varchar(255)" json:"issueId"` + LabelName string `gorm:"primaryKey;type:varchar(255)" json:"labelName"` + common.NoPKModel +} + +func (LinearIssueLabel) TableName() string { + return "_tool_linear_issue_labels" +} diff --git a/backend/plugins/linear/models/migrationscripts/20260601_add_init_tables.go b/backend/plugins/linear/models/migrationscripts/20260601_add_init_tables.go new file mode 100644 index 00000000000..9bb577343be --- /dev/null +++ b/backend/plugins/linear/models/migrationscripts/20260601_add_init_tables.go @@ -0,0 +1,51 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" + "github.com/apache/incubator-devlake/plugins/linear/models/migrationscripts/archived" +) + +type addInitTables struct{} + +func (*addInitTables) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &archived.LinearConnection{}, + &archived.LinearTeam{}, + &archived.LinearScopeConfig{}, + &archived.LinearAccount{}, + &archived.LinearIssue{}, + &archived.LinearComment{}, + &archived.LinearIssueLabel{}, + &archived.LinearWorkflowState{}, + &archived.LinearCycle{}, + &archived.LinearIssueHistory{}, + ) +} + +func (*addInitTables) Version() uint64 { + return 20260601000001 +} + +func (*addInitTables) Name() string { + return "linear init schemas" +} diff --git a/backend/plugins/linear/models/migrationscripts/archived/models.go b/backend/plugins/linear/models/migrationscripts/archived/models.go new file mode 100644 index 00000000000..a90c3217153 --- /dev/null +++ b/backend/plugins/linear/models/migrationscripts/archived/models.go @@ -0,0 +1,169 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package archived holds frozen snapshots of the tool-layer models as they +// existed at each migration. The live models in plugins/linear/models may +// evolve; these snapshots keep historical migrations stable. +package archived + +import ( + "time" + + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" +) + +type LinearConnection struct { + Name string `gorm:"type:varchar(100);uniqueIndex" json:"name"` + archived.Model + Endpoint string `mapstructure:"endpoint" json:"endpoint"` + Proxy string `mapstructure:"proxy" json:"proxy"` + RateLimitPerHour int `json:"rateLimitPerHour"` + Token string `mapstructure:"token" json:"token" gorm:"serializer:encdec"` +} + +func (LinearConnection) TableName() string { return "_tool_linear_connections" } + +type LinearTeam struct { + archived.NoPKModel + ConnectionId uint64 `json:"connectionId" gorm:"primaryKey"` + ScopeConfigId uint64 `json:"scopeConfigId,omitempty"` + TeamId string `json:"teamId" gorm:"primaryKey;type:varchar(255)"` + Name string `json:"name" gorm:"type:varchar(255)"` + Key string `json:"key" gorm:"type:varchar(255)"` + Description string `json:"description"` +} + +func (LinearTeam) TableName() string { return "_tool_linear_teams" } + +type LinearScopeConfig struct { + archived.ScopeConfig + ConnectionId uint64 `json:"connectionId" gorm:"index"` + Name string `gorm:"type:varchar(255);uniqueIndex" json:"name"` + IssueTypeRequirement string `json:"issueTypeRequirement" gorm:"type:varchar(255)"` + IssueTypeBug string `json:"issueTypeBug" gorm:"type:varchar(255)"` + IssueTypeIncident string `json:"issueTypeIncident" gorm:"type:varchar(255)"` +} + +func (LinearScopeConfig) TableName() string { return "_tool_linear_scope_configs" } + +type LinearAccount struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + DisplayName string `gorm:"type:varchar(255)"` + Email string `gorm:"type:varchar(255)"` + AvatarUrl string `gorm:"type:varchar(255)"` + Active bool + archived.NoPKModel +} + +func (LinearAccount) TableName() string { return "_tool_linear_accounts" } + +type LinearIssue struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + TeamId string `gorm:"index;type:varchar(255)"` + Identifier string `gorm:"type:varchar(255)"` + Number int + Title string + Description string + Url string + Priority int + PriorityLabel string `gorm:"type:varchar(100)"` + Estimate *float64 + StateId string `gorm:"index;type:varchar(255)"` + StateName string `gorm:"type:varchar(255)"` + StateType string `gorm:"type:varchar(100)"` + CreatorId string `gorm:"type:varchar(255)"` + AssigneeId string `gorm:"type:varchar(255)"` + CycleId string `gorm:"index;type:varchar(255)"` + ParentId string `gorm:"type:varchar(255)"` + CreatedAt time.Time + UpdatedAt time.Time `gorm:"index"` + StartedAt *time.Time + CompletedAt *time.Time + CanceledAt *time.Time + archived.NoPKModel +} + +func (LinearIssue) TableName() string { return "_tool_linear_issues" } + +type LinearComment struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + IssueId string `gorm:"index;type:varchar(255)"` + Body string + AuthorId string `gorm:"type:varchar(255)"` + CreatedAt time.Time + UpdatedAt time.Time `gorm:"index"` + archived.NoPKModel +} + +func (LinearComment) TableName() string { return "_tool_linear_comments" } + +type LinearIssueLabel struct { + ConnectionId uint64 `gorm:"primaryKey"` + IssueId string `gorm:"primaryKey;type:varchar(255)"` + LabelName string `gorm:"primaryKey;type:varchar(255)"` + archived.NoPKModel +} + +func (LinearIssueLabel) TableName() string { return "_tool_linear_issue_labels" } + +type LinearWorkflowState struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + TeamId string `gorm:"index;type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + Type string `gorm:"type:varchar(100)"` + Color string `gorm:"type:varchar(50)"` + Position float64 + archived.NoPKModel +} + +func (LinearWorkflowState) TableName() string { return "_tool_linear_workflow_states" } + +type LinearCycle struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + TeamId string `gorm:"index;type:varchar(255)"` + Number int + Name string `gorm:"type:varchar(255)"` + StartsAt *time.Time + EndsAt *time.Time + CompletedAt *time.Time + archived.NoPKModel +} + +func (LinearCycle) TableName() string { return "_tool_linear_cycles" } + +type LinearIssueHistory struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)"` + IssueId string `gorm:"index;type:varchar(255)"` + ActorId string `gorm:"type:varchar(255)"` + FromStateId string `gorm:"type:varchar(255)"` + FromStateName string `gorm:"type:varchar(255)"` + FromStateType string `gorm:"type:varchar(100)"` + ToStateId string `gorm:"type:varchar(255)"` + ToStateName string `gorm:"type:varchar(255)"` + ToStateType string `gorm:"type:varchar(100)"` + CreatedAt time.Time `gorm:"index"` + archived.NoPKModel +} + +func (LinearIssueHistory) TableName() string { return "_tool_linear_issue_history" } diff --git a/backend/plugins/linear/models/migrationscripts/register.go b/backend/plugins/linear/models/migrationscripts/register.go new file mode 100644 index 00000000000..ec054748c27 --- /dev/null +++ b/backend/plugins/linear/models/migrationscripts/register.go @@ -0,0 +1,29 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/plugin" +) + +// All return all the migration scripts +func All() []plugin.MigrationScript { + return []plugin.MigrationScript{ + new(addInitTables), + } +} diff --git a/backend/plugins/linear/models/scope_config.go b/backend/plugins/linear/models/scope_config.go new file mode 100644 index 00000000000..309ce749eae --- /dev/null +++ b/backend/plugins/linear/models/scope_config.go @@ -0,0 +1,44 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearScopeConfig keeps status mapping deterministic (Linear's +// WorkflowState.type maps to TODO/IN_PROGRESS/DONE without user input) but +// allows label-based issue-type mapping. Linear has no native issue "type", so +// each pattern below is a regular expression matched against an issue's label +// names to derive the domain ticket.Issue.Type. Precedence is +// INCIDENT > BUG > REQUIREMENT; an issue matching none defaults to REQUIREMENT. +type LinearScopeConfig struct { + common.ScopeConfig `mapstructure:",squash" json:",inline" gorm:"embedded"` + IssueTypeRequirement string `mapstructure:"issueTypeRequirement,omitempty" json:"issueTypeRequirement" gorm:"type:varchar(255)"` + IssueTypeBug string `mapstructure:"issueTypeBug,omitempty" json:"issueTypeBug" gorm:"type:varchar(255)"` + IssueTypeIncident string `mapstructure:"issueTypeIncident,omitempty" json:"issueTypeIncident" gorm:"type:varchar(255)"` +} + +func (LinearScopeConfig) TableName() string { + return "_tool_linear_scope_configs" +} + +func (sc *LinearScopeConfig) SetConnectionId(c *LinearScopeConfig, connectionId uint64) { + c.ConnectionId = connectionId + c.ScopeConfig.ConnectionId = connectionId +} diff --git a/backend/plugins/linear/models/team.go b/backend/plugins/linear/models/team.go new file mode 100644 index 00000000000..d6226208266 --- /dev/null +++ b/backend/plugins/linear/models/team.go @@ -0,0 +1,66 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" +) + +var _ plugin.ToolLayerScope = (*LinearTeam)(nil) + +// LinearTeam is the data-source scope for the Linear plugin. A Linear Team +// owns issues, cycles, workflow states and labels, mapping cleanly to a +// DevLake domain-layer ticket.Board. +type LinearTeam struct { + common.Scope `mapstructure:",squash"` + TeamId string `json:"teamId" mapstructure:"teamId" gorm:"primaryKey;type:varchar(255)"` + Name string `json:"name" mapstructure:"name" gorm:"type:varchar(255)"` + Key string `json:"key" mapstructure:"key" gorm:"type:varchar(255)"` + Description string `json:"description" mapstructure:"description"` +} + +func (t LinearTeam) ScopeId() string { + return t.TeamId +} + +func (t LinearTeam) ScopeName() string { + return t.Name +} + +func (t LinearTeam) ScopeFullName() string { + return t.Name +} + +func (t LinearTeam) ScopeParams() interface{} { + return &LinearApiParams{ + ConnectionId: t.ConnectionId, + TeamId: t.TeamId, + } +} + +func (LinearTeam) TableName() string { + return "_tool_linear_teams" +} + +// LinearApiParams identifies the scope a raw row belongs to. It is stored in +// the `params` column of every _raw_linear_* table. +type LinearApiParams struct { + ConnectionId uint64 + TeamId string +} diff --git a/backend/plugins/linear/models/workflow_state.go b/backend/plugins/linear/models/workflow_state.go new file mode 100644 index 00000000000..9273f183c91 --- /dev/null +++ b/backend/plugins/linear/models/workflow_state.go @@ -0,0 +1,39 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +// LinearWorkflowState is a Linear team's workflow state. Its Type +// (backlog|unstarted|started|completed|canceled) drives issue status mapping. +type LinearWorkflowState struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id string `gorm:"primaryKey;type:varchar(255)" json:"id"` + TeamId string `gorm:"index;type:varchar(255)" json:"teamId"` + Name string `gorm:"type:varchar(255)" json:"name"` + Type string `gorm:"type:varchar(100)" json:"type"` + Color string `gorm:"type:varchar(50)" json:"color"` + Position float64 `json:"position"` + common.NoPKModel +} + +func (LinearWorkflowState) TableName() string { + return "_tool_linear_workflow_states" +} diff --git a/backend/plugins/linear/tasks/account_collector.go b/backend/plugins/linear/tasks/account_collector.go new file mode 100644 index 00000000000..88fa181f002 --- /dev/null +++ b/backend/plugins/linear/tasks/account_collector.go @@ -0,0 +1,98 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/merico-ai/graphql" +) + +const RAW_ACCOUNTS_TABLE = "linear_accounts" + +// GraphqlQueryAccountWrapper is the paginated `users` query envelope. +type GraphqlQueryAccountWrapper struct { + Users struct { + Nodes []GraphqlQueryAccount + PageInfo *helper.GraphqlQueryPageInfo + } `graphql:"users(first: $pageSize, after: $skipCursor)"` +} + +type GraphqlQueryAccount struct { + Id string + Name string + DisplayName string + Email string + AvatarUrl string + Active bool +} + +var CollectAccountsMeta = plugin.SubTaskMeta{ + Name: "Collect Users", + EntryPoint: CollectAccounts, + EnabledByDefault: true, + Description: "Collect workspace users from the Linear GraphQL API", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, +} + +var _ plugin.SubTaskEntryPoint = CollectAccounts + +func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + collector, err := helper.NewGraphqlCollector(helper.GraphqlCollectorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ACCOUNTS_TABLE, + }, + GraphqlClient: data.GraphqlClient, + PageSize: 100, + BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) { + query := &GraphqlQueryAccountWrapper{} + if reqData == nil { + return query, map[string]interface{}{}, nil + } + variables := map[string]interface{}{ + "pageSize": graphql.Int(reqData.Pager.Size), + "skipCursor": (*graphql.String)(reqData.Pager.SkipCursor), + } + return query, variables, nil + }, + GetPageInfo: func(iQuery interface{}, args *helper.GraphqlCollectorArgs) (*helper.GraphqlQueryPageInfo, error) { + query := iQuery.(*GraphqlQueryAccountWrapper) + return query.Users.PageInfo, nil + }, + ResponseParser: func(queryWrapper interface{}) (messages []json.RawMessage, err errors.Error) { + query := queryWrapper.(*GraphqlQueryAccountWrapper) + for _, account := range query.Users.Nodes { + messages = append(messages, errors.Must1(json.Marshal(account))) + } + return + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/linear/tasks/account_convertor.go b/backend/plugins/linear/tasks/account_convertor.go new file mode 100644 index 00000000000..d3c8efd3686 --- /dev/null +++ b/backend/plugins/linear/tasks/account_convertor.go @@ -0,0 +1,97 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ConvertAccountsMeta = plugin.SubTaskMeta{ + Name: "Convert Users", + EntryPoint: ConvertAccounts, + EnabledByDefault: true, + Description: "Convert tool layer table _tool_linear_accounts into domain layer table accounts", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, + DependencyTables: []string{models.LinearAccount{}.TableName()}, + ProductTables: []string{crossdomain.Account{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertAccounts + +func ConvertAccounts(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + accountIdGen := didgen.NewDomainIdGenerator(&models.LinearAccount{}) + + cursor, err := db.Cursor( + dal.From(&models.LinearAccount{}), + dal.Where("connection_id = ?", data.Options.ConnectionId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ACCOUNTS_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearAccount{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + account := inputRow.(*models.LinearAccount) + status := 1 + if !account.Active { + status = 0 + } + fullName := account.Name + if account.DisplayName != "" { + fullName = account.DisplayName + } + domainAccount := &crossdomain.Account{ + DomainEntity: domainlayer.DomainEntity{ + Id: accountIdGen.Generate(data.Options.ConnectionId, account.Id), + }, + UserName: account.Name, + FullName: fullName, + Email: account.Email, + AvatarUrl: account.AvatarUrl, + Status: status, + } + return []interface{}{domainAccount}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/linear/tasks/account_extractor.go b/backend/plugins/linear/tasks/account_extractor.go new file mode 100644 index 00000000000..19cc86a04cc --- /dev/null +++ b/backend/plugins/linear/tasks/account_extractor.go @@ -0,0 +1,74 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ExtractAccountsMeta = plugin.SubTaskMeta{ + Name: "Extract Users", + EntryPoint: ExtractAccounts, + EnabledByDefault: true, + Description: "Extract raw user data into tool layer table _tool_linear_accounts", + DomainTypes: []string{plugin.DOMAIN_TYPE_CROSS}, +} + +var _ plugin.SubTaskEntryPoint = ExtractAccounts + +func ExtractAccounts(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ACCOUNTS_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiAccount := &GraphqlQueryAccount{} + if err := errors.Convert(json.Unmarshal(row.Data, apiAccount)); err != nil { + return nil, err + } + if apiAccount.Id == "" { + return nil, nil + } + account := &models.LinearAccount{ + ConnectionId: data.Options.ConnectionId, + Id: apiAccount.Id, + Name: apiAccount.Name, + DisplayName: apiAccount.DisplayName, + Email: apiAccount.Email, + AvatarUrl: apiAccount.AvatarUrl, + Active: apiAccount.Active, + } + return []interface{}{account}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/linear/tasks/api_client.go b/backend/plugins/linear/tasks/api_client.go new file mode 100644 index 00000000000..0ca6b8da25a --- /dev/null +++ b/backend/plugins/linear/tasks/api_client.go @@ -0,0 +1,108 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + gocontext "context" + "net/http" + "net/url" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/log" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" + "github.com/merico-ai/graphql" +) + +// linearTransport injects the Linear personal API key into every request. +// Linear expects the key verbatim in the Authorization header (no Bearer prefix). +type linearTransport struct { + token string + base http.RoundTripper +} + +func (t *linearTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req.Header.Set("Authorization", t.token) + return t.base.RoundTrip(req) +} + +// graphqlQueryViewer is a tiny probe used to validate connectivity / liveness. +type graphqlQueryViewer struct { + Viewer struct { + Id graphql.String + } +} + +// defaultRateLimitPerHour is Linear's documented per-API-key request budget. +// Used when the connection does not override RateLimitPerHour. +const defaultRateLimitPerHour = 1500 + +// NewLinearGraphqlClient builds a rate-limited async GraphQL client for the +// Linear API from the given connection. +func NewLinearGraphqlClient(taskCtx plugin.TaskContext, connection *models.LinearConnection) (*helper.GraphqlAsyncClient, errors.Error) { + httpClient, err := newLinearHttpClient(connection) + if err != nil { + return nil, err + } + + endpoint := connection.Endpoint + if endpoint == "" { + endpoint = "https://api.linear.app/graphql" + } + client := graphql.NewClient(endpoint, httpClient) + + rateLimitPerHour := connection.RateLimitPerHour + if rateLimitPerHour <= 0 { + rateLimitPerHour = defaultRateLimitPerHour + } + + return helper.CreateAsyncGraphqlClient(taskCtx, client, taskCtx.GetLogger(), + func(ctx gocontext.Context, c *graphql.Client, logger log.Logger) (rateRemaining int, resetAt *time.Time, e errors.Error) { + // Linear does not expose rate-limit info in the GraphQL body (it uses + // HTTP response headers), so we probe liveness and pace against the + // configured hourly budget. The async client self-throttles from here. + var q graphqlQueryViewer + dataErrors, queryErr := errors.Convert01(c.Query(ctx, &q, nil)) + if queryErr != nil { + return 0, nil, queryErr + } + if len(dataErrors) > 0 { + return 0, nil, errors.Default.Wrap(dataErrors[0], "linear graphql viewer query failed") + } + reset := time.Now().Add(1 * time.Hour) + logger.Info("linear graphql client initialized, pacing against %d req/hour", rateLimitPerHour) + return rateLimitPerHour, &reset, nil + }) +} + +func newLinearHttpClient(connection *models.LinearConnection) (*http.Client, errors.Error) { + base := http.DefaultTransport + if proxy := connection.Proxy; proxy != "" { + pu, err := url.Parse(proxy) + if err != nil { + return nil, errors.BadInput.Wrap(err, "malformed proxy url") + } + base = &http.Transport{Proxy: http.ProxyURL(pu)} + } + return &http.Client{ + Timeout: 60 * time.Second, + Transport: &linearTransport{token: connection.Token, base: base}, + }, nil +} diff --git a/backend/plugins/linear/tasks/board_convertor.go b/backend/plugins/linear/tasks/board_convertor.go new file mode 100644 index 00000000000..9366efed2ab --- /dev/null +++ b/backend/plugins/linear/tasks/board_convertor.go @@ -0,0 +1,93 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +// RAW_TEAMS_TABLE labels the raw-data lineage for the team-scope-derived board. +// Teams are added as scopes (no collector), so this is a logical tag only. +const RAW_TEAMS_TABLE = "linear_teams" + +var ConvertTeamsMeta = plugin.SubTaskMeta{ + Name: "Convert Teams", + EntryPoint: ConvertTeams, + EnabledByDefault: true, + Description: "Convert the Linear team scope (_tool_linear_teams) into the domain layer table boards", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.LinearTeam{}.TableName()}, + ProductTables: []string{ticket.Board{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertTeams + +func ConvertTeams(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + connectionId := data.Options.ConnectionId + + // boardId must be generated identically to the issue/sprint convertors so the + // board joins to the board_issues/sprint_issues that reference it. + boardIdGen := didgen.NewDomainIdGenerator(&models.LinearTeam{}) + + cursor, err := db.Cursor( + dal.From(&models.LinearTeam{}), + dal.Where("connection_id = ? AND team_id = ?", connectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: connectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_TEAMS_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearTeam{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + team := inputRow.(*models.LinearTeam) + board := &ticket.Board{ + DomainEntity: domainlayer.DomainEntity{Id: boardIdGen.Generate(connectionId, team.TeamId)}, + Name: team.Name, + Description: team.Description, + Type: "linear", + } + return []interface{}{board}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/linear/tasks/comment_collector.go b/backend/plugins/linear/tasks/comment_collector.go new file mode 100644 index 00000000000..14e801ff9e3 --- /dev/null +++ b/backend/plugins/linear/tasks/comment_collector.go @@ -0,0 +1,148 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "reflect" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/merico-ai/graphql" +) + +const RAW_COMMENTS_TABLE = "linear_comments" + +// SimpleLinearIssue is the iterator element used to drive per-issue collection +// of child resources (comments, history). Its JSON form is stored in the raw +// row's `input` column so extractors can recover the owning issue id. +type SimpleLinearIssue struct { + // Id is populated by the DalCursorIterator (the _tool_linear_issues.id column) + // when driving per-issue child collection. + Id string `json:"Id"` + // IssueId is populated when parsing a raw row's `input` column: the GraphQL + // collector stores the query variables there (which carry `issueId`), not the + // iterator element. OwningIssueId resolves whichever is present. + IssueId string `json:"issueId" gorm:"-"` +} + +// OwningIssueId returns the issue id this child row belongs to, tolerating both +// the iterator element shape ({"Id":...}) and the collector's stored variables +// shape ({"issueId":...}). +func (s SimpleLinearIssue) OwningIssueId() string { + if s.IssueId != "" { + return s.IssueId + } + return s.Id +} + +// GraphqlQueryCommentWrapper is the per-issue, paginated `comments` query. +type GraphqlQueryCommentWrapper struct { + Issue struct { + Comments struct { + Nodes []GraphqlQueryComment + PageInfo *helper.GraphqlQueryPageInfo + } `graphql:"comments(first: $pageSize, after: $skipCursor)"` + } `graphql:"issue(id: $issueId)"` +} + +type GraphqlQueryComment struct { + Id string + Body string + CreatedAt time.Time + UpdatedAt time.Time + User *struct{ Id string } +} + +var CollectCommentsMeta = plugin.SubTaskMeta{ + Name: "Collect Comments", + EntryPoint: CollectComments, + EnabledByDefault: true, + Description: "Collect comments for each collected Linear issue", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + Dependencies: []*plugin.SubTaskMeta{&ExtractIssuesMeta}, +} + +var _ plugin.SubTaskEntryPoint = CollectComments + +func CollectComments(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + + apiCollector, err := helper.NewStatefulApiCollector(helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_COMMENTS_TABLE, + }) + if err != nil { + return err + } + + // Only sweep issues updated since the last successful collection: an + // unchanged issue's comments cannot have changed, so re-fetching every + // issue each run wastes a request per issue against Linear's hourly budget. + since := apiCollector.GetSince() + cursor, err := db.Cursor(issuesToCollectChildrenClauses(data.Options.ConnectionId, data.Options.TeamId, since)...) + if err != nil { + return err + } + iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleLinearIssue{})) + if err != nil { + return err + } + + err = apiCollector.InitGraphQLCollector(helper.GraphqlCollectorArgs{ + GraphqlClient: data.GraphqlClient, + Input: iterator, + InputStep: 1, + PageSize: 100, + BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) { + query := &GraphqlQueryCommentWrapper{} + if reqData == nil { + return query, map[string]interface{}{}, nil + } + issue := reqData.Input.(*SimpleLinearIssue) + variables := map[string]interface{}{ + "pageSize": graphql.Int(reqData.Pager.Size), + "skipCursor": (*graphql.String)(reqData.Pager.SkipCursor), + "issueId": graphql.String(issue.Id), + } + return query, variables, nil + }, + GetPageInfo: func(iQuery interface{}, args *helper.GraphqlCollectorArgs) (*helper.GraphqlQueryPageInfo, error) { + query := iQuery.(*GraphqlQueryCommentWrapper) + return query.Issue.Comments.PageInfo, nil + }, + ResponseParser: func(queryWrapper interface{}) (messages []json.RawMessage, err errors.Error) { + query := queryWrapper.(*GraphqlQueryCommentWrapper) + for _, comment := range query.Issue.Comments.Nodes { + messages = append(messages, errors.Must1(json.Marshal(comment))) + } + return + }, + }) + if err != nil { + return err + } + return apiCollector.Execute() +} diff --git a/backend/plugins/linear/tasks/comment_convertor.go b/backend/plugins/linear/tasks/comment_convertor.go new file mode 100644 index 00000000000..8c4729f0e5c --- /dev/null +++ b/backend/plugins/linear/tasks/comment_convertor.go @@ -0,0 +1,97 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ConvertCommentsMeta = plugin.SubTaskMeta{ + Name: "Convert Comments", + EntryPoint: ConvertComments, + EnabledByDefault: true, + Description: "Convert tool layer table _tool_linear_comments into domain layer table issue_comments", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.LinearComment{}.TableName(), RAW_COMMENTS_TABLE}, + ProductTables: []string{ticket.IssueComment{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertComments + +func ConvertComments(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + connectionId := data.Options.ConnectionId + + issueIdGen := didgen.NewDomainIdGenerator(&models.LinearIssue{}) + commentIdGen := didgen.NewDomainIdGenerator(&models.LinearComment{}) + accountIdGen := didgen.NewDomainIdGenerator(&models.LinearAccount{}) + + cursor, err := db.Cursor( + dal.Select("c.*"), + dal.From("_tool_linear_comments c"), + dal.Join("LEFT JOIN _tool_linear_issues i ON (i.connection_id = c.connection_id AND i.id = c.issue_id)"), + dal.Where("c.connection_id = ? AND i.team_id = ?", connectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: connectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_COMMENTS_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearComment{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + comment := inputRow.(*models.LinearComment) + domainComment := &ticket.IssueComment{ + DomainEntity: domainlayer.DomainEntity{Id: commentIdGen.Generate(connectionId, comment.Id)}, + IssueId: issueIdGen.Generate(connectionId, comment.IssueId), + Body: comment.Body, + CreatedDate: comment.CreatedAt, + } + if comment.AuthorId != "" { + domainComment.AccountId = accountIdGen.Generate(connectionId, comment.AuthorId) + } + if !comment.UpdatedAt.IsZero() { + domainComment.UpdatedDate = &comment.UpdatedAt + } + return []interface{}{domainComment}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/linear/tasks/comment_extractor.go b/backend/plugins/linear/tasks/comment_extractor.go new file mode 100644 index 00000000000..15b28a3b969 --- /dev/null +++ b/backend/plugins/linear/tasks/comment_extractor.go @@ -0,0 +1,78 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ExtractCommentsMeta = plugin.SubTaskMeta{ + Name: "Extract Comments", + EntryPoint: ExtractComments, + EnabledByDefault: true, + Description: "Extract raw comment data into tool layer table _tool_linear_comments", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = ExtractComments + +func ExtractComments(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_COMMENTS_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiComment := &GraphqlQueryComment{} + if err := errors.Convert(json.Unmarshal(row.Data, apiComment)); err != nil { + return nil, err + } + // The owning issue id is carried in the raw row's input column. + issueRef := &SimpleLinearIssue{} + if err := errors.Convert(json.Unmarshal(row.Input, issueRef)); err != nil { + return nil, err + } + comment := &models.LinearComment{ + ConnectionId: data.Options.ConnectionId, + Id: apiComment.Id, + IssueId: issueRef.OwningIssueId(), + Body: apiComment.Body, + CreatedAt: apiComment.CreatedAt, + UpdatedAt: apiComment.UpdatedAt, + } + if apiComment.User != nil { + comment.AuthorId = apiComment.User.Id + } + return []interface{}{comment}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/linear/tasks/cycle_collector.go b/backend/plugins/linear/tasks/cycle_collector.go new file mode 100644 index 00000000000..88572650fcd --- /dev/null +++ b/backend/plugins/linear/tasks/cycle_collector.go @@ -0,0 +1,102 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/merico-ai/graphql" +) + +const RAW_CYCLES_TABLE = "linear_cycles" + +// GraphqlQueryCycleWrapper is the team-scoped, paginated `cycles` query. +type GraphqlQueryCycleWrapper struct { + Team struct { + Cycles struct { + Nodes []GraphqlQueryCycle + PageInfo *helper.GraphqlQueryPageInfo + } `graphql:"cycles(first: $pageSize, after: $skipCursor)"` + } `graphql:"team(id: $teamId)"` +} + +type GraphqlQueryCycle struct { + Id string + Number int + Name string + StartsAt *time.Time + EndsAt *time.Time + CompletedAt *time.Time +} + +var CollectCyclesMeta = plugin.SubTaskMeta{ + Name: "Collect Cycles", + EntryPoint: CollectCycles, + EnabledByDefault: true, + Description: "Collect cycles (sprints) for a Linear team", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = CollectCycles + +func CollectCycles(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + collector, err := helper.NewGraphqlCollector(helper.GraphqlCollectorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_CYCLES_TABLE, + }, + GraphqlClient: data.GraphqlClient, + PageSize: 100, + BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) { + query := &GraphqlQueryCycleWrapper{} + if reqData == nil { + return query, map[string]interface{}{}, nil + } + variables := map[string]interface{}{ + "pageSize": graphql.Int(reqData.Pager.Size), + "skipCursor": (*graphql.String)(reqData.Pager.SkipCursor), + "teamId": graphql.String(data.Options.TeamId), + } + return query, variables, nil + }, + GetPageInfo: func(iQuery interface{}, args *helper.GraphqlCollectorArgs) (*helper.GraphqlQueryPageInfo, error) { + query := iQuery.(*GraphqlQueryCycleWrapper) + return query.Team.Cycles.PageInfo, nil + }, + ResponseParser: func(queryWrapper interface{}) (messages []json.RawMessage, err errors.Error) { + query := queryWrapper.(*GraphqlQueryCycleWrapper) + for _, cycle := range query.Team.Cycles.Nodes { + messages = append(messages, errors.Must1(json.Marshal(cycle))) + } + return + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/linear/tasks/cycle_convertor.go b/backend/plugins/linear/tasks/cycle_convertor.go new file mode 100644 index 00000000000..ae4a0dd5a84 --- /dev/null +++ b/backend/plugins/linear/tasks/cycle_convertor.go @@ -0,0 +1,106 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ConvertCyclesMeta = plugin.SubTaskMeta{ + Name: "Convert Cycles", + EntryPoint: ConvertCycles, + EnabledByDefault: true, + Description: "Convert tool layer table _tool_linear_cycles into domain layer tables sprints and board_sprints", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.LinearCycle{}.TableName(), RAW_CYCLES_TABLE}, + ProductTables: []string{ticket.Sprint{}.TableName(), ticket.BoardSprint{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertCycles + +func ConvertCycles(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + connectionId := data.Options.ConnectionId + + cycleIdGen := didgen.NewDomainIdGenerator(&models.LinearCycle{}) + boardIdGen := didgen.NewDomainIdGenerator(&models.LinearTeam{}) + boardId := boardIdGen.Generate(connectionId, data.Options.TeamId) + + cursor, err := db.Cursor( + dal.From(&models.LinearCycle{}), + dal.Where("connection_id = ? AND team_id = ?", connectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: connectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_CYCLES_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearCycle{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + cycle := inputRow.(*models.LinearCycle) + sprintId := cycleIdGen.Generate(connectionId, cycle.Id) + name := cycle.Name + if name == "" { + name = fmt.Sprintf("Cycle %d", cycle.Number) + } + status := "ACTIVE" + if cycle.CompletedAt != nil { + status = "CLOSED" + } + sprint := &ticket.Sprint{ + DomainEntity: domainlayer.DomainEntity{Id: sprintId}, + Name: name, + Status: status, + StartedDate: cycle.StartsAt, + EndedDate: cycle.EndsAt, + CompletedDate: cycle.CompletedAt, + OriginalBoardID: boardId, + } + boardSprint := &ticket.BoardSprint{ + BoardId: boardId, + SprintId: sprintId, + } + return []interface{}{sprint, boardSprint}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/linear/tasks/cycle_extractor.go b/backend/plugins/linear/tasks/cycle_extractor.go new file mode 100644 index 00000000000..43a4233aa88 --- /dev/null +++ b/backend/plugins/linear/tasks/cycle_extractor.go @@ -0,0 +1,72 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ExtractCyclesMeta = plugin.SubTaskMeta{ + Name: "Extract Cycles", + EntryPoint: ExtractCycles, + EnabledByDefault: true, + Description: "Extract raw cycle data into tool layer table _tool_linear_cycles", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = ExtractCycles + +func ExtractCycles(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_CYCLES_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiCycle := &GraphqlQueryCycle{} + if err := errors.Convert(json.Unmarshal(row.Data, apiCycle)); err != nil { + return nil, err + } + cycle := &models.LinearCycle{ + ConnectionId: data.Options.ConnectionId, + Id: apiCycle.Id, + TeamId: data.Options.TeamId, + Number: apiCycle.Number, + Name: apiCycle.Name, + StartsAt: apiCycle.StartsAt, + EndsAt: apiCycle.EndsAt, + CompletedAt: apiCycle.CompletedAt, + } + return []interface{}{cycle}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/linear/tasks/issue_collector.go b/backend/plugins/linear/tasks/issue_collector.go new file mode 100644 index 00000000000..9f5a0aedd1d --- /dev/null +++ b/backend/plugins/linear/tasks/issue_collector.go @@ -0,0 +1,164 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/utils" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/merico-ai/graphql" +) + +const RAW_ISSUES_TABLE = "linear_issues" + +// GraphqlQueryIssueWrapper is the team-scoped, paginated `issues` query. +// Incremental runs filter server-side on updatedAt ($filter) rather than +// relying on result ordering, so collection no longer depends on an undocumented +// default sort direction. +type GraphqlQueryIssueWrapper struct { + Team struct { + Issues struct { + Nodes []GraphqlQueryIssue `graphql:"nodes"` + PageInfo *helper.GraphqlQueryPageInfo + } `graphql:"issues(first: $pageSize, after: $skipCursor, orderBy: updatedAt, filter: $filter)"` + } `graphql:"team(id: $teamId)"` +} + +// IssueFilter mirrors the subset of Linear's GraphQL IssueFilter input used to +// restrict collection to issues updated after a point in time. The Go type +// name is significant: the GraphQL client emits it as the variable's type +// ($filter:IssueFilter!). +type IssueFilter struct { + UpdatedAt *DateComparator `json:"updatedAt,omitempty"` +} + +// DateComparator mirrors Linear's DateComparator input (only the `gt` operator +// is needed here). +type DateComparator struct { + Gt *time.Time `json:"gt,omitempty"` +} + +// buildIssueFilter returns an IssueFilter restricting to issues updated after +// `since`. When `since` is nil (a full sync) it returns the empty filter, which +// Linear treats as "match all". +func buildIssueFilter(since *time.Time) IssueFilter { + if since == nil { + return IssueFilter{} + } + return IssueFilter{UpdatedAt: &DateComparator{Gt: since}} +} + +type GraphqlQueryIssue struct { + Id string + Identifier string + Number int + Title string + Description string + Url string + Priority int + Estimate *float64 + CreatedAt time.Time + UpdatedAt time.Time + StartedAt *time.Time + CompletedAt *time.Time + CanceledAt *time.Time + State *struct { + Id string + Name string + Type string + } + Assignee *struct{ Id string } + Creator *struct{ Id string } + Cycle *struct{ Id string } + Parent *struct{ Id string } + Labels struct { + Nodes []struct { + Id string + Name string + } + } `graphql:"labels(first: 50)"` +} + +var CollectIssuesMeta = plugin.SubTaskMeta{ + Name: "Collect Issues", + EntryPoint: CollectIssues, + EnabledByDefault: true, + Description: "Collect issues for a Linear team, supports incremental collection", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = CollectIssues + +func CollectIssues(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + apiCollector, err := helper.NewStatefulApiCollector(helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUES_TABLE, + }) + if err != nil { + return err + } + + since := apiCollector.GetSince() + err = apiCollector.InitGraphQLCollector(helper.GraphqlCollectorArgs{ + GraphqlClient: data.GraphqlClient, + PageSize: 100, + BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) { + query := &GraphqlQueryIssueWrapper{} + if reqData == nil { + return query, map[string]interface{}{}, nil + } + variables := map[string]interface{}{ + "pageSize": graphql.Int(reqData.Pager.Size), + "skipCursor": (*graphql.String)(reqData.Pager.SkipCursor), + "teamId": graphql.String(data.Options.TeamId), + "filter": buildIssueFilter(since), + } + return query, variables, nil + }, + GetPageInfo: func(iQuery interface{}, args *helper.GraphqlCollectorArgs) (*helper.GraphqlQueryPageInfo, error) { + query := iQuery.(*GraphqlQueryIssueWrapper) + return query.Team.Issues.PageInfo, nil + }, + ResponseParser: func(queryWrapper interface{}) (messages []json.RawMessage, err errors.Error) { + query := queryWrapper.(*GraphqlQueryIssueWrapper) + // The server-side $filter already restricts to issues updated after + // `since`, so every returned issue is in scope -- no client-side + // early-stop (and thus no dependency on sort direction) is needed. + for _, issue := range query.Team.Issues.Nodes { + issue.CompletedAt = utils.NilIfZeroTime(issue.CompletedAt) + issue.CanceledAt = utils.NilIfZeroTime(issue.CanceledAt) + issue.StartedAt = utils.NilIfZeroTime(issue.StartedAt) + messages = append(messages, errors.Must1(json.Marshal(issue))) + } + return + }, + }) + if err != nil { + return err + } + return apiCollector.Execute() +} diff --git a/backend/plugins/linear/tasks/issue_collector_test.go b/backend/plugins/linear/tasks/issue_collector_test.go new file mode 100644 index 00000000000..2dbbeecf586 --- /dev/null +++ b/backend/plugins/linear/tasks/issue_collector_test.go @@ -0,0 +1,41 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +// TestBuildIssueFilter pins the server-side incremental filter that replaces +// the previous reliance on result ordering plus a client-side early-stop. A +// full sync must produce an empty filter (match all); an incremental run must +// produce Linear's IssueFilter shape `{ updatedAt: { gt: } }`. +func TestBuildIssueFilter(t *testing.T) { + full, err := json.Marshal(buildIssueFilter(nil)) + assert.NoError(t, err) + assert.Equal(t, "{}", string(full)) + + since := time.Date(2026, 5, 1, 0, 0, 0, 0, time.UTC) + incremental, err := json.Marshal(buildIssueFilter(&since)) + assert.NoError(t, err) + assert.JSONEq(t, `{"updatedAt":{"gt":"2026-05-01T00:00:00Z"}}`, string(incremental)) +} diff --git a/backend/plugins/linear/tasks/issue_convertor.go b/backend/plugins/linear/tasks/issue_convertor.go new file mode 100644 index 00000000000..b124e59f369 --- /dev/null +++ b/backend/plugins/linear/tasks/issue_convertor.go @@ -0,0 +1,229 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + "regexp" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ConvertIssuesMeta = plugin.SubTaskMeta{ + Name: "Convert Issues", + EntryPoint: ConvertIssues, + EnabledByDefault: true, + Description: "Convert tool layer table _tool_linear_issues into domain layer tables issues and board_issues", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.LinearIssue{}.TableName(), RAW_ISSUES_TABLE}, + ProductTables: []string{ticket.Issue{}.TableName(), ticket.BoardIssue{}.TableName(), ticket.IssueAssignee{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertIssues + +func ConvertIssues(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + connectionId := data.Options.ConnectionId + + issueIdGen := didgen.NewDomainIdGenerator(&models.LinearIssue{}) + accountIdGen := didgen.NewDomainIdGenerator(&models.LinearAccount{}) + boardIdGen := didgen.NewDomainIdGenerator(&models.LinearTeam{}) + boardId := boardIdGen.Generate(connectionId, data.Options.TeamId) + + // Preload account display names so issues can carry assignee/creator names + // and emit issue_assignees rows, mirroring how the account convertor derives + // the domain account's full name (displayName, falling back to name). + var accounts []models.LinearAccount + if err := db.All(&accounts, dal.Where("connection_id = ?", connectionId)); err != nil { + return err + } + accountNames := make(map[string]string, len(accounts)) + for _, account := range accounts { + name := account.Name + if account.DisplayName != "" { + name = account.DisplayName + } + accountNames[account.Id] = name + } + + // Preload issue labels so the convertor can derive domain issue Type from a + // label-based mapping (Linear has no native issue type). + var issueLabels []models.LinearIssueLabel + if err := db.All(&issueLabels, dal.Where("connection_id = ?", connectionId)); err != nil { + return err + } + labelsByIssue := make(map[string][]string, len(issueLabels)) + for _, l := range issueLabels { + labelsByIssue[l.IssueId] = append(labelsByIssue[l.IssueId], l.LabelName) + } + + // Compile the label-matching patterns from the scope config. An empty + // pattern is treated as "no match" (nil), so issues default to REQUIREMENT. + typeMatcher, err := newIssueTypeMatcher(data.ScopeConfig) + if err != nil { + return err + } + + cursor, err := db.Cursor( + dal.From(&models.LinearIssue{}), + dal.Where("connection_id = ? AND team_id = ?", connectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: connectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUES_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearIssue{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + issue := inputRow.(*models.LinearIssue) + domainIssue := &ticket.Issue{ + DomainEntity: domainlayer.DomainEntity{Id: issueIdGen.Generate(connectionId, issue.Id)}, + IssueKey: issue.Identifier, + Title: issue.Title, + Description: issue.Description, + Url: issue.Url, + Type: typeMatcher.typeOf(labelsByIssue[issue.Id]), + Status: StatusFromStateType(issue.StateType), + OriginalStatus: issue.StateName, + StoryPoint: issue.Estimate, + Priority: issue.PriorityLabel, + CreatedDate: &issue.CreatedAt, + UpdatedDate: &issue.UpdatedAt, + } + if issue.CreatorId != "" { + domainIssue.CreatorId = accountIdGen.Generate(connectionId, issue.CreatorId) + domainIssue.CreatorName = accountNames[issue.CreatorId] + } + if issue.AssigneeId != "" { + domainIssue.AssigneeId = accountIdGen.Generate(connectionId, issue.AssigneeId) + domainIssue.AssigneeName = accountNames[issue.AssigneeId] + } + if issue.ParentId != "" { + domainIssue.ParentIssueId = issueIdGen.Generate(connectionId, issue.ParentId) + domainIssue.IsSubtask = true + } + // Resolution date: completedAt, falling back to canceledAt. + if issue.CompletedAt != nil { + domainIssue.ResolutionDate = issue.CompletedAt + } else if issue.CanceledAt != nil { + domainIssue.ResolutionDate = issue.CanceledAt + } + // Fallback lead time when no history-derived value is present. + // Guard against a resolution that precedes creation (clock skew or + // migrated/imported issues): a negative duration cast to uint yields + // platform-dependent garbage, so leave lead time unset instead. + if domainIssue.LeadTimeMinutes == nil && domainIssue.ResolutionDate != nil && + domainIssue.ResolutionDate.After(issue.CreatedAt) { + minutes := uint(domainIssue.ResolutionDate.Sub(issue.CreatedAt).Minutes()) + domainIssue.LeadTimeMinutes = &minutes + } + boardIssue := &ticket.BoardIssue{ + BoardId: boardId, + IssueId: domainIssue.Id, + } + results := []interface{}{domainIssue, boardIssue} + if domainIssue.AssigneeId != "" { + results = append(results, &ticket.IssueAssignee{ + IssueId: domainIssue.Id, + AssigneeId: domainIssue.AssigneeId, + AssigneeName: domainIssue.AssigneeName, + }) + } + return results, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} + +// issueTypeMatcher derives the domain ticket.Issue.Type from an issue's label +// names using the scope config's regex patterns. Precedence is +// INCIDENT > BUG > REQUIREMENT; an issue whose labels match none (or with no +// patterns configured) defaults to REQUIREMENT. +type issueTypeMatcher struct { + incident *regexp.Regexp + bug *regexp.Regexp + requirement *regexp.Regexp +} + +func newIssueTypeMatcher(sc *models.LinearScopeConfig) (*issueTypeMatcher, errors.Error) { + m := &issueTypeMatcher{} + if sc == nil { + return m, nil + } + for _, p := range []struct { + pattern string + field string + out **regexp.Regexp + }{ + {sc.IssueTypeIncident, "issueTypeIncident", &m.incident}, + {sc.IssueTypeBug, "issueTypeBug", &m.bug}, + {sc.IssueTypeRequirement, "issueTypeRequirement", &m.requirement}, + } { + if p.pattern == "" { + continue + } + re, err := regexp.Compile(p.pattern) + if err != nil { + return nil, errors.Default.Wrap(err, "invalid "+p.field+" pattern") + } + *p.out = re + } + return m, nil +} + +func (m *issueTypeMatcher) typeOf(labels []string) string { + for _, c := range []struct { + pattern *regexp.Regexp + typ string + }{ + {m.incident, ticket.INCIDENT}, + {m.bug, ticket.BUG}, + {m.requirement, ticket.REQUIREMENT}, + } { + if c.pattern == nil { + continue + } + for _, name := range labels { + if c.pattern.MatchString(name) { + return c.typ + } + } + } + return ticket.REQUIREMENT +} diff --git a/backend/plugins/linear/tasks/issue_extractor.go b/backend/plugins/linear/tasks/issue_extractor.go new file mode 100644 index 00000000000..e423076612d --- /dev/null +++ b/backend/plugins/linear/tasks/issue_extractor.go @@ -0,0 +1,108 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ExtractIssuesMeta = plugin.SubTaskMeta{ + Name: "Extract Issues", + EntryPoint: ExtractIssues, + EnabledByDefault: true, + Description: "Extract raw issue data into tool layer tables _tool_linear_issues and _tool_linear_issue_labels", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = ExtractIssues + +func ExtractIssues(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUES_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiIssue := &GraphqlQueryIssue{} + if err := errors.Convert(json.Unmarshal(row.Data, apiIssue)); err != nil { + return nil, err + } + connectionId := data.Options.ConnectionId + issue := &models.LinearIssue{ + ConnectionId: connectionId, + Id: apiIssue.Id, + TeamId: data.Options.TeamId, + Identifier: apiIssue.Identifier, + Number: apiIssue.Number, + Title: apiIssue.Title, + Description: apiIssue.Description, + Url: apiIssue.Url, + Priority: apiIssue.Priority, + PriorityLabel: PriorityLabel(apiIssue.Priority), + Estimate: apiIssue.Estimate, + CreatedAt: apiIssue.CreatedAt, + UpdatedAt: apiIssue.UpdatedAt, + StartedAt: apiIssue.StartedAt, + CompletedAt: apiIssue.CompletedAt, + CanceledAt: apiIssue.CanceledAt, + } + if apiIssue.State != nil { + issue.StateId = apiIssue.State.Id + issue.StateName = apiIssue.State.Name + issue.StateType = apiIssue.State.Type + } + if apiIssue.Assignee != nil { + issue.AssigneeId = apiIssue.Assignee.Id + } + if apiIssue.Creator != nil { + issue.CreatorId = apiIssue.Creator.Id + } + if apiIssue.Cycle != nil { + issue.CycleId = apiIssue.Cycle.Id + } + if apiIssue.Parent != nil { + issue.ParentId = apiIssue.Parent.Id + } + + results := make([]interface{}, 0, len(apiIssue.Labels.Nodes)+1) + results = append(results, issue) + for _, label := range apiIssue.Labels.Nodes { + results = append(results, &models.LinearIssueLabel{ + ConnectionId: connectionId, + IssueId: apiIssue.Id, + LabelName: label.Name, + }) + } + return results, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/linear/tasks/issue_history_collector.go b/backend/plugins/linear/tasks/issue_history_collector.go new file mode 100644 index 00000000000..11de7c218c7 --- /dev/null +++ b/backend/plugins/linear/tasks/issue_history_collector.go @@ -0,0 +1,137 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "reflect" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/merico-ai/graphql" +) + +const RAW_ISSUE_HISTORY_TABLE = "linear_issue_history" + +// GraphqlQueryHistoryWrapper is the per-issue, paginated `history` query. +type GraphqlQueryHistoryWrapper struct { + Issue struct { + History struct { + Nodes []GraphqlQueryHistory + PageInfo *helper.GraphqlQueryPageInfo + } `graphql:"history(first: $pageSize, after: $skipCursor)"` + } `graphql:"issue(id: $issueId)"` +} + +type GraphqlQueryHistory struct { + Id string + CreatedAt time.Time + Actor *struct{ Id string } + FromState *struct { + Id string + Name string + Type string + } + ToState *struct { + Id string + Name string + Type string + } +} + +var CollectIssueHistoryMeta = plugin.SubTaskMeta{ + Name: "Collect Issue History", + EntryPoint: CollectIssueHistory, + EnabledByDefault: true, + Description: "Collect history events for each collected Linear issue", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + Dependencies: []*plugin.SubTaskMeta{&ExtractIssuesMeta}, +} + +var _ plugin.SubTaskEntryPoint = CollectIssueHistory + +func CollectIssueHistory(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + + apiCollector, err := helper.NewStatefulApiCollector(helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUE_HISTORY_TABLE, + }) + if err != nil { + return err + } + + // Only sweep issues updated since the last successful collection: an + // unchanged issue's history cannot have changed, so re-fetching it every + // run wastes a request per issue against Linear's hourly budget. + since := apiCollector.GetSince() + cursor, err := db.Cursor(issuesToCollectChildrenClauses(data.Options.ConnectionId, data.Options.TeamId, since)...) + if err != nil { + return err + } + iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleLinearIssue{})) + if err != nil { + return err + } + + err = apiCollector.InitGraphQLCollector(helper.GraphqlCollectorArgs{ + GraphqlClient: data.GraphqlClient, + Input: iterator, + InputStep: 1, + PageSize: 100, + BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) { + query := &GraphqlQueryHistoryWrapper{} + if reqData == nil { + return query, map[string]interface{}{}, nil + } + issue := reqData.Input.(*SimpleLinearIssue) + variables := map[string]interface{}{ + "pageSize": graphql.Int(reqData.Pager.Size), + "skipCursor": (*graphql.String)(reqData.Pager.SkipCursor), + "issueId": graphql.String(issue.Id), + } + return query, variables, nil + }, + GetPageInfo: func(iQuery interface{}, args *helper.GraphqlCollectorArgs) (*helper.GraphqlQueryPageInfo, error) { + query := iQuery.(*GraphqlQueryHistoryWrapper) + return query.Issue.History.PageInfo, nil + }, + ResponseParser: func(queryWrapper interface{}) (messages []json.RawMessage, err errors.Error) { + query := queryWrapper.(*GraphqlQueryHistoryWrapper) + for _, event := range query.Issue.History.Nodes { + // Only state transitions are relevant to the status changelog. + if event.FromState == nil && event.ToState == nil { + continue + } + messages = append(messages, errors.Must1(json.Marshal(event))) + } + return + }, + }) + if err != nil { + return err + } + return apiCollector.Execute() +} diff --git a/backend/plugins/linear/tasks/issue_history_convertor.go b/backend/plugins/linear/tasks/issue_history_convertor.go new file mode 100644 index 00000000000..f1fa99d0519 --- /dev/null +++ b/backend/plugins/linear/tasks/issue_history_convertor.go @@ -0,0 +1,168 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ConvertIssueHistoryMeta = plugin.SubTaskMeta{ + Name: "Convert Issue History", + EntryPoint: ConvertIssueHistory, + EnabledByDefault: true, + Description: "Convert tool layer table _tool_linear_issue_history into domain layer table issue_changelogs", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.LinearIssueHistory{}.TableName(), models.LinearIssue{}.TableName(), RAW_ISSUE_HISTORY_TABLE}, + ProductTables: []string{ticket.IssueChangelogs{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertIssueHistory + +func ConvertIssueHistory(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + connectionId := data.Options.ConnectionId + + issueIdGen := didgen.NewDomainIdGenerator(&models.LinearIssue{}) + historyIdGen := didgen.NewDomainIdGenerator(&models.LinearIssueHistory{}) + accountIdGen := didgen.NewDomainIdGenerator(&models.LinearAccount{}) + + cursor, err := db.Cursor( + dal.Select("h.*"), + dal.From("_tool_linear_issue_history h"), + dal.Join("LEFT JOIN _tool_linear_issues i ON (i.connection_id = h.connection_id AND i.id = h.issue_id)"), + dal.Where("h.connection_id = ? AND i.team_id = ?", connectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: connectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUE_HISTORY_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearIssueHistory{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + event := inputRow.(*models.LinearIssueHistory) + changelog := &ticket.IssueChangelogs{ + DomainEntity: domainlayer.DomainEntity{Id: historyIdGen.Generate(connectionId, event.Id)}, + IssueId: issueIdGen.Generate(connectionId, event.IssueId), + FieldId: "state", + FieldName: "status", + OriginalFromValue: event.FromStateName, + OriginalToValue: event.ToStateName, + CreatedDate: event.CreatedAt, + } + if event.FromStateType != "" { + changelog.FromValue = StatusFromStateType(event.FromStateType) + } + if event.ToStateType != "" { + changelog.ToValue = StatusFromStateType(event.ToStateType) + } + if event.ActorId != "" { + changelog.AuthorId = accountIdGen.Generate(connectionId, event.ActorId) + } + return []interface{}{changelog}, nil + }, + }) + if err != nil { + return err + } + if err := converter.Execute(); err != nil { + return err + } + + return deriveLeadTimeFromHistory(db, connectionId, data.Options.TeamId, issueIdGen) +} + +// deriveLeadTimeFromHistory refines each issue's lead time from its recorded +// state transitions: the span from the issue's first transition into an +// in-progress state to its first transition into a done state thereafter (the +// active cycle time). This is the value that genuinely requires history and is +// more accurate than the coarse createdAt -> resolutionDate fallback set by +// ConvertIssues, so it overrides that fallback when the transitions exist. +// Issues whose history lacks an in-progress -> done sequence keep the fallback. +func deriveLeadTimeFromHistory(db dal.Dal, connectionId uint64, teamId string, issueIdGen *didgen.DomainIdGenerator) errors.Error { + var events []models.LinearIssueHistory + if err := db.All(&events, + dal.Select("h.*"), + dal.From("_tool_linear_issue_history h"), + dal.Join("LEFT JOIN _tool_linear_issues i ON (i.connection_id = h.connection_id AND i.id = h.issue_id)"), + dal.Where("h.connection_id = ? AND i.team_id = ?", connectionId, teamId), + dal.Orderby("h.issue_id, h.created_at"), + ); err != nil { + return err + } + + type leadWindow struct { + startedAt *time.Time + doneAt *time.Time + } + windows := map[string]*leadWindow{} + for i := range events { + event := events[i] + window := windows[event.IssueId] + if window == nil { + window = &leadWindow{} + windows[event.IssueId] = window + } + switch StatusFromStateType(event.ToStateType) { + case ticket.IN_PROGRESS: + if window.startedAt == nil { + createdAt := event.CreatedAt + window.startedAt = &createdAt + } + case ticket.DONE: + if window.startedAt != nil && window.doneAt == nil { + createdAt := event.CreatedAt + window.doneAt = &createdAt + } + } + } + + for issueId, window := range windows { + if window.startedAt == nil || window.doneAt == nil || !window.doneAt.After(*window.startedAt) { + continue + } + minutes := uint(window.doneAt.Sub(*window.startedAt).Minutes()) + if err := db.UpdateColumn( + &ticket.Issue{}, "lead_time_minutes", minutes, + dal.Where("id = ?", issueIdGen.Generate(connectionId, issueId)), + ); err != nil { + return err + } + } + return nil +} diff --git a/backend/plugins/linear/tasks/issue_history_extractor.go b/backend/plugins/linear/tasks/issue_history_extractor.go new file mode 100644 index 00000000000..006b92f9701 --- /dev/null +++ b/backend/plugins/linear/tasks/issue_history_extractor.go @@ -0,0 +1,85 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ExtractIssueHistoryMeta = plugin.SubTaskMeta{ + Name: "Extract Issue History", + EntryPoint: ExtractIssueHistory, + EnabledByDefault: true, + Description: "Extract raw issue history into tool layer table _tool_linear_issue_history", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = ExtractIssueHistory + +func ExtractIssueHistory(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUE_HISTORY_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiEvent := &GraphqlQueryHistory{} + if err := errors.Convert(json.Unmarshal(row.Data, apiEvent)); err != nil { + return nil, err + } + issueRef := &SimpleLinearIssue{} + if err := errors.Convert(json.Unmarshal(row.Input, issueRef)); err != nil { + return nil, err + } + event := &models.LinearIssueHistory{ + ConnectionId: data.Options.ConnectionId, + Id: apiEvent.Id, + IssueId: issueRef.OwningIssueId(), + CreatedAt: apiEvent.CreatedAt, + } + if apiEvent.Actor != nil { + event.ActorId = apiEvent.Actor.Id + } + if apiEvent.FromState != nil { + event.FromStateId = apiEvent.FromState.Id + event.FromStateName = apiEvent.FromState.Name + event.FromStateType = apiEvent.FromState.Type + } + if apiEvent.ToState != nil { + event.ToStateId = apiEvent.ToState.Id + event.ToStateName = apiEvent.ToState.Name + event.ToStateType = apiEvent.ToState.Type + } + return []interface{}{event}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/linear/tasks/label_convertor.go b/backend/plugins/linear/tasks/label_convertor.go new file mode 100644 index 00000000000..e94826d4775 --- /dev/null +++ b/backend/plugins/linear/tasks/label_convertor.go @@ -0,0 +1,85 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ConvertIssueLabelsMeta = plugin.SubTaskMeta{ + Name: "Convert Issue Labels", + EntryPoint: ConvertIssueLabels, + EnabledByDefault: true, + Description: "Convert tool layer table _tool_linear_issue_labels into domain layer table issue_labels", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.LinearIssueLabel{}.TableName(), models.LinearIssue{}.TableName(), RAW_ISSUES_TABLE}, + ProductTables: []string{ticket.IssueLabel{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertIssueLabels + +func ConvertIssueLabels(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + connectionId := data.Options.ConnectionId + issueIdGen := didgen.NewDomainIdGenerator(&models.LinearIssue{}) + + cursor, err := db.Cursor( + dal.Select("l.*"), + dal.From("_tool_linear_issue_labels l"), + dal.Join("LEFT JOIN _tool_linear_issues i ON (i.connection_id = l.connection_id AND i.id = l.issue_id)"), + dal.Where("l.connection_id = ? AND i.team_id = ?", connectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: connectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUES_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearIssueLabel{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + label := inputRow.(*models.LinearIssueLabel) + domainLabel := &ticket.IssueLabel{ + IssueId: issueIdGen.Generate(connectionId, label.IssueId), + LabelName: label.LabelName, + } + return []interface{}{domainLabel}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/linear/tasks/shared.go b/backend/plugins/linear/tasks/shared.go new file mode 100644 index 00000000000..0a8cfe9da7f --- /dev/null +++ b/backend/plugins/linear/tasks/shared.go @@ -0,0 +1,86 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "time" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +// issuesToCollectChildrenClauses builds the cursor clauses that drive per-issue +// child collection (comments, history). When `since` is non-nil (an incremental +// run), it restricts the sweep to issues updated since the last successful +// collection, so unchanged issues no longer trigger a request every run. On a +// full sync `since` is nil and all of the team's issues are swept. +func issuesToCollectChildrenClauses(connectionId uint64, teamId string, since *time.Time) []dal.Clause { + clauses := []dal.Clause{ + dal.Select("id"), + dal.From(&models.LinearIssue{}), + dal.Where("connection_id = ? AND team_id = ?", connectionId, teamId), + } + if since != nil { + clauses = append(clauses, dal.Where("updated_at > ?", *since)) + } + return clauses +} + +// priorityLabels maps Linear's integer priority to its human-readable label. +// Linear: 0 = No priority, 1 = Urgent, 2 = High, 3 = Medium, 4 = Low. +var priorityLabels = map[int]string{ + 0: "No priority", + 1: "Urgent", + 2: "High", + 3: "Medium", + 4: "Low", +} + +// PriorityLabel returns the human-readable label for a Linear priority value. +func PriorityLabel(priority int) string { + if label, ok := priorityLabels[priority]; ok { + return label + } + return "No priority" +} + +// StatusFromStateType maps a Linear WorkflowState.type to a DevLake standard +// issue status. Linear's state types are standardized, so no user-supplied +// mapping is required: +// +// triage, backlog, unstarted -> TODO +// started -> IN_PROGRESS +// completed, canceled -> DONE +// +// "triage" is the inbox state issues land in before they are accepted into a +// workflow; it is treated as not-yet-started (TODO). Any unrecognized type +// falls back to OTHER so unexpected API values surface rather than silently +// masquerading as a known status. +func StatusFromStateType(stateType string) string { + switch stateType { + case "triage", "backlog", "unstarted": + return ticket.TODO + case "started": + return ticket.IN_PROGRESS + case "completed", "canceled": + return ticket.DONE + default: + return ticket.OTHER + } +} diff --git a/backend/plugins/linear/tasks/shared_clauses_test.go b/backend/plugins/linear/tasks/shared_clauses_test.go new file mode 100644 index 00000000000..ad2fc7bc6d0 --- /dev/null +++ b/backend/plugins/linear/tasks/shared_clauses_test.go @@ -0,0 +1,40 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +// TestIssuesToCollectChildrenClauses pins the incremental behaviour of the +// per-issue child collectors (comments, history): a full sync sweeps every +// issue, while an incremental run adds an updated_at filter so unchanged issues +// are skipped instead of triggering a request each run. +func TestIssuesToCollectChildrenClauses(t *testing.T) { + // full sync: no `since` -> select/from/where(connection,team) only + full := issuesToCollectChildrenClauses(1, "team-1", nil) + assert.Len(t, full, 3) + + // incremental: a `since` adds the updated_at filter clause + since := time.Date(2026, 5, 1, 0, 0, 0, 0, time.UTC) + incremental := issuesToCollectChildrenClauses(1, "team-1", &since) + assert.Len(t, incremental, 4) +} diff --git a/backend/plugins/linear/tasks/shared_test.go b/backend/plugins/linear/tasks/shared_test.go new file mode 100644 index 00000000000..eaf737c9022 --- /dev/null +++ b/backend/plugins/linear/tasks/shared_test.go @@ -0,0 +1,45 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/stretchr/testify/assert" +) + +// TestStatusFromStateType pins the mapping for every Linear WorkflowState.type +// value. Linear's state types are standardized; "triage" is the inbox state +// issues land in before they are accepted, so it maps to TODO. Any genuinely +// unknown type falls back to OTHER. +func TestStatusFromStateType(t *testing.T) { + cases := map[string]string{ + "backlog": ticket.TODO, + "unstarted": ticket.TODO, + "triage": ticket.TODO, + "started": ticket.IN_PROGRESS, + "completed": ticket.DONE, + "canceled": ticket.DONE, + "": ticket.OTHER, + "something": ticket.OTHER, + } + for stateType, want := range cases { + assert.Equal(t, want, StatusFromStateType(stateType), "state type %q", stateType) + } +} diff --git a/backend/plugins/linear/tasks/sprint_issue_convertor.go b/backend/plugins/linear/tasks/sprint_issue_convertor.go new file mode 100644 index 00000000000..7ebe467a908 --- /dev/null +++ b/backend/plugins/linear/tasks/sprint_issue_convertor.go @@ -0,0 +1,107 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ConvertSprintIssuesMeta = plugin.SubTaskMeta{ + Name: "Convert Sprint Issues", + EntryPoint: ConvertSprintIssues, + EnabledByDefault: true, + Description: "Link issues to their cycle (sprint) in the domain layer table sprint_issues", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, + DependencyTables: []string{models.LinearIssue{}.TableName(), RAW_ISSUES_TABLE}, + ProductTables: []string{ticket.SprintIssue{}.TableName()}, +} + +var _ plugin.SubTaskEntryPoint = ConvertSprintIssues + +func ConvertSprintIssues(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*LinearTaskData) + connectionId := data.Options.ConnectionId + + issueIdGen := didgen.NewDomainIdGenerator(&models.LinearIssue{}) + cycleIdGen := didgen.NewDomainIdGenerator(&models.LinearCycle{}) + + // Sprint membership is derived from each issue's cycle_id. Clear this team's + // existing sprint_issues up front so issues that have since left their cycle + // leave no stale rows: the batch divider only deletes outdated records when + // it produces at least one row of the type, which misses the case where + // every issue has been removed from its cycle. + var teamIssues []models.LinearIssue + if err := db.All(&teamIssues, + dal.Select("id"), + dal.Where("connection_id = ? AND team_id = ?", connectionId, data.Options.TeamId), + ); err != nil { + return err + } + if len(teamIssues) > 0 { + issueIds := make([]string, len(teamIssues)) + for i, issue := range teamIssues { + issueIds[i] = issueIdGen.Generate(connectionId, issue.Id) + } + if err := db.Delete(&ticket.SprintIssue{}, dal.Where("issue_id IN ?", issueIds)); err != nil { + return err + } + } + + cursor, err := db.Cursor( + dal.From(&models.LinearIssue{}), + dal.Where("connection_id = ? AND team_id = ? AND cycle_id != ''", connectionId, data.Options.TeamId), + ) + if err != nil { + return err + } + defer cursor.Close() + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: connectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_ISSUES_TABLE, + }, + InputRowType: reflect.TypeOf(models.LinearIssue{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + issue := inputRow.(*models.LinearIssue) + sprintIssue := &ticket.SprintIssue{ + SprintId: cycleIdGen.Generate(connectionId, issue.CycleId), + IssueId: issueIdGen.Generate(connectionId, issue.Id), + } + return []interface{}{sprintIssue}, nil + }, + }) + if err != nil { + return err + } + return converter.Execute() +} diff --git a/backend/plugins/linear/tasks/task_data.go b/backend/plugins/linear/tasks/task_data.go new file mode 100644 index 00000000000..19f991d5f75 --- /dev/null +++ b/backend/plugins/linear/tasks/task_data.go @@ -0,0 +1,46 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "time" + + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +// LinearOptions are the per-scope options passed to a pipeline task. +type LinearOptions struct { + ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId,omitempty"` + TeamId string `json:"teamId" mapstructure:"teamId,omitempty"` + ScopeConfigId uint64 `json:"scopeConfigId" mapstructure:"scopeConfigId,omitempty"` + // TimeAfter limits collection to data created/updated after this time. + TimeAfter string `json:"timeAfter" mapstructure:"timeAfter,omitempty"` +} + +// LinearTaskData is the shared context handed to every Linear subtask. +type LinearTaskData struct { + Options *LinearOptions + GraphqlClient *api.GraphqlAsyncClient + TimeAfter *time.Time + // ScopeConfig carries the resolved scope config (e.g. label-based issue-type + // mapping). Never nil: PrepareTaskData defaults it to an empty config. + ScopeConfig *models.LinearScopeConfig +} + +type LinearApiParams models.LinearApiParams diff --git a/backend/plugins/linear/tasks/workflow_state_collector.go b/backend/plugins/linear/tasks/workflow_state_collector.go new file mode 100644 index 00000000000..94c22d1d3a5 --- /dev/null +++ b/backend/plugins/linear/tasks/workflow_state_collector.go @@ -0,0 +1,100 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/merico-ai/graphql" +) + +const RAW_WORKFLOW_STATES_TABLE = "linear_workflow_states" + +// GraphqlQueryWorkflowStateWrapper is the team-scoped paginated `states` query. +type GraphqlQueryWorkflowStateWrapper struct { + Team struct { + States struct { + Nodes []GraphqlQueryWorkflowState + PageInfo *helper.GraphqlQueryPageInfo + } `graphql:"states(first: $pageSize, after: $skipCursor)"` + } `graphql:"team(id: $teamId)"` +} + +type GraphqlQueryWorkflowState struct { + Id string + Name string + Type string + Color string + Position float64 +} + +var CollectWorkflowStatesMeta = plugin.SubTaskMeta{ + Name: "Collect Workflow States", + EntryPoint: CollectWorkflowStates, + EnabledByDefault: true, + Description: "Collect workflow states for a Linear team", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = CollectWorkflowStates + +func CollectWorkflowStates(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + collector, err := helper.NewGraphqlCollector(helper.GraphqlCollectorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_WORKFLOW_STATES_TABLE, + }, + GraphqlClient: data.GraphqlClient, + PageSize: 100, + BuildQuery: func(reqData *helper.GraphqlRequestData) (interface{}, map[string]interface{}, error) { + query := &GraphqlQueryWorkflowStateWrapper{} + if reqData == nil { + return query, map[string]interface{}{}, nil + } + variables := map[string]interface{}{ + "pageSize": graphql.Int(reqData.Pager.Size), + "skipCursor": (*graphql.String)(reqData.Pager.SkipCursor), + "teamId": graphql.String(data.Options.TeamId), + } + return query, variables, nil + }, + GetPageInfo: func(iQuery interface{}, args *helper.GraphqlCollectorArgs) (*helper.GraphqlQueryPageInfo, error) { + query := iQuery.(*GraphqlQueryWorkflowStateWrapper) + return query.Team.States.PageInfo, nil + }, + ResponseParser: func(queryWrapper interface{}) (messages []json.RawMessage, err errors.Error) { + query := queryWrapper.(*GraphqlQueryWorkflowStateWrapper) + for _, state := range query.Team.States.Nodes { + messages = append(messages, errors.Must1(json.Marshal(state))) + } + return + }, + }) + if err != nil { + return err + } + return collector.Execute() +} diff --git a/backend/plugins/linear/tasks/workflow_state_extractor.go b/backend/plugins/linear/tasks/workflow_state_extractor.go new file mode 100644 index 00000000000..75a2099aeaf --- /dev/null +++ b/backend/plugins/linear/tasks/workflow_state_extractor.go @@ -0,0 +1,71 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/linear/models" +) + +var ExtractWorkflowStatesMeta = plugin.SubTaskMeta{ + Name: "Extract Workflow States", + EntryPoint: ExtractWorkflowStates, + EnabledByDefault: true, + Description: "Extract raw workflow state data into tool layer table _tool_linear_workflow_states", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +var _ plugin.SubTaskEntryPoint = ExtractWorkflowStates + +func ExtractWorkflowStates(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*LinearTaskData) + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: LinearApiParams{ + ConnectionId: data.Options.ConnectionId, + TeamId: data.Options.TeamId, + }, + Table: RAW_WORKFLOW_STATES_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, errors.Error) { + apiState := &GraphqlQueryWorkflowState{} + if err := errors.Convert(json.Unmarshal(row.Data, apiState)); err != nil { + return nil, err + } + state := &models.LinearWorkflowState{ + ConnectionId: data.Options.ConnectionId, + Id: apiState.Id, + TeamId: data.Options.TeamId, + Name: apiState.Name, + Type: apiState.Type, + Color: apiState.Color, + Position: apiState.Position, + } + return []interface{}{state}, nil + }, + }) + if err != nil { + return err + } + return extractor.Execute() +} diff --git a/backend/plugins/table_info_test.go b/backend/plugins/table_info_test.go index 686b99816b0..0d4482ca6c9 100644 --- a/backend/plugins/table_info_test.go +++ b/backend/plugins/table_info_test.go @@ -28,8 +28,8 @@ import ( bamboo "github.com/apache/incubator-devlake/plugins/bamboo/impl" bitbucket "github.com/apache/incubator-devlake/plugins/bitbucket/impl" bitbucket_server "github.com/apache/incubator-devlake/plugins/bitbucket_server/impl" - claudeCode "github.com/apache/incubator-devlake/plugins/claude_code/impl" circleci "github.com/apache/incubator-devlake/plugins/circleci/impl" + claudeCode "github.com/apache/incubator-devlake/plugins/claude_code/impl" customize "github.com/apache/incubator-devlake/plugins/customize/impl" dbt "github.com/apache/incubator-devlake/plugins/dbt/impl" dora "github.com/apache/incubator-devlake/plugins/dora/impl" @@ -44,6 +44,7 @@ import ( issueTrace "github.com/apache/incubator-devlake/plugins/issue_trace/impl" jenkins "github.com/apache/incubator-devlake/plugins/jenkins/impl" jira "github.com/apache/incubator-devlake/plugins/jira/impl" + linear "github.com/apache/incubator-devlake/plugins/linear/impl" linker "github.com/apache/incubator-devlake/plugins/linker/impl" opsgenie "github.com/apache/incubator-devlake/plugins/opsgenie/impl" org "github.com/apache/incubator-devlake/plugins/org/impl" @@ -88,6 +89,7 @@ func Test_GetPluginTablesInfo(t *testing.T) { checker.FeedIn("icla/models", icla.Icla{}.GetTablesInfo) checker.FeedIn("jenkins/models", jenkins.Jenkins{}.GetTablesInfo) checker.FeedIn("jira/models", jira.Jira{}.GetTablesInfo) + checker.FeedIn("linear/models", linear.Linear{}.GetTablesInfo) checker.FeedIn("org", org.Org{}.GetTablesInfo) checker.FeedIn("pagerduty/models", pagerduty.PagerDuty{}.GetTablesInfo) checker.FeedIn("refdiff/models", refdiff.RefDiff{}.GetTablesInfo) diff --git a/backend/plugins/webhook/api/issues.go b/backend/plugins/webhook/api/issues.go index 49ea72b51f1..9487cf6877a 100644 --- a/backend/plugins/webhook/api/issues.go +++ b/backend/plugins/webhook/api/issues.go @@ -19,11 +19,12 @@ package api import ( "fmt" - "github.com/apache/incubator-devlake/core/log" - "github.com/apache/incubator-devlake/helpers/dbhelper" "net/http" "time" + "github.com/apache/incubator-devlake/core/log" + "github.com/apache/incubator-devlake/helpers/dbhelper" + "github.com/apache/incubator-devlake/core/dal" "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/models/domainlayer" @@ -234,6 +235,42 @@ func CloseIssue(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, erro return closeIssue(input, err, connection) } +// CloseIssueByBodyRequest is the body for the body-based close endpoint +type CloseIssueByBodyRequest struct { + IssueKey string `mapstructure:"issueKey" validate:"required,max=255"` + ResolutionDate *time.Time `mapstructure:"resolutionDate"` + OriginalStatus string `mapstructure:"originalStatus"` +} + +// CloseIssueByBody +// @Summary close an issue (body-based) +// @Description Close an incident by passing issueKey in the request body. +// @Description Use this when the client (e.g. Kibana) cannot construct a dynamic URL. +// @Tags plugins/webhook +// @Param connectionId path int true "connection ID" +// @Param body body CloseIssueByBodyRequest true "close request" +// @Success 200 {string} noResponse "" +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/webhook/connections/{connectionId}/issue/close [POST] +func CloseIssueByBody(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.WebhookConnection{} + err := connectionHelper.First(connection, input.Params) + if err != nil { + return nil, err + } + request := &CloseIssueByBodyRequest{} + if err2 := helper.DecodeMapStruct(input.Body, request, true); err2 != nil { + return &plugin.ApiResourceOutput{Body: err2.Error(), Status: http.StatusBadRequest}, nil + } + vld = validator.New() + if err2 := errors.Convert(vld.Struct(request)); err2 != nil { + return &plugin.ApiResourceOutput{Body: err2.Error(), Status: http.StatusBadRequest}, nil + } + input.Params["issueKey"] = request.IssueKey + return closeIssue(input, err, connection) +} + // CloseIssueByName // @Summary set issue's status to DONE // @Description set issue's status to DONE @@ -248,6 +285,34 @@ func CloseIssueByName(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput return closeIssue(input, err, connection) } +// CloseIssueByBodyByName +// @Summary close an issue by connection name (body-based) +// @Description Close an incident using connection name + issueKey in request body. +// @Tags plugins/webhook +// @Param connectionName path string true "connection name" +// @Param body body CloseIssueByBodyRequest true "close request" +// @Success 200 {string} noResponse "" +// @Failure 400 {string} errcode.Error "Bad Request" +// @Failure 500 {string} errcode.Error "Internal Error" +// @Router /plugins/webhook/connections/by-name/{connectionName}/issue/close [POST] +func CloseIssueByBodyByName(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + connection := &models.WebhookConnection{} + err := connectionHelper.FirstByName(connection, input.Params) + if err != nil { + return nil, err + } + request := &CloseIssueByBodyRequest{} + if err2 := helper.DecodeMapStruct(input.Body, request, true); err2 != nil { + return &plugin.ApiResourceOutput{Body: err2.Error(), Status: http.StatusBadRequest}, nil + } + vld = validator.New() + if err2 := errors.Convert(vld.Struct(request)); err2 != nil { + return &plugin.ApiResourceOutput{Body: err2.Error(), Status: http.StatusBadRequest}, nil + } + input.Params["issueKey"] = request.IssueKey + return closeIssue(input, err, connection) +} + func closeIssue(input *plugin.ApiResourceInput, err errors.Error, connection *models.WebhookConnection) (*plugin.ApiResourceOutput, errors.Error) { if err != nil { return nil, err @@ -265,6 +330,10 @@ func closeIssue(input *plugin.ApiResourceInput, err errors.Error, connection *mo } domainIssue.Status = ticket.DONE domainIssue.OriginalStatus = `` + now := time.Now() + if domainIssue.ResolutionDate == nil { + domainIssue.ResolutionDate = &now + } // save err = tx.Update(domainIssue) if err != nil { @@ -278,6 +347,9 @@ func closeIssue(input *plugin.ApiResourceInput, err errors.Error, connection *mo if err == nil { domainIncident.Status = ticket.DONE domainIncident.OriginalStatus = `` + if domainIncident.ResolutionDate == nil { + domainIncident.ResolutionDate = &now + } // save err = tx.Update(domainIncident) if err != nil { diff --git a/backend/plugins/webhook/impl/impl.go b/backend/plugins/webhook/impl/impl.go index 9a67683584e..daf07b53526 100644 --- a/backend/plugins/webhook/impl/impl.go +++ b/backend/plugins/webhook/impl/impl.go @@ -99,6 +99,9 @@ func (p Webhook) ApiResources() map[string]map[string]plugin.ApiResourceHandler "connections/:connectionId/issue/:issueKey/close": { "POST": api.CloseIssue, }, + "connections/:connectionId/issue/close": { + "POST": api.CloseIssueByBody, + }, ":connectionId/deployments": { "POST": api.PostDeployments, }, @@ -111,6 +114,9 @@ func (p Webhook) ApiResources() map[string]map[string]plugin.ApiResourceHandler ":connectionId/issue/:issueKey/close": { "POST": api.CloseIssue, }, + ":connectionId/issue/close": { + "POST": api.CloseIssueByBody, + }, "connections/by-name/:connectionName": { "GET": api.GetConnectionByName, "PATCH": api.PatchConnectionByName, @@ -128,6 +134,9 @@ func (p Webhook) ApiResources() map[string]map[string]plugin.ApiResourceHandler "connections/by-name/:connectionName/issue/:issueKey/close": { "POST": api.CloseIssueByName, }, + "connections/by-name/:connectionName/issue/close": { + "POST": api.CloseIssueByBodyByName, + }, "projects/:projectName/deployments": { "POST": api.PostDeploymentsByProjectName, }, diff --git a/config-ui/src/plugins/components/connection-form/index.tsx b/config-ui/src/plugins/components/connection-form/index.tsx index 26432cee70f..b61c1d5d076 100644 --- a/config-ui/src/plugins/components/connection-form/index.tsx +++ b/config-ui/src/plugins/components/connection-form/index.tsx @@ -110,7 +110,7 @@ export const ConnectionForm = ({ plugin, connectionId, onSuccess }: Props) => { const { name, connection: { docLink, fields, initialValues }, - } = getPluginConfig(plugin); + } = getPluginConfig(plugin) ?? {}; const disabled = useMemo(() => { return Object.values(errors).some(Boolean); @@ -118,6 +118,10 @@ export const ConnectionForm = ({ plugin, connectionId, onSuccess }: Props) => { const sanitizedCustomHeaders = useMemo(() => sanitizeCustomHeaders(values.customHeaders), [values.customHeaders]); + if (!plugin || !name) { + return null; + } + const handleTest = async () => { const isUpdate = type === 'update' && !!connectionId; await operator( diff --git a/config-ui/src/plugins/register/index.ts b/config-ui/src/plugins/register/index.ts index b8bd6672f41..d1fff9c79e2 100644 --- a/config-ui/src/plugins/register/index.ts +++ b/config-ui/src/plugins/register/index.ts @@ -31,6 +31,7 @@ import { GhCopilotConfig } from './gh-copilot'; import { GitLabConfig } from './gitlab'; import { JenkinsConfig } from './jenkins'; import { JiraConfig } from './jira'; +import { LinearConfig } from './linear'; import { PagerDutyConfig } from './pagerduty'; import { RootlyConfig } from './rootly'; import { SonarQubeConfig } from './sonarqube'; @@ -59,6 +60,7 @@ export const pluginConfigs: IPluginConfig[] = [ GitLabConfig, JenkinsConfig, JiraConfig, + LinearConfig, PagerDutyConfig, RootlyConfig, SlackConfig, diff --git a/config-ui/src/plugins/register/jira/config.tsx b/config-ui/src/plugins/register/jira/config.tsx index c6544d6e9e7..d82186577d5 100644 --- a/config-ui/src/plugins/register/jira/config.tsx +++ b/config-ui/src/plugins/register/jira/config.tsx @@ -49,7 +49,7 @@ export const JiraConfig: IPluginConfig = { 'By default, DevLake uses dynamic rate limit for optimized data collection for Jira. But you can adjust the collection speed by setting up your desirable rate limit.', learnMore: DOC_URL.PLUGIN.JIRA.RATE_LIMIT, externalInfo: - 'Jira Cloud does not specify a maximum value of rate limit. For Jira Server, please contact your admin for more information.', + 'Jira Cloud does not specify a maximum value of rate limit. For Jira Server / Jira Data Center, please contact your admin for more information.', defaultValue: 10000, }, ], diff --git a/config-ui/src/plugins/register/jira/connection-fields/auth.tsx b/config-ui/src/plugins/register/jira/connection-fields/auth.tsx index a85999ba3b0..bc43a6acec3 100644 --- a/config-ui/src/plugins/register/jira/connection-fields/auth.tsx +++ b/config-ui/src/plugins/register/jira/connection-fields/auth.tsx @@ -124,7 +124,7 @@ export const Auth = ({ type, initialValues, values, setValues, setErrors }: Prop Jira Cloud - Jira Server + Jira Server / Jira Data Center } diff --git a/config-ui/src/plugins/register/linear/assets/icon.svg b/config-ui/src/plugins/register/linear/assets/icon.svg new file mode 100644 index 00000000000..64cddfdea54 --- /dev/null +++ b/config-ui/src/plugins/register/linear/assets/icon.svg @@ -0,0 +1,19 @@ + + + + diff --git a/config-ui/src/plugins/register/linear/config.tsx b/config-ui/src/plugins/register/linear/config.tsx new file mode 100644 index 00000000000..28cf9f98a95 --- /dev/null +++ b/config-ui/src/plugins/register/linear/config.tsx @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +import { IPluginConfig } from '@/types'; + +import Icon from './assets/icon.svg?react'; + +export const LinearConfig: IPluginConfig = { + plugin: 'linear', + name: 'Linear', + icon: ({ color }) => , + sort: 13, + connection: { + docLink: 'https://developers.linear.app/docs', + initialValues: { + endpoint: 'https://api.linear.app/graphql', + }, + fields: [ + 'name', + { + key: 'endpoint', + label: 'Endpoint', + subLabel: 'Linear GraphQL API base URL.', + }, + { + key: 'token', + label: 'API Key', + subLabel: 'Your Linear personal API key (Settings → Security & access → Personal API keys).', + }, + 'proxy', + { + key: 'rateLimitPerHour', + subLabel: 'Maximum number of API requests per hour. Leave blank for the default (1500).', + defaultValue: 1500, + }, + ], + }, + dataScope: { + title: 'Teams', + searchPlaceholder: 'Search teams...', + }, +}; diff --git a/config-ui/src/plugins/register/linear/index.ts b/config-ui/src/plugins/register/linear/index.ts new file mode 100644 index 00000000000..de415db39ab --- /dev/null +++ b/config-ui/src/plugins/register/linear/index.ts @@ -0,0 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +export * from './config'; diff --git a/config-ui/src/plugins/utils.ts b/config-ui/src/plugins/utils.ts index e820f72cdd4..34c6ed63b14 100644 --- a/config-ui/src/plugins/utils.ts +++ b/config-ui/src/plugins/utils.ts @@ -45,6 +45,8 @@ export const getPluginScopeId = (plugin: string, scope: any) => { return `${scope.name}`; case 'asana': return `${scope.gid}`; + case 'linear': + return `${scope.teamId}`; default: return `${scope.id}`; } diff --git a/config-ui/src/routes/connection/connections.tsx b/config-ui/src/routes/connection/connections.tsx index 62427c45ccd..87b358b9ee9 100644 --- a/config-ui/src/routes/connection/connections.tsx +++ b/config-ui/src/routes/connection/connections.tsx @@ -61,8 +61,9 @@ export const Connections = () => { setPlugin(plugin); }; - const handleShowFormDialog = () => { + const handleShowFormDialog = (pluginName?: string) => { setType('form'); + if (pluginName) setPlugin(pluginName); }; const handleHideDialog = () => { @@ -168,7 +169,7 @@ export const Connections = () => { )} - {type === 'form' && pluginConfig && ( + {type === 'form' && plugin && pluginConfig && ( \n\nThis dashboard is created based on this [data schema](https://devlake.apache.org/docs/DataModels/DevLakeDomainLayerSchema). Want to add more metrics? Please follow the [guide](https://devlake.apache.org/docs/Configuration/Dashboards/GrafanaUserGuide).", + "mode": "markdown" + }, + "pluginVersion": "9.5.15", + "targets": [ + { + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "queryType": "randomWalk", + "refId": "A" + } + ], + "type": "text" + } + ], + "refresh": "", + "schemaVersion": 38, + "style": "dark", + "tags": [ + "Data Source Dashboard" + ], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": "mysql", + "definition": "select concat(name, '--', id) from boards where id like 'linear%'", + "hide": 0, + "includeAll": true, + "label": "Choose Board", + "multi": true, + "name": "board_id", + "options": [], + "query": "select concat(name, '--', id) from boards where id like 'linear%'", + "refresh": 1, + "regex": "/^(?.*)--(?.*)$/", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": "mysql", + "definition": "select distinct type from issues", + "hide": 0, + "includeAll": true, + "label": "Issue Type", + "multi": false, + "name": "type", + "options": [], + "query": "select distinct type from issues", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6M", + "to": "now" + }, + "timepicker": {}, + "timezone": "utc", + "title": "Linear", + "uid": "linear-dashboard", + "version": 1, + "weekStart": "" +}