From 693f85ba26e2dc090d28d8dc116d6f6aed4f17a2 Mon Sep 17 00:00:00 2001 From: Gabriel Gordon-Hall Date: Wed, 23 Jul 2025 12:05:41 +0100 Subject: [PATCH] feat: environment toggle (#325) * - add git clone logic - add logic to list Github repos - toggle between local and cloud envs * ci --- CLAUDE.md | 117 ++++++++ backend/Cargo.toml | 1 + backend/src/app_state.rs | 40 ++- backend/src/bin/generate_types.rs | 5 +- backend/src/main.rs | 32 ++- backend/src/models/config.rs | 43 ++- backend/src/models/mod.rs | 2 +- backend/src/models/project.rs | 11 + backend/src/routes/config.rs | 8 +- backend/src/routes/github.rs | 207 ++++++++++++++ backend/src/routes/mod.rs | 1 + backend/src/services/git_service.rs | 52 ++++ backend/src/services/github_service.rs | 173 ++++++++---- .../projects/github-repository-picker.tsx | 256 ++++++++++++++++++ .../src/components/projects/project-form.tsx | 223 +++++++++++---- frontend/src/lib/api.ts | 38 +++ shared/types.ts | 10 +- 17 files changed, 1092 insertions(+), 127 deletions(-) create mode 100644 CLAUDE.md create mode 100644 backend/src/routes/github.rs create mode 100644 frontend/src/components/projects/github-repository-picker.tsx diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..9b1273ae --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,117 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Development Commands + +### Core Development +- `pnpm run dev` - Start both frontend (port 3000) and backend (port 3001) with live reload +- `pnpm run check` - Run cargo check and TypeScript type checking - **always run this before committing** +- `pnpm run generate-types` - Generate TypeScript types from Rust structs (run after modifying Rust types) + +### Testing and Validation +- `pnpm run frontend:dev` - Start frontend development server only +- `pnpm run backend:dev` - Start Rust backend only +- `cargo test` - Run Rust unit tests from backend directory +- `cargo fmt` - Format Rust code +- `cargo clippy` - Run Rust linter + +### Building +- `./build-npm-package.sh` - Build production package for distribution +- `cargo build --release` - Build optimized Rust binary + +## Architecture Overview + +### Tech Stack +- **Backend**: Rust with Axum web framework, SQLite + SQLX, Tokio async runtime +- **Frontend**: React 18 + TypeScript, Vite, Tailwind CSS, Radix UI +- **Package Management**: pnpm workspace monorepo +- **Type Sharing**: Rust types exported to TypeScript via `ts-rs` + +### Core Concepts + +**Vibe Kanban** is an AI coding agent orchestration platform that manages multiple coding agents (Claude Code, Gemini CLI, Amp, etc.) through a unified interface. + +**Project Structure**: +- `/backend/src/` - Rust backend with API endpoints, database, and agent executors +- `/frontend/src/` - React frontend with task management UI +- `/backend/migrations/` - SQLite database schema migrations +- `/shared-types/` - Generated TypeScript types from Rust structs + +**Executor System**: Each AI agent is implemented as an executor in `/backend/src/executors/`: +- `claude.rs` - Claude Code integration +- `gemini.rs` - Google Gemini CLI +- `amp.rs` - Amp coding agent +- `dev_server.rs` - Development server management +- `echo.rs` - Test/debug executor + +**Key Backend Modules**: +- `/backend/src/api/` - REST API endpoints +- `/backend/src/db/` - Database models and queries +- `/backend/src/github/` - GitHub OAuth and API integration +- `/backend/src/git/` - Git operations and worktree management +- `/backend/src/mcp/` - Model Context Protocol server implementation + +### Database Schema +SQLite database with core entities: +- `projects` - Coding projects with GitHub repo integration +- `tasks` - Individual tasks assigned to executors +- `processes` - Execution processes with streaming logs +- `github_users`, `github_repos` - GitHub integration data + +### API Architecture +- RESTful endpoints at `/api/` prefix +- WebSocket streaming for real-time task updates at `/api/stream/:process_id` +- GitHub OAuth flow with PKCE +- MCP server exposed for external tool integration + +## Development Guidelines + +### Type management +- First ensure that `src/bin/generate_types.rs` is up to date with the types in the project +- **Always regenerate types after modifying Rust structs**: Run `pnpm run generate-types` +- Backend-first development: Define data structures in Rust, export to frontend +- Use `#[derive(Serialize, Deserialize, PartialEq, Debug, Clone, TS)]` for shared types + +### Code Style +- **Rust**: Use rustfmt, follow snake_case naming, leverage tokio for async operations +- **TypeScript**: Strict mode enabled, use `@/` path aliases for imports +- **React**: Functional components with hooks, avoid class components + +### Git Integration Features +- Automatic branch creation per task +- Git worktree management for concurrent development +- GitHub PR creation and monitoring +- Commit streaming and real-time git status updates + +### MCP Server Integration +Built-in MCP server provides task management tools: +- `create_task`, `update_task`, `delete_task` +- `list_tasks`, `get_task`, `list_projects` +- Requires `project_id` for most operations + +### Process Execution +- All agent executions run as managed processes with streaming logs +- Process lifecycle: queued → running → completed/failed +- Real-time updates via WebSocket connections +- Automatic cleanup of completed processes + +### Environment Configuration +- Backend runs on port 3001, frontend proxies API calls in development +- GitHub OAuth requires `GITHUB_CLIENT_ID` and `GITHUB_CLIENT_SECRET` +- Optional PostHog analytics integration +- Rust nightly toolchain required (version 2025-05-18 or later) + +## Testing Strategy +- Run `pnpm run check` to validate both Rust and TypeScript code +- Use `cargo test` for backend unit tests +- Frontend testing focuses on component integration +- Process execution testing via echo executor + +## Key Dependencies +- **axum** - Web framework and routing +- **sqlx** - Database operations with compile-time query checking +- **octocrab** - GitHub API client +- **rmcp** - MCP server implementation +- **@dnd-kit** - Drag-and-drop task management +- **react-router-dom** - Frontend routing diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 4e1dbbff..3e8dba78 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -55,6 +55,7 @@ lazy_static = "1.4" futures-util = "0.3" async-stream = "0.3" json-patch = "2.0" +backon = "1.5.1" [dev-dependencies] tempfile = "3.8" diff --git a/backend/src/app_state.rs b/backend/src/app_state.rs index 187f86db..21d763d5 100644 --- a/backend/src/app_state.rs +++ b/backend/src/app_state.rs @@ -1,11 +1,14 @@ -use std::{collections::HashMap, sync::Arc, time::Duration}; +use std::{collections::HashMap, path::PathBuf, sync::Arc, time::Duration}; #[cfg(unix)] use nix::{sys::signal::Signal, unistd::Pid}; use tokio::sync::{Mutex, RwLock as TokioRwLock}; use uuid::Uuid; -use crate::services::{generate_user_id, AnalyticsConfig, AnalyticsService}; +use crate::{ + models::Environment, + services::{generate_user_id, AnalyticsConfig, AnalyticsService}, +}; #[derive(Debug)] pub enum ExecutionType { @@ -29,12 +32,14 @@ pub struct AppState { config: Arc>, pub analytics: Arc>, user_id: String, + pub mode: Environment, } impl AppState { pub async fn new( db_pool: sqlx::SqlitePool, config: Arc>, + mode: Environment, ) -> Self { // Initialize analytics with user preferences let user_enabled = { @@ -51,6 +56,7 @@ impl AppState { config, analytics, user_id: generate_user_id(), + mode, } } @@ -215,4 +221,34 @@ impl AppState { scope.set_user(Some(sentry_user)); }); } + + /// Get the workspace directory path, creating it if it doesn't exist in cloud mode + pub async fn get_workspace_path( + &self, + ) -> Result> { + if !self.mode.is_cloud() { + return Err("Workspace directory only available in cloud mode".into()); + } + + let workspace_path = { + let config = self.config.read().await; + match &config.workspace_dir { + Some(dir) => PathBuf::from(dir), + None => { + // Use default workspace directory + let home_dir = dirs::home_dir().ok_or("Could not find home directory")?; + home_dir.join(".vibe-kanban").join("projects") + } + } + }; + + // Create the workspace directory if it doesn't exist + if !workspace_path.exists() { + std::fs::create_dir_all(&workspace_path) + .map_err(|e| format!("Failed to create workspace directory: {}", e))?; + tracing::info!("Created workspace directory: {}", workspace_path.display()); + } + + Ok(workspace_path) + } } diff --git a/backend/src/bin/generate_types.rs b/backend/src/bin/generate_types.rs index b2bea18b..52c1f223 100644 --- a/backend/src/bin/generate_types.rs +++ b/backend/src/bin/generate_types.rs @@ -74,11 +74,12 @@ fn generate_types_content() -> String { // Do not edit this file manually.\n\ // Auto-generated from Rust backend types using ts-rs\n\n"; - // 5. Add `export` if it’s missing, then join + // 5. Add `export` if it's missing, then join let decls = [ vibe_kanban::models::ApiResponse::<()>::decl(), vibe_kanban::models::config::Config::decl(), vibe_kanban::models::config::EnvironmentInfo::decl(), + vibe_kanban::models::config::Environment::decl(), vibe_kanban::models::config::ThemeMode::decl(), vibe_kanban::models::config::EditorConfig::decl(), vibe_kanban::models::config::GitHubConfig::decl(), @@ -90,6 +91,7 @@ fn generate_types_content() -> String { vibe_kanban::executor::ExecutorConfig::decl(), vibe_kanban::executor::ExecutorConstants::decl(), vibe_kanban::models::project::CreateProject::decl(), + vibe_kanban::models::project::CreateProjectFromGitHub::decl(), vibe_kanban::models::project::Project::decl(), vibe_kanban::models::project::ProjectWithBranch::decl(), vibe_kanban::models::project::UpdateProject::decl(), @@ -114,6 +116,7 @@ fn generate_types_content() -> String { vibe_kanban::routes::filesystem::DirectoryEntry::decl(), vibe_kanban::routes::filesystem::DirectoryListResponse::decl(), vibe_kanban::routes::auth::DeviceStartResponse::decl(), + vibe_kanban::services::github_service::RepositoryInfo::decl(), vibe_kanban::routes::task_attempts::ProcessLogsResponse::decl(), vibe_kanban::models::task_attempt::DiffChunkType::decl(), vibe_kanban::models::task_attempt::DiffChunk::decl(), diff --git a/backend/src/main.rs b/backend/src/main.rs index 861f07e5..c5d57749 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -33,9 +33,10 @@ use middleware::{ load_execution_process_simple_middleware, load_project_middleware, load_task_attempt_middleware, load_task_middleware, load_task_template_middleware, }; -use models::{ApiResponse, Config}; +use models::{ApiResponse, Config, Environment}; use routes::{ - auth, config, filesystem, health, projects, stream, task_attempts, task_templates, tasks, + auth, config, filesystem, github, health, projects, stream, task_attempts, task_templates, + tasks, }; use services::PrMonitorService; @@ -167,8 +168,13 @@ fn main() -> anyhow::Result<()> { let config = Config::load(&config_path)?; let config_arc = Arc::new(RwLock::new(config)); + let env = std::env::var("ENVIRONMENT") + .unwrap_or_else(|_| "local".to_string()); + let mode = env.parse().unwrap_or(Environment::Local); + tracing::info!("Running in {mode} mode" ); + // Create app state - let app_state = AppState::new(pool.clone(), config_arc.clone()).await; + let app_state = AppState::new(pool.clone(), config_arc.clone(), mode).await; app_state.update_sentry_scope().await; @@ -245,16 +251,24 @@ fn main() -> anyhow::Result<()> { .merge(task_attempts::task_attempts_with_id_router(app_state.clone()) .layer(from_fn_with_state(app_state.clone(), load_task_attempt_middleware))); + // Conditionally add GitHub routes for cloud mode + let mut api_routes = Router::new() + .merge(base_routes) + .merge(template_routes) + .merge(project_routes) + .merge(task_routes) + .merge(task_attempt_routes); + + if mode.is_cloud() { + api_routes = api_routes.merge(github::github_router()); + tracing::info!("GitHub repository routes enabled (cloud mode)"); + } + // All routes (no auth required) let app_routes = Router::new() .nest( "/api", - Router::new() - .merge(base_routes) - .merge(template_routes) - .merge(project_routes) - .merge(task_routes) - .merge(task_attempt_routes) + api_routes .layer(from_fn_with_state(app_state.clone(), auth::sentry_user_context_middleware)), ); diff --git a/backend/src/models/config.rs b/backend/src/models/config.rs index 0c95babf..a34b68e7 100644 --- a/backend/src/models/config.rs +++ b/backend/src/models/config.rs @@ -1,10 +1,49 @@ -use std::path::PathBuf; +use std::{path::PathBuf, str::FromStr}; use serde::{Deserialize, Serialize}; use ts_rs::TS; use crate::executor::ExecutorConfig; +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, TS)] +#[ts(export)] +#[serde(rename_all = "lowercase")] +pub enum Environment { + Local, + Cloud, +} + +impl FromStr for Environment { + type Err = String; + + fn from_str(s: &str) -> Result { + match s.to_lowercase().as_str() { + "local" => Ok(Environment::Local), + "cloud" => Ok(Environment::Cloud), + _ => Err(format!("Invalid environment: {}", s)), + } + } +} + +impl Environment { + pub fn is_cloud(&self) -> bool { + matches!(self, Environment::Cloud) + } + + pub fn is_local(&self) -> bool { + matches!(self, Environment::Local) + } +} + +impl std::fmt::Display for Environment { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Environment::Local => write!(f, "local"), + Environment::Cloud => write!(f, "cloud"), + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize, TS)] #[ts(export)] pub struct EnvironmentInfo { @@ -30,6 +69,7 @@ pub struct Config { pub github: GitHubConfig, pub analytics_enabled: Option, pub environment: EnvironmentInfo, + pub workspace_dir: Option, } #[derive(Debug, Clone, Serialize, Deserialize, TS)] @@ -186,6 +226,7 @@ impl Default for Config { architecture: info.architecture().unwrap_or("unknown").to_string(), bitness: info.bitness().to_string(), }, + workspace_dir: None, } } } diff --git a/backend/src/models/mod.rs b/backend/src/models/mod.rs index 79b25be0..e1907e0c 100644 --- a/backend/src/models/mod.rs +++ b/backend/src/models/mod.rs @@ -9,4 +9,4 @@ pub mod task_attempt; pub mod task_template; pub use api_response::ApiResponse; -pub use config::Config; +pub use config::{Config, Environment}; diff --git a/backend/src/models/project.rs b/backend/src/models/project.rs index 55e18579..23752211 100644 --- a/backend/src/models/project.rs +++ b/backend/src/models/project.rs @@ -42,6 +42,17 @@ pub struct UpdateProject { pub cleanup_script: Option, } +#[derive(Debug, Deserialize, TS)] +#[ts(export)] +pub struct CreateProjectFromGitHub { + pub repository_id: i64, + pub name: String, + pub clone_url: String, + pub setup_script: Option, + pub dev_script: Option, + pub cleanup_script: Option, +} + #[derive(Debug, Serialize, TS)] #[ts(export)] pub struct ProjectWithBranch { diff --git a/backend/src/routes/config.rs b/backend/src/routes/config.rs index b5ad28d3..434cc307 100644 --- a/backend/src/routes/config.rs +++ b/backend/src/routes/config.rs @@ -16,7 +16,7 @@ use crate::{ executor::ExecutorConfig, models::{ config::{Config, EditorConstants, SoundConstants}, - ApiResponse, + ApiResponse, Environment, }, utils, }; @@ -70,12 +70,16 @@ async fn update_config( pub struct ConfigConstants { pub editor: EditorConstants, pub sound: SoundConstants, + pub mode: Environment, } -async fn get_config_constants() -> ResponseJson> { +async fn get_config_constants( + State(app_state): State, +) -> ResponseJson> { let constants = ConfigConstants { editor: EditorConstants::new(), sound: SoundConstants::new(), + mode: app_state.mode, }; ResponseJson(ApiResponse::success(constants)) diff --git a/backend/src/routes/github.rs b/backend/src/routes/github.rs new file mode 100644 index 00000000..014fcb0d --- /dev/null +++ b/backend/src/routes/github.rs @@ -0,0 +1,207 @@ +use axum::{ + extract::{Query, State}, + http::StatusCode, + response::Json as ResponseJson, + routing::{get, post}, + Json, Router, +}; +use uuid::Uuid; + +use crate::{ + app_state::AppState, + models::{ + project::{CreateProject, CreateProjectFromGitHub, Project}, + ApiResponse, + }, + services::{ + git_service::GitService, + github_service::{GitHubService, RepositoryInfo}, + GitHubServiceError, + }, +}; + +#[derive(serde::Deserialize)] +pub struct RepositoryQuery { + pub page: Option, +} + +/// List GitHub repositories for the authenticated user +pub async fn list_repositories( + State(app_state): State, + Query(params): Query, +) -> Result>>, StatusCode> { + // Only available in cloud mode + if app_state.mode.is_local() { + return Err(StatusCode::NOT_FOUND); + } + + let page = params.page.unwrap_or(1); + + // Get GitHub configuration + let github_config = { + let config = app_state.get_config().read().await; + config.github.clone() + }; + + // Check if GitHub is configured + if github_config.token.is_none() { + return Ok(ResponseJson(ApiResponse::error( + "GitHub token not configured. Please authenticate with GitHub first.", + ))); + } + + // Create GitHub service with token + let github_token = github_config.token.as_deref().unwrap(); + let github_service = match GitHubService::new(github_token) { + Ok(service) => service, + Err(e) => { + tracing::error!("Failed to create GitHub service: {}", e); + return Err(StatusCode::INTERNAL_SERVER_ERROR); + } + }; + + // List repositories + match github_service.list_repositories(page).await { + Ok(repositories) => { + tracing::info!( + "Retrieved {} repositories from GitHub (page {})", + repositories.len(), + page + ); + Ok(ResponseJson(ApiResponse::success(repositories))) + } + Err(GitHubServiceError::TokenInvalid) => Ok(ResponseJson(ApiResponse::error( + "GitHub token is invalid or expired. Please re-authenticate with GitHub.", + ))), + Err(e) => { + tracing::error!("Failed to list GitHub repositories: {}", e); + Ok(ResponseJson(ApiResponse::error(&format!( + "Failed to retrieve repositories: {}", + e + )))) + } + } +} + +/// Create a project from a GitHub repository +pub async fn create_project_from_github( + State(app_state): State, + Json(payload): Json, +) -> Result>, StatusCode> { + // Only available in cloud mode + if app_state.mode.is_local() { + return Err(StatusCode::NOT_FOUND); + } + + tracing::debug!("Creating project '{}' from GitHub repository", payload.name); + + // Get workspace path + let workspace_path = match app_state.get_workspace_path().await { + Ok(path) => path, + Err(e) => { + tracing::error!("Failed to get workspace path: {}", e); + return Err(StatusCode::INTERNAL_SERVER_ERROR); + } + }; + + let target_path = workspace_path.join(&payload.name); + + // Check if project directory already exists + if target_path.exists() { + return Ok(ResponseJson(ApiResponse::error( + "A project with this name already exists in the workspace", + ))); + } + + // Check if git repo path is already used by another project + match Project::find_by_git_repo_path(&app_state.db_pool, &target_path.to_string_lossy()).await { + Ok(Some(_)) => { + return Ok(ResponseJson(ApiResponse::error( + "A project with this git repository path already exists", + ))); + } + Ok(None) => { + // Path is available, continue + } + Err(e) => { + tracing::error!("Failed to check for existing git repo path: {}", e); + return Err(StatusCode::INTERNAL_SERVER_ERROR); + } + } + + // Get GitHub token + let github_token = { + let config = app_state.get_config().read().await; + config.github.token.clone() + }; + + // Clone the repository + match GitService::clone_repository(&payload.clone_url, &target_path, github_token.as_deref()) { + Ok(_) => { + tracing::info!( + "Successfully cloned repository {} to {}", + payload.clone_url, + target_path.display() + ); + } + Err(e) => { + tracing::error!("Failed to clone repository: {}", e); + return Ok(ResponseJson(ApiResponse::error(&format!( + "Failed to clone repository: {}", + e + )))); + } + } + + // Create project record in database + let has_setup_script = payload.setup_script.is_some(); + let has_dev_script = payload.dev_script.is_some(); + let project_data = CreateProject { + name: payload.name.clone(), + git_repo_path: target_path.to_string_lossy().to_string(), + use_existing_repo: true, // Since we just cloned it + setup_script: payload.setup_script, + dev_script: payload.dev_script, + cleanup_script: payload.cleanup_script, + }; + + let project_id = Uuid::new_v4(); + match Project::create(&app_state.db_pool, &project_data, project_id).await { + Ok(project) => { + // Track project creation event + app_state + .track_analytics_event( + "project_created_from_github", + Some(serde_json::json!({ + "project_id": project.id.to_string(), + "repository_id": payload.repository_id, + "clone_url": payload.clone_url, + "has_setup_script": has_setup_script, + "has_dev_script": has_dev_script, + })), + ) + .await; + + Ok(ResponseJson(ApiResponse::success(project))) + } + Err(e) => { + tracing::error!("Failed to create project: {}", e); + + // Clean up cloned repository if project creation failed + if target_path.exists() { + if let Err(cleanup_err) = std::fs::remove_dir_all(&target_path) { + tracing::error!("Failed to cleanup cloned repository: {}", cleanup_err); + } + } + + Err(StatusCode::INTERNAL_SERVER_ERROR) + } + } +} + +/// Create router for GitHub-related endpoints (only registered in cloud mode) +pub fn github_router() -> Router { + Router::new() + .route("/github/repositories", get(list_repositories)) + .route("/projects/from-github", post(create_project_from_github)) +} diff --git a/backend/src/routes/mod.rs b/backend/src/routes/mod.rs index f282c90b..f1f259ce 100644 --- a/backend/src/routes/mod.rs +++ b/backend/src/routes/mod.rs @@ -1,6 +1,7 @@ pub mod auth; pub mod config; pub mod filesystem; +pub mod github; pub mod health; pub mod projects; pub mod stream; diff --git a/backend/src/services/git_service.rs b/backend/src/services/git_service.rs index 4f8211aa..79ae3f93 100644 --- a/backend/src/services/git_service.rs +++ b/backend/src/services/git_service.rs @@ -1279,6 +1279,58 @@ impl GitService { Ok(()) } + + /// Clone a repository to the specified directory + pub fn clone_repository( + clone_url: &str, + target_path: &Path, + token: Option<&str>, + ) -> Result { + if let Some(parent) = target_path.parent() { + std::fs::create_dir_all(parent)?; + } + + // Set up callbacks for authentication if token is provided + let mut callbacks = RemoteCallbacks::new(); + if let Some(token) = token { + callbacks.credentials(|_url, username_from_url, _allowed_types| { + Cred::userpass_plaintext(username_from_url.unwrap_or("git"), token) + }); + } else { + // Fallback to SSH agent and key file authentication + callbacks.credentials(|_url, username_from_url, _| { + // Try SSH agent first + if let Some(username) = username_from_url { + if let Ok(cred) = Cred::ssh_key_from_agent(username) { + return Ok(cred); + } + } + // Fallback to key file (~/.ssh/id_rsa) + let home = dirs::home_dir() + .ok_or_else(|| git2::Error::from_str("Could not find home directory"))?; + let key_path = home.join(".ssh").join("id_rsa"); + Cred::ssh_key(username_from_url.unwrap_or("git"), None, &key_path, None) + }); + } + + // Set up fetch options with our callbacks + let mut fetch_opts = FetchOptions::new(); + fetch_opts.remote_callbacks(callbacks); + + // Create a repository builder with fetch options + let mut builder = git2::build::RepoBuilder::new(); + builder.fetch_options(fetch_opts); + + let repo = builder.clone(clone_url, target_path)?; + + tracing::info!( + "Successfully cloned repository from {} to {}", + clone_url, + target_path.display() + ); + + Ok(repo) + } } #[cfg(test)] diff --git a/backend/src/services/github_service.rs b/backend/src/services/github_service.rs index b8a49085..f98c0b24 100644 --- a/backend/src/services/github_service.rs +++ b/backend/src/services/github_service.rs @@ -1,9 +1,10 @@ use std::time::Duration; +use backon::{ExponentialBuilder, Retryable}; use octocrab::{Octocrab, OctocrabBuilder}; use serde::{Deserialize, Serialize}; -use tokio::time::sleep; -use tracing::{info, warn}; +use tracing::info; +use ts_rs::TS; #[derive(Debug)] pub enum GitHubServiceError { @@ -75,27 +76,23 @@ pub struct PullRequestInfo { pub merge_commit_sha: Option, } +#[derive(Debug, Clone, Serialize, Deserialize, TS)] +#[ts(export)] +pub struct RepositoryInfo { + pub id: i64, + pub name: String, + pub full_name: String, + pub owner: String, + pub description: Option, + pub clone_url: String, + pub ssh_url: String, + pub default_branch: String, + pub private: bool, +} + #[derive(Debug, Clone)] pub struct GitHubService { client: Octocrab, - retry_config: RetryConfig, -} - -#[derive(Debug, Clone)] -pub struct RetryConfig { - pub max_retries: u32, - pub base_delay: Duration, - pub max_delay: Duration, -} - -impl Default for RetryConfig { - fn default() -> Self { - Self { - max_retries: 3, - base_delay: Duration::from_secs(1), - max_delay: Duration::from_secs(30), - } - } } impl GitHubService { @@ -108,10 +105,7 @@ impl GitHubService { GitHubServiceError::Auth(format!("Failed to create GitHub client: {}", e)) })?; - Ok(Self { - client, - retry_config: RetryConfig::default(), - }) + Ok(Self { client }) } /// Create a pull request on GitHub @@ -120,7 +114,22 @@ impl GitHubService { repo_info: &GitHubRepoInfo, request: &CreatePrRequest, ) -> Result { - self.with_retry(|| async { self.create_pr_internal(repo_info, request).await }) + (|| async { self.create_pr_internal(repo_info, request).await }) + .retry( + &ExponentialBuilder::default() + .with_min_delay(Duration::from_secs(1)) + .with_max_delay(Duration::from_secs(30)) + .with_max_times(3) + .with_jitter(), + ) + .when(|e| !matches!(e, GitHubServiceError::TokenInvalid)) + .notify(|err: &GitHubServiceError, dur: Duration| { + tracing::warn!( + "GitHub API call failed, retrying after {:.2}s: {}", + dur.as_secs_f64(), + err + ); + }) .await } @@ -225,7 +234,22 @@ impl GitHubService { repo_info: &GitHubRepoInfo, pr_number: i64, ) -> Result { - self.with_retry(|| async { self.update_pr_status_internal(repo_info, pr_number).await }) + (|| async { self.update_pr_status_internal(repo_info, pr_number).await }) + .retry( + &ExponentialBuilder::default() + .with_min_delay(Duration::from_secs(1)) + .with_max_delay(Duration::from_secs(30)) + .with_max_times(3) + .with_jitter(), + ) + .when(|e| !matches!(e, GitHubServiceError::TokenInvalid)) + .notify(|err: &GitHubServiceError, dur: Duration| { + tracing::warn!( + "GitHub API call failed, retrying after {:.2}s: {}", + dur.as_secs_f64(), + err + ); + }) .await } @@ -268,40 +292,73 @@ impl GitHubService { Ok(pr_info) } - /// Retry wrapper for GitHub API calls with exponential backoff - async fn with_retry(&self, operation: F) -> Result - where - F: Fn() -> Fut, - Fut: std::future::Future>, - { - let mut last_error = None; + /// List repositories for the authenticated user with pagination + pub async fn list_repositories( + &self, + page: u8, + ) -> Result, GitHubServiceError> { + (|| async { self.list_repositories_internal(page).await }) + .retry( + &ExponentialBuilder::default() + .with_min_delay(Duration::from_secs(1)) + .with_max_delay(Duration::from_secs(30)) + .with_max_times(3) + .with_jitter(), + ) + .when(|e| !matches!(e, GitHubServiceError::TokenInvalid)) + .notify(|err: &GitHubServiceError, dur: Duration| { + tracing::warn!( + "GitHub API call failed, retrying after {:.2}s: {}", + dur.as_secs_f64(), + err + ); + }) + .await + } - for attempt in 0..=self.retry_config.max_retries { - match operation().await { - Ok(result) => return Ok(result), - Err(e) => { - last_error = Some(e); + async fn list_repositories_internal( + &self, + page: u8, + ) -> Result, GitHubServiceError> { + let repos_page = self + .client + .current() + .list_repos_for_authenticated_user() + .type_("all") + .sort("updated") + .direction("desc") + .per_page(50) + .page(page) + .send() + .await + .map_err(|e| { + GitHubServiceError::Repository(format!("Failed to list repositories: {}", e)) + })?; - if attempt < self.retry_config.max_retries { - let delay = std::cmp::min( - self.retry_config.base_delay * 2_u32.pow(attempt), - self.retry_config.max_delay, - ); + let repositories: Vec = repos_page + .items + .into_iter() + .map(|repo| RepositoryInfo { + id: repo.id.0 as i64, + name: repo.name, + full_name: repo.full_name.unwrap_or_default(), + owner: repo.owner.map(|o| o.login).unwrap_or_default(), + description: repo.description, + clone_url: repo + .clone_url + .map(|url| url.to_string()) + .unwrap_or_default(), + ssh_url: repo.ssh_url.unwrap_or_default(), + default_branch: repo.default_branch.unwrap_or_else(|| "main".to_string()), + private: repo.private.unwrap_or(false), + }) + .collect(); - warn!( - "GitHub API call failed (attempt {}/{}), retrying in {:?}: {}", - attempt + 1, - self.retry_config.max_retries + 1, - delay, - last_error.as_ref().unwrap() - ); - - sleep(delay).await; - } - } - } - } - - Err(last_error.unwrap()) + tracing::info!( + "Retrieved {} repositories from GitHub (page {})", + repositories.len(), + page + ); + Ok(repositories) } } diff --git a/frontend/src/components/projects/github-repository-picker.tsx b/frontend/src/components/projects/github-repository-picker.tsx new file mode 100644 index 00000000..427fd461 --- /dev/null +++ b/frontend/src/components/projects/github-repository-picker.tsx @@ -0,0 +1,256 @@ +import { useState, useEffect, useCallback, useRef } from 'react'; +import { Button } from '@/components/ui/button'; +import { Input } from '@/components/ui/input'; +import { Label } from '@/components/ui/label'; +import { Alert, AlertDescription } from '@/components/ui/alert'; +import { Loader2, Github } from 'lucide-react'; +import { githubApi, RepositoryInfo } from '@/lib/api'; + +interface GitHubRepositoryPickerProps { + selectedRepository: RepositoryInfo | null; + onRepositorySelect: (repository: RepositoryInfo | null) => void; + onNameChange: (name: string) => void; + name: string; + error: string; +} + +// Simple in-memory cache for repositories +const repositoryCache = new Map(); +const CACHE_DURATION = 5 * 60 * 1000; // 5 minutes +const cacheTimestamps = new Map(); + +function isCacheValid(page: number): boolean { + const timestamp = cacheTimestamps.get(page); + return timestamp ? Date.now() - timestamp < CACHE_DURATION : false; +} + +export function GitHubRepositoryPicker({ + selectedRepository, + onRepositorySelect, + onNameChange, + name, + error, +}: GitHubRepositoryPickerProps) { + const [repositories, setRepositories] = useState([]); + const [loading, setLoading] = useState(false); + const [loadError, setLoadError] = useState(''); + const [page, setPage] = useState(1); + const [hasMorePages, setHasMorePages] = useState(true); + const [loadingMore, setLoadingMore] = useState(false); + const scrollContainerRef = useRef(null); + + const loadRepositories = useCallback( + async (pageNum: number = 1, isLoadingMore: boolean = false) => { + if (isLoadingMore) { + setLoadingMore(true); + } else { + setLoading(true); + } + setLoadError(''); + + try { + // Check cache first + if (isCacheValid(pageNum)) { + const cachedRepos = repositoryCache.get(pageNum); + if (cachedRepos) { + if (pageNum === 1) { + setRepositories(cachedRepos); + } else { + setRepositories((prev) => [...prev, ...cachedRepos]); + } + setPage(pageNum); + return; + } + } + + const repos = await githubApi.listRepositories(pageNum); + + // Cache the results + repositoryCache.set(pageNum, repos); + cacheTimestamps.set(pageNum, Date.now()); + + if (pageNum === 1) { + setRepositories(repos); + } else { + setRepositories((prev) => [...prev, ...repos]); + } + setPage(pageNum); + + // If we got fewer than expected results, we've reached the end + if (repos.length < 30) { + // GitHub typically returns 30 repos per page + setHasMorePages(false); + } + } catch (err) { + setLoadError( + err instanceof Error ? err.message : 'Failed to load repositories' + ); + } finally { + if (isLoadingMore) { + setLoadingMore(false); + } else { + setLoading(false); + } + } + }, + [] + ); + + useEffect(() => { + loadRepositories(1); + }, [loadRepositories]); + + const handleRepositorySelect = (repository: RepositoryInfo) => { + onRepositorySelect(repository); + // Auto-populate project name from repository name if name is empty + if (!name) { + const cleanName = repository.name + .replace(/[-_]/g, ' ') + .replace(/\b\w/g, (l) => l.toUpperCase()); + onNameChange(cleanName); + } + }; + + const loadMoreRepositories = useCallback(() => { + if (!loading && !loadingMore && hasMorePages) { + loadRepositories(page + 1, true); + } + }, [loading, loadingMore, hasMorePages, page, loadRepositories]); + + // Infinite scroll handler + const handleScroll = useCallback( + (e: React.UIEvent) => { + const { scrollTop, scrollHeight, clientHeight } = e.currentTarget; + const isNearBottom = scrollHeight - scrollTop <= clientHeight + 100; // 100px threshold + + if (isNearBottom && !loading && !loadingMore && hasMorePages) { + loadMoreRepositories(); + } + }, + [loading, loadingMore, hasMorePages, loadMoreRepositories] + ); + + if (loadError) { + return ( + + + {loadError} + + + + ); + } + + return ( +
+
+ + {loading && repositories.length === 0 ? ( +
+ + Loading repositories... +
+ ) : ( +
+ {repositories.map((repository) => ( +
handleRepositorySelect(repository)} + > +
+ +
+
+ {repository.name} + {repository.private && ( + + Private + + )} +
+
+
{repository.full_name}
+ {repository.description && ( +
{repository.description}
+ )} +
+
+
+
+ ))} + + {repositories.length === 0 && !loading && ( +
+ No repositories found +
+ )} + + {/* Loading more indicator */} + {loadingMore && ( +
+ + + Loading more repositories... + +
+ )} + + {/* Manual load more button (fallback if infinite scroll doesn't work) */} + {hasMorePages && !loadingMore && repositories.length > 0 && ( +
+ +
+ )} + + {/* End of results indicator */} + {!hasMorePages && repositories.length > 0 && ( +
+ All repositories loaded +
+ )} +
+ )} +
+ + {selectedRepository && ( +
+ + onNameChange(e.target.value)} + /> +
+ )} + + {error && ( + + {error} + + )} +
+ ); +} diff --git a/frontend/src/components/projects/project-form.tsx b/frontend/src/components/projects/project-form.tsx index 54259310..7f377c89 100644 --- a/frontend/src/components/projects/project-form.tsx +++ b/frontend/src/components/projects/project-form.tsx @@ -1,5 +1,7 @@ import { useEffect, useState } from 'react'; import { Button } from '@/components/ui/button'; +import { Label } from '@/components/ui/label'; +import { Loader2 } from 'lucide-react'; import { Dialog, DialogContent, @@ -12,8 +14,15 @@ import { Tabs, TabsContent, TabsList, TabsTrigger } from '@/components/ui/tabs'; import { FolderPicker } from '@/components/ui/folder-picker'; import { TaskTemplateManager } from '@/components/TaskTemplateManager'; import { ProjectFormFields } from './project-form-fields'; -import { CreateProject, Project, UpdateProject } from 'shared/types'; -import { projectsApi } from '@/lib/api'; +import { GitHubRepositoryPicker } from './github-repository-picker'; +import { + CreateProject, + CreateProjectFromGitHub, + Project, + UpdateProject, + Environment, +} from 'shared/types'; +import { projectsApi, configApi, githubApi, RepositoryInfo } from '@/lib/api'; interface ProjectFormProps { open: boolean; @@ -42,8 +51,34 @@ export function ProjectForm({ const [parentPath, setParentPath] = useState(''); const [folderName, setFolderName] = useState(''); + // Environment and GitHub repository state + const [environment, setEnvironment] = useState('local'); + const [selectedRepository, setSelectedRepository] = + useState(null); + const [modeLoading, setModeLoading] = useState(true); + const isEditing = !!project; + // Load cloud mode configuration + useEffect(() => { + const loadMode = async () => { + try { + const constants = await configApi.getConstants(); + setEnvironment(constants.mode); + } catch (err) { + console.error('Failed to load config constants:', err); + } finally { + setModeLoading(false); + } + }; + + if (!isEditing) { + loadMode(); + } else { + setModeLoading(false); + } + }, [isEditing]); + // Update form fields when project prop changes useEffect(() => { if (project) { @@ -58,6 +93,7 @@ export function ProjectForm({ setSetupScript(''); setDevScript(''); setCleanupScript(''); + setSelectedRepository(null); } }, [project]); @@ -85,14 +121,13 @@ export function ProjectForm({ setLoading(true); try { - let finalGitRepoPath = gitRepoPath; - - // For new repo mode, construct the full path - if (!isEditing && repoMode === 'new') { - finalGitRepoPath = `${parentPath}/${folderName}`.replace(/\/+/g, '/'); - } - if (isEditing) { + // Editing existing project (local mode only) + let finalGitRepoPath = gitRepoPath; + if (repoMode === 'new') { + finalGitRepoPath = `${parentPath}/${folderName}`.replace(/\/+/g, '/'); + } + const updateData: UpdateProject = { name, git_repo_path: finalGitRepoPath, @@ -101,37 +136,59 @@ export function ProjectForm({ cleanup_script: cleanupScript.trim() || null, }; - try { - await projectsApi.update(project.id, updateData); - } catch (error) { - setError('Failed to update project'); - return; - } + await projectsApi.update(project.id, updateData); } else { - const createData: CreateProject = { - name, - git_repo_path: finalGitRepoPath, - use_existing_repo: repoMode === 'existing', - setup_script: setupScript.trim() || null, - dev_script: devScript.trim() || null, - cleanup_script: cleanupScript.trim() || null, - }; + // Creating new project + if (environment === 'cloud') { + // Cloud mode: Create project from GitHub repository + if (!selectedRepository) { + setError('Please select a GitHub repository'); + return; + } + + const githubData: CreateProjectFromGitHub = { + repository_id: BigInt(selectedRepository.id), + name, + clone_url: selectedRepository.clone_url, + setup_script: setupScript.trim() || null, + dev_script: devScript.trim() || null, + cleanup_script: cleanupScript.trim() || null, + }; + + await githubApi.createProjectFromRepository(githubData); + } else { + // Local mode: Create local project + let finalGitRepoPath = gitRepoPath; + if (repoMode === 'new') { + finalGitRepoPath = `${parentPath}/${folderName}`.replace( + /\/+/g, + '/' + ); + } + + const createData: CreateProject = { + name, + git_repo_path: finalGitRepoPath, + use_existing_repo: repoMode === 'existing', + setup_script: setupScript.trim() || null, + dev_script: devScript.trim() || null, + cleanup_script: cleanupScript.trim() || null, + }; - try { await projectsApi.create(createData); - } catch (error) { - setError('Failed to create project'); - return; } } onSuccess(); + // Reset form setName(''); setGitRepoPath(''); setSetupScript(''); + setDevScript(''); setCleanupScript(''); setParentPath(''); setFolderName(''); + setSelectedRepository(null); } catch (error) { setError(error instanceof Error ? error.message : 'An error occurred'); } finally { @@ -226,27 +283,89 @@ export function ProjectForm({ ) : (
- + {modeLoading ? ( +
+ + Loading... +
+ ) : environment === 'cloud' ? ( + // Cloud mode: Show only GitHub repositories + <> + + + {/* Show script fields for GitHub source */} +
+
+ +