daonanchao/vibe-kanban
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Store binaries in Cloudflare R2 instead of bundling in npm package (Vibe Kanban) (#1554)

Browse Source 
* All changes are complete. Let me provide a summary of what was implemented:

## Summary

I've implemented the Cloudflare R2 binary storage feature. Here are the changes:

### Files Created
- **`npx-cli/bin/download.js`** - New module that handles downloading binaries from R2 with:
  - SHA-256 checksum verification
  - Progress reporting
  - Redirect handling
  - Local caching in `~/.vibe-kanban/bin/{version}/{platform}/`

### Files Modified

1. **`npx-cli/bin/cli.js`**
   - Converted to async for on-demand downloads
   - Downloads binaries from R2 instead of using bundled files
   - Non-blocking update check shows notice if newer version available
   - Caches binaries locally per version/platform

2. **`npx-cli/package.json`**
   - Removed `"dist"` from `files` array (package now ~15KB instead of ~150MB)

3. **`.github/workflows/pre-release.yml`**
   - Added new `upload-to-r2` job that:
     - Downloads all platform packages from artifacts
     - Generates manifest.json with SHA-256 checksums for each binary
     - Uploads binaries to R2 at `binaries/v{version}/{platform}/{binary}.zip`
     - Updates global manifest with latest version
   - Modified `create-prerelease` job to:
     - Depend on `upload-to-r2` instead of `package-npx-cli`
     - Inject R2 public URL into download.js via `sed`
     - No longer download/bundle binaries in the npm package

### Required GitHub Secrets

You'll need to add these secrets to your repository:
- `R2_BINARIES_ACCESS_KEY_ID`
- `R2_BINARIES_SECRET_ACCESS_KEY`
- `R2_BINARIES_ENDPOINT` (e.g., `https://<account_id>.r2.cloudflarestorage.com`)
- `R2_BINARIES_BUCKET`
- `R2_BINARIES_PUBLIC_URL` (the public URL for your R2 bucket, e.g., custom domain or public R2 URL)

### R2 Bucket Structure
```
binaries/
  manifest.json                    # {"latest": "0.0.135"}
  v0.0.135/
    manifest.json                  # Per-version manifest with checksums
    linux-x64/
      vibe-kanban.zip
      vibe-kanban-mcp.zip
      vibe-kanban-review.zip
    ...
```

* Done! I've updated the implementation to use full tags (with timestamps) instead of just version numbers. Here's a summary of what changed:

## Changes Made

1. **`npx-cli/bin/download.js`**:
   - Added `BINARY_TAG` constant (placeholder `__BINARY_TAG__`)
   - Changed `ensureBinary()` to use `BINARY_TAG` instead of accepting version as parameter
   - Cache paths now use the tag: `~/.vibe-kanban/bin/{tag}/{platform}/`

2. **`npx-cli/bin/cli.js`**:
   - Updated to import `BINARY_TAG` from download.js
   - Cache directory now uses `BINARY_TAG` instead of `CLI_VERSION`
   - Updated `ensureBinary()` call to not pass version

3. **`.github/workflows/pre-release.yml`**:
   - `upload-to-r2` job now uses `new_tag` instead of `new_version` for R2 paths
   - Binaries are stored at `binaries/{tag}/{platform}/{binary}.zip`
   - Global manifest stores the full tag: `{"latest": "v0.0.135-20251215122030"}`
   - `create-prerelease` job now injects both `__R2_PUBLIC_URL__` and `__BINARY_TAG__`

This allows multiple pre-releases to coexist in R2 (e.g., `v0.0.135-20251215122030` and `v0.0.135-20251215100000`), making rollbacks easy.

* chore: bump version to 0.0.136

---------

Co-authored-by: GitHub Action <action@github.com>
This commit is contained in:
Louis Knight-Webb
2025-12-15 23:48:27 +00:00
committed by GitHub
parent e16602cc0f
commit 7f9f1433af
16 changed files with 335 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

109
.github/workflows/pre-release.yml vendored
View File

@@ -477,8 +477,101 @@ jobs:
path: npx-cli/dist/
retention-days: 1
upload-to-r2:
needs: [bump-version, package-npx-cli]
runs-on: ubuntu-latest-m
steps:
- uses: actions/checkout@v4
with:
ref: ${{ needs.bump-version.outputs.new_tag }}
- name: Download all platform packages
uses: actions/download-artifact@v4
with:
pattern: npx-platform-*
path: binaries/
merge-multiple: true
- name: List downloaded binaries
run: |
echo "Downloaded binaries:"
find binaries/
- name: Configure AWS CLI for R2
run: |
aws configure set aws_access_key_id ${{ secrets.R2_BINARIES_ACCESS_KEY_ID }}
aws configure set aws_secret_access_key ${{ secrets.R2_BINARIES_SECRET_ACCESS_KEY }}
aws configure set default.region auto
- name: Generate manifest and upload to R2
run: |
TAG="${{ needs.bump-version.outputs.new_tag }}"
ENDPOINT="${{ secrets.R2_BINARIES_ENDPOINT }}"
BUCKET="${{ secrets.R2_BINARIES_BUCKET }}"
# Generate version manifest with checksums
node -e "
const fs = require('fs');
const crypto = require('crypto');
const manifest = { version: '$TAG', platforms: {} };
const platforms = ['linux-x64', 'linux-arm64', 'windows-x64', 'windows-arm64', 'macos-x64', 'macos-arm64'];
const binaries = ['vibe-kanban', 'vibe-kanban-mcp', 'vibe-kanban-review'];
for (const platform of platforms) {
manifest.platforms[platform] = {};
for (const binary of binaries) {
const zipPath = \`binaries/\${platform}/\${binary}.zip\`;
if (fs.existsSync(zipPath)) {
const data = fs.readFileSync(zipPath);
manifest.platforms[platform][binary] = {
sha256: crypto.createHash('sha256').update(data).digest('hex'),
size: data.length
};
}
}
}
fs.writeFileSync('version-manifest.json', JSON.stringify(manifest, null, 2));
console.log('Generated manifest:');
console.log(JSON.stringify(manifest, null, 2));
"
# Upload binaries (use full tag for path, allows multiple pre-releases to coexist)
for platform in linux-x64 linux-arm64 windows-x64 windows-arm64 macos-x64 macos-arm64; do
for binary in vibe-kanban vibe-kanban-mcp vibe-kanban-review; do
if [ -f "binaries/$platform/$binary.zip" ]; then
echo "Uploading binaries/$platform/$binary.zip..."
aws s3 cp "binaries/$platform/$binary.zip" \
"s3://$BUCKET/binaries/$TAG/$platform/$binary.zip" \
--endpoint-url "$ENDPOINT"
fi
done
done
# Upload version manifest
echo "Uploading version manifest..."
aws s3 cp version-manifest.json \
"s3://$BUCKET/binaries/$TAG/manifest.json" \
--endpoint-url "$ENDPOINT" --content-type "application/json"
# Update global manifest
echo "Updating global manifest..."
echo "{\"latest\": \"$TAG\"}" | aws s3 cp - \
"s3://$BUCKET/binaries/manifest.json" \
--endpoint-url "$ENDPOINT" --content-type "application/json"
- name: Verify upload
run: |
TAG="${{ needs.bump-version.outputs.new_tag }}"
ENDPOINT="${{ secrets.R2_BINARIES_ENDPOINT }}"
BUCKET="${{ secrets.R2_BINARIES_BUCKET }}"
echo "Listing uploaded files..."
aws s3 ls "s3://$BUCKET/binaries/$TAG/" \
--endpoint-url "$ENDPOINT" \
--recursive
create-prerelease:
needs: [bump-version, build-frontend, build-backend, package-npx-cli]
needs: [bump-version, build-frontend, upload-to-r2]
runs-on: ubuntu-latest-m
steps:
- uses: actions/checkout@v4
@@ -491,17 +584,8 @@ jobs:
name: frontend-dist
path: frontend/dist/
- name: Download backend npx-cli zips
uses: actions/download-artifact@v4
with:
pattern: npx-platform-*
path: npx-cli/dist/
merge-multiple: true
- name: List downloaded artifacts
run: |
echo "Backend dist:"
find npx-cli/dist
echo "Frontend dist:"
find frontend/dist
@@ -514,9 +598,12 @@ jobs:
- name: Setup Node for npm pack
uses: ./.github/actions/setup-node
- name: Pack
- name: Inject R2 URL and tag, then Pack
run: |
cd npx-cli
# Replace placeholders with actual values
sed -i "s|__R2_PUBLIC_URL__|${{ secrets.R2_BINARIES_PUBLIC_URL }}|g" bin/download.js
sed -i "s|__BINARY_TAG__|${{ needs.bump-version.outputs.new_tag }}|g" bin/download.js
npm pack
- name: Create GitHub Pre-Release

18
Cargo.lock generated
View File

@@ -1616,7 +1616,7 @@ checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476"
[[package]]
name = "db"
version = "0.0.135"
version = "0.0.136"
dependencies = [
"anyhow",
"chrono",
@@ -1645,7 +1645,7 @@ dependencies = [
[[package]]
name = "deployment"
version = "0.0.135"
version = "0.0.136"
dependencies = [
"anyhow",
"async-trait",
@@ -2015,7 +2015,7 @@ dependencies = [
[[package]]
name = "executors"
version = "0.0.135"
version = "0.0.136"
dependencies = [
"agent-client-protocol",
"async-trait",
@@ -3401,7 +3401,7 @@ dependencies = [
[[package]]
name = "local-deployment"
version = "0.0.135"
version = "0.0.136"
dependencies = [
"anyhow",
"async-trait",
@@ -4511,7 +4511,7 @@ checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58"
[[package]]
name = "remote"
version = "0.0.135"
version = "0.0.136"
dependencies = [
"aes-gcm",
"anyhow",
@@ -4605,7 +4605,7 @@ dependencies = [
[[package]]
name = "review"
version = "0.0.134"
version = "0.0.136"
dependencies = [
"anyhow",
"chrono",
@@ -5296,7 +5296,7 @@ dependencies = [
[[package]]
name = "server"
version = "0.0.135"
version = "0.0.136"
dependencies = [
"anyhow",
"axum",
@@ -5343,7 +5343,7 @@ dependencies = [
[[package]]
name = "services"
version = "0.0.135"
version = "0.0.136"
dependencies = [
"anyhow",
"async-trait",
@@ -6588,7 +6588,7 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]]
name = "utils"
version = "0.0.135"
version = "0.0.136"
dependencies = [
"axum",
"bytes",

2
crates/db/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "db"
version = "0.0.135"
version = "0.0.136"
edition = "2024"
[dependencies]

2
crates/deployment/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "deployment"
version = "0.0.135"
version = "0.0.136"
edition = "2024"
[dependencies]

2
crates/executors/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "executors"
version = "0.0.135"
version = "0.0.136"
edition = "2024"
[dependencies]

2
crates/local-deployment/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "local-deployment"
version = "0.0.135"
version = "0.0.136"
edition = "2024"
[dependencies]

2
crates/remote/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "remote"
version = "0.0.135"
version = "0.0.136"
edition = "2024"
publish = false

2
crates/review/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "review"
version = "0.0.134"
version = "0.0.136"
edition = "2024"
publish = false

2
crates/server/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "server"
version = "0.0.135"
version = "0.0.136"
edition = "2024"
default-run = "server"

2
crates/services/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "services"
version = "0.0.135"
version = "0.0.136"
edition = "2024"
[features]

2
crates/utils/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "utils"
version = "0.0.135"
version = "0.0.136"
edition = "2024"
[dependencies]

2
frontend/package.json
View File

@@ -1,7 +1,7 @@
{
"name": "vibe-kanban",
"private": true,
"version": "0.0.135",
"version": "0.0.136",
"type": "module",
"scripts": {
"dev": "VITE_OPEN=${VITE_OPEN:-false} vite",

169
npx-cli/bin/cli.js
View File

@@ -4,6 +4,9 @@ const { execSync, spawn } = require("child_process");
const AdmZip = require("adm-zip");
const path = require("path");
const fs = require("fs");
const { ensureBinary, BINARY_TAG, CACHE_DIR, getLatestVersion } = require("./download");
const CLI_VERSION = require("../package.json").version;
// Resolve effective arch for our published 64-bit binaries only.
// Any ARM → arm64; anything else → x64. On macOS, handle Rosetta.
@@ -12,7 +15,7 @@ function getEffectiveArch() {
const nodeArch = process.arch;
if (platform === "darwin") {
// If Node itself is arm64, we’re natively on Apple silicon
// If Node itself is arm64, we're natively on Apple silicon
if (nodeArch === "arm64") return "arm64";
// Otherwise check for Rosetta translation
@@ -52,7 +55,7 @@ function getPlatformDir() {
if (platform === "darwin" && arch === "x64") return "macos-x64";
if (platform === "darwin" && arch === "arm64") return "macos-arm64";
console.error(`❌ Unsupported platform: ${platform}-${arch}`);
console.error(`Unsupported platform: ${platform}-${arch}`);
console.error("Supported platforms:");
console.error(" - Linux x64");
console.error(" - Linux ARM64");
@@ -68,101 +71,131 @@ function getBinaryName(base) {
}
const platformDir = getPlatformDir();
const extractDir = path.join(__dirname, "..", "dist", platformDir);
const args = process.argv.slice(2);
const isMcpMode = process.argv.includes("--mcp");
const isReviewMode = args[0] === "review";
const versionCacheDir = path.join(CACHE_DIR, BINARY_TAG, platformDir);
// ensure output dir
fs.mkdirSync(extractDir, { recursive: true });
function showProgress(downloaded, total) {
const percent = total ? Math.round((downloaded / total) * 100) : 0;
const mb = (downloaded / (1024 * 1024)).toFixed(1);
const totalMb = total ? (total / (1024 * 1024)).toFixed(1) : "?";
process.stdout.write(`\r Downloading: ${mb}MB / ${totalMb}MB (${percent}%)`);
}
function extractAndRun(baseName, launch) {
async function extractAndRun(baseName, launch) {
const binName = getBinaryName(baseName);
const binPath = path.join(extractDir, binName);
const zipName = `${baseName}.zip`;
const zipPath = path.join(extractDir, zipName);
const binPath = path.join(versionCacheDir, binName);
const zipPath = path.join(versionCacheDir, `${baseName}.zip`);
// clean old binary
// Clean old binary if exists
try {
if (fs.existsSync(binPath)) {
fs.unlinkSync(binPath);
}
} catch (err) {
// If the binary is in use, we can't delete it.
// We'll skip extraction and try to use the existing one.
if (process.env.VIBE_KANBAN_DEBUG) {
console.warn(`⚠️ Could not delete existing binary (likely in use): ${err.message}`);
console.warn(`Warning: Could not delete existing binary: ${err.message}`);
}
}
if (!fs.existsSync(zipPath)) {
console.error(`❌ ${zipName} not found at: ${zipPath}`);
console.error(`Current platform: ${platform}-${arch} (${platformDir})`);
process.exit(1);
}
// extract
try {
if (!fs.existsSync(binPath)) {
const zip = new AdmZip(zipPath);
zip.extractAllTo(extractDir, true);
// Download if not cached
if (!fs.existsSync(zipPath)) {
console.log(`Downloading ${baseName}...`);
try {
await ensureBinary(platformDir, baseName, showProgress);
console.log(""); // newline after progress
} catch (err) {
console.error(`\nDownload failed: ${err.message}`);
process.exit(1);
}
} catch (err) {
console.error("❌ Failed to extract vibe-kanban archive:", err.message);
if (process.env.VIBE_KANBAN_DEBUG) {
console.error(err.stack);
}
// Extract
if (!fs.existsSync(binPath)) {
try {
const zip = new AdmZip(zipPath);
zip.extractAllTo(versionCacheDir, true);
} catch (err) {
console.error("Extraction failed:", err.message);
try {
fs.unlinkSync(zipPath);
} catch {}
process.exit(1);
}
process.exit(1);
}
if (!fs.existsSync(binPath)) {
console.error(`❌ Extracted binary not found at: ${binPath}`);
console.error("This usually indicates a corrupt download. Please reinstall the package.");
console.error(`Extracted binary not found at: ${binPath}`);
console.error("This usually indicates a corrupt download. Please try again.");
process.exit(1);
}
// perms & launch
// Set permissions (non-Windows)
if (platform !== "win32") {
try {
fs.chmodSync(binPath, 0o755);
} catch { }
} catch {}
}
return launch(binPath);
}
if (isMcpMode) {
extractAndRun("vibe-kanban-mcp", (bin) => {
const proc = spawn(bin, [], { stdio: "inherit" });
proc.on("exit", (c) => process.exit(c || 0));
proc.on("error", (e) => {
console.error("❌ MCP server error:", e.message);
process.exit(1);
async function main() {
fs.mkdirSync(versionCacheDir, { recursive: true });
// Non-blocking update check
getLatestVersion()
.then((latest) => {
if (latest && latest !== CLI_VERSION) {
setTimeout(() => {
console.log(`\nUpdate available: ${CLI_VERSION} -> ${latest}`);
console.log(`Run: npx vibe-kanban@latest`);
}, 2000);
}
})
.catch(() => {});
const args = process.argv.slice(2);
const isMcpMode = args.includes("--mcp");
const isReviewMode = args[0] === "review";
if (isMcpMode) {
await extractAndRun("vibe-kanban-mcp", (bin) => {
const proc = spawn(bin, [], { stdio: "inherit" });
proc.on("exit", (c) => process.exit(c || 0));
proc.on("error", (e) => {
console.error("MCP server error:", e.message);
process.exit(1);
});
process.on("SIGINT", () => {
proc.kill("SIGINT");
});
process.on("SIGTERM", () => proc.kill("SIGTERM"));
});
process.on("SIGINT", () => {
console.error("\n🛑 Shutting down MCP server...");
proc.kill("SIGINT");
} else if (isReviewMode) {
await extractAndRun("vibe-kanban-review", (bin) => {
const reviewArgs = args.slice(1);
const proc = spawn(bin, reviewArgs, { stdio: "inherit" });
proc.on("exit", (c) => process.exit(c || 0));
proc.on("error", (e) => {
console.error("Review CLI error:", e.message);
process.exit(1);
});
});
process.on("SIGTERM", () => proc.kill("SIGTERM"));
});
} else if (isReviewMode) {
extractAndRun("vibe-kanban-review", (bin) => {
// Pass all args except 'review' to the binary
const reviewArgs = args.slice(1);
const proc = spawn(bin, reviewArgs, { stdio: "inherit" });
proc.on("exit", (c) => process.exit(c || 0));
proc.on("error", (e) => {
console.error("❌ Review CLI error:", e.message);
process.exit(1);
} else {
console.log(`Starting vibe-kanban v${CLI_VERSION}...`);
await extractAndRun("vibe-kanban", (bin) => {
if (platform === "win32") {
execSync(`"${bin}"`, { stdio: "inherit" });
} else {
execSync(`"${bin}"`, { stdio: "inherit" });
}
});
});
} else {
console.log(`📦 Extracting vibe-kanban...`);
extractAndRun("vibe-kanban", (bin) => {
console.log(`🚀 Launching vibe-kanban...`);
if (platform === "win32") {
execSync(`"${bin}"`, { stdio: "inherit" });
} else {
execSync(`"${bin}"`, { stdio: "inherit" });
}
});
}
}
main().catch((err) => {
console.error("Fatal error:", err.message);
if (process.env.VIBE_KANBAN_DEBUG) {
console.error(err.stack);
}
process.exit(1);
});

115
npx-cli/bin/download.js Normal file
View File

@@ -0,0 +1,115 @@
const https = require("https");
const fs = require("fs");
const path = require("path");
const crypto = require("crypto");
// Replaced during npm pack by workflow
const R2_BASE_URL = "__R2_PUBLIC_URL__";
const BINARY_TAG = "__BINARY_TAG__"; // e.g., v0.0.135-20251215122030
const CACHE_DIR = path.join(require("os").homedir(), ".vibe-kanban", "bin");
async function fetchJson(url) {
return new Promise((resolve, reject) => {
https.get(url, (res) => {
if (res.statusCode === 301 || res.statusCode === 302) {
return fetchJson(res.headers.location).then(resolve).catch(reject);
}
if (res.statusCode !== 200) {
return reject(new Error(`HTTP ${res.statusCode} fetching ${url}`));
}
let data = "";
res.on("data", (chunk) => (data += chunk));
res.on("end", () => {
try {
resolve(JSON.parse(data));
} catch (e) {
reject(new Error(`Failed to parse JSON from ${url}`));
}
});
}).on("error", reject);
});
}
async function downloadFile(url, destPath, expectedSha256, onProgress) {
return new Promise((resolve, reject) => {
const file = fs.createWriteStream(destPath);
const hash = crypto.createHash("sha256");
https.get(url, (res) => {
if (res.statusCode === 301 || res.statusCode === 302) {
file.close();
try {
fs.unlinkSync(destPath);
} catch {}
return downloadFile(res.headers.location, destPath, expectedSha256, onProgress)
.then(resolve)
.catch(reject);
}
if (res.statusCode !== 200) {
file.close();
try {
fs.unlinkSync(destPath);
} catch {}
return reject(new Error(`HTTP ${res.statusCode} downloading ${url}`));
}
const totalSize = parseInt(res.headers["content-length"], 10);
let downloadedSize = 0;
res.on("data", (chunk) => {
downloadedSize += chunk.length;
hash.update(chunk);
if (onProgress) onProgress(downloadedSize, totalSize);
});
res.pipe(file);
file.on("finish", () => {
file.close();
const actualSha256 = hash.digest("hex");
if (expectedSha256 && actualSha256 !== expectedSha256) {
try {
fs.unlinkSync(destPath);
} catch {}
reject(new Error(`Checksum mismatch: expected ${expectedSha256}, got ${actualSha256}`));
} else {
resolve(destPath);
}
});
}).on("error", (err) => {
file.close();
try {
fs.unlinkSync(destPath);
} catch {}
reject(err);
});
});
}
async function ensureBinary(platform, binaryName, onProgress) {
const cacheDir = path.join(CACHE_DIR, BINARY_TAG, platform);
const zipPath = path.join(cacheDir, `${binaryName}.zip`);
if (fs.existsSync(zipPath)) return zipPath;
fs.mkdirSync(cacheDir, { recursive: true });
const manifest = await fetchJson(`${R2_BASE_URL}/binaries/${BINARY_TAG}/manifest.json`);
const binaryInfo = manifest.platforms?.[platform]?.[binaryName];
if (!binaryInfo) {
throw new Error(`Binary ${binaryName} not available for ${platform}`);
}
const url = `${R2_BASE_URL}/binaries/${BINARY_TAG}/${platform}/${binaryName}.zip`;
await downloadFile(url, zipPath, binaryInfo.sha256, onProgress);
return zipPath;
}
async function getLatestVersion() {
const manifest = await fetchJson(`${R2_BASE_URL}/binaries/manifest.json`);
return manifest.latest;
}
module.exports = { R2_BASE_URL, BINARY_TAG, CACHE_DIR, ensureBinary, getLatestVersion };

3
npx-cli/package.json
View File

@@ -1,7 +1,7 @@
{
"name": "vibe-kanban",
"private": false,
"version": "0.0.135",
"version": "0.0.136",
"main": "index.js",
"bin": {
"vibe-kanban": "bin/cli.js"
@@ -14,7 +14,6 @@
"adm-zip": "^0.5.16"
},
"files": [
"dist",
"bin"
]
}

2
package.json
View File

@@ -1,6 +1,6 @@
{
"name": "vibe-kanban",
"version": "0.0.135",
"version": "0.0.136",
"private": true,
"bin": {
"vibe-kanban": "npx-cli/bin/cli.js"