diff options
| author | Paul Buetow <paul@buetow.org> | 2025-07-03 22:38:37 +0300 |
|---|---|---|
| committer | Paul Buetow <paul@buetow.org> | 2025-07-03 22:38:37 +0300 |
| commit | 64095a2c8d5a3a72c55d7bd0737c5542a5aeee09 (patch) | |
| tree | 0af2501374550e8fdadd4df00d245c6260c0305d | |
| parent | 0c072d964d4d07e69d1c0af1f3b09f9adc543571 (diff) | |
feat: add SSH backup locations with --backup flagv0.2.0
- Add support for SSH backup locations (e.g., paul@server:git/)
- Backup locations are one-way only (push only, never pull)
- Automatic bare repository creation on SSH servers
- Add --backup flag to opt-in to backup syncing
- Backup locations are disabled by default for offline resilience
- Update version to 0.2.0
This allows users to maintain private backups on home servers that may
be offline without affecting regular sync operations.
🤖 Generated with [Claude Code](https://claude.ai/code)
Co-Authored-By: Claude <noreply@anthropic.com>
| -rw-r--r-- | README.md | 87 | ||||
| -rw-r--r-- | gitsyncer.example.json | 4 | ||||
| -rw-r--r-- | internal/cli/flags.go | 2 | ||||
| -rw-r--r-- | internal/cli/sync_handlers.go | 4 | ||||
| -rw-r--r-- | internal/config/config.go | 24 | ||||
| -rw-r--r-- | internal/sync/branch_sync.go | 8 | ||||
| -rw-r--r-- | internal/sync/git_operations.go | 117 | ||||
| -rw-r--r-- | internal/sync/repository_setup.go | 39 | ||||
| -rw-r--r-- | internal/sync/sync.go | 65 | ||||
| -rw-r--r-- | internal/version/version.go | 2 |
10 files changed, 341 insertions, 11 deletions
@@ -20,9 +20,12 @@ GitSyncer is a tool for synchronizing git repositories between multiple organiza - Sync all public repositories from Codeberg to GitHub - Sync all public repositories from GitHub to Codeberg - Automatic repository creation on GitHub and Codeberg +- SSH backup locations with automatic bare repository creation +- One-way backup to private SSH servers (e.g., home NAS) - Merge conflict detection with clear error messages - Never deletes branches (only adds/updates) - GitHub token validation tool +- Opt-in backup mode with --backup flag for resilient offline backups ## Installation @@ -44,6 +47,10 @@ Create a `gitsyncer.json` file: { "host": "git@github.com", "name": "yourusername" + }, + { + "host": "user@nas.local:git", + "backupLocation": true } ], "repositories": [ @@ -58,11 +65,17 @@ Create a `gitsyncer.json` file: ### Sync a single repository ```bash ./gitsyncer --sync repo-name + +# Include backup locations +./gitsyncer --sync repo-name --backup ``` ### Sync all configured repositories ```bash ./gitsyncer --sync-all + +# Include backup locations +./gitsyncer --sync-all --backup ``` ### Sync all public Codeberg repositories to GitHub @@ -112,6 +125,24 @@ Create a `gitsyncer.json` file: ./gitsyncer --version ``` +### The --backup Flag + +The `--backup` flag enables syncing to backup locations configured in your `gitsyncer.json`. This is particularly useful when: +- Your backup server might be offline (e.g., home NAS) +- You want to control when backups happen +- You need to separate regular syncing from backup operations + +Without `--backup`: GitSyncer only syncs between primary git hosts (GitHub, Codeberg, etc.) +With `--backup`: GitSyncer also pushes to backup locations marked with `"backupLocation": true` + +```bash +# Regular sync (backup locations ignored) +./gitsyncer --sync myrepo + +# Sync with backup enabled +./gitsyncer --sync myrepo --backup +``` + ## How It Works 1. GitSyncer clones the repository from the first configured organization @@ -140,6 +171,62 @@ You can exclude branches from synchronization using regex patterns in your confi Excluded branches will be reported during sync but not synchronized. +## SSH Backup Locations + +You can configure SSH backup locations for one-way repository backups to private servers: + +```json +{ + "organizations": [ + { + "host": "git@github.com", + "name": "yourusername" + }, + { + "host": "paul@t450:git", + "backupLocation": true + } + ] +} +``` + +### How SSH Backup Works + +1. **Opt-in feature**: Backup locations are disabled by default. Use the `--backup` flag to enable syncing to them +2. **One-way sync**: Repositories are only pushed TO backup locations, never pulled FROM them +3. **Automatic repository creation**: If a repository doesn't exist on the SSH server, GitSyncer will: + - SSH into the server + - Create the directory structure + - Initialize a bare git repository +4. **Archive functionality**: Repositories that exist only on the backup location are considered archived and won't be synced to other organizations +5. **All branches and tags**: Every branch and tag is pushed to the backup location when `--backup` is used + +### SSH Backup Example + +```bash +# Configure your gitsyncer.json with an SSH backup location +# Backup locations are DISABLED by default to handle offline servers + +# Sync without backup (default behavior) +./gitsyncer --sync myrepo + +# Sync WITH backup enabled +./gitsyncer --sync myrepo --backup + +# Sync all repositories with backup +./gitsyncer --sync-all --backup + +# Full sync with backup +./gitsyncer --full --backup +``` + +The backup location path format is: `user@host:path/REPONAME.git` +- `user@host`: SSH connection string +- `path`: Base directory for repositories +- `REPONAME.git`: Automatically appended repository name + +**Note**: The `--backup` flag is required to sync to backup locations. This allows GitSyncer to work normally even when backup servers are offline or unreachable. + ## Example Workflows ### Sync specific repositories diff --git a/gitsyncer.example.json b/gitsyncer.example.json index 9116f35..abe3d78 100644 --- a/gitsyncer.example.json +++ b/gitsyncer.example.json @@ -7,6 +7,10 @@ { "host": "git@github.com", "name": "snonux" + }, + { + "host": "paul@t450:git", + "backupLocation": true } ], "repositories": [ diff --git a/internal/cli/flags.go b/internal/cli/flags.go index 71708a2..322ad5a 100644 --- a/internal/cli/flags.go +++ b/internal/cli/flags.go @@ -24,6 +24,7 @@ type Flags struct { TestGitHubToken bool Clean bool DeleteRepo string + Backup bool } // ParseFlags parses command-line flags and returns the flags struct @@ -48,6 +49,7 @@ func ParseFlags() *Flags { flag.BoolVar(&f.TestGitHubToken, "test-github-token", false, "test GitHub token authentication") flag.BoolVar(&f.Clean, "clean", false, "delete all repositories in work directory (with confirmation)") flag.StringVar(&f.DeleteRepo, "delete-repo", "", "delete specified repository from all configured organizations (with confirmation)") + flag.BoolVar(&f.Backup, "backup", false, "enable syncing to backup locations") flag.Parse() diff --git a/internal/cli/sync_handlers.go b/internal/cli/sync_handlers.go index c1fefb6..619b01e 100644 --- a/internal/cli/sync_handlers.go +++ b/internal/cli/sync_handlers.go @@ -22,6 +22,7 @@ func HandleSync(cfg *config.Config, flags *Flags) int { } syncer := sync.New(cfg, flags.WorkDir) + syncer.SetBackupEnabled(flags.Backup) if err := syncer.SyncRepository(flags.SyncRepo); err != nil { log.Fatal("Sync failed:", err) return 1 @@ -47,6 +48,7 @@ func HandleSyncAll(cfg *config.Config, flags *Flags) int { } syncer := sync.New(cfg, flags.WorkDir) + syncer.SetBackupEnabled(flags.Backup) successCount := 0 for i, repo := range cfg.Repositories { @@ -264,6 +266,7 @@ func syncCodebergRepos(cfg *config.Config, flags *Flags, repos []codeberg.Reposi fmt.Printf("\nStarting sync of %d repositories...\n", len(repoNames)) syncer := sync.New(cfg, flags.WorkDir) + syncer.SetBackupEnabled(flags.Backup) successCount := 0 // Create map for descriptions @@ -329,6 +332,7 @@ func syncGitHubRepos(cfg *config.Config, flags *Flags, repos []github.Repository fmt.Printf("\nStarting sync of %d repositories...\n", len(repoNames)) syncer := sync.New(cfg, flags.WorkDir) + syncer.SetBackupEnabled(flags.Backup) successCount := 0 // Create map for descriptions diff --git a/internal/config/config.go b/internal/config/config.go index ef8e3b6..7dedee7 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -10,10 +10,11 @@ import ( // Organization represents a git organization with its host and name type Organization struct { - Host string `json:"host"` - Name string `json:"name"` - GitHubToken string `json:"github_token,omitempty"` - CodebergToken string `json:"codeberg_token,omitempty"` + Host string `json:"host"` + Name string `json:"name"` + GitHubToken string `json:"github_token,omitempty"` + CodebergToken string `json:"codeberg_token,omitempty"` + BackupLocation bool `json:"backupLocation,omitempty"` // Mark this as a backup-only destination } // Config holds the application configuration @@ -83,8 +84,8 @@ func (c *Config) Validate() error { if org.Host == "" { return fmt.Errorf("organization %d: missing host", i) } - // Name can be empty for file:// URLs - if org.Name == "" && !strings.HasPrefix(org.Host, "file://") { + // Name can be empty for file:// URLs or SSH backup locations + if org.Name == "" && !strings.HasPrefix(org.Host, "file://") && !org.IsSSH() { return fmt.Errorf("organization %d: missing name", i) } } @@ -94,6 +95,10 @@ func (c *Config) Validate() error { // GetGitURL returns the git URL for an organization func (o *Organization) GetGitURL() string { + // For SSH backup locations with empty name, just return the host + if o.IsSSH() && o.Name == "" { + return o.Host + } return fmt.Sprintf("%s:%s", o.Host, o.Name) } @@ -137,3 +142,10 @@ func (c *Config) FindGitHubOrg() *Organization { return nil } +// IsSSH checks if the organization is a plain SSH location +func (o *Organization) IsSSH() bool { + // Check if it's not a known git hosting service and contains SSH-like syntax + return !o.IsGitHub() && !o.IsCodeberg() && !strings.HasPrefix(o.Host, "file://") && + (strings.Contains(o.Host, "@") || strings.Contains(o.Host, ":")) +} + diff --git a/internal/sync/branch_sync.go b/internal/sync/branch_sync.go index 1bf8b79..02f8964 100644 --- a/internal/sync/branch_sync.go +++ b/internal/sync/branch_sync.go @@ -10,7 +10,11 @@ import ( func (s *Syncer) trackRemotesWithBranch(branch string, remotes map[string]*config.Organization) map[string]bool { remotesWithBranch := make(map[string]bool) - for remoteName := range remotes { + for remoteName, org := range remotes { + // Skip checking backup locations as we don't sync from them + if org.BackupLocation { + continue + } if s.remoteBranchExists(remoteName, branch) { remotesWithBranch[remoteName] = true } @@ -48,7 +52,7 @@ func pushToAllRemotes(branch string, remotes map[string]*config.Organization, re fmt.Printf(" Pushing to %s (%s)...\n", remoteName, org.Host) } - if err := pushBranch(remoteName, branch, remoteHasBranch); err != nil { + if err := pushBranchWithBackupSupport(remoteName, branch, remoteHasBranch, org); err != nil { return err } } diff --git a/internal/sync/git_operations.go b/internal/sync/git_operations.go index 7664113..3bd6618 100644 --- a/internal/sync/git_operations.go +++ b/internal/sync/git_operations.go @@ -7,6 +7,8 @@ import ( "os/exec" "regexp" "strings" + + "codeberg.org/snonux/gitsyncer/internal/config" ) // checkForMergeConflicts checks if the repository has merge conflicts @@ -233,3 +235,118 @@ func getAllUniqueBranches(output []byte) []string { return branches } + +// createSSHBareRepository creates a bare repository on an SSH server +func createSSHBareRepository(sshHost, repoPath string) error { + // Extract user@host and path components + parts := strings.Split(sshHost, ":") + if len(parts) != 2 { + return fmt.Errorf("invalid SSH host format: %s", sshHost) + } + + userHost := parts[0] + basePath := parts[1] + + // Full path to the repository + fullRepoPath := fmt.Sprintf("%s/%s.git", basePath, repoPath) + + fmt.Printf("Creating bare repository at %s:%s\n", userHost, fullRepoPath) + + // Create the repository directory and initialize as bare + commands := fmt.Sprintf("mkdir -p %s && cd %s && git init --bare", fullRepoPath, fullRepoPath) + cmd := exec.Command("ssh", userHost, commands) + output, err := cmd.CombinedOutput() + + if err != nil { + return fmt.Errorf("failed to create bare repository: %w\n%s", err, string(output)) + } + + fmt.Printf("Successfully created bare repository at %s:%s\n", userHost, fullRepoPath) + return nil +} + +// pushBranchWithBackupSupport pushes a branch to a remote, creating SSH repos if needed +func pushBranchWithBackupSupport(remoteName, branch string, remoteHasBranch bool, org *config.Organization) error { + cmd := exec.Command("git", "push", remoteName, branch, "--tags") + output, err := cmd.CombinedOutput() + + if err != nil { + outputStr := string(output) + // Check if it's because the repository doesn't exist + if isRepositoryMissing(outputStr) { + // If it's an SSH backup location, try to create the repository + if org.BackupLocation && org.IsSSH() { + // Get the repository name from the remote URL + remoteURL, err := getRemoteURL(remoteName) + if err != nil { + return fmt.Errorf("failed to get remote URL: %w", err) + } + + // Extract repo name from URL + repoName := extractRepoName(remoteURL) + if repoName == "" { + return fmt.Errorf("failed to extract repository name from URL: %s", remoteURL) + } + + // Create the bare repository + if err := createSSHBareRepository(org.Host, repoName); err != nil { + return fmt.Errorf("failed to create SSH repository: %w", err) + } + + // Try pushing again + cmd = exec.Command("git", "push", remoteName, branch, "--tags") + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to push after creating repository: %w", err) + } + fmt.Printf(" Successfully pushed to newly created backup repository\n") + return nil + } + + fmt.Printf(" Note: Remote repository %s does not exist - must be created manually\n", remoteName) + fmt.Printf(" Skipping push to %s\n", remoteName) + return nil // Not an error, just skip + } + + // Check if it's because the branch doesn't exist on the remote + if isBranchMissing(outputStr) { + fmt.Printf(" Creating new branch on %s\n", remoteName) + // Try again with -u flag to set upstream + cmd = exec.Command("git", "push", "-u", remoteName, branch, "--tags") + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to push to %s: %w", remoteName, err) + } + return nil + } + + return fmt.Errorf("failed to push to %s: %w\n%s", remoteName, err, outputStr) + } + + if !remoteHasBranch { + fmt.Printf(" Successfully created branch %s on %s\n", branch, remoteName) + } + + return nil +} + +// getRemoteURL gets the URL for a given remote +func getRemoteURL(remoteName string) (string, error) { + cmd := exec.Command("git", "remote", "get-url", remoteName) + output, err := cmd.Output() + if err != nil { + return "", err + } + return strings.TrimSpace(string(output)), nil +} + +// extractRepoName extracts the repository name from a git URL +func extractRepoName(url string) string { + // Remove .git suffix if present + url = strings.TrimSuffix(url, ".git") + + // Extract the last component of the path + parts := strings.Split(url, "/") + if len(parts) > 0 { + return parts[len(parts)-1] + } + return "" +} diff --git a/internal/sync/repository_setup.go b/internal/sync/repository_setup.go index 15d436e..3ebafbd 100644 --- a/internal/sync/repository_setup.go +++ b/internal/sync/repository_setup.go @@ -22,7 +22,21 @@ func (s *Syncer) setupNewRepository(repoPath string) error { return fmt.Errorf("no organizations configured") } - firstOrg := &s.config.Organizations[0] + // Find first non-backup organization to clone from + var firstOrg *config.Organization + var firstOrgIndex int + for i := range s.config.Organizations { + if !s.config.Organizations[i].BackupLocation { + firstOrg = &s.config.Organizations[i] + firstOrgIndex = i + break + } + } + + if firstOrg == nil { + return fmt.Errorf("no non-backup organizations configured to clone from") + } + if err := s.cloneRepository(firstOrg, repoPath); err != nil { return fmt.Errorf("failed to clone repository: %w", err) } @@ -35,8 +49,17 @@ func (s *Syncer) setupNewRepository(repoPath string) error { } // Add other organizations as remotes - for i := 1; i < len(s.config.Organizations); i++ { + for i := range s.config.Organizations { + if i == firstOrgIndex { + continue // Skip the first org we already cloned from + } org := &s.config.Organizations[i] + + // Skip backup locations if backup is not enabled + if org.BackupLocation && !s.backupEnabled { + continue + } + if err := s.addRemote(repoPath, org); err != nil { return fmt.Errorf("failed to add remote %s: %w", s.getRemoteName(org), err) } @@ -52,6 +75,12 @@ func (s *Syncer) setupExistingRepository(repoPath string) error { // Check and add any missing remotes for i := range s.config.Organizations { org := &s.config.Organizations[i] + + // Skip backup locations if backup is not enabled + if org.BackupLocation && !s.backupEnabled { + continue + } + remoteName := s.getRemoteName(org) // Check if remote exists @@ -86,6 +115,12 @@ func (s *Syncer) getRemotesMap() map[string]*config.Organization { remotes := make(map[string]*config.Organization) for i := range s.config.Organizations { org := &s.config.Organizations[i] + + // Skip backup locations if backup is not enabled + if org.BackupLocation && !s.backupEnabled { + continue + } + remoteName := s.getRemoteName(org) remotes[remoteName] = org } diff --git a/internal/sync/sync.go b/internal/sync/sync.go index b413318..af0feba 100644 --- a/internal/sync/sync.go +++ b/internal/sync/sync.go @@ -17,6 +17,7 @@ type Syncer struct { repoName string abandonedReports map[string]*AbandonedBranchReport // Collects reports across repos branchFilter *BranchFilter // Filter for excluding branches + backupEnabled bool // Whether to sync to backup locations } // CLAUDE: Is there a reason, we return a pointer to Syncer? @@ -35,9 +36,15 @@ func New(cfg *config.Config, workDir string) *Syncer { workDir: workDir, abandonedReports: make(map[string]*AbandonedBranchReport), branchFilter: branchFilter, + backupEnabled: false, // Default to false, will be set via SetBackupEnabled } } +// SetBackupEnabled enables or disables syncing to backup locations +func (s *Syncer) SetBackupEnabled(enabled bool) { + s.backupEnabled = enabled +} + // SyncRepository synchronizes a repository across all configured organizations func (s *Syncer) SyncRepository(repoName string) error { s.repoName = repoName @@ -109,11 +116,19 @@ func (s *Syncer) SyncRepository(repoName string) error { // cloneRepository clones a repository from an organization func (s *Syncer) cloneRepository(org *config.Organization, repoPath string) error { + // Skip cloning from backup locations + if org.BackupLocation { + return fmt.Errorf("cannot clone from backup location %s", org.Host) + } + // For file:// URLs, we need special handling var cloneURL string if strings.HasPrefix(org.Host, "file://") { // For local file paths, the format is: file:///path/to/repo.git cloneURL = fmt.Sprintf("%s/%s.git", org.Host, s.repoName) + } else if org.IsSSH() && org.Name == "" { + // For SSH backup locations: user@host:path/repo.git + cloneURL = fmt.Sprintf("%s/%s.git", org.Host, s.repoName) } else { // For SSH URLs, the format is: git@host:org/repo.git cloneURL = fmt.Sprintf("%s/%s.git", org.GetGitURL(), s.repoName) @@ -140,6 +155,9 @@ func (s *Syncer) addRemote(repoPath string, org *config.Organization) error { var remoteURL string if strings.HasPrefix(org.Host, "file://") { remoteURL = fmt.Sprintf("%s/%s.git", org.Host, s.repoName) + } else if org.IsSSH() && org.Name == "" { + // For SSH backup locations: user@host:path/repo.git + remoteURL = fmt.Sprintf("%s/%s.git", org.Host, s.repoName) } else { remoteURL = fmt.Sprintf("%s/%s.git", org.GetGitURL(), s.repoName) } @@ -163,8 +181,17 @@ func (s *Syncer) fetchAll() error { return err } + // Get remotes map to check if it's a backup location + remotesMap := s.getRemotesMap() + // Fetch from each remote for remote := range remotes { + // Skip backup locations - we don't fetch from them + if org, exists := remotesMap[remote]; exists && org.BackupLocation { + fmt.Printf("Skipping fetch from backup location %s\n", remote) + continue + } + if err := fetchRemote(remote); err != nil { return err } @@ -181,6 +208,12 @@ func (s *Syncer) getAllBranches() ([]string, error) { return nil, err } + // If backup is disabled, filter out branches from backup locations + if !s.backupEnabled { + filteredOutput := s.filterBackupBranches(output) + return getAllUniqueBranches(filteredOutput), nil + } + return getAllUniqueBranches(output), nil } @@ -287,3 +320,35 @@ func (s *Syncer) getRemoteName(org *config.Organization) string { return host } + +// filterBackupBranches filters out branches from backup locations +func (s *Syncer) filterBackupBranches(output []byte) []byte { + lines := strings.Split(string(output), "\n") + var filtered []string + + for _, line := range lines { + line = strings.TrimSpace(line) + if line == "" { + continue + } + + // Check if this branch is from a backup remote + isBackup := false + for i := range s.config.Organizations { + org := &s.config.Organizations[i] + if org.BackupLocation { + remoteName := s.getRemoteName(org) + if strings.HasPrefix(line, remoteName+"/") { + isBackup = true + break + } + } + } + + if !isBackup { + filtered = append(filtered, line) + } + } + + return []byte(strings.Join(filtered, "\n")) +} diff --git a/internal/version/version.go b/internal/version/version.go index 79661ce..fe2cd61 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -7,7 +7,7 @@ import ( var ( // Version is the current version of gitsyncer - Version = "0.1.0" + Version = "0.2.0" // GitCommit is the git commit hash at build time GitCommit = "unknown" |
