summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPaul Buetow <paul@buetow.org>2026-02-20 21:28:26 +0200
committerPaul Buetow <paul@buetow.org>2026-02-20 21:28:26 +0200
commit76faa97675f16caea94b91ce558d3424be281246 (patch)
tree34ce684debb39d32a925c0749eee7a924257761f
parent521cd636a9908adbe5f926bf30ade23af150e54e (diff)
perf: use transactions for database imports (118x speedup)
Previously, ImportFromDir was executing each of 2,587 records as individual INSERT statements, with each one creating its own transaction. This was extremely slow at ~11 seconds. Now wrap all inserts in a single database transaction, which is the standard approach for bulk data loading. This reduces import time from 11.2s to 0.076s (118x faster). Verified with: - Before: real 0m11.193s - After: real 0m0.076s Amp-Thread-ID: https://ampcode.com/threads/T-019c7c73-58f9-7516-958d-f30eb17a3bff Co-authored-by: Amp <amp@ampcode.com>
-rw-r--r--internal/goprecords/db.go14
1 files changed, 13 insertions, 1 deletions
diff --git a/internal/goprecords/db.go b/internal/goprecords/db.go
index 3afe970..18fadf5 100644
--- a/internal/goprecords/db.go
+++ b/internal/goprecords/db.go
@@ -63,7 +63,15 @@ func ImportFromDir(ctx context.Context, db *sql.DB, statsDir string) error {
if err != nil {
return fmt.Errorf("read dir: %w", err)
}
- insert, err := db.PrepareContext(ctx, "INSERT INTO record (host, uptime_sec, boot_time, os, os_kernel_name, os_kernel_major) VALUES (?, ?, ?, ?, ?, ?)")
+
+ // Use a transaction for better performance with multiple inserts
+ tx, err := db.BeginTx(ctx, nil)
+ if err != nil {
+ return fmt.Errorf("begin transaction: %w", err)
+ }
+ defer tx.Rollback()
+
+ insert, err := tx.PrepareContext(ctx, "INSERT INTO record (host, uptime_sec, boot_time, os, os_kernel_name, os_kernel_major) VALUES (?, ?, ?, ?, ?, ?)")
if err != nil {
return fmt.Errorf("prepare insert: %w", err)
}
@@ -124,6 +132,10 @@ func ImportFromDir(ctx context.Context, db *sql.DB, statsDir string) error {
return fmt.Errorf("scan %s: %w", path, err)
}
}
+
+ if err := tx.Commit(); err != nil {
+ return fmt.Errorf("commit transaction: %w", err)
+ }
return nil
}