Initial commit: email-openclaw bridge v1

Full pipeline: IMAP ingress -> OpenClaw dispatch -> callback -> SMTP reply.
SQLite stateful storage with idempotency, threading, and retry logic.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
2026-04-27 18:16:57 +07:00
co-authored by Claude Opus 4.7
commit 8815c20c13
35 changed files with 2808 additions and 0 deletions
+188
View File
@@ -0,0 +1,188 @@
package ai_client
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"time"
"thuanle.me/claw-email-bridge/internal/config"
"thuanle.me/claw-email-bridge/internal/database"
"thuanle.me/claw-email-bridge/internal/logging"
"gorm.io/gorm"
)
// Retry backoff schedule: 1s, 5s, 15s.
var retryBackoffs = []time.Duration{1 * time.Second, 5 * time.Second, 15 * time.Second}
// openClawRequest is the payload sent to OpenClaw API.
type openClawRequest struct {
Input string `json:"input"`
SessionID string `json:"session_id"`
History []historyEntry `json:"history,omitempty"`
CallbackURL string `json:"callback_url"`
Metadata map[string]string `json:"metadata"`
}
type historyEntry struct {
Role string `json:"role"`
Content string `json:"content"`
}
// Dispatcher sends tasks to OpenClaw and handles retry logic.
type Dispatcher struct {
cfg *config.Config
db *gorm.DB
httpClient *http.Client
}
// NewDispatcher creates a dispatcher with a direct HTTP client (no system proxy).
func NewDispatcher(cfg *config.Config, db *gorm.DB) *Dispatcher {
return &Dispatcher{
cfg: cfg,
db: db,
httpClient: &http.Client{
Timeout: 30 * time.Second,
Transport: &http.Transport{
Proxy: nil, // Direct connection, no proxy inheritance.
},
},
}
}
// Dispatch sends a task to OpenClaw with retry.
// This should be called asynchronously (go dispatcher.Dispatch(task)).
func (d *Dispatcher) Dispatch(task *database.Task) {
log := logging.TaskLogger(task.TaskUUID, task.ThreadID, task.MessageID)
// Build history from completed tasks in the same thread.
history, err := d.buildHistory(task.ThreadID)
if err != nil {
log.Error("openclaw: failed to build history", "error", err)
// Continue without history — non-fatal.
}
// Build callback URL.
callbackURL := fmt.Sprintf("http://%s/callback", d.cfg.ListenAddr)
payload := openClawRequest{
Input: task.BodyPlain,
SessionID: task.ThreadID,
History: history,
CallbackURL: callbackURL,
Metadata: map[string]string{"task_uuid": task.TaskUUID},
}
// Update status to AI_PROCESSING before first attempt.
task.Status = database.StatusAIProcessing
if err := database.UpdateTask(d.db, task); err != nil {
log.Error("openclaw: failed to update status to AI_PROCESSING", "error", err)
return
}
// Retry loop: up to 3 attempts.
for attempt := 1; attempt <= len(retryBackoffs); attempt++ {
task.AttemptOpenClaw = attempt
log.Info("openclaw: dispatching",
"attempt", attempt,
"status", task.Status,
)
err := d.callAPI(payload)
if err == nil {
// Success — task stays AI_PROCESSING, waiting for callback.
log.Info("openclaw: dispatch successful", "attempt", attempt)
if err := database.UpdateTask(d.db, task); err != nil {
log.Error("openclaw: failed to save attempt count", "error", err)
}
return
}
log.Warn("openclaw: dispatch failed",
"attempt", attempt,
"error", err,
)
// Last attempt — mark as FAILED.
if attempt == len(retryBackoffs) {
errMsg := err.Error()
task.Status = database.StatusFailed
task.LastError = &errMsg
if err := database.UpdateTask(d.db, task); err != nil {
log.Error("openclaw: failed to save FAILED status", "error", err)
}
log.Error("openclaw: all retries exhausted",
"attempt", attempt,
"status", database.StatusFailed,
"error", errMsg,
)
return
}
// Wait before retry.
backoff := retryBackoffs[attempt-1]
log.Info("openclaw: retrying after backoff", "backoff", backoff)
time.Sleep(backoff)
}
}
// callAPI makes a single HTTP POST to OpenClaw.
func (d *Dispatcher) callAPI(payload openClawRequest) error {
body, err := json.Marshal(payload)
if err != nil {
return fmt.Errorf("marshal payload: %w", err)
}
req, err := http.NewRequest(http.MethodPost, d.cfg.OpenClawURL, bytes.NewReader(body))
if err != nil {
return fmt.Errorf("create request: %w", err)
}
req.Header.Set("Content-Type", "application/json")
if d.cfg.OpenClawAPIKey != "" {
req.Header.Set("Authorization", "Bearer "+d.cfg.OpenClawAPIKey)
}
resp, err := d.httpClient.Do(req)
if err != nil {
return fmt.Errorf("http request: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode >= 200 && resp.StatusCode < 300 {
return nil
}
// Read response body for error details (limit to 1KB).
respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 1024))
return fmt.Errorf("openclaw returned %d: %s", resp.StatusCode, string(respBody))
}
// buildHistory fetches completed tasks in the thread and formats them as history entries.
func (d *Dispatcher) buildHistory(threadID string) ([]historyEntry, error) {
tasks, err := database.ThreadHistory(d.db, threadID)
if err != nil {
return nil, err
}
var history []historyEntry
for _, t := range tasks {
// User message.
history = append(history, historyEntry{
Role: "user",
Content: t.BodyPlain,
})
// AI response.
if t.AIResponse != "" {
history = append(history, historyEntry{
Role: "assistant",
Content: t.AIResponse,
})
}
}
return history, nil
}
+114
View File
@@ -0,0 +1,114 @@
package ai_client_test
import (
"net/http"
"net/http/httptest"
"sync/atomic"
"testing"
"thuanle.me/claw-email-bridge/internal/ai_client"
"thuanle.me/claw-email-bridge/internal/config"
"thuanle.me/claw-email-bridge/internal/database"
)
// Test matrix #2: OpenClaw fail 2 lần, lần 3 thành công → COMPLETED, attempt_openclaw = 3.
func TestDispatch_RetryThenSuccess(t *testing.T) {
tdb := database.NewTestDB(t)
var callCount atomic.Int32
mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
n := callCount.Add(1)
if n <= 2 {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte("temporary error"))
return
}
w.WriteHeader(http.StatusOK)
w.Write([]byte(`{"status":"ok"}`))
}))
defer mockServer.Close()
cfg := &config.Config{
OpenClawURL: mockServer.URL,
ListenAddr: ":9999",
}
task := &database.Task{
TaskUUID: "uuid-retry-test",
ThreadID: "thread-1",
MessageID: "msg-retry@test.com",
Sender: "user@test.com",
Subject: "Retry test",
BodyPlain: "Hello",
Status: database.StatusReceived,
}
if err := database.CreateTask(tdb.DB, task); err != nil {
t.Fatalf("create task: %v", err)
}
dispatcher := ai_client.NewDispatcher(cfg, tdb.DB)
dispatcher.Dispatch(task) // Synchronous call for testing.
// Verify 3 attempts were made.
if callCount.Load() != 3 {
t.Errorf("expected 3 API calls, got %d", callCount.Load())
}
// Reload task from DB.
updated, err := database.FindByTaskUUID(tdb.DB, "uuid-retry-test")
if err != nil {
t.Fatalf("find task: %v", err)
}
if updated.Status != database.StatusAIProcessing {
t.Errorf("expected AI_PROCESSING (waiting for callback), got %s", updated.Status)
}
if updated.AttemptOpenClaw != 3 {
t.Errorf("expected attempt_openclaw = 3, got %d", updated.AttemptOpenClaw)
}
}
// OpenClaw fail all 3 retries → FAILED with last_error.
func TestDispatch_AllRetriesFailed(t *testing.T) {
tdb := database.NewTestDB(t)
mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusInternalServerError)
w.Write([]byte("server down"))
}))
defer mockServer.Close()
cfg := &config.Config{
OpenClawURL: mockServer.URL,
ListenAddr: ":9999",
}
task := &database.Task{
TaskUUID: "uuid-fail-all",
ThreadID: "thread-1",
MessageID: "msg-fail@test.com",
Sender: "user@test.com",
Subject: "Fail test",
BodyPlain: "Hello",
Status: database.StatusReceived,
}
if err := database.CreateTask(tdb.DB, task); err != nil {
t.Fatalf("create task: %v", err)
}
dispatcher := ai_client.NewDispatcher(cfg, tdb.DB)
dispatcher.Dispatch(task)
updated, err := database.FindByTaskUUID(tdb.DB, "uuid-fail-all")
if err != nil {
t.Fatalf("find task: %v", err)
}
if updated.Status != database.StatusFailed {
t.Errorf("expected FAILED, got %s", updated.Status)
}
if updated.LastError == nil {
t.Error("expected last_error to be set")
}
if updated.AttemptOpenClaw != 3 {
t.Errorf("expected attempt_openclaw = 3, got %d", updated.AttemptOpenClaw)
}
}
+137
View File
@@ -0,0 +1,137 @@
package api
import (
"log/slog"
"net/http"
"github.com/gin-gonic/gin"
"thuanle.me/claw-email-bridge/internal/database"
"thuanle.me/claw-email-bridge/internal/logging"
"gorm.io/gorm"
)
// CallbackRequest is the payload received from OpenClaw.
type CallbackRequest struct {
Metadata struct {
TaskUUID string `json:"task_uuid" binding:"required"`
} `json:"metadata" binding:"required"`
Result string `json:"result" binding:"required"`
Status string `json:"status"`
}
// OnCallbackDone is called after a task transitions to CALLBACK_DONE.
// This will be wired to SMTP egress.
type OnCallbackDone func(task *database.Task)
// NewRouter creates the Gin engine with health and callback routes.
func NewRouter(db *gorm.DB, callbackToken string, onDone OnCallbackDone) *gin.Engine {
gin.SetMode(gin.ReleaseMode)
r := gin.New()
r.Use(gin.Recovery())
r.GET("/healthz", func(c *gin.Context) {
c.JSON(http.StatusOK, gin.H{"status": "ok"})
})
r.GET("/readyz", readyzHandler(db, callbackToken))
r.POST("/callback", tokenAuthMiddleware(callbackToken), callbackHandler(db, onDone))
return r
}
// tokenAuthMiddleware validates X-Bridge-Token header.
func tokenAuthMiddleware(expected string) gin.HandlerFunc {
return func(c *gin.Context) {
token := c.GetHeader("X-Bridge-Token")
if token == "" || token != expected {
slog.Warn("callback: unauthorized request",
"remote_addr", c.ClientIP(),
"has_token", token != "",
)
c.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{
"error": "unauthorized",
})
return
}
c.Next()
}
}
// callbackHandler processes the callback from OpenClaw.
func callbackHandler(db *gorm.DB, onDone OnCallbackDone) gin.HandlerFunc {
return func(c *gin.Context) {
var req CallbackRequest
if err := c.ShouldBindJSON(&req); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"error": "invalid payload: " + err.Error()})
return
}
taskUUID := req.Metadata.TaskUUID
log := logging.TaskLogger(taskUUID, "", "")
// Find task.
task, err := database.FindByTaskUUID(db, taskUUID)
if err != nil {
log.Error("callback: db lookup failed", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "internal error"})
return
}
if task == nil {
log.Warn("callback: task not found")
c.JSON(http.StatusNotFound, gin.H{"error": "task not found"})
return
}
log = logging.TaskLogger(task.TaskUUID, task.ThreadID, task.MessageID)
// Idempotent: if already COMPLETED, return 200 without changes.
if task.Status == database.StatusCompleted {
log.Info("callback: task already completed, idempotent response")
c.JSON(http.StatusOK, gin.H{"status": "already_completed"})
return
}
// Update task with AI response.
task.AIResponse = req.Result
task.Status = database.StatusCallbackDone
if err := database.UpdateTask(db, task); err != nil {
log.Error("callback: failed to update task", "error", err)
c.JSON(http.StatusInternalServerError, gin.H{"error": "internal error"})
return
}
log.Info("callback: task updated",
"status", database.StatusCallbackDone,
)
c.JSON(http.StatusOK, gin.H{"status": "ok"})
// Trigger SMTP egress (async, after response is sent).
if onDone != nil {
go onDone(task)
}
}
}
// readyzHandler checks DB writable + required config present.
func readyzHandler(db *gorm.DB, callbackToken string) gin.HandlerFunc {
return func(c *gin.Context) {
if err := database.Healthy(db); err != nil {
c.JSON(http.StatusServiceUnavailable, gin.H{
"status": "error",
"detail": "database not writable",
})
return
}
if callbackToken == "" {
c.JSON(http.StatusServiceUnavailable, gin.H{
"status": "error",
"detail": "missing BRIDGE_CALLBACK_TOKEN",
})
return
}
c.JSON(http.StatusOK, gin.H{"status": "ok"})
}
}
+198
View File
@@ -0,0 +1,198 @@
package api_test
import (
"bytes"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"thuanle.me/claw-email-bridge/internal/api"
"thuanle.me/claw-email-bridge/internal/database"
)
const testToken = "test-secret-token"
func setupTestRouter(t *testing.T) (*database.TestDB, *httptest.Server) {
t.Helper()
tdb := database.NewTestDB(t)
router := api.NewRouter(tdb.DB, testToken, nil)
srv := httptest.NewServer(router)
t.Cleanup(func() { srv.Close() })
return tdb, srv
}
// Test matrix #4: Callback sai token → 401, state không đổi.
func TestCallback_WrongToken_Returns401(t *testing.T) {
tdb, srv := setupTestRouter(t)
// Create a task in AI_PROCESSING state.
task := &database.Task{
TaskUUID: "uuid-wrong-token",
ThreadID: "thread-1",
MessageID: "msg-1",
Sender: "user@test.com",
Subject: "Test",
Status: database.StatusAIProcessing,
}
if err := database.CreateTask(tdb.DB, task); err != nil {
t.Fatalf("create task: %v", err)
}
payload := map[string]any{
"metadata": map[string]string{"task_uuid": "uuid-wrong-token"},
"result": "AI response",
"status": "ok",
}
body, _ := json.Marshal(payload)
// No token.
resp, err := http.Post(srv.URL+"/callback", "application/json", bytes.NewReader(body))
if err != nil {
t.Fatalf("request failed: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusUnauthorized {
t.Errorf("expected 401, got %d", resp.StatusCode)
}
// Wrong token.
req, _ := http.NewRequest("POST", srv.URL+"/callback", bytes.NewReader(body))
req.Header.Set("Content-Type", "application/json")
req.Header.Set("X-Bridge-Token", "wrong-token")
resp2, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("request failed: %v", err)
}
defer resp2.Body.Close()
if resp2.StatusCode != http.StatusUnauthorized {
t.Errorf("expected 401, got %d", resp2.StatusCode)
}
// Verify state unchanged.
updated, err := database.FindByTaskUUID(tdb.DB, "uuid-wrong-token")
if err != nil {
t.Fatalf("find task: %v", err)
}
if updated.Status != database.StatusAIProcessing {
t.Errorf("expected status AI_PROCESSING, got %s", updated.Status)
}
if updated.AIResponse != "" {
t.Errorf("expected empty ai_response, got %q", updated.AIResponse)
}
}
// Test matrix #1 (partial): callback hợp lệ → CALLBACK_DONE.
func TestCallback_ValidToken_UpdatesTask(t *testing.T) {
tdb, srv := setupTestRouter(t)
task := &database.Task{
TaskUUID: "uuid-valid",
ThreadID: "thread-1",
MessageID: "msg-1",
Sender: "user@test.com",
Subject: "Test",
Status: database.StatusAIProcessing,
}
if err := database.CreateTask(tdb.DB, task); err != nil {
t.Fatalf("create task: %v", err)
}
payload := map[string]any{
"metadata": map[string]string{"task_uuid": "uuid-valid"},
"result": "Hello from AI",
"status": "ok",
}
body, _ := json.Marshal(payload)
req, _ := http.NewRequest("POST", srv.URL+"/callback", bytes.NewReader(body))
req.Header.Set("Content-Type", "application/json")
req.Header.Set("X-Bridge-Token", testToken)
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("request failed: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Errorf("expected 200, got %d", resp.StatusCode)
}
updated, _ := database.FindByTaskUUID(tdb.DB, "uuid-valid")
if updated.Status != database.StatusCallbackDone {
t.Errorf("expected CALLBACK_DONE, got %s", updated.Status)
}
if updated.AIResponse != "Hello from AI" {
t.Errorf("expected 'Hello from AI', got %q", updated.AIResponse)
}
}
// Idempotent callback: task already COMPLETED → 200, no changes.
func TestCallback_AlreadyCompleted_Idempotent(t *testing.T) {
tdb, srv := setupTestRouter(t)
task := &database.Task{
TaskUUID: "uuid-completed",
ThreadID: "thread-1",
MessageID: "msg-1",
Sender: "user@test.com",
Subject: "Test",
Status: database.StatusCompleted,
AIResponse: "Original response",
}
if err := database.CreateTask(tdb.DB, task); err != nil {
t.Fatalf("create task: %v", err)
}
payload := map[string]any{
"metadata": map[string]string{"task_uuid": "uuid-completed"},
"result": "New response that should be ignored",
"status": "ok",
}
body, _ := json.Marshal(payload)
req, _ := http.NewRequest("POST", srv.URL+"/callback", bytes.NewReader(body))
req.Header.Set("Content-Type", "application/json")
req.Header.Set("X-Bridge-Token", testToken)
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatalf("request failed: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Errorf("expected 200, got %d", resp.StatusCode)
}
updated, _ := database.FindByTaskUUID(tdb.DB, "uuid-completed")
if updated.AIResponse != "Original response" {
t.Errorf("expected original response preserved, got %q", updated.AIResponse)
}
}
// Health endpoints.
func TestHealthz(t *testing.T) {
_, srv := setupTestRouter(t)
resp, err := http.Get(srv.URL + "/healthz")
if err != nil {
t.Fatalf("request failed: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Errorf("expected 200, got %d", resp.StatusCode)
}
}
func TestReadyz(t *testing.T) {
_, srv := setupTestRouter(t)
resp, err := http.Get(srv.URL + "/readyz")
if err != nil {
t.Fatalf("request failed: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
t.Errorf("expected 200, got %d", resp.StatusCode)
}
}
+111
View File
@@ -0,0 +1,111 @@
package config
import (
"fmt"
"os"
"strings"
"github.com/joho/godotenv"
)
// Config holds all application configuration loaded from environment variables.
type Config struct {
// IMAP settings
IMAPHost string
IMAPPort string
IMAPUser string
IMAPPass string
// SMTP settings
SMTPHost string
SMTPPort string
SMTPUser string
SMTPPass string
// OpenClaw settings
OpenClawURL string
OpenClawAPIKey string
// Bridge settings
BridgeCallbackToken string
SystemEmail string
WhitelistEmails []string
// Optional
IMAPProxyURL string
// Server
ListenAddr string
}
// Load reads .env (if present) and populates Config from environment variables.
// Returns an error if any required variable is missing.
func Load() (*Config, error) {
// Best-effort load .env — ignore error if file doesn't exist.
_ = godotenv.Load()
cfg := &Config{
IMAPHost: os.Getenv("IMAP_HOST"),
IMAPPort: os.Getenv("IMAP_PORT"),
IMAPUser: os.Getenv("IMAP_USER"),
IMAPPass: os.Getenv("IMAP_PASS"),
SMTPHost: os.Getenv("SMTP_HOST"),
SMTPPort: os.Getenv("SMTP_PORT"),
SMTPUser: os.Getenv("SMTP_USER"),
SMTPPass: os.Getenv("SMTP_PASS"),
OpenClawURL: os.Getenv("OPENCLAW_URL"),
OpenClawAPIKey: os.Getenv("OPENCLAW_API_KEY"),
BridgeCallbackToken: os.Getenv("BRIDGE_CALLBACK_TOKEN"),
SystemEmail: os.Getenv("SYSTEM_EMAIL"),
IMAPProxyURL: os.Getenv("IMAP_PROXY_URL"),
ListenAddr: os.Getenv("LISTEN_ADDR"),
}
if raw := os.Getenv("WHITELIST_EMAILS"); raw != "" {
for _, email := range strings.Split(raw, ",") {
if trimmed := strings.TrimSpace(email); trimmed != "" {
cfg.WhitelistEmails = append(cfg.WhitelistEmails, trimmed)
}
}
}
if cfg.ListenAddr == "" {
cfg.ListenAddr = ":8080"
}
if err := cfg.validate(); err != nil {
return nil, err
}
return cfg, nil
}
// validate checks that all required configuration values are present.
func (c *Config) validate() error {
required := map[string]string{
"IMAP_HOST": c.IMAPHost,
"IMAP_PORT": c.IMAPPort,
"IMAP_USER": c.IMAPUser,
"IMAP_PASS": c.IMAPPass,
"SMTP_HOST": c.SMTPHost,
"SMTP_PORT": c.SMTPPort,
"SMTP_USER": c.SMTPUser,
"SMTP_PASS": c.SMTPPass,
"OPENCLAW_URL": c.OpenClawURL,
"BRIDGE_CALLBACK_TOKEN": c.BridgeCallbackToken,
"SYSTEM_EMAIL": c.SystemEmail,
}
var missing []string
for name, val := range required {
if val == "" {
missing = append(missing, name)
}
}
if len(missing) > 0 {
return fmt.Errorf("missing required config: %s", strings.Join(missing, ", "))
}
return nil
}
+35
View File
@@ -0,0 +1,35 @@
package database
import (
"fmt"
"github.com/glebarez/sqlite"
"gorm.io/gorm"
"gorm.io/gorm/logger"
)
// Open initializes the SQLite database and runs auto-migration.
func Open(dbPath string) (*gorm.DB, error) {
db, err := gorm.Open(sqlite.Open(dbPath), &gorm.Config{
Logger: logger.Default.LogMode(logger.Silent), // We use our own structured logger.
})
if err != nil {
return nil, fmt.Errorf("database open: %w", err)
}
// Enable WAL mode for better concurrent read performance.
if err := db.Exec("PRAGMA journal_mode=WAL").Error; err != nil {
return nil, fmt.Errorf("database pragma WAL: %w", err)
}
if err := db.AutoMigrate(&Task{}); err != nil {
return nil, fmt.Errorf("database migrate: %w", err)
}
return db, nil
}
// Healthy checks if the database is writable.
func Healthy(db *gorm.DB) error {
return db.Exec("SELECT 1").Error
}
+35
View File
@@ -0,0 +1,35 @@
package database
import (
"time"
)
// Task status constants matching the requirement's state machine.
const (
StatusIgnored = "IGNORED"
StatusReceived = "RECEIVED"
StatusAIProcessing = "AI_PROCESSING"
StatusCallbackDone = "CALLBACK_DONE"
StatusSMTPRetrying = "SMTP_RETRYING"
StatusCompleted = "COMPLETED"
StatusFailed = "FAILED"
)
// Task represents a single email processing unit tracked through the pipeline.
type Task struct {
ID uint `gorm:"primaryKey;autoIncrement"`
TaskUUID string `gorm:"uniqueIndex;not null"`
ThreadID string `gorm:"index"`
MessageID string `gorm:"index"`
Sender string
Subject string
BodyPlain string
AIResponse string
Status string `gorm:"index;not null"`
AttemptOpenClaw int `gorm:"default:0"`
AttemptSMTP int `gorm:"default:0"`
LastError *string
NextRetryAt *time.Time
CreatedAt time.Time
UpdatedAt time.Time
}
+65
View File
@@ -0,0 +1,65 @@
package database
import (
"errors"
"gorm.io/gorm"
)
// FindByMessageID returns the task with the given message_id, or nil if not found.
func FindByMessageID(db *gorm.DB, messageID string) (*Task, error) {
var task Task
err := db.Where("message_id = ?", messageID).First(&task).Error
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, nil
}
if err != nil {
return nil, err
}
return &task, nil
}
// FindLatestByThreadID returns the most recent COMPLETED task in the thread.
func FindLatestByThreadID(db *gorm.DB, threadID string) (*Task, error) {
var task Task
err := db.Where("thread_id = ? AND status = ?", threadID, StatusCompleted).
Order("created_at DESC").First(&task).Error
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, nil
}
if err != nil {
return nil, err
}
return &task, nil
}
// FindByTaskUUID returns the task with the given task_uuid.
func FindByTaskUUID(db *gorm.DB, taskUUID string) (*Task, error) {
var task Task
err := db.Where("task_uuid = ?", taskUUID).First(&task).Error
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, nil
}
if err != nil {
return nil, err
}
return &task, nil
}
// ThreadHistory returns all COMPLETED tasks for a thread, ordered chronologically.
func ThreadHistory(db *gorm.DB, threadID string) ([]Task, error) {
var tasks []Task
err := db.Where("thread_id = ? AND status = ?", threadID, StatusCompleted).
Order("created_at ASC").Find(&tasks).Error
return tasks, err
}
// CreateTask inserts a new task record.
func CreateTask(db *gorm.DB, task *Task) error {
return db.Create(task).Error
}
// UpdateTask saves changes to an existing task record.
func UpdateTask(db *gorm.DB, task *Task) error {
return db.Save(task).Error
}
+105
View File
@@ -0,0 +1,105 @@
package database_test
import (
"testing"
"thuanle.me/claw-email-bridge/internal/database"
)
// Test matrix #5: Duplicate message_id → không tạo task mới.
func TestDuplicateMessageID(t *testing.T) {
tdb := database.NewTestDB(t)
task1 := &database.Task{
TaskUUID: "uuid-1",
ThreadID: "thread-1",
MessageID: "msg-duplicate@test.com",
Sender: "user@test.com",
Subject: "First email",
Status: database.StatusReceived,
}
if err := database.CreateTask(tdb.DB, task1); err != nil {
t.Fatalf("create first task: %v", err)
}
// Lookup should find the existing task.
found, err := database.FindByMessageID(tdb.DB, "msg-duplicate@test.com")
if err != nil {
t.Fatalf("find by message_id: %v", err)
}
if found == nil {
t.Fatal("expected to find existing task")
}
if found.TaskUUID != "uuid-1" {
t.Errorf("expected uuid-1, got %s", found.TaskUUID)
}
// Attempting to create another task with same task_uuid should fail (unique constraint).
task2 := &database.Task{
TaskUUID: "uuid-1", // Same UUID — unique constraint violation.
ThreadID: "thread-1",
MessageID: "msg-duplicate@test.com",
Sender: "user@test.com",
Subject: "Duplicate",
Status: database.StatusReceived,
}
err = database.CreateTask(tdb.DB, task2)
if err == nil {
t.Error("expected error creating duplicate task_uuid, got nil")
}
}
func TestFindByMessageID_NotFound(t *testing.T) {
tdb := database.NewTestDB(t)
found, err := database.FindByMessageID(tdb.DB, "nonexistent@test.com")
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if found != nil {
t.Error("expected nil for nonexistent message_id")
}
}
func TestThreadHistory(t *testing.T) {
tdb := database.NewTestDB(t)
// Create two completed tasks in same thread.
for i, uuid := range []string{"uuid-a", "uuid-b"} {
task := &database.Task{
TaskUUID: uuid,
ThreadID: "thread-history",
MessageID: uuid + "@test.com",
Sender: "user@test.com",
Subject: "Thread test",
BodyPlain: "Message " + uuid,
AIResponse: "Reply " + uuid,
Status: database.StatusCompleted,
}
_ = i
if err := database.CreateTask(tdb.DB, task); err != nil {
t.Fatalf("create task %s: %v", uuid, err)
}
}
// Create one non-completed task — should not appear in history.
task3 := &database.Task{
TaskUUID: "uuid-c",
ThreadID: "thread-history",
MessageID: "uuid-c@test.com",
Sender: "user@test.com",
Subject: "Pending",
Status: database.StatusAIProcessing,
}
if err := database.CreateTask(tdb.DB, task3); err != nil {
t.Fatalf("create task: %v", err)
}
history, err := database.ThreadHistory(tdb.DB, "thread-history")
if err != nil {
t.Fatalf("thread history: %v", err)
}
if len(history) != 2 {
t.Errorf("expected 2 completed tasks in history, got %d", len(history))
}
}
+35
View File
@@ -0,0 +1,35 @@
package database
import (
"os"
"path/filepath"
"testing"
"gorm.io/gorm"
)
// TestDB holds a test database instance with automatic cleanup.
type TestDB struct {
DB *gorm.DB
cleanup func()
}
// NewTestDB creates a temporary SQLite database for testing.
func NewTestDB(t *testing.T) *TestDB {
t.Helper()
dir := t.TempDir()
dbPath := filepath.Join(dir, "test.db")
db, err := Open(dbPath)
if err != nil {
t.Fatalf("open test db: %v", err)
}
return &TestDB{
DB: db,
cleanup: func() {
os.RemoveAll(dir)
},
}
}
+25
View File
@@ -0,0 +1,25 @@
package logging
import (
"log/slog"
"os"
)
// Setup initialises the application-wide structured logger.
// All output goes to stdout as JSON for easy parsing by Docker / log aggregators.
func Setup() {
handler := slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{
Level: slog.LevelInfo,
})
slog.SetDefault(slog.New(handler))
}
// TaskLogger returns a logger pre-populated with common task fields.
// Usage: logging.TaskLogger(taskUUID, threadID, messageID).Info("message", ...)
func TaskLogger(taskUUID, threadID, messageID string) *slog.Logger {
return slog.With(
"task_uuid", taskUUID,
"thread_id", threadID,
"message_id", messageID,
)
}
+362
View File
@@ -0,0 +1,362 @@
package mail
import (
"context"
"fmt"
"io"
"log/slog"
"strings"
"time"
gomessage "github.com/emersion/go-message/mail"
"github.com/emersion/go-imap/v2"
"github.com/emersion/go-imap/v2/imapclient"
"github.com/google/uuid"
"thuanle.me/claw-email-bridge/internal/config"
"thuanle.me/claw-email-bridge/internal/database"
"thuanle.me/claw-email-bridge/internal/logging"
"gorm.io/gorm"
)
// IMAPWatcher monitors an IMAP mailbox via IDLE and processes new emails.
type IMAPWatcher struct {
cfg *config.Config
db *gorm.DB
client *imapclient.Client
// onReceived is called after a task is saved as RECEIVED.
// This will be wired to the OpenClaw dispatch in a future step.
OnReceived func(task *database.Task)
}
// NewIMAPWatcher creates a new IMAP watcher.
func NewIMAPWatcher(cfg *config.Config, db *gorm.DB) *IMAPWatcher {
return &IMAPWatcher{cfg: cfg, db: db}
}
// Run connects to IMAP, selects INBOX, and enters the IDLE loop.
// It blocks until ctx is cancelled. Reconnects automatically on error.
func (w *IMAPWatcher) Run(ctx context.Context) {
for {
if err := w.connectAndWatch(ctx); err != nil {
slog.Error("imap watcher error, reconnecting in 10s", "error", err)
}
select {
case <-ctx.Done():
slog.Info("imap watcher stopped")
return
case <-time.After(10 * time.Second):
// Reconnect after delay.
}
}
}
// connectAndWatch handles a single IMAP session lifecycle.
func (w *IMAPWatcher) connectAndWatch(ctx context.Context) error {
numMessages := uint32(0)
options := &imapclient.Options{
UnilateralDataHandler: &imapclient.UnilateralDataHandler{
Mailbox: func(data *imapclient.UnilateralDataMailbox) {
if data.NumMessages != nil && *data.NumMessages > numMessages {
slog.Info("imap: new message notification", "num_messages", *data.NumMessages)
numMessages = *data.NumMessages
}
},
},
}
addr := fmt.Sprintf("%s:%s", w.cfg.IMAPHost, w.cfg.IMAPPort)
slog.Info("imap: connecting", "addr", addr)
c, err := imapclient.DialTLS(addr, options)
if err != nil {
return fmt.Errorf("dial TLS: %w", err)
}
w.client = c
defer func() {
_ = c.Close()
w.client = nil
}()
if err := c.Login(w.cfg.IMAPUser, w.cfg.IMAPPass).Wait(); err != nil {
return fmt.Errorf("login: %w", err)
}
slog.Info("imap: logged in", "user", w.cfg.IMAPUser)
selectedMbox, err := c.Select("INBOX", nil).Wait()
if err != nil {
return fmt.Errorf("select INBOX: %w", err)
}
numMessages = selectedMbox.NumMessages
slog.Info("imap: INBOX selected", "messages", numMessages)
// Fetch any unseen messages on startup.
if err := w.fetchUnseen(c); err != nil {
slog.Error("imap: initial fetch unseen failed", "error", err)
}
// IDLE loop: wait for new messages, then fetch unseen.
for {
select {
case <-ctx.Done():
return nil
default:
}
idleCmd, err := c.Idle()
if err != nil {
return fmt.Errorf("idle: %w", err)
}
done := make(chan error, 1)
go func() {
done <- idleCmd.Wait()
}()
// Re-check every 28 minutes (RFC recommends <29 min IDLE timeout).
idleTimer := time.NewTimer(28 * time.Minute)
select {
case <-ctx.Done():
_ = idleCmd.Close()
idleTimer.Stop()
return nil
case err := <-done:
idleTimer.Stop()
if err != nil {
return fmt.Errorf("idle wait: %w", err)
}
case <-idleTimer.C:
// Stop IDLE and restart it.
if err := idleCmd.Close(); err != nil {
return fmt.Errorf("idle close: %w", err)
}
<-done
}
// After exiting IDLE, fetch unseen messages.
if err := w.fetchUnseen(c); err != nil {
slog.Error("imap: fetch unseen failed", "error", err)
}
}
}
// fetchUnseen searches for UNSEEN messages and processes each one.
func (w *IMAPWatcher) fetchUnseen(c *imapclient.Client) error {
criteria := &imap.SearchCriteria{
NotFlag: []imap.Flag{imap.FlagSeen},
}
searchData, err := c.UIDSearch(criteria, nil).Wait()
if err != nil {
return fmt.Errorf("uid search: %w", err)
}
uids := searchData.AllUIDs()
if len(uids) == 0 {
return nil
}
slog.Info("imap: unseen messages found", "count", len(uids))
uidSet := imap.UIDSet{}
for _, uid := range uids {
uidSet.AddNum(uid)
}
bodySection := &imap.FetchItemBodySection{}
fetchOptions := &imap.FetchOptions{
Envelope: true,
UID: true,
BodySection: []*imap.FetchItemBodySection{bodySection},
}
fetchCmd := c.Fetch(uidSet, fetchOptions)
defer fetchCmd.Close()
for {
msg := fetchCmd.Next()
if msg == nil {
break
}
buf, err := msg.Collect()
if err != nil {
slog.Error("imap: collect message failed", "error", err)
continue
}
w.processMessage(c, buf, bodySection)
}
if err := fetchCmd.Close(); err != nil {
return fmt.Errorf("fetch close: %w", err)
}
return nil
}
// processMessage handles a single fetched email message.
func (w *IMAPWatcher) processMessage(c *imapclient.Client, buf *imapclient.FetchMessageBuffer, bodySection *imap.FetchItemBodySection) {
env := buf.Envelope
if env == nil {
slog.Warn("imap: message has no envelope, skipping")
return
}
messageID := env.MessageID
subject := env.Subject
// Extract sender email.
sender := ""
if len(env.From) > 0 {
sender = env.From[0].Addr()
}
log := logging.TaskLogger("", "", messageID)
// 1. Anti-loop: skip if sender is the system email.
if strings.EqualFold(sender, w.cfg.SystemEmail) {
log.Info("imap: anti-loop, ignoring own email", "sender", sender)
w.saveIgnored(messageID, sender, subject, "anti-loop: system email")
w.markSeen(c, buf.UID)
return
}
// 2. Whitelist: skip if sender not in allowed list.
if !w.isWhitelisted(sender) {
log.Info("imap: sender not whitelisted, ignoring", "sender", sender)
w.saveIgnored(messageID, sender, subject, "not whitelisted")
w.markSeen(c, buf.UID)
return
}
// 3. Idempotency: skip if message_id already exists.
existing, err := database.FindByMessageID(w.db, messageID)
if err != nil {
log.Error("imap: db lookup failed", "error", err)
return
}
if existing != nil {
log.Info("imap: duplicate message_id, skipping")
w.markSeen(c, buf.UID)
return
}
// 4. Threading: determine thread_id.
threadID := messageID
inReplyTo := ""
// Parse body to get In-Reply-To / References headers and plain text.
bodyPlain := ""
rawBody := buf.FindBodySection(bodySection)
if rawBody != nil {
mr, err := gomessage.CreateReader(io.NopCloser(strings.NewReader(string(rawBody))))
if err == nil {
// Extract In-Reply-To header.
if irt, err := mr.Header.Text("In-Reply-To"); err == nil && irt != "" {
inReplyTo = strings.TrimSpace(irt)
}
// Read body parts for plain text.
for {
p, err := mr.NextPart()
if err != nil {
break
}
if _, ok := p.Header.(*gomessage.InlineHeader); ok {
b, _ := io.ReadAll(p.Body)
if bodyPlain == "" {
bodyPlain = string(b)
}
}
}
}
}
if inReplyTo != "" {
parent, err := database.FindByMessageID(w.db, inReplyTo)
if err == nil && parent != nil {
threadID = parent.ThreadID
}
}
// 5. Save task with status RECEIVED.
taskUUID := uuid.New().String()
task := &database.Task{
TaskUUID: taskUUID,
ThreadID: threadID,
MessageID: messageID,
Sender: sender,
Subject: subject,
BodyPlain: bodyPlain,
Status: database.StatusReceived,
}
taskLog := logging.TaskLogger(taskUUID, threadID, messageID)
if err := database.CreateTask(w.db, task); err != nil {
taskLog.Error("imap: failed to save task", "error", err)
return
}
taskLog.Info("imap: task created",
"sender", sender,
"subject", subject,
"status", database.StatusReceived,
)
// Mark as seen in IMAP so we don't re-process.
w.markSeen(c, buf.UID)
// Trigger dispatch (will be wired later).
if w.OnReceived != nil {
w.OnReceived(task)
}
}
// saveIgnored records an ignored email in the DB for auditability.
func (w *IMAPWatcher) saveIgnored(messageID, sender, subject, reason string) {
taskUUID := uuid.New().String()
errMsg := reason
task := &database.Task{
TaskUUID: taskUUID,
ThreadID: messageID,
MessageID: messageID,
Sender: sender,
Subject: subject,
Status: database.StatusIgnored,
LastError: &errMsg,
}
if err := database.CreateTask(w.db, task); err != nil {
slog.Error("imap: failed to save ignored task", "error", err, "message_id", messageID)
}
}
// isWhitelisted checks if the sender is in the allowed list.
// Empty whitelist means all senders are allowed.
func (w *IMAPWatcher) isWhitelisted(sender string) bool {
if len(w.cfg.WhitelistEmails) == 0 {
return true
}
lower := strings.ToLower(sender)
for _, email := range w.cfg.WhitelistEmails {
if strings.ToLower(email) == lower {
return true
}
}
return false
}
// markSeen flags a message as \Seen in IMAP.
func (w *IMAPWatcher) markSeen(c *imapclient.Client, uid imap.UID) {
storeFlags := imap.StoreFlags{
Op: imap.StoreFlagsAdd,
Flags: []imap.Flag{imap.FlagSeen},
Silent: true,
}
if err := c.Store(imap.UIDSetNum(uid), &storeFlags, nil).Close(); err != nil {
slog.Error("imap: failed to mark message as seen", "error", err, "uid", uid)
}
}
+134
View File
@@ -0,0 +1,134 @@
package mail
import (
"fmt"
"net/smtp"
"strings"
"time"
"thuanle.me/claw-email-bridge/internal/config"
"thuanle.me/claw-email-bridge/internal/database"
"thuanle.me/claw-email-bridge/internal/logging"
"gorm.io/gorm"
)
// Retry backoff schedule: 1s, 5s, 15s.
var smtpRetryBackoffs = []time.Duration{1 * time.Second, 5 * time.Second, 15 * time.Second}
// SMTPSender sends reply emails via SMTP with retry logic.
type SMTPSender struct {
cfg *config.Config
db *gorm.DB
}
// NewSMTPSender creates a new SMTP sender.
func NewSMTPSender(cfg *config.Config, db *gorm.DB) *SMTPSender {
return &SMTPSender{cfg: cfg, db: db}
}
// SendReply sends an AI-generated reply to the original sender.
// This should be called asynchronously (go sender.SendReply(task)).
func (s *SMTPSender) SendReply(task *database.Task) {
log := logging.TaskLogger(task.TaskUUID, task.ThreadID, task.MessageID)
// Build the email message with threading headers.
msg := s.buildMessage(task)
for attempt := 1; attempt <= len(smtpRetryBackoffs); attempt++ {
task.AttemptSMTP = attempt
log.Info("smtp: sending reply",
"attempt", attempt,
"to", task.Sender,
)
err := s.send(task.Sender, msg)
if err == nil {
task.Status = database.StatusCompleted
if err := database.UpdateTask(s.db, task); err != nil {
log.Error("smtp: failed to save COMPLETED status", "error", err)
}
log.Info("smtp: reply sent successfully",
"attempt", attempt,
"status", database.StatusCompleted,
)
return
}
log.Warn("smtp: send failed",
"attempt", attempt,
"error", err,
)
// Update status to SMTP_RETRYING.
task.Status = database.StatusSMTPRetrying
if err := database.UpdateTask(s.db, task); err != nil {
log.Error("smtp: failed to save retry status", "error", err)
}
// Last attempt — mark as FAILED.
if attempt == len(smtpRetryBackoffs) {
errMsg := err.Error()
task.Status = database.StatusFailed
task.LastError = &errMsg
if err := database.UpdateTask(s.db, task); err != nil {
log.Error("smtp: failed to save FAILED status", "error", err)
}
log.Error("smtp: all retries exhausted",
"attempt", attempt,
"status", database.StatusFailed,
"error", errMsg,
)
return
}
backoff := smtpRetryBackoffs[attempt-1]
log.Info("smtp: retrying after backoff", "backoff", backoff)
time.Sleep(backoff)
}
}
// buildMessage constructs the RFC 2822 email with threading headers.
func (s *SMTPSender) buildMessage(task *database.Task) string {
subject := task.Subject
if !strings.HasPrefix(strings.ToLower(subject), "re:") {
subject = "Re: " + subject
}
var b strings.Builder
b.WriteString(fmt.Sprintf("From: %s\r\n", s.cfg.SystemEmail))
b.WriteString(fmt.Sprintf("To: %s\r\n", task.Sender))
b.WriteString(fmt.Sprintf("Subject: %s\r\n", subject))
// Threading headers — required by requirements 4.3.
b.WriteString(fmt.Sprintf("In-Reply-To: %s\r\n", task.MessageID))
b.WriteString(fmt.Sprintf("References: %s\r\n", task.MessageID))
b.WriteString("MIME-Version: 1.0\r\n")
b.WriteString("Content-Type: text/plain; charset=UTF-8\r\n")
b.WriteString("\r\n")
b.WriteString(task.AIResponse)
return b.String()
}
// send performs the actual SMTP delivery.
func (s *SMTPSender) send(to string, msg string) error {
addr := fmt.Sprintf("%s:%s", s.cfg.SMTPHost, s.cfg.SMTPPort)
auth := smtp.PlainAuth("", s.cfg.SMTPUser, s.cfg.SMTPPass, s.cfg.SMTPHost)
err := smtp.SendMail(addr, auth, s.cfg.SystemEmail, []string{to}, []byte(msg))
if err != nil {
return fmt.Errorf("smtp sendmail: %w", err)
}
return nil
}
// SendReplyFunc returns a function compatible with api.OnCallbackDone.
func (s *SMTPSender) SendReplyFunc() func(task *database.Task) {
return func(task *database.Task) {
s.SendReply(task)
}
}