job monitoring

internal/api/handlers.go

···
 	"time"
 
 	"github.com/atscan/atscanner/internal/log"
+	"github.com/atscan/atscanner/internal/monitor"
 	"github.com/atscan/atscanner/internal/plc"
 	"github.com/atscan/atscanner/internal/storage"
 	"github.com/gorilla/mux"
···
 
 func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) {
 	newResponse(w).json(map[string]string{"status": "ok"})
+}
+
+func (s *Server) handleGetJobStatus(w http.ResponseWriter, r *http.Request) {
+	resp := newResponse(w)
+	tracker := monitor.GetTracker()
+
+	jobs := tracker.GetAllJobs()
+
+	result := make(map[string]interface{})
+	for name, job := range jobs {
+		jobData := map[string]interface{}{
+			"name":          job.Name,
+			"status":        job.Status,
+			"run_count":     job.RunCount,
+			"success_count": job.SuccessCount,
+			"error_count":   job.ErrorCount,
+		}
+
+		if !job.LastRun.IsZero() {
+			jobData["last_run"] = job.LastRun
+			jobData["last_duration"] = job.Duration.String()
+		}
+
+		if !job.NextRun.IsZero() {
+			jobData["next_run"] = job.NextRun
+			jobData["next_run_in"] = time.Until(job.NextRun).Round(time.Second).String()
+		}
+
+		if job.Status == "running" {
+			jobData["running_for"] = job.Duration.Round(time.Second).String()
+
+			if job.Progress != nil {
+				jobData["progress"] = job.Progress
+			}
+
+			// Add worker status
+			workers := tracker.GetWorkers(name)
+			if len(workers) > 0 {
+				jobData["workers"] = workers
+			}
+		}
+
+		if job.Error != "" {
+			jobData["error"] = job.Error
+		}
+
+		result[name] = jobData
+	}
+
+	resp.json(result)
 }
 
 // ===== UTILITY FUNCTIONS =====

internal/api/server.go

···
 	// Metrics routes
 	api.HandleFunc("/metrics/plc", s.handleGetPLCMetrics).Methods("GET")
 
+	// Job status endpoint
+	api.HandleFunc("/jobs", s.handleGetJobStatus).Methods("GET")
+
 	// Health check
 	s.router.HandleFunc("/health", s.handleHealth).Methods("GET")
 }

internal/monitor/tracker.go

···
-package monitor
 package monitor
 
 import (
···
 }
 
 type Progress struct {
-	Current int    `json:"current"`
-	Total   int    `json:"total"`
+	Current int     `json:"current"`
+	Total   int     `json:"total"`
 	Percent float64 `json:"percent"`
-	Message string `json:"message,omitempty"`
+	Message string  `json:"message,omitempty"`
 }
 
 type WorkerStatus struct {
-	ID          int       `json:"id"`
-	Status      string    `json:"status"` // "idle", "working"
-	CurrentTask string    `json:"current_task,omitempty"`
-	StartedAt   time.Time `json:"started_at,omitempty"`
+	ID          int           `json:"id"`
+	Status      string        `json:"status"` // "idle", "working"
+	CurrentTask string        `json:"current_task,omitempty"`
+	StartedAt   time.Time     `json:"started_at,omitempty"`
 	Duration    time.Duration `json:"duration,omitempty"`
 }
 
···
 func (t *Tracker) RegisterJob(name string) {
 	t.mu.Lock()
 	defer t.mu.Unlock()
-	
+
 	t.jobs[name] = &JobStatus{
 		Name:   name,
 		Status: "idle",
···
 func (t *Tracker) StartJob(name string) {
 	t.mu.Lock()
 	defer t.mu.Unlock()
-	
+
 	if job, exists := t.jobs[name]; exists {
 		job.Status = "running"
 		job.StartTime = time.Now()
···
 func (t *Tracker) CompleteJob(name string, err error) {
 	t.mu.Lock()
 	defer t.mu.Unlock()
-	
+
 	if job, exists := t.jobs[name]; exists {
 		job.LastRun = time.Now()
 		job.Duration = time.Since(job.StartTime)
-		
+
 		if err != nil {
 			job.Status = "error"
 			job.Error = err.Error()
···
 			job.Status = "completed"
 			job.SuccessCount++
 		}
-		
+
 		job.Progress = nil // Clear progress
 	}
 }
···
 func (t *Tracker) UpdateProgress(name string, current, total int, message string) {
 	t.mu.Lock()
 	defer t.mu.Unlock()
-	
+
 	if job, exists := t.jobs[name]; exists {
 		var percent float64
 		if total > 0 {
 			percent = float64(current) / float64(total) * 100
 		}
-		
+
 		job.Progress = &Progress{
 			Current: current,
 			Total:   total,
···
 func (t *Tracker) SetNextRun(name string, nextRun time.Time) {
 	t.mu.Lock()
 	defer t.mu.Unlock()
-	
+
 	if job, exists := t.jobs[name]; exists {
 		job.NextRun = nextRun
 	}
···
 func (t *Tracker) GetJobStatus(name string) *JobStatus {
 	t.mu.RLock()
 	defer t.mu.RUnlock()
-	
+
 	if job, exists := t.jobs[name]; exists {
 		// Create a copy
 		jobCopy := *job
···
 			progressCopy := *job.Progress
 			jobCopy.Progress = &progressCopy
 		}
-		
+
 		// Calculate duration for running jobs
 		if jobCopy.Status == "running" {
 			jobCopy.Duration = time.Since(jobCopy.StartTime)
 		}
-		
+
 		return &jobCopy
 	}
 	return nil
···
 func (t *Tracker) GetAllJobs() map[string]*JobStatus {
 	t.mu.RLock()
 	defer t.mu.RUnlock()
-	
+
 	result := make(map[string]*JobStatus)
 	for name, job := range t.jobs {
 		jobCopy := *job
···
 			progressCopy := *job.Progress
 			jobCopy.Progress = &progressCopy
 		}
-		
+
 		// Calculate duration for running jobs
 		if jobCopy.Status == "running" {
 			jobCopy.Duration = time.Since(jobCopy.StartTime)
 		}
-		
+
 		result[name] = &jobCopy
 	}
 	return result
···
 func (t *Tracker) InitWorkers(jobName string, count int) {
 	t.mu.Lock()
 	defer t.mu.Unlock()
-	
+
 	workers := make([]WorkerStatus, count)
 	for i := 0; i < count; i++ {
 		workers[i] = WorkerStatus{
···
 func (t *Tracker) StartWorker(jobName string, workerID int, task string) {
 	t.mu.Lock()
 	defer t.mu.Unlock()
-	
+
 	if workers, exists := t.workers[jobName]; exists && workerID > 0 && workerID <= len(workers) {
 		workers[workerID-1].Status = "working"
 		workers[workerID-1].CurrentTask = task
···
 func (t *Tracker) CompleteWorker(jobName string, workerID int) {
 	t.mu.Lock()
 	defer t.mu.Unlock()
-	
+
 	if workers, exists := t.workers[jobName]; exists && workerID > 0 && workerID <= len(workers) {
 		workers[workerID-1].Status = "idle"
 		workers[workerID-1].CurrentTask = ""
···
 func (t *Tracker) GetWorkers(jobName string) []WorkerStatus {
 	t.mu.RLock()
 	defer t.mu.RUnlock()
-	
+
 	if workers, exists := t.workers[jobName]; exists {
 		// Create a copy with calculated durations
 		result := make([]WorkerStatus, len(workers))

internal/pds/scanner.go

···
 	"fmt"
 	"math/rand"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/acarl005/stripansi"
 	"github.com/atscan/atscanner/internal/config"
 	"github.com/atscan/atscanner/internal/ipinfo"
 	"github.com/atscan/atscanner/internal/log"
+	"github.com/atscan/atscanner/internal/monitor"
 	"github.com/atscan/atscanner/internal/storage"
 )
 
···
 	startTime := time.Now()
 	log.Info("Starting PDS availability scan...")
 
-	// Get only PDS endpoints that need checking (stale or never checked)
+	// Get only PDS endpoints that need checking
 	servers, err := s.db.GetEndpoints(ctx, &storage.EndpointFilter{
 		Type:            "pds",
-		OnlyStale:       true,                     // NEW: Only get stale endpoints
-		RecheckInterval: s.config.RecheckInterval, // NEW: Use recheck interval from config
+		OnlyStale:       true,
+		RecheckInterval: s.config.RecheckInterval,
 	})
 	if err != nil {
 		return err
···
 
 	if len(servers) == 0 {
 		log.Info("No endpoints need scanning at this time")
+		monitor.GetTracker().UpdateProgress("pds_scan", 0, 0, "No endpoints need scanning")
 		return nil
 	}
 
-	log.Info("Found %d endpoints that need scanning (not checked in last %s)", len(servers), s.config.RecheckInterval)
+	log.Info("Found %d endpoints that need scanning", len(servers))
+	monitor.GetTracker().UpdateProgress("pds_scan", 0, len(servers), "Preparing to scan")
 
-	// Shuffle the servers slice in place
+	// Shuffle servers
 	if len(servers) > 0 {
 		r := rand.New(rand.NewSource(time.Now().UnixNano()))
 		r.Shuffle(len(servers), func(i, j int) {
 			servers[i], servers[j] = servers[j], servers[i]
 		})
-		log.Info("Randomized scan order for %d PDS servers...", len(servers))
 	}
 
-	// Worker pool
-	jobs := make(chan *storage.Endpoint, len(servers))
+	// Initialize workers in tracker
+	monitor.GetTracker().InitWorkers("pds_scan", s.config.Workers)
+
+	// Worker pool with progress tracking
+	jobs := make(chan *workerJob, len(servers))
 	var wg sync.WaitGroup
+	var completed int32
 
 	for i := 0; i < s.config.Workers; i++ {
 		wg.Add(1)
-		go func() {
+		workerID := i + 1
+		go func(id int) {
 			defer wg.Done()
-			s.worker(ctx, jobs)
-		}()
+			s.workerWithProgress(ctx, id, jobs, &completed, len(servers))
+		}(workerID)
 	}
 
 	// Send jobs
 	for _, server := range servers {
-		jobs <- server
+		jobs <- &workerJob{endpoint: server}
 	}
 	close(jobs)
 
···
 	wg.Wait()
 
 	log.Info("PDS scan completed in %v", time.Since(startTime))
+	monitor.GetTracker().UpdateProgress("pds_scan", len(servers), len(servers), "Completed")
 
 	return nil
+}
+
+type workerJob struct {
+	endpoint *storage.Endpoint
+}
+
+func (s *Scanner) workerWithProgress(ctx context.Context, workerID int, jobs <-chan *workerJob, completed *int32, total int) {
+	for job := range jobs {
+		select {
+		case <-ctx.Done():
+			return
+		default:
+			// Update worker status
+			monitor.GetTracker().StartWorker("pds_scan", workerID, job.endpoint.Endpoint)
+
+			// Scan endpoint
+			s.scanAndSaveEndpoint(ctx, job.endpoint)
+
+			// Update progress
+			atomic.AddInt32(completed, 1)
+			current := atomic.LoadInt32(completed)
+			monitor.GetTracker().UpdateProgress("pds_scan", int(current), total,
+				fmt.Sprintf("Scanned %d/%d endpoints", current, total))
+
+			// Mark worker as idle
+			monitor.GetTracker().CompleteWorker("pds_scan", workerID)
+		}
+	}
 }
 
 func (s *Scanner) worker(ctx context.Context, jobs <-chan *storage.Endpoint) {

internal/worker/scheduler.go

···
 	"time"
 
 	"github.com/atscan/atscanner/internal/log"
+	"github.com/atscan/atscanner/internal/monitor"
 )
 
 type Job struct {
···
 		Interval: interval,
 		Fn:       fn,
 	})
+
+	// Register job with tracker
+	monitor.GetTracker().RegisterJob(name)
 }
 
 func (s *Scheduler) Start(ctx context.Context) {
···
 
 	// Run immediately
 	log.Info("Starting job: %s", job.Name)
-	job.Fn()
+	s.executeJob(job)
 
 	for {
+		// Set next run time
+		monitor.GetTracker().SetNextRun(job.Name, time.Now().Add(job.Interval))
+
 		select {
 		case <-ctx.Done():
 			log.Info("Stopping job: %s", job.Name)
 			return
 		case <-ticker.C:
 			log.Info("Running job: %s", job.Name)
-			job.Fn()
+			s.executeJob(job)
 		}
 	}
 }
+
+func (s *Scheduler) executeJob(job *Job) {
+	monitor.GetTracker().StartJob(job.Name)
+
+	// Run job and capture any panic
+	func() {
+		defer func() {
+			if r := recover(); r != nil {
+				log.Error("Job %s panicked: %v", job.Name, r)
+				monitor.GetTracker().CompleteJob(job.Name, nil)
+			}
+		}()
+
+		job.Fn()
+		monitor.GetTracker().CompleteJob(job.Name, nil)
+	}()
+}