update

cmd/atscanner.go

···
 	}
 	defer db.Close()
 
+	// Set scan retention from config
+	db.SetScanRetention(cfg.PDS.ScanRetention)
+
 	// Run migrations
 	if err := db.Migrate(); err != nil {
 		log.Fatal("Failed to run migrations: %v", err)

config.yaml

···
   timeout: "30s"
   workers: 20
   recheck_interval: "5m"
+  scan_retention: 3
 
 api:
   host: "0.0.0.0"

internal/api/handlers.go

···
 	if pds.LatestScan != nil {
 		response["user_count"] = pds.LatestScan.UserCount
 		response["response_time"] = pds.LatestScan.ResponseTime
+		if pds.LatestScan.Version != "" { // NEW: Add this block
+			response["version"] = pds.LatestScan.Version
+		}
 		if !pds.LatestScan.ScannedAt.IsZero() {
 			response["last_scan"] = pds.LatestScan.ScannedAt
 		}
···
 	// Start with list item formatting
 	response := formatPDSListItem(&pds.PDSListItem)
 
-	// Add server_info from latest scan
-	if pds.LatestScan != nil && pds.LatestScan.ServerInfo != nil {
-		response["server_info"] = pds.LatestScan.ServerInfo
+	// Add server_info and version from latest scan (PDSDetail's LatestScan takes precedence)
+	if pds.LatestScan != nil {
+		// Override with detail-specific scan data
+		response["user_count"] = pds.LatestScan.UserCount
+		response["response_time"] = pds.LatestScan.ResponseTime
+
+		if pds.LatestScan.Version != "" {
+			response["version"] = pds.LatestScan.Version
+		}
+
+		if !pds.LatestScan.ScannedAt.IsZero() {
+			response["last_scan"] = pds.LatestScan.ScannedAt
+		}
+
+		if pds.LatestScan.ServerInfo != nil {
+			response["server_info"] = pds.LatestScan.ServerInfo
+		}
 	}
 
 	// Add full IP info
 	if pds.IPInfo != nil {
-		response["ip_info"] = pds.IPInfo // This now includes raw_data
+		response["ip_info"] = pds.IPInfo
 	}
 
 	return response
···
 			scanMap["response_time"] = scan.ResponseTime
 		}
 
-		// NEW: Use the top-level UserCount field first
+		// NEW: Add version if available
+		if scan.Version != "" {
+			scanMap["version"] = scan.Version
+		}
+
+		// Use the top-level UserCount field first
 		if scan.UserCount > 0 {
 			scanMap["user_count"] = scan.UserCount
 		} else if scan.ScanData != nil && scan.ScanData.Metadata != nil {
···
 		}
 
 		if scan.ScanData != nil {
-			// Metadata is already map[string]interface{}, no type assertion needed
-			// if scan.ScanData.Metadata != nil {
-			// 	// Extract user_count from metadata
-			// 	if userCount, ok := scan.ScanData.Metadata["user_count"].(int); ok {
-			// 		scanMap["user_count"] = userCount
-			// 	} else if userCount, ok := scan.ScanData.Metadata["user_count"].(float64); ok {
-			// 		scanMap["user_count"] = int(userCount)
-			// 	}
-			// } -- OLD Block (replaced by above)
-
 			// Include DID count if available
 			if scan.ScanData.DIDCount > 0 {
 				scanMap["did_count"] = scan.ScanData.DIDCount

internal/config/config.go

···
 
 type DatabaseConfig struct {
 	Path string `yaml:"path"`
-	Type string `yaml:"type"` // sqlite, postgres
+	Type string `yaml:"type"` // postgres
 }
 
 type PLCConfig struct {
···
 	Timeout         time.Duration `yaml:"timeout"`
 	Workers         int           `yaml:"workers"`
 	RecheckInterval time.Duration `yaml:"recheck_interval"`
+	ScanRetention   int           `yaml:"scan_retention"`
 }
 
 type APIConfig struct {
···
 	}
 	if cfg.PDS.Workers == 0 {
 		cfg.PDS.Workers = 10
+	}
+	if cfg.PDS.ScanRetention == 0 {
+		cfg.PDS.ScanRetention = 3
 	}
 	if cfg.API.Port == 0 {
 		cfg.API.Port = 8080

internal/pds/client.go

···
 	return &desc, nil
 }
 
-// CheckHealth performs a basic health check
-func (c *Client) CheckHealth(ctx context.Context, endpoint string) (bool, time.Duration, error) {
+// CheckHealth performs a basic health check, ensuring the endpoint returns JSON with a "version"
+func (c *Client) CheckHealth(ctx context.Context, endpoint string) (bool, time.Duration, string, error) {
 	startTime := time.Now()
 
 	url := fmt.Sprintf("%s/xrpc/_health", endpoint)
 	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
 	if err != nil {
-		return false, 0, err
+		return false, 0, "", err
 	}
 
 	resp, err := c.httpClient.Do(req)
 	duration := time.Since(startTime)
 
 	if err != nil {
-		return false, duration, err
+		return false, duration, "", err
 	}
 	defer resp.Body.Close()
 
-	return resp.StatusCode == http.StatusOK, duration, nil
+	if resp.StatusCode != http.StatusOK {
+		return false, duration, "", fmt.Errorf("health check returned status %d", resp.StatusCode)
+	}
+
+	// Decode the JSON response and check for "version"
+	var healthResponse struct {
+		Version string `json:"version"`
+	}
+
+	if err := json.NewDecoder(resp.Body).Decode(&healthResponse); err != nil {
+		return false, duration, "", fmt.Errorf("failed to decode health JSON: %w", err)
+	}
+
+	if healthResponse.Version == "" {
+		return false, duration, "", fmt.Errorf("health JSON response missing 'version' field")
+	}
+
+	// All checks passed
+	return true, duration, healthResponse.Version, nil
 }

internal/pds/scanner.go

···
 		return err
 	}
 
+	// 2. ADD THIS BLOCK TO SHUFFLE THE LIST
 	if len(servers) > 0 {
 		// Create a new random source to avoid using the global one
 		r := rand.New(rand.NewSource(time.Now().UnixNano()))
···
 	s.db.UpdateEndpointIP(ctx, ep.ID, ip, time.Now().UTC())
 
 	// STEP 2: Health check
-	available, responseTime, err := s.client.CheckHealth(ctx, ep.Endpoint)
+	available, responseTime, version, err := s.client.CheckHealth(ctx, ep.Endpoint) // CHANGED: receive version
 	if err != nil || !available {
 		errMsg := "health check failed"
 		if err != nil {
···
 		ResponseTime: responseTime,
 		Description:  desc,
 		DIDs:         dids,
+		Version:      version, // CHANGED: Pass version
 	})
 
 	// STEP 5: Fetch IP info if needed (async, with backoff)
···
 		EndpointID:   endpointID,
 		Status:       result.Status,
 		ResponseTime: result.ResponseTime.Seconds() * 1000, // Convert to ms
-		UserCount:    userCount,                            // NEW: Set the top-level field
+		UserCount:    userCount,
+		Version:      result.Version, // NEW: Set the version field
 		ScanData:     scanData,
 		ScannedAt:    time.Now().UTC(),
 	}

internal/pds/types.go

···
 	ErrorMessage string
 	Description  *ServerDescription
 	DIDs         []string
+	Version      string // NEW: Add this field to pass the version
 }

internal/storage/db.go

···
 	GetEndpointScans(ctx context.Context, endpointID int64, limit int) ([]*EndpointScan, error)
 	UpdateEndpointIP(ctx context.Context, endpointID int64, ip string, resolvedAt time.Time) error
 	SaveEndpointScan(ctx context.Context, scan *EndpointScan) error
+	SetScanRetention(retention int)
 	UpdateEndpointStatus(ctx context.Context, endpointID int64, update *EndpointUpdate) error
 
 	// PDS virtual endpoints (created via JOINs)

internal/storage/postgres.go

···
 )
 
 type PostgresDB struct {
-	db   *sql.DB
-	pool *pgxpool.Pool // Add this for COPY support
+	db            *sql.DB
+	pool          *pgxpool.Pool
+	scanRetention int
 }
 
 func NewPostgresDB(connString string) (*PostgresDB, error) {
···
 		return nil, fmt.Errorf("failed to create pgx pool: %w", err)
 	}
 
-	return &PostgresDB{db: db, pool: pool}, nil
+	return &PostgresDB{
+		db:            db,
+		pool:          pool,
+		scanRetention: 3,
+	}, nil
 }
 
 func (p *PostgresDB) Close() error {
···
 		endpoint_id BIGINT NOT NULL,
 		status INTEGER NOT NULL,
 		response_time DOUBLE PRECISION,
-		user_count BIGINT, -- NEW: Add user_count column
+		user_count BIGINT,
+		version TEXT,
 		scan_data JSONB,
 		scanned_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
 		FOREIGN KEY (endpoint_id) REFERENCES endpoints(id) ON DELETE CASCADE
···
 
 	CREATE INDEX IF NOT EXISTS idx_endpoint_scans_endpoint_status_scanned ON endpoint_scans(endpoint_id, status, scanned_at DESC);
 	CREATE INDEX IF NOT EXISTS idx_endpoint_scans_scanned_at ON endpoint_scans(scanned_at);
-	CREATE INDEX IF NOT EXISTS idx_endpoint_scans_user_count ON endpoint_scans(user_count DESC NULLS LAST); -- NEW: Index for sorting
+	CREATE INDEX IF NOT EXISTS idx_endpoint_scans_user_count ON endpoint_scans(user_count DESC NULLS LAST);
 
 	CREATE TABLE IF NOT EXISTS plc_metrics (
         id BIGSERIAL PRIMARY KEY,
···
 
 // ===== SCAN OPERATIONS =====
 
+func (p *PostgresDB) SetScanRetention(retention int) {
+	p.scanRetention = retention
+}
+
 func (p *PostgresDB) SaveEndpointScan(ctx context.Context, scan *EndpointScan) error {
 	var scanDataJSON []byte
 	if scan.ScanData != nil {
 		scanDataJSON, _ = json.Marshal(scan.ScanData)
 	}
 
+	tx, err := p.db.BeginTx(ctx, nil)
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+
 	query := `
-		INSERT INTO endpoint_scans (endpoint_id, status, response_time, user_count, scan_data, scanned_at)
-		VALUES ($1, $2, $3, $4, $5, $6)
+		INSERT INTO endpoint_scans (endpoint_id, status, response_time, user_count, version, scan_data, scanned_at)
+		VALUES ($1, $2, $3, $4, $5, $6, $7)
+	`
+	_, err = tx.ExecContext(ctx, query, scan.EndpointID, scan.Status, scan.ResponseTime, scan.UserCount, scan.Version, scanDataJSON, scan.ScannedAt)
+	if err != nil {
+		return err
+	}
+
+	// Use configured retention value
+	cleanupQuery := `
+		DELETE FROM endpoint_scans
+		WHERE endpoint_id = $1
+		AND id NOT IN (
+			SELECT id
+			FROM endpoint_scans
+			WHERE endpoint_id = $1
+			ORDER BY scanned_at DESC
+			LIMIT $2
+		)
 	`
-	_, err := p.db.ExecContext(ctx, query, scan.EndpointID, scan.Status, scan.ResponseTime, scan.UserCount, scanDataJSON, scan.ScannedAt)
-	return err
+	_, err = tx.ExecContext(ctx, cleanupQuery, scan.EndpointID, p.scanRetention)
+	if err != nil {
+		return err
+	}
+
+	return tx.Commit()
 }
 
 func (p *PostgresDB) GetEndpointScans(ctx context.Context, endpointID int64, limit int) ([]*EndpointScan, error) {
 	query := `
-		SELECT id, endpoint_id, status, response_time, user_count, scan_data, scanned_at
-		FROM endpoint_scans
-		WHERE endpoint_id = $1
+        SELECT id, endpoint_id, status, response_time, user_count, version, scan_data, scanned_at
+        FROM endpoint_scans
+        WHERE endpoint_id = $1
         ORDER BY scanned_at DESC
         LIMIT $2
     `
···
 	for rows.Next() {
 		var scan EndpointScan
 		var responseTime sql.NullFloat64
-		var userCount sql.NullInt64 // NEW: Use NullInt64
+		var userCount sql.NullInt64
+		var version sql.NullString // NEW
 		var scanDataJSON []byte
 
-		err := rows.Scan(&scan.ID, &scan.EndpointID, &scan.Status, &responseTime, &userCount, &scanDataJSON, &scan.ScannedAt)
+		err := rows.Scan(&scan.ID, &scan.EndpointID, &scan.Status, &responseTime, &userCount, &version, &scanDataJSON, &scan.ScannedAt)
 		if err != nil {
 			return nil, err
 		}
···
 			scan.ResponseTime = responseTime.Float64
 		}
 
-		if userCount.Valid { // NEW: Check validity and set
+		if userCount.Valid {
 			scan.UserCount = userCount.Int64
 		}
 
+		if version.Valid { // NEW
+			scan.Version = version.String
+		}
+
 		if len(scanDataJSON) > 0 {
 			var scanData EndpointScanData
 			if err := json.Unmarshal(scanDataJSON, &scanData); err == nil {
···
 
 func (p *PostgresDB) GetPDSList(ctx context.Context, filter *EndpointFilter) ([]*PDSListItem, error) {
 	query := `
-		SELECT 
-			e.id, e.endpoint, e.discovered_at, e.last_checked, e.status, e.ip,
-			latest.user_count, latest.response_time, latest.scanned_at,
-			i.city, i.country, i.country_code, i.asn, i.asn_org,
+        SELECT 
+            e.id, e.endpoint, e.discovered_at, e.last_checked, e.status, e.ip,
+            latest.user_count, latest.response_time, latest.version, latest.scanned_at,
+            i.city, i.country, i.country_code, i.asn, i.asn_org,
             i.is_datacenter, i.is_vpn, i.latitude, i.longitude
         FROM endpoints e
-		LEFT JOIN LATERAL (
-			SELECT 
-				-- COALESCE((scan_data->'metadata'->>'user_count')::int, 0) as user_count, -- OLD
-				user_count, -- NEW: Use the column directly
-				response_time,
-				scanned_at
+        LEFT JOIN LATERAL (
+            SELECT 
+                user_count,
+                response_time,
+                version,
+                scanned_at
             FROM endpoint_scans
             WHERE endpoint_id = e.id AND status = 1
             ORDER BY scanned_at DESC
···
 		var lat, lon sql.NullFloat64
 		var userCount sql.NullInt32
 		var responseTime sql.NullFloat64
+		var version sql.NullString // ADD THIS LINE
 		var scannedAt sql.NullTime
 
 		err := rows.Scan(
 			&item.ID, &item.Endpoint, &item.DiscoveredAt, &item.LastChecked, &item.Status, &ip,
-			&userCount, &responseTime, &scannedAt,
+			&userCount, &responseTime, &version, &scannedAt, // ADD &version HERE
 			&city, &country, &countryCode, &asn, &asnOrg,
 			&isDatacenter, &isVPN, &lat, &lon,
 		)
···
 			item.LatestScan = &struct {
 				UserCount    int
 				ResponseTime float64
+				Version      string
 				ScannedAt    time.Time
 			}{
 				UserCount:    int(userCount.Int32),
 				ResponseTime: responseTime.Float64,
+				Version:      version.String,
 				ScannedAt:    scannedAt.Time,
 			}
 		}
···
 
 func (p *PostgresDB) GetPDSDetail(ctx context.Context, endpoint string) (*PDSDetail, error) {
 	query := `
-		SELECT 
-			e.id, e.endpoint, e.discovered_at, e.last_checked, e.status, e.ip,
-			latest.user_count,
-			latest.response_time,
-			latest.scan_data->'metadata'->'server_info' as server_info,
-			latest.scanned_at,
-			i.city, i.country, i.country_code, i.asn, i.asn_org,
-			i.is_datacenter, i.is_vpn, i.latitude, i.longitude,
-			i.raw_data -- NEW: Select the raw_data column
-		FROM endpoints e
-		LEFT JOIN LATERAL (
-			SELECT scan_data, response_time, scanned_at, user_count -- NEW: Select user_count
-			FROM endpoint_scans
-			WHERE endpoint_id = e.id
+        SELECT 
+            e.id, e.endpoint, e.discovered_at, e.last_checked, e.status, e.ip,
+            latest.user_count,
+            latest.response_time,
+            latest.version,  -- ADD THIS LINE
+            latest.scan_data->'metadata'->'server_info' as server_info,
+            latest.scanned_at,
+            i.city, i.country, i.country_code, i.asn, i.asn_org,
+            i.is_datacenter, i.is_vpn, i.latitude, i.longitude,
+            i.raw_data
+        FROM endpoints e
+        LEFT JOIN LATERAL (
+            SELECT scan_data, response_time, version, scanned_at, user_count  -- ADD version HERE
+            FROM endpoint_scans
+            WHERE endpoint_id = e.id
             ORDER BY scanned_at DESC
             LIMIT 1
         ) latest ON true
···
 	var lat, lon sql.NullFloat64
 	var userCount sql.NullInt32
 	var responseTime sql.NullFloat64
+	var version sql.NullString // ADD THIS LINE
 	var serverInfoJSON []byte
 	var scannedAt sql.NullTime
-	var rawDataJSON []byte // NEW: Add a variable for raw_data
+	var rawDataJSON []byte
 
 	err := p.db.QueryRowContext(ctx, query, endpoint).Scan(
 		&detail.ID, &detail.Endpoint, &detail.DiscoveredAt, &detail.LastChecked, &detail.Status, &ip,
-		&userCount, &responseTime, &serverInfoJSON, &scannedAt,
+		&userCount, &responseTime, &version, &serverInfoJSON, &scannedAt, // ADD &version HERE
 		&city, &country, &countryCode, &asn, &asnOrg,
 		&isDatacenter, &isVPN, &lat, &lon,
-		&rawDataJSON, // NEW: Scan into the variable
+		&rawDataJSON,
 	)
 	if err != nil {
 		return nil, err
···
 		detail.LatestScan = &struct {
 			UserCount    int
 			ResponseTime float64
+			Version      string // ADD THIS LINE
 			ServerInfo   interface{}
 			ScannedAt    time.Time
 		}{
 			UserCount:    int(userCount.Int32),
 			ResponseTime: responseTime.Float64,
+			Version:      version.String, // ADD THIS LINE
 			ServerInfo:   serverInfo,
 			ScannedAt:    scannedAt.Time,
 		}
···
 		WITH latest_scans AS (
 			SELECT DISTINCT ON (endpoint_id)
 				endpoint_id,
-				-- COALESCE((scan_data->'metadata'->>'user_count')::int, 0) as user_count, -- OLD
-				user_count, -- NEW: Use the column directly
+				user_count,
 				status
 			FROM endpoint_scans
 			WHERE endpoint_id IN (SELECT id FROM endpoints WHERE endpoint_type = 'pds')
···
 		WITH latest_pds_scans AS (
 			SELECT DISTINCT ON (endpoint_id)
 				endpoint_id,
-				-- COALESCE((scan_data->'metadata'->>'user_count')::int, 0) as user_count -- OLD
-				user_count -- NEW: Use the column directly
+				user_count
 			FROM endpoint_scans
 			WHERE endpoint_id IN (SELECT id FROM endpoints WHERE endpoint_type = 'pds')
 			ORDER BY endpoint_id, scanned_at DESC

internal/storage/types.go

···
 	Status       int
 	ResponseTime float64
 	UserCount    int64
+	Version      string // NEW: Add this field
 	ScanData     *EndpointScanData
 	ScannedAt    time.Time
 }
···
 	LatestScan *struct {
 		UserCount    int
 		ResponseTime float64
+		Version      string // NEW: Add this
 		ScannedAt    time.Time
 	}
 
···
 	LatestScan *struct {
 		UserCount    int
 		ResponseTime float64
+		Version      string      // ADD THIS LINE
 		ServerInfo   interface{} // Full server description
 		ScannedAt    time.Time
 	}