sundial: scraping logic from nationalrail

Signed-off-by: oppiliappan <[e-mail]>

config.go

···
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"gopkg.in/yaml.v3"
+)
+
+type Config struct {
+	Server struct {
+		Port            int `yaml:"port"`
+		RefreshInterval int `yaml:"refresh_interval"`
+	} `yaml:"server"`
+	Stations []Station `yaml:"stations"`
+}
+
+type Station struct {
+	Name string `yaml:"name"`
+	Code string `yaml:"code"`
+}
+
+func LoadConfig(filename string) (*Config, error) {
+	data, err := os.ReadFile(filename)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read config file: %w", err)
+	}
+
+	var config Config
+	err = yaml.Unmarshal(data, &config)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse config file: %w", err)
+	}
+
+	return &config, nil
+}

config.yaml

···
+server:
+  port: 8080
+  refresh_interval: 60 # seconds
+
+stations:
+  - name: "Lewisham"
+    code: "lewisham"
+  - name: "Ladywell"
+    code: "ladywell"

departure.go

···
+package main
+
+import (
+	"regexp"
+	"strconv"
+	"strings"
+)
+
+type Departure struct {
+	ScheduledTime    string `json:"scheduled_time"`
+	ExpectedTime     string `json:"expected_time,omitempty"`
+	Status           string `json:"status"` // "On time", "Expected", "Delayed", etc.
+	Destination      string `json:"destination"`
+	Via              string `json:"via,omitempty"`
+	Platform         string `json:"platform"`
+	Duration         string `json:"duration"`
+	Stops            int    `json:"stops"`
+	Operator         string `json:"operator"`
+	DelayReason      string `json:"delay_reason,omitempty"`
+	ServiceID        string `json:"service_id,omitempty"`
+}
+
+// ParseAriaLabel extracts departure information from the aria-label attribute
+func ParseAriaLabel(ariaLabel string) (*Departure, error) {
+	departure := &Departure{}
+
+	// Handle delayed trains first
+	delayPattern := regexp.MustCompile(`^(.*?), (\d{2}:\d{2}), Expected (\d{2}:\d{2}), service for ([^,]+)(?:, via ([^,]+))?, calling at [^,]+, from platform (\d+), duration (\d+) minutes, (\d+) stops, operated by (.+)$`)
+	if matches := delayPattern.FindStringSubmatch(ariaLabel); matches != nil {
+		departure.DelayReason = matches[1]
+		departure.ScheduledTime = matches[2]
+		departure.ExpectedTime = matches[3]
+		departure.Status = "Expected " + matches[3]
+		departure.Destination = strings.TrimSpace(matches[4])
+		departure.Via = strings.TrimSpace(matches[5])
+		departure.Platform = matches[6]
+		departure.Duration = matches[7] + " minutes"
+		stops, _ := strconv.Atoi(matches[8])
+		departure.Stops = stops
+		departure.Operator = strings.TrimSpace(matches[9])
+		return departure, nil
+	}
+
+	// Handle normal trains
+	normalPattern := regexp.MustCompile(`^(\d{2}:\d{2}), (On time|Cancelled), service for ([^,]+)(?:, via ([^,]+))?, calling at [^,]+, from platform (\d+), duration (\d+) minutes, (\d+) stops, operated by (.+)$`)
+	if matches := normalPattern.FindStringSubmatch(ariaLabel); matches != nil {
+		departure.ScheduledTime = matches[1]
+		departure.Status = matches[2]
+		departure.Destination = strings.TrimSpace(matches[3])
+		departure.Via = strings.TrimSpace(matches[4])
+		departure.Platform = matches[5]
+		departure.Duration = matches[6] + " minutes"
+		stops, _ := strconv.Atoi(matches[7])
+		departure.Stops = stops
+		departure.Operator = strings.TrimSpace(matches[8])
+		return departure, nil
+	}
+
+	return nil, nil // Unable to parse
+}

fetcher.go

···
+package main
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+)
+
+const BaseURL = "https://www.nationalrail.co.uk/live-trains/departures/"
+
+// FetchStationDepartures fetches the HTML content for a station's departures
+func FetchStationDepartures(stationCode string) (string, error) {
+	client := &http.Client{
+		Timeout: 30 * time.Second,
+	}
+
+	url := BaseURL + stationCode + "/"
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return "", fmt.Errorf("failed to create request: %w", err)
+	}
+
+	// Add headers to mimic a browser request
+	req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36")
+	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", fmt.Errorf("failed to fetch data: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+	}
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return "", fmt.Errorf("failed to read response body: %w", err)
+	}
+
+	return string(body), nil
+}
+
+type StationData struct {
+	Station    Station     `json:"station"`
+	Departures []Departure `json:"departures"`
+	LastUpdate time.Time   `json:"last_update"`
+	Error      string      `json:"error,omitempty"`
+}
+
+// FetchAllStationsData fetches departure data for all configured stations
+func FetchAllStationsData(stations []Station) []StationData {
+	results := make([]StationData, len(stations))
+
+	for i, station := range stations {
+		results[i] = StationData{
+			Station:    station,
+			LastUpdate: time.Now(),
+		}
+
+		htmlContent, err := FetchStationDepartures(station.Code)
+		if err != nil {
+			results[i].Error = err.Error()
+			continue
+		}
+
+		departures, err := ParseDeparturesFromHTML(htmlContent)
+		if err != nil {
+			results[i].Error = err.Error()
+			continue
+		}
+
+		results[i].Departures = departures
+	}
+
+	return results
+}

json_parser.go

···
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"regexp"
+	"strings"
+	"time"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+// NextData represents the structure of the __NEXT_DATA__ JSON
+type NextData struct {
+	Props struct {
+		PageProps struct {
+			LiveTrainsState struct {
+				Queries []struct {
+					State struct {
+						Data struct {
+							Pages []struct {
+								Services []JSONService `json:"services"`
+							} `json:"pages"`
+						} `json:"data"`
+					} `json:"state"`
+				} `json:"queries"`
+			} `json:"liveTrainsState"`
+		} `json:"pageProps"`
+	} `json:"props"`
+}
+
+type JSONService struct {
+	RID         string `json:"rid"`
+	TrainUID    string `json:"trainUid"`
+	Origin      []struct {
+		LocationName string `json:"locationName"`
+		CRS          string `json:"crs"`
+		Via          *string `json:"via"`
+	} `json:"origin"`
+	Destination []struct {
+		LocationName string `json:"locationName"`
+		CRS          string `json:"crs"`
+		Via          *string `json:"via"`
+	} `json:"destination"`
+	JourneyDetails struct {
+		From struct {
+			LocationName string `json:"locationName"`
+			CRS          string `json:"crs"`
+		} `json:"from"`
+		To struct {
+			LocationName string `json:"locationName"`
+			CRS          string `json:"crs"`
+		} `json:"to"`
+		Stops int `json:"stops"`
+		DepartureInfo struct {
+			Scheduled string  `json:"scheduled"`
+			Estimated *string `json:"estimated"`
+			Actual    *string `json:"actual"`
+		} `json:"departureInfo"`
+		ArrivalInfo struct {
+			Scheduled string  `json:"scheduled"`
+			Estimated *string `json:"estimated"`
+			Actual    *string `json:"actual"`
+		} `json:"arrivalInfo"`
+	} `json:"journeyDetails"`
+	Operator struct {
+		Name string `json:"name"`
+		Code string `json:"code"`
+	} `json:"operator"`
+	Status struct {
+		Status       string  `json:"status"`
+		DelayReason  *string `json:"delayReason"`
+		CancelReason *string `json:"cancelReason"`
+	} `json:"status"`
+	Platform string `json:"platform"`
+}
+
+// ParseDeparturesFromJSONHTML extracts departure information from HTML containing __NEXT_DATA__ JSON
+func ParseDeparturesFromJSONHTML(htmlContent string) ([]Departure, error) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(htmlContent))
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse HTML: %w", err)
+	}
+
+	// Find the __NEXT_DATA__ script tag
+	var jsonContent string
+	doc.Find("script#__NEXT_DATA__").Each(func(i int, s *goquery.Selection) {
+		rawContent := s.Text()
+		jsonContent = cleanJSONString(rawContent)
+	})
+
+	if jsonContent == "" {
+		return nil, fmt.Errorf("no __NEXT_DATA__ script tag found")
+	}
+
+	// Parse the JSON
+	var nextData NextData
+	err = json.Unmarshal([]byte(jsonContent), &nextData)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse JSON: %w", err)
+	}
+
+	// Extract services from the nested structure
+	var departures []Departure
+	if len(nextData.Props.PageProps.LiveTrainsState.Queries) > 0 &&
+		len(nextData.Props.PageProps.LiveTrainsState.Queries[0].State.Data.Pages) > 0 {
+
+		services := nextData.Props.PageProps.LiveTrainsState.Queries[0].State.Data.Pages[0].Services
+
+		for _, service := range services {
+			departure := convertJSONServiceToDeparture(service)
+			departures = append(departures, departure)
+		}
+	}
+
+	return departures, nil
+}
+
+func convertJSONServiceToDeparture(service JSONService) Departure {
+	departure := Departure{
+		ServiceID: service.RID,
+		Platform:  service.Platform,
+		Stops:     service.JourneyDetails.Stops,
+		Operator:  service.Operator.Name,
+	}
+
+	// Extract destination
+	if len(service.Destination) > 0 {
+		departure.Destination = service.Destination[0].LocationName
+		if service.Destination[0].Via != nil && *service.Destination[0].Via != "" {
+			departure.Via = *service.Destination[0].Via
+		}
+	}
+
+	// Parse scheduled time
+	if scheduledTime, err := time.Parse(time.RFC3339, service.JourneyDetails.DepartureInfo.Scheduled); err == nil {
+		departure.ScheduledTime = scheduledTime.Format("15:04")
+	}
+
+	// Parse estimated/actual time and set status
+	if service.JourneyDetails.DepartureInfo.Actual != nil {
+		// Train has already departed
+		if actualTime, err := time.Parse(time.RFC3339, *service.JourneyDetails.DepartureInfo.Actual); err == nil {
+			departure.ExpectedTime = actualTime.Format("15:04")
+			departure.Status = "Departed " + departure.ExpectedTime
+		}
+	} else if service.JourneyDetails.DepartureInfo.Estimated != nil {
+		// Train has estimated departure time
+		if estimatedTime, err := time.Parse(time.RFC3339, *service.JourneyDetails.DepartureInfo.Estimated); err == nil {
+			departure.ExpectedTime = estimatedTime.Format("15:04")
+			if departure.ExpectedTime == departure.ScheduledTime {
+				departure.Status = "On time"
+			} else {
+				departure.Status = "Expected " + departure.ExpectedTime
+			}
+		}
+	} else {
+		// Use service status
+		switch service.Status.Status {
+		case "OnTime":
+			departure.Status = "On time"
+		case "Late":
+			departure.Status = "Late"
+		case "Cancelled":
+			departure.Status = "Cancelled"
+		default:
+			departure.Status = service.Status.Status
+		}
+	}
+
+	// Calculate duration (rough estimate)
+	if scheduledDep, err1 := time.Parse(time.RFC3339, service.JourneyDetails.DepartureInfo.Scheduled); err1 == nil {
+		if scheduledArr, err2 := time.Parse(time.RFC3339, service.JourneyDetails.ArrivalInfo.Scheduled); err2 == nil {
+			duration := scheduledArr.Sub(scheduledDep)
+			departure.Duration = fmt.Sprintf("%d minutes", int(duration.Minutes()))
+		}
+	}
+
+	// Add delay reason if available
+	if service.Status.DelayReason != nil && *service.Status.DelayReason != "" {
+		departure.DelayReason = *service.Status.DelayReason
+	}
+
+	return departure
+}
+
+// cleanJSONString fixes common JSON issues from HTML extraction
+func cleanJSONString(s string) string {
+	s = strings.TrimSpace(s)
+
+	// Fix newlines within string literals
+	// This is a simplified approach - we replace newlines within quoted strings
+	re := regexp.MustCompile(`"([^"]*\n[^"]*)"`)
+	s = re.ReplaceAllStringFunc(s, func(match string) string {
+		// Remove quotes, replace newlines with spaces, add quotes back
+		inner := match[1 : len(match)-1] // Remove surrounding quotes
+		inner = regexp.MustCompile(`\s*\n\s*`).ReplaceAllString(inner, " ")
+		return `"` + inner + `"`
+	})
+
+	return s
+}

main.go

···
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+)
+
+func main() {
+	if len(os.Args) < 2 {
+		fmt.Println("Sundial - Live Train Departures")
+		fmt.Println("Usage:")
+		fmt.Println("  go run . server              - Start web server")
+		fmt.Println("  go run . cli                 - Start interactive CLI")
+		fmt.Println("  go run . parse <html_file>   - Parse HTML file")
+		fmt.Println("Examples:")
+		fmt.Println("  go run . server")
+		fmt.Println("  go run . cli")
+		fmt.Println("  go run . parse sample.html")
+		os.Exit(1)
+	}
+
+	command := os.Args[1]
+
+	switch command {
+	case "server":
+		runServer()
+	case "cli":
+		err := runCLI()
+		if err != nil {
+			log.Fatalf("CLI error: %v", err)
+		}
+	case "parse":
+		if len(os.Args) < 3 {
+			fmt.Println("Usage: go run . parse <html_file>")
+			os.Exit(1)
+		}
+		parseFile(os.Args[2])
+	default:
+		// Backward compatibility - treat first arg as filename
+		parseFile(command)
+	}
+}
+
+func runServer() {
+	config, err := LoadConfig("config.yaml")
+	if err != nil {
+		log.Fatalf("Error loading config: %v", err)
+	}
+
+	server := NewServer(config)
+	log.Fatal(server.Start())
+}
+
+func parseFile(filename string) {
+	departures, err := ParseDeparturesFromFile(filename)
+	if err != nil {
+		log.Fatalf("Error parsing departures: %v", err)
+	}
+
+	fmt.Printf("Found %d departures:\n\n", len(departures))
+
+	// Print as formatted JSON
+	jsonData, err := json.MarshalIndent(departures, "", "  ")
+	if err != nil {
+		log.Fatalf("Error marshaling to JSON: %v", err)
+	}
+
+	fmt.Println(string(jsonData))
+}
+

parser.go

···
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+)
+
+// ParseDeparturesFromHTML extracts departure information from the National Rail HTML
+func ParseDeparturesFromHTML(htmlContent string) ([]Departure, error) {
+	// Try JSON approach first (for new React-based site)
+	if departures, err := ParseDeparturesFromJSONHTML(htmlContent); err == nil && len(departures) > 0 {
+		return departures, nil
+	}
+
+	// Fallback to old HTML parsing approach
+	return parseFromAriaLabels(htmlContent)
+}
+
+func parseFromAriaLabels(htmlContent string) ([]Departure, error) {
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(htmlContent))
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse HTML: %w", err)
+	}
+
+	seenServices := make(map[string]bool)
+	var departures []Departure
+
+	// Find all anchor tags with aria-label containing service information
+	doc.Find("a[aria-label*='service for']").Each(func(i int, s *goquery.Selection) {
+		ariaLabel, exists := s.Attr("aria-label")
+		if !exists {
+			return
+		}
+
+		// Extract service ID from href if available
+		href, _ := s.Attr("href")
+		serviceID := extractServiceID(href)
+
+		// Skip if we've already processed this service
+		if seenServices[serviceID] {
+			return
+		}
+		seenServices[serviceID] = true
+
+		departure, err := ParseAriaLabel(ariaLabel)
+		if err != nil || departure == nil {
+			return
+		}
+
+		departure.ServiceID = serviceID
+		departures = append(departures, *departure)
+	})
+
+	return departures, nil
+}
+
+// extractServiceID extracts the service ID from the href URL
+func extractServiceID(href string) string {
+	if strings.Contains(href, "sid=") {
+		parts := strings.Split(href, "sid=")
+		if len(parts) > 1 {
+			sidPart := strings.Split(parts[1], "&")[0]
+			return sidPart
+		}
+	}
+	return ""
+}
+
+// ParseDeparturesFromFile reads an HTML file and extracts departure information
+func ParseDeparturesFromFile(filename string) ([]Departure, error) {
+	content, err := os.ReadFile(filename)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read file %s: %w", filename, err)
+	}
+
+	return ParseDeparturesFromHTML(string(content))
+}