smokesignal / utils / clean_ftl_duplicates.sh

#!/bin/bash

# Script to detect and remove duplicate keys in FTL files
# Usage: ./clean_ftl_duplicates.sh [directory]

set -e

# Colors for display
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Help function
show_help() {
    echo "Usage: $0 [OPTIONS] [DIRECTORY]"
    echo ""
    echo "Detects and removes duplicate keys in FTL files"
    echo ""
    echo "OPTIONS:"
    echo "  -h, --help     Show this help"
    echo "  -l, --list     List mode (does not modify files, only shows duplicates)"
    echo "  -v, --verbose  Verbose mode"
    echo "  -i, --interactive  Interactive mode (choose which occurrences to keep)"
    echo ""
    echo "ARGUMENTS:"
    echo "  DIRECTORY      Directory containing FTL files (default: current directory)"
    echo ""
    echo "Examples:"
    echo "  $0                    # Process current directory"
    echo "  $0 ./translations     # Process ./translations directory"
    echo "  $0 -l ./translations  # List mode"
}

# Default variables
DIRECTORY="."
LIST_ONLY=false
VERBOSE=false
INTERACTIVE=true  # Default to interactive mode

# Parse arguments
while [[ $# -gt 0 ]]; do
    case $1 in
        -h|--help)
            show_help
            exit 0
            ;;
        -l|--list)
            LIST_ONLY=true
            shift
            ;;
        -v|--verbose)
            VERBOSE=true
            shift
            ;;
        -i|--interactive)
            INTERACTIVE=true
            shift
            ;;
        -*)
            echo -e "${RED}Unknown option: $1${NC}" >&2
            show_help
            exit 1
            ;;
        *)
            DIRECTORY="$1"
            shift
            ;;
    esac
done

# Check that directory exists
if [[ ! -d "$DIRECTORY" ]]; then
    echo -e "${RED}Error: Directory '$DIRECTORY' does not exist${NC}" >&2
    exit 1
fi

# Function to extract keys from an FTL file
extract_keys() {
    local file="$1"
    # Extract keys (lines that start with non-space characters followed by =)
    grep -E "^[^#[:space:]][^=]*=" "$file" 2>/dev/null | cut -d'=' -f1 | sed 's/[[:space:]]*$//' || true
}

# Function to get all occurrences of a key with their context
get_key_occurrences() {
    local key="$1"
    shift
    local files=("$@")
    local -a occurrences
    local index=0
    
    for file in "${files[@]}"; do
        local line_num=1
        while IFS= read -r line; do
            if [[ "$line" =~ ^[[:space:]]*"$key"[[:space:]]*= ]]; then
                occurrences[$index]="$file:$line_num:$line"
                ((index++))
            fi
            ((line_num++))
        done < "$file"
    done
    
    printf '%s\n' "${occurrences[@]}"
}

# Function to display occurrences and let user choose
choose_occurrence_to_keep() {
    local key="$1"
    shift
    local files=("$@")
    
    echo -e "${RED}Duplicate key found:${NC} '$key'"
    echo ""
    
    # Get all occurrences
    local -a occurrences
    mapfile -t occurrences < <(get_key_occurrences "$key" "${files[@]}")
    
    if [[ ${#occurrences[@]} -eq 0 ]]; then
        echo -e "${RED}Error: No occurrences found${NC}"
        return 1
    fi
    
    # Display all occurrences
    echo "Found ${#occurrences[@]} occurrences:"
    for i in "${!occurrences[@]}"; do
        local occurrence="${occurrences[$i]}"
        IFS=':' read -r file line_num content <<< "$occurrence"
        echo -e "  ${YELLOW}[$((i+1))]${NC} $(basename "$file"):$line_num"
        echo -e "      $content"
        echo ""
    done
    
    # Ask user to choose
    while true; do
        if [[ "$LIST_ONLY" == true ]]; then
            echo -e "${YELLOW}LIST MODE: Showing duplicates only${NC}"
            echo ""
            return 0
        fi
        
        echo -e "${BLUE}Which occurrence do you want to KEEP?${NC} [1-${#occurrences[@]}] (or 's' to skip): "
        read -r choice
        
        if [[ "$choice" == "s" || "$choice" == "S" ]]; then
            echo -e "${YELLOW}Skipping '$key'${NC}"
            echo ""
            return 0
        fi
        
        if [[ "$choice" =~ ^[0-9]+$ ]] && [[ "$choice" -ge 1 ]] && [[ "$choice" -le ${#occurrences[@]} ]]; then
            local keep_index=$((choice-1))
            break
        else
            echo -e "${RED}Invalid choice. Please enter a number between 1 and ${#occurrences[@]}, or 's' to skip.${NC}"
        fi
    done
    
    # Remove all occurrences except the chosen one
    echo -e "Keeping occurrence #$choice, removing others..."
    
    for i in "${!occurrences[@]}"; do
        if [[ $i -ne $keep_index ]]; then
            local occurrence="${occurrences[$i]}"
            IFS=':' read -r file line_num content <<< "$occurrence"
            remove_occurrence_at_line "$file" "$line_num" "$key"
        fi
    done
    
    echo -e "${GREEN}✓${NC} Cleaned up duplicates for '$key'"
    echo ""
}

# Function to remove a specific occurrence at a given line number
remove_occurrence_at_line() {
    local file="$1"
    local target_line="$2"
    local key="$3"
    
    if [[ "$LIST_ONLY" == true ]]; then
        echo -e "${YELLOW}[LIST MODE]${NC} Would remove line $target_line from $(basename "$file")"
        return
    fi
    
    # Create temporary copy
    local temp_file=$(mktemp)
    local current_line=1
    
    while IFS= read -r line; do
        if [[ $current_line -eq $target_line ]]; then
            if [[ "$VERBOSE" == true ]]; then
                echo -e "    ${RED}Removed:${NC} $(basename "$file"):$target_line: $line"
            fi
            # Skip this line (don't write it to temp file)
        else
            echo "$line" >> "$temp_file"
        fi
        ((current_line++))
    done < "$file"
    
    # Replace original file
    mv "$temp_file" "$file"
    echo -e "    ${GREEN}✓${NC} Removed line $target_line from $(basename "$file")"
}

# Main function
main() {
    echo -e "${BLUE}=== FTL Duplicate Key Cleanup ===${NC}"
    echo -e "Directory: ${YELLOW}$DIRECTORY${NC}"
    
    if [[ "$LIST_ONLY" == true ]]; then
        echo -e "Mode: ${YELLOW}LIST ONLY${NC} (no files will be modified)"
    fi
    
    echo ""
    
    # Find all FTL files
    local ftl_files=($(find "$DIRECTORY" -maxdepth 1 -name "*.ftl" -type f))
    
    if [[ ${#ftl_files[@]} -eq 0 ]]; then
        echo -e "${YELLOW}No FTL files found in '$DIRECTORY'${NC}"
        exit 0
    fi
    
    echo -e "${BLUE}FTL files found: ${#ftl_files[@]}${NC}"
    for file in "${ftl_files[@]}"; do
        echo "  - $(basename "$file")"
    done
    echo ""
    
    # Collect all keys with their source files
    declare -A key_files
    declare -A key_counts
    
    echo -e "${BLUE}Analyzing keys...${NC}"
    
    for file in "${ftl_files[@]}"; do
        if [[ "$VERBOSE" == true ]]; then
            echo "  Analyzing $(basename "$file")..."
        fi
        
        while IFS= read -r key; do
            [[ -z "$key" ]] && continue
            
            if [[ -z "${key_files[$key]}" ]]; then
                key_files[$key]="$file"
                key_counts[$key]=1
            else
                key_files[$key]="${key_files[$key]}|$file"
                ((key_counts[$key]++))
            fi
        done < <(extract_keys "$file")
    done
    
    # Find and process duplicates
    local duplicates_found=false
    
    echo -e "${BLUE}Searching for duplicates...${NC}"
    echo ""
    
    for key in "${!key_counts[@]}"; do
        if [[ ${key_counts[$key]} -gt 1 ]]; then
            duplicates_found=true
            
            # Split files for this key
            IFS='|' read -ra files <<< "${key_files[$key]}"
            
            # Let user choose which occurrence to keep
            choose_occurrence_to_keep "$key" "${files[@]}"
        fi
    done
    
    if [[ "$duplicates_found" == false ]]; then
        echo -e "${GREEN}✓ No duplicate keys found!${NC}"
    else
        if [[ "$LIST_ONLY" == true ]]; then
            echo -e "${YELLOW}List mode completed. Run again without -l to apply changes.${NC}"
        else
            echo -e "${GREEN}✓ Cleanup completed!${NC}"
        fi
    fi
}

# Check dependencies
command -v grep >/dev/null 2>&1 || { echo -e "${RED}Error: 'grep' is not installed${NC}" >&2; exit 1; }
command -v cut >/dev/null 2>&1 || { echo -e "${RED}Error: 'cut' is not installed${NC}" >&2; exit 1; }
command -v sed >/dev/null 2>&1 || { echo -e "${RED}Error: 'sed' is not installed${NC}" >&2; exit 1; }

# Main execution
main