#![deny(clippy::unwrap_used)] use chromiumoxide::{Browser, Page}; use color_eyre::eyre::{self, Context, ContextCompat, eyre}; use futures::StreamExt; use indicatif::ProgressBar; use scraper::{Html, Selector}; use sha2::{Digest, Sha256}; use std::{ io::{self, Read}, path::PathBuf, process::Stdio, }; use tokio::process::Command; fn get_chrome_path() -> PathBuf { if cfg!(target_os = "windows") { let program_files = std::env::var("PROGRAMFILES").unwrap_or_default(); let program_files_x86 = std::env::var("PROGRAMFILES(X86)").unwrap_or_default(); let path64 = PathBuf::from(program_files).join(r"Google\Chrome\Application\chrome.exe"); let path32 = PathBuf::from(program_files_x86).join(r"Google\Chrome\Application\chrome.exe"); if path64.exists() { path64 } else { path32 } } else if cfg!(target_os = "macos") { PathBuf::from("/Applications/Google Chrome.app/Contents/MacOS/Google Chrome") } else { PathBuf::from("chromium") } } pub async fn launch_chrome() -> eyre::Result { let chrome_path = get_chrome_path(); let user_data_dir = std::env::temp_dir().join("scraping_chrome_profile"); let child = Command::new(chrome_path) .arg("--remote-debugging-port=9222") .arg(format!( "--user-data-dir={}", user_data_dir.to_string_lossy() )) .arg("--no-first-run") .arg("--no-default-browser-check") .stdout(Stdio::null()) .stderr(Stdio::null()) .spawn() .context("Failed to open browser")?; // Give Chrome time to bind to port 9222 tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; Ok(child) } #[tokio::main] async fn main() -> eyre::Result<()> { let current_fonds = read_xlsx()?; let _ = launch_chrome().await?; let (mut browser, mut handler) = Browser::connect("http://127.0.0.1:9222") .await .context("Failed to connect to browser")?; tokio::spawn(async move { while let Some(h) = handler.next().await { if h.is_err() { break; } } }); let page = browser .new_page("https://www.fondsweb.com/".to_string()) .await .context("Failed to create Page")?; let progress_bar = ProgressBar::new(current_fonds.len() as u64); progress_bar.tick(); let mut fonds = vec![]; for fond in ¤t_fonds { tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; fonds.push( scrape_data(&page, &fond.id) .await .context(format!("Failed to scarpe - {}", fond.id)), ); progress_bar.inc(1); } progress_bar.finish_and_clear(); if let Err(err) = page.close().await.context("Failed to close Page") { eprintln!("{:?}", err); }; if let Err(err) = browser.close().await.context("Failed to close Browser") { eprintln!("{:?}", err); }; let fonds: Vec<_> = fonds .into_iter() .filter_map(|fond| match fond { Ok(data) => Some(data), Err(err) => { eprintln!("{:?}\n", err); None } }) .collect(); let mut has_changes = false; for old_fond in current_fonds { let Some(new_fond) = fonds.iter().find(|fond| fond.id == old_fond.id) else { continue; }; let changes = old_fond.diff(new_fond); if changes.is_empty() { continue; } has_changes = true; println!("{} has changed!", new_fond.id); for change in changes { match change { FondChange::Name => println!(" Name: {} -> {}", old_fond.name, new_fond.name), FondChange::SRI => println!(" SRI: {} -> {}", old_fond.sri, new_fond.sri), FondChange::Info => println!(" Basic Information pdf changed!"), } } println!() } if has_changes { write_xlsx(fonds)?; } else { println!("nothing changed!\n") } println!("Press Enter to exit..."); let _ = io::stdin().read(&mut [0u8]); Ok(()) } fn read_xlsx() -> eyre::Result> { let mut fonds = vec![]; let mut book = umya_spreadsheet::reader::xlsx::read("./data.xlsx") .context("Failed not Read ./data.xlsx")?; let sheet = book .sheet_mut(0) .context("Failed to access Sheet index 0")?; let mut index = 2; while let Some(cell) = sheet.cell((1, index)) { let id = cell.value().to_string(); let name = sheet .cell((2, index)) .context("Failed to read Name from column 2")? .value() .to_string(); let sri = sheet .cell((3, index)) .context("Failed to read SRI from column 3")? .value() .to_string(); let info_hash = sheet .cell((5, index)) .context("Failed to read InfoHash from column 5")? .value() .to_string(); fonds.push(FondData { id, name, sri: sri.parse().context("Failed to parse SRI from column 3")?, info_hash, }); index += 1; } for (i, fond) in fonds.iter().enumerate() { sheet .cell_mut((1, i as u32 + 2)) .set_value(fond.id.to_string()); sheet .cell_mut((2, i as u32 + 2)) .set_value(fond.name.to_string()); sheet .cell_mut((3, i as u32 + 2)) .set_value(fond.sri.to_string()); sheet .cell_mut((5, i as u32 + 2)) .set_value(fond.info_hash.to_string()); } Ok(fonds) } fn write_xlsx(fonds: Vec) -> eyre::Result<()> { let mut book = umya_spreadsheet::reader::xlsx::read("./data.xlsx") .context("Failed not Read ./data.xlsx")?; let sheet = book .sheet_mut(0) .context("Failed to access Sheet index 0")?; for (i, fond) in fonds.iter().enumerate() { sheet .cell_mut((1, i as u32 + 2)) .set_value(fond.id.to_string()); sheet .cell_mut((2, i as u32 + 2)) .set_value(fond.name.to_string()); sheet .cell_mut((3, i as u32 + 2)) .set_value(fond.sri.to_string()); sheet .cell_mut((5, i as u32 + 2)) .set_value(fond.info_hash.to_string()); } umya_spreadsheet::writer::xlsx::write(&book, "./data.xlsx") .context("Failed to Write ./data.xlsx")?; Ok(()) } #[derive(Debug)] pub struct FondData { id: String, name: String, sri: u8, info_hash: String, } pub enum FondChange { Name, SRI, Info, } impl FondData { pub fn diff(&self, other: &FondData) -> Vec { let mut changes = vec![]; if self.name != other.name { changes.push(FondChange::Name); } if self.sri != other.sri { changes.push(FondChange::SRI); } if self.info_hash != other.info_hash { changes.push(FondChange::Info); } changes } } pub async fn scrape_data(page: &Page, id: &str) -> eyre::Result { page.goto(format!("https://www.fondsweb.com/at/{}", id)) .await .context("Failed to access Page")?; loop { let title = page.get_title().await.context("Failed to read title")?; if let Some(title) = title && title.contains(id) { break; } tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; } let content = page .content() .await .context("Failed to read page content")?; let document = Html::parse_document(&content); let mut name = None; for header in document.select(&(Selector::parse("h1").map_err(|_| eyre!("Failed to build Selector"))?)) { name = Some(header.inner_html()); } let mut sri = None; for rating in document.select( &Selector::parse(".fw--ratingNumbers").map_err(|_| eyre!("Failed to build Selector"))?, ) { for i in 1..7 { if rating.value().has_class( &format!("fw--ratingNumbers-value_{}", i), scraper::CaseSensitivity::AsciiCaseInsensitive, ) { sri = Some(i); } } } let mut pdf_url = None; for list in document.select( &Selector::parse("ul.f-pdf-list li a").map_err(|_| eyre!("Failed to build Selector"))?, ) { let link = list .value() .attr("href") .context("Failed to Parse pdf url")?; let text = list .first_child() .context("Failed to Parse pdf url")? .value() .as_text() .context("Failed to Parse pdf url")?; if text.text.to_string() == "Basisinformationsblatt" { pdf_url = Some(link) } } let hash; if let Some(pdf_url) = pdf_url { hash = get_pdf_hash(pdf_url).await; } else { return Err(eyre!("Failed to find Info Page")); } Ok(FondData { id: id.to_string(), name: name.context("Failed to find Name")?, sri: sri.context("Failed to find SRI")?, info_hash: hash.context("Failed to read Info Page")?, }) } async fn get_pdf_hash(url: &str) -> eyre::Result { let response = reqwest::get(url) .await .context(format!("Failed to fetch - {}", url))?; if !response.status().is_success() { return Err(eyre!(format!("Failed to read - {}", url))); } let mut stream = response.bytes_stream(); let mut hasher = Sha256::new(); while let Some(chunk) = stream.next().await { hasher.update(chunk.context("Failed to calculate Hash")?); } Ok(hex::encode(hasher.finalize())) }