Something went wrong. Try again.
Something went wrong. Try again.
10 kB · 368 lines
Rust
at main
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369#![deny(clippy::unwrap_used)]
use chromiumoxide::{Browser, Page};use color_eyre::eyre::{self, Context, ContextCompat, eyre};use futures::StreamExt;use indicatif::ProgressBar;use scraper::{Html, Selector};use sha2::{Digest, Sha256};use std::{ io::{self, Read}, path::PathBuf, process::Stdio,};use tokio::process::Command;
fn get_chrome_path() -> PathBuf { if cfg!(target_os = "windows") { let program_files = std::env::var("PROGRAMFILES").unwrap_or_default(); let program_files_x86 = std::env::var("PROGRAMFILES(X86)").unwrap_or_default();
let path64 = PathBuf::from(program_files).join(r"Google\Chrome\Application\chrome.exe"); let path32 = PathBuf::from(program_files_x86).join(r"Google\Chrome\Application\chrome.exe");
if path64.exists() { path64 } else { path32 } } else if cfg!(target_os = "macos") { PathBuf::from("/Applications/Google Chrome.app/Contents/MacOS/Google Chrome") } else { PathBuf::from("chromium") }}
pub async fn launch_chrome() -> eyre::Result<tokio::process::Child> { let chrome_path = get_chrome_path(); let user_data_dir = std::env::temp_dir().join("scraping_chrome_profile");
let child = Command::new(chrome_path) .arg("--remote-debugging-port=9222") .arg(format!( "--user-data-dir={}", user_data_dir.to_string_lossy() )) .arg("--no-first-run") .arg("--no-default-browser-check") .stdout(Stdio::null()) .stderr(Stdio::null()) .spawn() .context("Failed to open browser")?;
// Give Chrome time to bind to port 9222 tokio::time::sleep(tokio::time::Duration::from_secs(3)).await;
Ok(child)}
#[tokio::main]async fn main() -> eyre::Result<()> { let current_fonds = read_xlsx()?;
let _ = launch_chrome().await?; let (mut browser, mut handler) = Browser::connect("http://127.0.0.1:9222") .await .context("Failed to connect to browser")?;
tokio::spawn(async move { while let Some(h) = handler.next().await { if h.is_err() { break; } } });
let page = browser .new_page("https://www.fondsweb.com/".to_string()) .await .context("Failed to create Page")?;
let progress_bar = ProgressBar::new(current_fonds.len() as u64); progress_bar.tick();
let mut fonds = vec![]; for fond in ¤t_fonds { tokio::time::sleep(tokio::time::Duration::from_secs(1)).await;
fonds.push( scrape_data(&page, &fond.id) .await .context(format!("Failed to scarpe - {}", fond.id)), );
progress_bar.inc(1); }
progress_bar.finish_and_clear();
if let Err(err) = page.close().await.context("Failed to close Page") { eprintln!("{:?}", err); }; if let Err(err) = browser.close().await.context("Failed to close Browser") { eprintln!("{:?}", err); };
let fonds: Vec<_> = fonds .into_iter() .filter_map(|fond| match fond { Ok(data) => Some(data), Err(err) => { eprintln!("{:?}\n", err); None } }) .collect();
let mut has_changes = false;
for old_fond in current_fonds { let Some(new_fond) = fonds.iter().find(|fond| fond.id == old_fond.id) else { continue; };
let changes = old_fond.diff(new_fond); if changes.is_empty() { continue; }
has_changes = true; println!("{} has changed!", new_fond.id);
for change in changes { match change { FondChange::Name => println!(" Name: {} -> {}", old_fond.name, new_fond.name), FondChange::SRI => println!(" SRI: {} -> {}", old_fond.sri, new_fond.sri), FondChange::Info => println!(" Basic Information pdf changed!"), } }
println!() }
if has_changes { write_xlsx(fonds)?; } else { println!("nothing changed!\n") }
println!("Press Enter to exit..."); let _ = io::stdin().read(&mut [0u8]);
Ok(())}
fn read_xlsx() -> eyre::Result<Vec<FondData>> { let mut fonds = vec![];
let mut book = umya_spreadsheet::reader::xlsx::read("./data.xlsx") .context("Failed not Read ./data.xlsx")?; let sheet = book .sheet_mut(0) .context("Failed to access Sheet index 0")?;
let mut index = 2; while let Some(cell) = sheet.cell((1, index)) { let id = cell.value().to_string(); let name = sheet .cell((2, index)) .context("Failed to read Name from column 2")? .value() .to_string(); let sri = sheet .cell((3, index)) .context("Failed to read SRI from column 3")? .value() .to_string(); let info_hash = sheet .cell((5, index)) .context("Failed to read InfoHash from column 5")? .value() .to_string();
fonds.push(FondData { id, name, sri: sri.parse().context("Failed to parse SRI from column 3")?, info_hash, });
index += 1; }
for (i, fond) in fonds.iter().enumerate() { sheet .cell_mut((1, i as u32 + 2)) .set_value(fond.id.to_string());
sheet .cell_mut((2, i as u32 + 2)) .set_value(fond.name.to_string());
sheet .cell_mut((3, i as u32 + 2)) .set_value(fond.sri.to_string());
sheet .cell_mut((5, i as u32 + 2)) .set_value(fond.info_hash.to_string()); }
Ok(fonds)}
fn write_xlsx(fonds: Vec<FondData>) -> eyre::Result<()> { let mut book = umya_spreadsheet::reader::xlsx::read("./data.xlsx") .context("Failed not Read ./data.xlsx")?; let sheet = book .sheet_mut(0) .context("Failed to access Sheet index 0")?;
for (i, fond) in fonds.iter().enumerate() { sheet .cell_mut((1, i as u32 + 2)) .set_value(fond.id.to_string());
sheet .cell_mut((2, i as u32 + 2)) .set_value(fond.name.to_string());
sheet .cell_mut((3, i as u32 + 2)) .set_value(fond.sri.to_string());
sheet .cell_mut((5, i as u32 + 2)) .set_value(fond.info_hash.to_string()); }
umya_spreadsheet::writer::xlsx::write(&book, "./data.xlsx") .context("Failed to Write ./data.xlsx")?;
Ok(())}
#[derive(Debug)]pub struct FondData { id: String, name: String, sri: u8, info_hash: String,}
pub enum FondChange { Name, SRI, Info,}
impl FondData { pub fn diff(&self, other: &FondData) -> Vec<FondChange> { let mut changes = vec![]; if self.name != other.name { changes.push(FondChange::Name); } if self.sri != other.sri { changes.push(FondChange::SRI); } if self.info_hash != other.info_hash { changes.push(FondChange::Info); }
changes }}
pub async fn scrape_data(page: &Page, id: &str) -> eyre::Result<FondData> { page.goto(format!("https://www.fondsweb.com/at/{}", id)) .await .context("Failed to access Page")?;
loop { let title = page.get_title().await.context("Failed to read title")?;
if let Some(title) = title && title.contains(id) { break; }
tokio::time::sleep(tokio::time::Duration::from_secs(1)).await; }
let content = page .content() .await .context("Failed to read page content")?; let document = Html::parse_document(&content);
let mut name = None; for header in document.select(&(Selector::parse("h1").map_err(|_| eyre!("Failed to build Selector"))?)) { name = Some(header.inner_html()); }
let mut sri = None; for rating in document.select( &Selector::parse(".fw--ratingNumbers").map_err(|_| eyre!("Failed to build Selector"))?, ) { for i in 1..7 { if rating.value().has_class( &format!("fw--ratingNumbers-value_{}", i), scraper::CaseSensitivity::AsciiCaseInsensitive, ) { sri = Some(i); } } }
let mut pdf_url = None; for list in document.select( &Selector::parse("ul.f-pdf-list li a").map_err(|_| eyre!("Failed to build Selector"))?, ) { let link = list .value() .attr("href") .context("Failed to Parse pdf url")?;
let text = list .first_child() .context("Failed to Parse pdf url")? .value() .as_text() .context("Failed to Parse pdf url")?; if text.text.to_string() == "Basisinformationsblatt" { pdf_url = Some(link) } }
let hash; if let Some(pdf_url) = pdf_url { hash = get_pdf_hash(pdf_url).await; } else { return Err(eyre!("Failed to find Info Page")); }
Ok(FondData { id: id.to_string(), name: name.context("Failed to find Name")?, sri: sri.context("Failed to find SRI")?, info_hash: hash.context("Failed to read Info Page")?, })}
async fn get_pdf_hash(url: &str) -> eyre::Result<String> { let response = reqwest::get(url) .await .context(format!("Failed to fetch - {}", url))?;
if !response.status().is_success() { return Err(eyre!(format!("Failed to read - {}", url))); }
let mut stream = response.bytes_stream(); let mut hasher = Sha256::new();
while let Some(chunk) = stream.next().await { hasher.update(chunk.context("Failed to calculate Hash")?); }
Ok(hex::encode(hasher.finalize()))}