arhivach-downloader

Download arhivach.vc threads
git clone https://git.ea.contact/arhivach-downloader
Log | Files | Refs | README

lib.rs (3456B)

      1 pub mod config;
      2 pub mod error;
      3 pub mod event;
      4 pub mod export;
      5 
      6 mod download;
      7 mod post;
      8 
      9 use crate::error::{Error, Result};
     10 use crate::post::{Post, File};
     11 use crate::export::Exporter;
     12 
     13 use std::sync::mpsc::Sender;
     14 
     15 pub const BASE_URL: &str = "https://arhivach.vc";
     16 
     17 pub fn run(config: &config::Config, tx: Sender<event::Event>) -> Result<()> {
     18     tx.send(event::Event::GetStarted)?;
     19     let html = download::download(&config.url, config.download_retries)?
     20         .body_mut()
     21         .read_to_string()?;
     22     let posts = Post::parse_posts(&html)
     23         .inspect_err(|e| { let _ = tx.send(event::Event::GetFailed { error: e.to_string() }); })?;
     24     tx.send(event::Event::GetDone)?;
     25 
     26     tx.send(event::Event::DownloadAllStarted)?;
     27     run_download(&posts, &config, tx.clone())
     28         .inspect_err(|e| { let _ = tx.send(event::Event::DownloadAllFailed { error: e.to_string() }); })?;
     29     tx.send(event::Event::DownloadAllDone)?;
     30 
     31     tx.send(event::Event::ExportStarted)?;
     32     config.exporter.export(&posts, config)
     33         .inspect_err(|e| { let _ = tx.send(event::Event::ExportFailed { error: e.to_string() }); })?;
     34     tx.send(event::Event::ExportDone)?;
     35 
     36     Ok(())
     37 }
     38 
     39 /// Download files and thumbnails. Send DownloadStarted, DownloadDone and DownloadFailed events
     40 fn run_download(posts: &[Post], config: &config::Config, tx: Sender<event::Event>) -> Result<()> {
     41     std::fs::create_dir_all(&config.dir)?;
     42 
     43     let download_item = |url: &str, filepath: &std::path::PathBuf| -> Result<()> {
     44         let result = download::download(url, config.download_retries)?;
     45         let mut bytes = Vec::new();
     46         std::io::Read::read_to_end(&mut result.into_body().as_reader(), &mut bytes)?;
     47         if bytes.is_empty() {
     48             return Err(Error::EmptyFile(url.to_string()));
     49         }
     50         std::fs::write(filepath, bytes)?;
     51         Ok(())
     52     };
     53 
     54     let download_section = |
     55         subdir: &str,
     56         get_url: fn(&File) -> (&str, &str),
     57     | -> Result<()> {
     58         let dir = config.dir.join(subdir);
     59         std::fs::create_dir_all(&dir)?;
     60 
     61         let mut index: usize = 1;
     62         let max_index: usize = posts.iter().map(|p| p.files.len()).sum();
     63         for f in posts.iter().flat_map(|p| &p.files) {
     64             tx.send(event::Event::DownloadStarted { index, max_index })?;
     65             let (url, fallback) = get_url(f);
     66             let filename = url.rsplit("/").next().unwrap_or(fallback).trim();
     67             let filepath = dir.join(filename);
     68             if config.resume && filepath.exists() {
     69                 tx.send(event::Event::DownloadSkipped { index, max_index })?;
     70                 index += 1;
     71                 continue
     72             }
     73             match download_item(url, &filepath) {
     74                 Ok(()) => tx.send(event::Event::DownloadDone{ index, max_index })?,
     75                 Err(e) => tx.send(event::Event::DownloadFailed {
     76                     url: url.to_string(),
     77                     error: e.to_string()
     78                 })?
     79             };
     80             index += 1;
     81         }
     82         Ok(())
     83     };
     84 
     85     if config.files {
     86         tx.send(event::Event::DownloadFilesStarted)?;
     87         download_section("files", |f| (&f.url, &f.name_timestamp))?;
     88         tx.send(event::Event::DownloadFilesDone)?;
     89     }
     90     if config.thumb {
     91         tx.send(event::Event::DownloadThumbStarted)?;
     92         download_section("thumb", |f| (&f.url_thumb, &f.name_timestamp))?;
     93         tx.send(event::Event::DownloadThumbDone)?;
     94     }
     95 
     96     Ok(())
     97 }