Add support for Accept-Encoding header (gzip, deflate, br)

Author: chr4

src/cli.rs

@@ -9,6 +9,7 @@ pub struct Args {
     pub uri_file: String,
     pub user_agent: String,
     pub keep_alive: bool,
+    pub gzip: bool,
     pub quiet: bool,
     pub progress_bar: bool,
     pub captcha_string: String,
@@ -47,6 +48,12 @@ pub fn get_args() -> Args {
                 .long("no-keep-alive")
                 .help("Do not use keep-alive"),
         )
+        .arg(
+            Arg::with_name("no-gzip")
+                .short("g")
+                .long("no-gzip")
+                .help("Do not set 'Accept-Encoding: br,gzip,deflate' header"),
+        )
         .arg(
             Arg::with_name("delay")
                 .short("d")
@@ -131,6 +138,7 @@ pub fn get_args() -> Args {
             0
         },
         cookies: cookies,
+        gzip: !args.is_present("no-gzip"),
         keep_alive: !args.is_present("no-keep-alive"),
         quiet: args.is_present("quiet"),
         progress_bar: !args.is_present("no-progress-bar") && !args.is_present("quiet"),

src/loader.rs

@@ -9,7 +9,7 @@ use futures::Future;
 use futures::stream::Stream;
 
 use hyper::{Client, Method, Request, StatusCode, Uri};
-use hyper::header::{Cookie, UserAgent};
+use hyper::header::{Cookie, UserAgent, AcceptEncoding, Encoding, qitem};
 use hyper_tls::HttpsConnector;
 
 use tokio_core::reactor::Core;
@@ -40,6 +40,7 @@ pub struct Loader {
     cookie: Cookie,
     captcha_string: String,
     keep_alive: bool,
+    gzip: bool,
     delay: u64,
 }
 
@@ -95,6 +96,7 @@ impl Loader {
             user_agent: user_agent,
             cookie: cookie_jar,
             keep_alive: args.keep_alive,
+            gzip: args.gzip,
             delay: args.delay,
             captcha_string: args.captcha_string.to_string(),
         }))
@@ -158,6 +160,13 @@ impl Loader {
             let mut req: Request = Request::new(Method::Get, cache_resource.uri.clone());
             req.headers_mut().set(self.user_agent.clone());
             req.headers_mut().set(self.cookie.clone());
+            if self.gzip {
+                req.headers_mut().set(AcceptEncoding(vec![
+                    qitem(Encoding::Brotli),
+                    qitem(Encoding::Gzip),
+                    qitem(Encoding::Deflate),
+                ]));
+            }
 
             let task = client.request(req).and_then(|res| {
                 cache_resource.cache_status = match res.headers().get::<XCacheStatus>() {