crates/crates-io/lib.rs

   1 #![allow(clippy::all)]
   2
   3 use std::collections::BTreeMap;
   4 use std::fmt;
   5 use std::fs::File;
   6 use std::io::prelude::*;
   7 use std::io::{Cursor, SeekFrom};
   8 use std::time::Instant;
   9
  10 use anyhow::{bail, format_err, Context, Result};
  11 use curl::easy::{Easy, List};
  12 use percent_encoding::{percent_encode, NON_ALPHANUMERIC};
  13 use serde::{Deserialize, Serialize};
  14 use url::Url;
  15
  16 pub struct Registry {
  17     /// The base URL for issuing API requests.
  18     host: String,
  19     /// Optional authorization token.
  20     /// If None, commands requiring authorization will fail.
  21     token: Option<String>,
  22     /// Curl handle for issuing requests.
  23     handle: Easy,
  24 }
  25
  26 #[derive(PartialEq, Clone, Copy)]
  27 pub enum Auth {
  28     Authorized,
  29     Unauthorized,
  30 }
  31
  32 #[derive(Deserialize)]
  33 pub struct Crate {
  34     pub name: String,
  35     pub description: Option<String>,
  36     pub max_version: String,
  37 }
  38
  39 #[derive(Serialize)]
  40 pub struct NewCrate {
  41     pub name: String,
  42     pub vers: String,
  43     pub deps: Vec<NewCrateDependency>,
  44     pub features: BTreeMap<String, Vec<String>>,
  45     pub authors: Vec<String>,
  46     pub description: Option<String>,
  47     pub documentation: Option<String>,
  48     pub homepage: Option<String>,
  49     pub readme: Option<String>,
  50     pub readme_file: Option<String>,
  51     pub keywords: Vec<String>,
  52     pub categories: Vec<String>,
  53     pub license: Option<String>,
  54     pub license_file: Option<String>,
  55     pub repository: Option<String>,
  56     pub badges: BTreeMap<String, BTreeMap<String, String>>,
  57     pub links: Option<String>,
  58 }
  59
  60 #[derive(Serialize)]
  61 pub struct NewCrateDependency {
  62     pub optional: bool,
  63     pub default_features: bool,
  64     pub name: String,
  65     pub features: Vec<String>,
  66     pub version_req: String,
  67     pub target: Option<String>,
  68     pub kind: String,
  69     #[serde(skip_serializing_if = "Option::is_none")]
  70     pub registry: Option<String>,
  71     #[serde(skip_serializing_if = "Option::is_none")]
  72     pub explicit_name_in_toml: Option<String>,
  73 }
  74
  75 #[derive(Deserialize)]
  76 pub struct User {
  77     pub id: u32,
  78     pub login: String,
  79     pub avatar: Option<String>,
  80     pub email: Option<String>,
  81     pub name: Option<String>,
  82 }
  83
  84 pub struct Warnings {
  85     pub invalid_categories: Vec<String>,
  86     pub invalid_badges: Vec<String>,
  87     pub other: Vec<String>,
  88 }
  89
  90 #[derive(Deserialize)]
  91 struct R {
  92     ok: bool,
  93 }
  94 #[derive(Deserialize)]
  95 struct OwnerResponse {
  96     ok: bool,
  97     msg: String,
  98 }
  99 #[derive(Deserialize)]
 100 struct ApiErrorList {
 101     errors: Vec<ApiError>,
 102 }
 103 #[derive(Deserialize)]
 104 struct ApiError {
 105     detail: String,
 106 }
 107 #[derive(Serialize)]
 108 struct OwnersReq<'a> {
 109     users: &'a [&'a str],
 110 }
 111 #[derive(Deserialize)]
 112 struct Users {
 113     users: Vec<User>,
 114 }
 115 #[derive(Deserialize)]
 116 struct TotalCrates {
 117     total: u32,
 118 }
 119 #[derive(Deserialize)]
 120 struct Crates {
 121     crates: Vec<Crate>,
 122     meta: TotalCrates,
 123 }
 124
 125 #[derive(Debug)]
 126 pub enum ResponseError {
 127     Curl(curl::Error),
 128     Api {
 129         code: u32,
 130         errors: Vec<String>,
 131     },
 132     Code {
 133         code: u32,
 134         headers: Vec<String>,
 135         body: String,
 136     },
 137     Other(anyhow::Error),
 138 }
 139
 140 impl std::error::Error for ResponseError {
 141     fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
 142         match self {
 143             ResponseError::Curl(..) => None,
 144             ResponseError::Api { .. } => None,
 145             ResponseError::Code { .. } => None,
 146             ResponseError::Other(e) => Some(e.as_ref()),
 147         }
 148     }
 149 }
 150
 151 impl fmt::Display for ResponseError {
 152     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
 153         match self {
 154             ResponseError::Curl(e) => write!(f, "{}", e),
 155             ResponseError::Api { code, errors } => {
 156                 f.write_str("the remote server responded with an error")?;
 157                 if *code != 200 {
 158                     write!(f, " (status {} {})", code, reason(*code))?;
 159                 };
 160                 write!(f, ": {}", errors.join(", "))
 161             }
 162             ResponseError::Code {
 163                 code,
 164                 headers,
 165                 body,
 166             } => write!(
 167                 f,
 168                 "failed to get a 200 OK response, got {}\n\
 169                  headers:\n\
 170                  \t{}\n\
 171                  body:\n\
 172                  {}",
 173                 code,
 174                 headers.join("\n\t"),
 175                 body
 176             ),
 177             ResponseError::Other(..) => write!(f, "invalid response from server"),
 178         }
 179     }
 180 }
 181
 182 impl From<curl::Error> for ResponseError {
 183     fn from(error: curl::Error) -> Self {
 184         ResponseError::Curl(error)
 185     }
 186 }
 187
 188 impl Registry {
 189     /// Creates a new `Registry`.
 190     ///
 191     /// ## Example
 192     ///
 193     /// ```rust
 194     /// use curl::easy::Easy;
 195     /// use crates_io::Registry;
 196     ///
 197     /// let mut handle = Easy::new();
 198     /// // If connecting to crates.io, a user-agent is required.
 199     /// handle.useragent("my_crawler (example.com/info)");
 200     /// let mut reg = Registry::new_handle(String::from("https://crates.io"), None, handle);
 201     /// ```
 202     pub fn new_handle(host: String, token: Option<String>, handle: Easy) -> Registry {
 203         Registry {
 204             host,
 205             token,
 206             handle,
 207         }
 208     }
 209
 210     pub fn host(&self) -> &str {
 211         &self.host
 212     }
 213
 214     pub fn host_is_crates_io(&self) -> bool {
 215         is_url_crates_io(&self.host)
 216     }
 217
 218     pub fn add_owners(&mut self, krate: &str, owners: &[&str]) -> Result<String> {
 219         let body = serde_json::to_string(&OwnersReq { users: owners })?;
 220         let body = self.put(&format!("/crates/{}/owners", krate), body.as_bytes())?;
 221         assert!(serde_json::from_str::<OwnerResponse>(&body)?.ok);
 222         Ok(serde_json::from_str::<OwnerResponse>(&body)?.msg)
 223     }
 224
 225     pub fn remove_owners(&mut self, krate: &str, owners: &[&str]) -> Result<()> {
 226         let body = serde_json::to_string(&OwnersReq { users: owners })?;
 227         let body = self.delete(&format!("/crates/{}/owners", krate), Some(body.as_bytes()))?;
 228         assert!(serde_json::from_str::<OwnerResponse>(&body)?.ok);
 229         Ok(())
 230     }
 231
 232     pub fn list_owners(&mut self, krate: &str) -> Result<Vec<User>> {
 233         let body = self.get(&format!("/crates/{}/owners", krate))?;
 234         Ok(serde_json::from_str::<Users>(&body)?.users)
 235     }
 236
 237     pub fn publish(&mut self, krate: &NewCrate, mut tarball: &File) -> Result<Warnings> {
 238         let json = serde_json::to_string(krate)?;
 239         // Prepare the body. The format of the upload request is:
 240         //
 241         //      <le u32 of json>
 242         //      <json request> (metadata for the package)
 243         //      <le u32 of tarball>
 244         //      <source tarball>
 245
 246         // NOTE: This can be replaced with `stream_len` if it is ever stabilized.
 247         //
 248         // This checks the length using seeking instead of metadata, because
 249         // on some filesystems, getting the metadata will fail because
 250         // the file was renamed in ops::package.
 251         let tarball_len = tarball
 252             .seek(SeekFrom::End(0))
 253             .with_context(|| "failed to seek tarball")?;
 254         tarball
 255             .seek(SeekFrom::Start(0))
 256             .with_context(|| "failed to seek tarball")?;
 257         let header = {
 258             let mut w = Vec::new();
 259             w.extend(&(json.len() as u32).to_le_bytes());
 260             w.extend(json.as_bytes().iter().cloned());
 261             w.extend(&(tarball_len as u32).to_le_bytes());
 262             w
 263         };
 264         let size = tarball_len as usize + header.len();
 265         let mut body = Cursor::new(header).chain(tarball);
 266
 267         let url = format!("{}/api/v1/crates/new", self.host);
 268
 269         let token = match self.token.as_ref() {
 270             Some(s) => s,
 271             None => bail!("no upload token found, please run `cargo login`"),
 272         };
 273         self.handle.put(true)?;
 274         self.handle.url(&url)?;
 275         self.handle.in_filesize(size as u64)?;
 276         let mut headers = List::new();
 277         headers.append("Accept: application/json")?;
 278         headers.append(&format!("Authorization: {}", token))?;
 279         self.handle.http_headers(headers)?;
 280
 281         let started = Instant::now();
 282         let body = self
 283             .handle(&mut |buf| body.read(buf).unwrap_or(0))
 284             .map_err(|e| match e {
 285                 ResponseError::Code { code, .. }
 286                     if code == 503
 287                         && started.elapsed().as_secs() >= 29
 288                         && self.host_is_crates_io() =>
 289                 {
 290                     format_err!(
 291                         "Request timed out after 30 seconds. If you're trying to \
 292                          upload a crate it may be too large. If the crate is under \
 293                          10MB in size, you can email help@crates.io for assistance.\n\
 294                          Total size was {}.",
 295                         tarball_len
 296                     )
 297                 }
 298                 _ => e.into(),
 299             })?;
 300
 301         let response = if body.is_empty() {
 302             "{}".parse()?
 303         } else {
 304             body.parse::<serde_json::Value>()?
 305         };
 306
 307         let invalid_categories: Vec<String> = response
 308             .get("warnings")
 309             .and_then(|j| j.get("invalid_categories"))
 310             .and_then(|j| j.as_array())
 311             .map(|x| x.iter().flat_map(|j| j.as_str()).map(Into::into).collect())
 312             .unwrap_or_else(Vec::new);
 313
 314         let invalid_badges: Vec<String> = response
 315             .get("warnings")
 316             .and_then(|j| j.get("invalid_badges"))
 317             .and_then(|j| j.as_array())
 318             .map(|x| x.iter().flat_map(|j| j.as_str()).map(Into::into).collect())
 319             .unwrap_or_else(Vec::new);
 320
 321         let other: Vec<String> = response
 322             .get("warnings")
 323             .and_then(|j| j.get("other"))
 324             .and_then(|j| j.as_array())
 325             .map(|x| x.iter().flat_map(|j| j.as_str()).map(Into::into).collect())
 326             .unwrap_or_else(Vec::new);
 327
 328         Ok(Warnings {
 329             invalid_categories,
 330             invalid_badges,
 331             other,
 332         })
 333     }
 334
 335     pub fn search(&mut self, query: &str, limit: u32) -> Result<(Vec<Crate>, u32)> {
 336         let formatted_query = percent_encode(query.as_bytes(), NON_ALPHANUMERIC);
 337         let body = self.req(
 338             &format!("/crates?q={}&per_page={}", formatted_query, limit),
 339             None,
 340             Auth::Unauthorized,
 341         )?;
 342
 343         let crates = serde_json::from_str::<Crates>(&body)?;
 344         Ok((crates.crates, crates.meta.total))
 345     }
 346
 347     pub fn yank(&mut self, krate: &str, version: &str) -> Result<()> {
 348         let body = self.delete(&format!("/crates/{}/{}/yank", krate, version), None)?;
 349         assert!(serde_json::from_str::<R>(&body)?.ok);
 350         Ok(())
 351     }
 352
 353     pub fn unyank(&mut self, krate: &str, version: &str) -> Result<()> {
 354         let body = self.put(&format!("/crates/{}/{}/unyank", krate, version), &[])?;
 355         assert!(serde_json::from_str::<R>(&body)?.ok);
 356         Ok(())
 357     }
 358
 359     fn put(&mut self, path: &str, b: &[u8]) -> Result<String> {
 360         self.handle.put(true)?;
 361         self.req(path, Some(b), Auth::Authorized)
 362     }
 363
 364     fn get(&mut self, path: &str) -> Result<String> {
 365         self.handle.get(true)?;
 366         self.req(path, None, Auth::Authorized)
 367     }
 368
 369     fn delete(&mut self, path: &str, b: Option<&[u8]>) -> Result<String> {
 370         self.handle.custom_request("DELETE")?;
 371         self.req(path, b, Auth::Authorized)
 372     }
 373
 374     fn req(&mut self, path: &str, body: Option<&[u8]>, authorized: Auth) -> Result<String> {
 375         self.handle.url(&format!("{}/api/v1{}", self.host, path))?;
 376         let mut headers = List::new();
 377         headers.append("Accept: application/json")?;
 378         headers.append("Content-Type: application/json")?;
 379
 380         if authorized == Auth::Authorized {
 381             let token = match self.token.as_ref() {
 382                 Some(s) => s,
 383                 None => bail!("no upload token found, please run `cargo login`"),
 384             };
 385             headers.append(&format!("Authorization: {}", token))?;
 386         }
 387         self.handle.http_headers(headers)?;
 388         match body {
 389             Some(mut body) => {
 390                 self.handle.upload(true)?;
 391                 self.handle.in_filesize(body.len() as u64)?;
 392                 self.handle(&mut |buf| body.read(buf).unwrap_or(0))
 393                     .map_err(|e| e.into())
 394             }
 395             None => self.handle(&mut |_| 0).map_err(|e| e.into()),
 396         }
 397     }
 398
 399     fn handle(
 400         &mut self,
 401         read: &mut dyn FnMut(&mut [u8]) -> usize,
 402     ) -> std::result::Result<String, ResponseError> {
 403         let mut headers = Vec::new();
 404         let mut body = Vec::new();
 405         {
 406             let mut handle = self.handle.transfer();
 407             handle.read_function(|buf| Ok(read(buf)))?;
 408             handle.write_function(|data| {
 409                 body.extend_from_slice(data);
 410                 Ok(data.len())
 411             })?;
 412             handle.header_function(|data| {
 413                 // Headers contain trailing \r\n, trim them to make it easier
 414                 // to work with.
 415                 let s = String::from_utf8_lossy(data).trim().to_string();
 416                 headers.push(s);
 417                 true
 418             })?;
 419             handle.perform()?;
 420         }
 421
 422         let body = match String::from_utf8(body) {
 423             Ok(body) => body,
 424             Err(..) => {
 425                 return Err(ResponseError::Other(format_err!(
 426                     "response body was not valid utf-8"
 427                 )))
 428             }
 429         };
 430         let errors = serde_json::from_str::<ApiErrorList>(&body)
 431             .ok()
 432             .map(|s| s.errors.into_iter().map(|s| s.detail).collect::<Vec<_>>());
 433
 434         match (self.handle.response_code()?, errors) {
 435             (0, None) | (200, None) => Ok(body),
 436             (code, Some(errors)) => Err(ResponseError::Api { code, errors }),
 437             (code, None) => Err(ResponseError::Code {
 438                 code,
 439                 headers,
 440                 body,
 441             }),
 442         }
 443     }
 444 }
 445
 446 fn reason(code: u32) -> &'static str {
 447     // Taken from https://developer.mozilla.org/en-US/docs/Web/HTTP/Status
 448     match code {
 449         100 => "Continue",
 450         101 => "Switching Protocol",
 451         103 => "Early Hints",
 452         200 => "OK",
 453         201 => "Created",
 454         202 => "Accepted",
 455         203 => "Non-Authoritative Information",
 456         204 => "No Content",
 457         205 => "Reset Content",
 458         206 => "Partial Content",
 459         300 => "Multiple Choice",
 460         301 => "Moved Permanently",
 461         302 => "Found",
 462         303 => "See Other",
 463         304 => "Not Modified",
 464         307 => "Temporary Redirect",
 465         308 => "Permanent Redirect",
 466         400 => "Bad Request",
 467         401 => "Unauthorized",
 468         402 => "Payment Required",
 469         403 => "Forbidden",
 470         404 => "Not Found",
 471         405 => "Method Not Allowed",
 472         406 => "Not Acceptable",
 473         407 => "Proxy Authentication Required",
 474         408 => "Request Timeout",
 475         409 => "Conflict",
 476         410 => "Gone",
 477         411 => "Length Required",
 478         412 => "Precondition Failed",
 479         413 => "Payload Too Large",
 480         414 => "URI Too Long",
 481         415 => "Unsupported Media Type",
 482         416 => "Request Range Not Satisfiable",
 483         417 => "Expectation Failed",
 484         429 => "Too Many Requests",
 485         431 => "Request Header Fields Too Large",
 486         500 => "Internal Server Error",
 487         501 => "Not Implemented",
 488         502 => "Bad Gateway",
 489         503 => "Service Unavailable",
 490         504 => "Gateway Timeout",
 491         _ => "<unknown>",
 492     }
 493 }
 494
 495 /// Returns `true` if the host of the given URL is "crates.io".
 496 pub fn is_url_crates_io(url: &str) -> bool {
 497     Url::parse(url)
 498         .map(|u| u.host_str() == Some("crates.io"))
 499         .unwrap_or(false)
 500 }