src/tools/cargo/crates/crates-io/lib.rs

   1 #![allow(clippy::all)]
   2
   3 use std::collections::BTreeMap;
   4 use std::fs::File;
   5 use std::io::prelude::*;
   6 use std::io::{Cursor, SeekFrom};
   7 use std::time::Instant;
   8
   9 use curl::easy::{Easy, List};
  10 use percent_encoding::{percent_encode, NON_ALPHANUMERIC};
  11 use serde::{Deserialize, Serialize};
  12 use url::Url;
  13
  14 pub type Result<T> = std::result::Result<T, Error>;
  15
  16 pub struct Registry {
  17     /// The base URL for issuing API requests.
  18     host: String,
  19     /// Optional authorization token.
  20     /// If None, commands requiring authorization will fail.
  21     token: Option<String>,
  22     /// Curl handle for issuing requests.
  23     handle: Easy,
  24     /// Whether to include the authorization token with all requests.
  25     auth_required: bool,
  26 }
  27
  28 #[derive(PartialEq, Clone, Copy)]
  29 pub enum Auth {
  30     Authorized,
  31     Unauthorized,
  32 }
  33
  34 #[derive(Deserialize)]
  35 pub struct Crate {
  36     pub name: String,
  37     pub description: Option<String>,
  38     pub max_version: String,
  39 }
  40
  41 #[derive(Serialize, Deserialize)]
  42 pub struct NewCrate {
  43     pub name: String,
  44     pub vers: String,
  45     pub deps: Vec<NewCrateDependency>,
  46     pub features: BTreeMap<String, Vec<String>>,
  47     pub authors: Vec<String>,
  48     pub description: Option<String>,
  49     pub documentation: Option<String>,
  50     pub homepage: Option<String>,
  51     pub readme: Option<String>,
  52     pub readme_file: Option<String>,
  53     pub keywords: Vec<String>,
  54     pub categories: Vec<String>,
  55     pub license: Option<String>,
  56     pub license_file: Option<String>,
  57     pub repository: Option<String>,
  58     pub badges: BTreeMap<String, BTreeMap<String, String>>,
  59     pub links: Option<String>,
  60     pub rust_version: Option<String>,
  61 }
  62
  63 #[derive(Serialize, Deserialize)]
  64 pub struct NewCrateDependency {
  65     pub optional: bool,
  66     pub default_features: bool,
  67     pub name: String,
  68     pub features: Vec<String>,
  69     pub version_req: String,
  70     pub target: Option<String>,
  71     pub kind: String,
  72     #[serde(skip_serializing_if = "Option::is_none")]
  73     pub registry: Option<String>,
  74     #[serde(skip_serializing_if = "Option::is_none")]
  75     pub explicit_name_in_toml: Option<String>,
  76     #[serde(skip_serializing_if = "Option::is_none")]
  77     pub artifact: Option<Vec<String>>,
  78     #[serde(skip_serializing_if = "Option::is_none")]
  79     pub bindep_target: Option<String>,
  80     #[serde(default, skip_serializing_if = "is_false")]
  81     pub lib: bool,
  82 }
  83
  84 fn is_false(x: &bool) -> bool {
  85     *x == false
  86 }
  87
  88 #[derive(Deserialize)]
  89 pub struct User {
  90     pub id: u32,
  91     pub login: String,
  92     pub avatar: Option<String>,
  93     pub email: Option<String>,
  94     pub name: Option<String>,
  95 }
  96
  97 pub struct Warnings {
  98     pub invalid_categories: Vec<String>,
  99     pub invalid_badges: Vec<String>,
 100     pub other: Vec<String>,
 101 }
 102
 103 #[derive(Deserialize)]
 104 struct R {
 105     ok: bool,
 106 }
 107 #[derive(Deserialize)]
 108 struct OwnerResponse {
 109     ok: bool,
 110     msg: String,
 111 }
 112 #[derive(Deserialize)]
 113 struct ApiErrorList {
 114     errors: Vec<ApiError>,
 115 }
 116 #[derive(Deserialize)]
 117 struct ApiError {
 118     detail: String,
 119 }
 120 #[derive(Serialize)]
 121 struct OwnersReq<'a> {
 122     users: &'a [&'a str],
 123 }
 124 #[derive(Deserialize)]
 125 struct Users {
 126     users: Vec<User>,
 127 }
 128 #[derive(Deserialize)]
 129 struct TotalCrates {
 130     total: u32,
 131 }
 132 #[derive(Deserialize)]
 133 struct Crates {
 134     crates: Vec<Crate>,
 135     meta: TotalCrates,
 136 }
 137
 138 /// Error returned when interacting with a registry.
 139 #[derive(Debug, thiserror::Error)]
 140 pub enum Error {
 141     /// Error from libcurl.
 142     #[error(transparent)]
 143     Curl(#[from] curl::Error),
 144
 145     /// Error from seriailzing the request payload and deserializing the
 146     /// response body (like response body didn't match expected structure).
 147     #[error(transparent)]
 148     Json(#[from] serde_json::Error),
 149
 150     /// Error from IO. Mostly from reading the tarball to upload.
 151     #[error("failed to seek tarball")]
 152     Io(#[from] std::io::Error),
 153
 154     /// Response body was not valid utf8.
 155     #[error("invalid response body from server")]
 156     Utf8(#[from] std::string::FromUtf8Error),
 157
 158     /// Error from API response containing JSON field `errors.details`.
 159     #[error(
 160         "the remote server responded with an error{}: {}",
 161         status(*code),
 162         errors.join(", "),
 163     )]
 164     Api {
 165         code: u32,
 166         headers: Vec<String>,
 167         errors: Vec<String>,
 168     },
 169
 170     /// Error from API response which didn't have pre-programmed `errors.details`.
 171     #[error(
 172         "failed to get a 200 OK response, got {code}\nheaders:\n\t{}\nbody:\n{body}",
 173         headers.join("\n\t"),
 174     )]
 175     Code {
 176         code: u32,
 177         headers: Vec<String>,
 178         body: String,
 179     },
 180
 181     /// Reason why the token was invalid.
 182     #[error("{0}")]
 183     InvalidToken(&'static str),
 184
 185     /// Server was unavailable and timeouted. Happened when uploading a way
 186     /// too large tarball to crates.io.
 187     #[error(
 188         "Request timed out after 30 seconds. If you're trying to \
 189          upload a crate it may be too large. If the crate is under \
 190          10MB in size, you can email help@crates.io for assistance.\n\
 191          Total size was {0}."
 192     )]
 193     Timeout(u64),
 194 }
 195
 196 impl Registry {
 197     /// Creates a new `Registry`.
 198     ///
 199     /// ## Example
 200     ///
 201     /// ```rust
 202     /// use curl::easy::Easy;
 203     /// use crates_io::Registry;
 204     ///
 205     /// let mut handle = Easy::new();
 206     /// // If connecting to crates.io, a user-agent is required.
 207     /// handle.useragent("my_crawler (example.com/info)");
 208     /// let mut reg = Registry::new_handle(String::from("https://crates.io"), None, handle, true);
 209     /// ```
 210     pub fn new_handle(
 211         host: String,
 212         token: Option<String>,
 213         handle: Easy,
 214         auth_required: bool,
 215     ) -> Registry {
 216         Registry {
 217             host,
 218             token,
 219             handle,
 220             auth_required,
 221         }
 222     }
 223
 224     pub fn set_token(&mut self, token: Option<String>) {
 225         self.token = token;
 226     }
 227
 228     fn token(&self) -> Result<&str> {
 229         let token = self.token.as_ref().ok_or_else(|| {
 230             Error::InvalidToken("no upload token found, please run `cargo login`")
 231         })?;
 232         check_token(token)?;
 233         Ok(token)
 234     }
 235
 236     pub fn host(&self) -> &str {
 237         &self.host
 238     }
 239
 240     pub fn host_is_crates_io(&self) -> bool {
 241         is_url_crates_io(&self.host)
 242     }
 243
 244     pub fn add_owners(&mut self, krate: &str, owners: &[&str]) -> Result<String> {
 245         let body = serde_json::to_string(&OwnersReq { users: owners })?;
 246         let body = self.put(&format!("/crates/{}/owners", krate), body.as_bytes())?;
 247         assert!(serde_json::from_str::<OwnerResponse>(&body)?.ok);
 248         Ok(serde_json::from_str::<OwnerResponse>(&body)?.msg)
 249     }
 250
 251     pub fn remove_owners(&mut self, krate: &str, owners: &[&str]) -> Result<()> {
 252         let body = serde_json::to_string(&OwnersReq { users: owners })?;
 253         let body = self.delete(&format!("/crates/{}/owners", krate), Some(body.as_bytes()))?;
 254         assert!(serde_json::from_str::<OwnerResponse>(&body)?.ok);
 255         Ok(())
 256     }
 257
 258     pub fn list_owners(&mut self, krate: &str) -> Result<Vec<User>> {
 259         let body = self.get(&format!("/crates/{}/owners", krate))?;
 260         Ok(serde_json::from_str::<Users>(&body)?.users)
 261     }
 262
 263     pub fn publish(&mut self, krate: &NewCrate, mut tarball: &File) -> Result<Warnings> {
 264         let json = serde_json::to_string(krate)?;
 265         // Prepare the body. The format of the upload request is:
 266         //
 267         //      <le u32 of json>
 268         //      <json request> (metadata for the package)
 269         //      <le u32 of tarball>
 270         //      <source tarball>
 271
 272         // NOTE: This can be replaced with `stream_len` if it is ever stabilized.
 273         //
 274         // This checks the length using seeking instead of metadata, because
 275         // on some filesystems, getting the metadata will fail because
 276         // the file was renamed in ops::package.
 277         let tarball_len = tarball.seek(SeekFrom::End(0))?;
 278         tarball.seek(SeekFrom::Start(0))?;
 279         let header = {
 280             let mut w = Vec::new();
 281             w.extend(&(json.len() as u32).to_le_bytes());
 282             w.extend(json.as_bytes().iter().cloned());
 283             w.extend(&(tarball_len as u32).to_le_bytes());
 284             w
 285         };
 286         let size = tarball_len as usize + header.len();
 287         let mut body = Cursor::new(header).chain(tarball);
 288
 289         let url = format!("{}/api/v1/crates/new", self.host);
 290
 291         self.handle.put(true)?;
 292         self.handle.url(&url)?;
 293         self.handle.in_filesize(size as u64)?;
 294         let mut headers = List::new();
 295         headers.append("Accept: application/json")?;
 296         headers.append(&format!("Authorization: {}", self.token()?))?;
 297         self.handle.http_headers(headers)?;
 298
 299         let started = Instant::now();
 300         let body = self
 301             .handle(&mut |buf| body.read(buf).unwrap_or(0))
 302             .map_err(|e| match e {
 303                 Error::Code { code, .. }
 304                     if code == 503
 305                         && started.elapsed().as_secs() >= 29
 306                         && self.host_is_crates_io() =>
 307                 {
 308                     Error::Timeout(tarball_len)
 309                 }
 310                 _ => e.into(),
 311             })?;
 312
 313         let response = if body.is_empty() {
 314             "{}".parse()?
 315         } else {
 316             body.parse::<serde_json::Value>()?
 317         };
 318
 319         let invalid_categories: Vec<String> = response
 320             .get("warnings")
 321             .and_then(|j| j.get("invalid_categories"))
 322             .and_then(|j| j.as_array())
 323             .map(|x| x.iter().flat_map(|j| j.as_str()).map(Into::into).collect())
 324             .unwrap_or_else(Vec::new);
 325
 326         let invalid_badges: Vec<String> = response
 327             .get("warnings")
 328             .and_then(|j| j.get("invalid_badges"))
 329             .and_then(|j| j.as_array())
 330             .map(|x| x.iter().flat_map(|j| j.as_str()).map(Into::into).collect())
 331             .unwrap_or_else(Vec::new);
 332
 333         let other: Vec<String> = response
 334             .get("warnings")
 335             .and_then(|j| j.get("other"))
 336             .and_then(|j| j.as_array())
 337             .map(|x| x.iter().flat_map(|j| j.as_str()).map(Into::into).collect())
 338             .unwrap_or_else(Vec::new);
 339
 340         Ok(Warnings {
 341             invalid_categories,
 342             invalid_badges,
 343             other,
 344         })
 345     }
 346
 347     pub fn search(&mut self, query: &str, limit: u32) -> Result<(Vec<Crate>, u32)> {
 348         let formatted_query = percent_encode(query.as_bytes(), NON_ALPHANUMERIC);
 349         let body = self.req(
 350             &format!("/crates?q={}&per_page={}", formatted_query, limit),
 351             None,
 352             Auth::Unauthorized,
 353         )?;
 354
 355         let crates = serde_json::from_str::<Crates>(&body)?;
 356         Ok((crates.crates, crates.meta.total))
 357     }
 358
 359     pub fn yank(&mut self, krate: &str, version: &str) -> Result<()> {
 360         let body = self.delete(&format!("/crates/{}/{}/yank", krate, version), None)?;
 361         assert!(serde_json::from_str::<R>(&body)?.ok);
 362         Ok(())
 363     }
 364
 365     pub fn unyank(&mut self, krate: &str, version: &str) -> Result<()> {
 366         let body = self.put(&format!("/crates/{}/{}/unyank", krate, version), &[])?;
 367         assert!(serde_json::from_str::<R>(&body)?.ok);
 368         Ok(())
 369     }
 370
 371     fn put(&mut self, path: &str, b: &[u8]) -> Result<String> {
 372         self.handle.put(true)?;
 373         self.req(path, Some(b), Auth::Authorized)
 374     }
 375
 376     fn get(&mut self, path: &str) -> Result<String> {
 377         self.handle.get(true)?;
 378         self.req(path, None, Auth::Authorized)
 379     }
 380
 381     fn delete(&mut self, path: &str, b: Option<&[u8]>) -> Result<String> {
 382         self.handle.custom_request("DELETE")?;
 383         self.req(path, b, Auth::Authorized)
 384     }
 385
 386     fn req(&mut self, path: &str, body: Option<&[u8]>, authorized: Auth) -> Result<String> {
 387         self.handle.url(&format!("{}/api/v1{}", self.host, path))?;
 388         let mut headers = List::new();
 389         headers.append("Accept: application/json")?;
 390         headers.append("Content-Type: application/json")?;
 391
 392         if self.auth_required || authorized == Auth::Authorized {
 393             headers.append(&format!("Authorization: {}", self.token()?))?;
 394         }
 395         self.handle.http_headers(headers)?;
 396         match body {
 397             Some(mut body) => {
 398                 self.handle.upload(true)?;
 399                 self.handle.in_filesize(body.len() as u64)?;
 400                 self.handle(&mut |buf| body.read(buf).unwrap_or(0))
 401                     .map_err(|e| e.into())
 402             }
 403             None => self.handle(&mut |_| 0).map_err(|e| e.into()),
 404         }
 405     }
 406
 407     fn handle(&mut self, read: &mut dyn FnMut(&mut [u8]) -> usize) -> Result<String> {
 408         let mut headers = Vec::new();
 409         let mut body = Vec::new();
 410         {
 411             let mut handle = self.handle.transfer();
 412             handle.read_function(|buf| Ok(read(buf)))?;
 413             handle.write_function(|data| {
 414                 body.extend_from_slice(data);
 415                 Ok(data.len())
 416             })?;
 417             handle.header_function(|data| {
 418                 // Headers contain trailing \r\n, trim them to make it easier
 419                 // to work with.
 420                 let s = String::from_utf8_lossy(data).trim().to_string();
 421                 // Don't let server sneak extra lines anywhere.
 422                 if s.contains('\n') {
 423                     return true;
 424                 }
 425                 headers.push(s);
 426                 true
 427             })?;
 428             handle.perform()?;
 429         }
 430
 431         let body = String::from_utf8(body)?;
 432         let errors = serde_json::from_str::<ApiErrorList>(&body)
 433             .ok()
 434             .map(|s| s.errors.into_iter().map(|s| s.detail).collect::<Vec<_>>());
 435
 436         match (self.handle.response_code()?, errors) {
 437             (0, None) | (200, None) => Ok(body),
 438             (code, Some(errors)) => Err(Error::Api {
 439                 code,
 440                 headers,
 441                 errors,
 442             }),
 443             (code, None) => Err(Error::Code {
 444                 code,
 445                 headers,
 446                 body,
 447             }),
 448         }
 449     }
 450 }
 451
 452 fn status(code: u32) -> String {
 453     if code == 200 {
 454         String::new()
 455     } else {
 456         let reason = reason(code);
 457         format!(" (status {code} {reason})")
 458     }
 459 }
 460
 461 fn reason(code: u32) -> &'static str {
 462     // Taken from https://developer.mozilla.org/en-US/docs/Web/HTTP/Status
 463     match code {
 464         100 => "Continue",
 465         101 => "Switching Protocol",
 466         103 => "Early Hints",
 467         200 => "OK",
 468         201 => "Created",
 469         202 => "Accepted",
 470         203 => "Non-Authoritative Information",
 471         204 => "No Content",
 472         205 => "Reset Content",
 473         206 => "Partial Content",
 474         300 => "Multiple Choice",
 475         301 => "Moved Permanently",
 476         302 => "Found",
 477         303 => "See Other",
 478         304 => "Not Modified",
 479         307 => "Temporary Redirect",
 480         308 => "Permanent Redirect",
 481         400 => "Bad Request",
 482         401 => "Unauthorized",
 483         402 => "Payment Required",
 484         403 => "Forbidden",
 485         404 => "Not Found",
 486         405 => "Method Not Allowed",
 487         406 => "Not Acceptable",
 488         407 => "Proxy Authentication Required",
 489         408 => "Request Timeout",
 490         409 => "Conflict",
 491         410 => "Gone",
 492         411 => "Length Required",
 493         412 => "Precondition Failed",
 494         413 => "Payload Too Large",
 495         414 => "URI Too Long",
 496         415 => "Unsupported Media Type",
 497         416 => "Request Range Not Satisfiable",
 498         417 => "Expectation Failed",
 499         429 => "Too Many Requests",
 500         431 => "Request Header Fields Too Large",
 501         500 => "Internal Server Error",
 502         501 => "Not Implemented",
 503         502 => "Bad Gateway",
 504         503 => "Service Unavailable",
 505         504 => "Gateway Timeout",
 506         _ => "<unknown>",
 507     }
 508 }
 509
 510 /// Returns `true` if the host of the given URL is "crates.io".
 511 pub fn is_url_crates_io(url: &str) -> bool {
 512     Url::parse(url)
 513         .map(|u| u.host_str() == Some("crates.io"))
 514         .unwrap_or(false)
 515 }
 516
 517 /// Checks if a token is valid or malformed.
 518 ///
 519 /// This check is necessary to prevent sending tokens which create an invalid HTTP request.
 520 /// It would be easier to check just for alphanumeric tokens, but we can't be sure that all
 521 /// registries only create tokens in that format so that is as less restricted as possible.
 522 pub fn check_token(token: &str) -> Result<()> {
 523     if token.is_empty() {
 524         return Err(Error::InvalidToken("please provide a non-empty token"));
 525     }
 526     if token.bytes().all(|b| {
 527         // This is essentially the US-ASCII limitation of
 528         // https://www.rfc-editor.org/rfc/rfc9110#name-field-values. That is,
 529         // visible ASCII characters (0x21-0x7e), space, and tab. We want to be
 530         // able to pass this in an HTTP header without encoding.
 531         b >= 32 && b < 127 || b == b'\t'
 532     }) {
 533         Ok(())
 534     } else {
 535         Err(Error::InvalidToken(
 536             "token contains invalid characters.\nOnly printable ISO-8859-1 characters \
 537              are allowed as it is sent in a HTTPS header.",
 538         ))
 539     }
 540 }