crates/crates-io/lib.rs

   1 #![allow(unknown_lints)]
   2 #![allow(clippy::identity_op)] // used for vertical alignment
   3
   4 use std::collections::BTreeMap;
   5 use std::fs::File;
   6 use std::io::prelude::*;
   7 use std::io::{Cursor, SeekFrom};
   8 use std::time::Instant;
   9
  10 use anyhow::{bail, Context, Result};
  11 use curl::easy::{Easy, List};
  12 use percent_encoding::{percent_encode, NON_ALPHANUMERIC};
  13 use serde::{Deserialize, Serialize};
  14 use url::Url;
  15
  16 pub struct Registry {
  17     /// The base URL for issuing API requests.
  18     host: String,
  19     /// Optional authorization token.
  20     /// If None, commands requiring authorization will fail.
  21     token: Option<String>,
  22     /// Curl handle for issuing requests.
  23     handle: Easy,
  24 }
  25
  26 #[derive(PartialEq, Clone, Copy)]
  27 pub enum Auth {
  28     Authorized,
  29     Unauthorized,
  30 }
  31
  32 #[derive(Deserialize)]
  33 pub struct Crate {
  34     pub name: String,
  35     pub description: Option<String>,
  36     pub max_version: String,
  37 }
  38
  39 #[derive(Serialize)]
  40 pub struct NewCrate {
  41     pub name: String,
  42     pub vers: String,
  43     pub deps: Vec<NewCrateDependency>,
  44     pub features: BTreeMap<String, Vec<String>>,
  45     pub authors: Vec<String>,
  46     pub description: Option<String>,
  47     pub documentation: Option<String>,
  48     pub homepage: Option<String>,
  49     pub readme: Option<String>,
  50     pub readme_file: Option<String>,
  51     pub keywords: Vec<String>,
  52     pub categories: Vec<String>,
  53     pub license: Option<String>,
  54     pub license_file: Option<String>,
  55     pub repository: Option<String>,
  56     pub badges: BTreeMap<String, BTreeMap<String, String>>,
  57     pub links: Option<String>,
  58 }
  59
  60 #[derive(Serialize)]
  61 pub struct NewCrateDependency {
  62     pub optional: bool,
  63     pub default_features: bool,
  64     pub name: String,
  65     pub features: Vec<String>,
  66     pub version_req: String,
  67     pub target: Option<String>,
  68     pub kind: String,
  69     #[serde(skip_serializing_if = "Option::is_none")]
  70     pub registry: Option<String>,
  71     #[serde(skip_serializing_if = "Option::is_none")]
  72     pub explicit_name_in_toml: Option<String>,
  73 }
  74
  75 #[derive(Deserialize)]
  76 pub struct User {
  77     pub id: u32,
  78     pub login: String,
  79     pub avatar: Option<String>,
  80     pub email: Option<String>,
  81     pub name: Option<String>,
  82 }
  83
  84 pub struct Warnings {
  85     pub invalid_categories: Vec<String>,
  86     pub invalid_badges: Vec<String>,
  87     pub other: Vec<String>,
  88 }
  89
  90 #[derive(Deserialize)]
  91 struct R {
  92     ok: bool,
  93 }
  94 #[derive(Deserialize)]
  95 struct OwnerResponse {
  96     ok: bool,
  97     msg: String,
  98 }
  99 #[derive(Deserialize)]
 100 struct ApiErrorList {
 101     errors: Vec<ApiError>,
 102 }
 103 #[derive(Deserialize)]
 104 struct ApiError {
 105     detail: String,
 106 }
 107 #[derive(Serialize)]
 108 struct OwnersReq<'a> {
 109     users: &'a [&'a str],
 110 }
 111 #[derive(Deserialize)]
 112 struct Users {
 113     users: Vec<User>,
 114 }
 115 #[derive(Deserialize)]
 116 struct TotalCrates {
 117     total: u32,
 118 }
 119 #[derive(Deserialize)]
 120 struct Crates {
 121     crates: Vec<Crate>,
 122     meta: TotalCrates,
 123 }
 124 impl Registry {
 125     /// Creates a new `Registry`.
 126     ///
 127     /// ## Example
 128     ///
 129     /// ```rust
 130     /// use curl::easy::Easy;
 131     /// use crates_io::Registry;
 132     ///
 133     /// let mut handle = Easy::new();
 134     /// // If connecting to crates.io, a user-agent is required.
 135     /// handle.useragent("my_crawler (example.com/info)");
 136     /// let mut reg = Registry::new_handle(String::from("https://crates.io"), None, handle);
 137     /// ```
 138     pub fn new_handle(host: String, token: Option<String>, handle: Easy) -> Registry {
 139         Registry {
 140             host,
 141             token,
 142             handle,
 143         }
 144     }
 145
 146     pub fn host(&self) -> &str {
 147         &self.host
 148     }
 149
 150     pub fn host_is_crates_io(&self) -> bool {
 151         is_url_crates_io(&self.host)
 152     }
 153
 154     pub fn add_owners(&mut self, krate: &str, owners: &[&str]) -> Result<String> {
 155         let body = serde_json::to_string(&OwnersReq { users: owners })?;
 156         let body = self.put(&format!("/crates/{}/owners", krate), body.as_bytes())?;
 157         assert!(serde_json::from_str::<OwnerResponse>(&body)?.ok);
 158         Ok(serde_json::from_str::<OwnerResponse>(&body)?.msg)
 159     }
 160
 161     pub fn remove_owners(&mut self, krate: &str, owners: &[&str]) -> Result<()> {
 162         let body = serde_json::to_string(&OwnersReq { users: owners })?;
 163         let body = self.delete(&format!("/crates/{}/owners", krate), Some(body.as_bytes()))?;
 164         assert!(serde_json::from_str::<OwnerResponse>(&body)?.ok);
 165         Ok(())
 166     }
 167
 168     pub fn list_owners(&mut self, krate: &str) -> Result<Vec<User>> {
 169         let body = self.get(&format!("/crates/{}/owners", krate))?;
 170         Ok(serde_json::from_str::<Users>(&body)?.users)
 171     }
 172
 173     pub fn publish(&mut self, krate: &NewCrate, mut tarball: &File) -> Result<Warnings> {
 174         let json = serde_json::to_string(krate)?;
 175         // Prepare the body. The format of the upload request is:
 176         //
 177         //      <le u32 of json>
 178         //      <json request> (metadata for the package)
 179         //      <le u32 of tarball>
 180         //      <source tarball>
 181
 182         // NOTE: This can be replaced with `stream_len` if it is ever stabilized.
 183         //
 184         // This checks the length using seeking instead of metadata, because
 185         // on some filesystems, getting the metadata will fail because
 186         // the file was renamed in ops::package.
 187         let tarball_len = tarball
 188             .seek(SeekFrom::End(0))
 189             .with_context(|| "failed to seek tarball")?;
 190         tarball
 191             .seek(SeekFrom::Start(0))
 192             .with_context(|| "failed to seek tarball")?;
 193         let header = {
 194             let mut w = Vec::new();
 195             w.extend(&(json.len() as u32).to_le_bytes());
 196             w.extend(json.as_bytes().iter().cloned());
 197             w.extend(&(tarball_len as u32).to_le_bytes());
 198             w
 199         };
 200         let size = tarball_len as usize + header.len();
 201         let mut body = Cursor::new(header).chain(tarball);
 202
 203         let url = format!("{}/api/v1/crates/new", self.host);
 204
 205         let token = match self.token.as_ref() {
 206             Some(s) => s,
 207             None => bail!("no upload token found, please run `cargo login`"),
 208         };
 209         self.handle.put(true)?;
 210         self.handle.url(&url)?;
 211         self.handle.in_filesize(size as u64)?;
 212         let mut headers = List::new();
 213         headers.append("Accept: application/json")?;
 214         headers.append(&format!("Authorization: {}", token))?;
 215         self.handle.http_headers(headers)?;
 216
 217         let body = self.handle(&mut |buf| body.read(buf).unwrap_or(0))?;
 218
 219         let response = if body.is_empty() {
 220             "{}".parse()?
 221         } else {
 222             body.parse::<serde_json::Value>()?
 223         };
 224
 225         let invalid_categories: Vec<String> = response
 226             .get("warnings")
 227             .and_then(|j| j.get("invalid_categories"))
 228             .and_then(|j| j.as_array())
 229             .map(|x| x.iter().flat_map(|j| j.as_str()).map(Into::into).collect())
 230             .unwrap_or_else(Vec::new);
 231
 232         let invalid_badges: Vec<String> = response
 233             .get("warnings")
 234             .and_then(|j| j.get("invalid_badges"))
 235             .and_then(|j| j.as_array())
 236             .map(|x| x.iter().flat_map(|j| j.as_str()).map(Into::into).collect())
 237             .unwrap_or_else(Vec::new);
 238
 239         let other: Vec<String> = response
 240             .get("warnings")
 241             .and_then(|j| j.get("other"))
 242             .and_then(|j| j.as_array())
 243             .map(|x| x.iter().flat_map(|j| j.as_str()).map(Into::into).collect())
 244             .unwrap_or_else(Vec::new);
 245
 246         Ok(Warnings {
 247             invalid_categories,
 248             invalid_badges,
 249             other,
 250         })
 251     }
 252
 253     pub fn search(&mut self, query: &str, limit: u32) -> Result<(Vec<Crate>, u32)> {
 254         let formatted_query = percent_encode(query.as_bytes(), NON_ALPHANUMERIC);
 255         let body = self.req(
 256             &format!("/crates?q={}&per_page={}", formatted_query, limit),
 257             None,
 258             Auth::Unauthorized,
 259         )?;
 260
 261         let crates = serde_json::from_str::<Crates>(&body)?;
 262         Ok((crates.crates, crates.meta.total))
 263     }
 264
 265     pub fn yank(&mut self, krate: &str, version: &str) -> Result<()> {
 266         let body = self.delete(&format!("/crates/{}/{}/yank", krate, version), None)?;
 267         assert!(serde_json::from_str::<R>(&body)?.ok);
 268         Ok(())
 269     }
 270
 271     pub fn unyank(&mut self, krate: &str, version: &str) -> Result<()> {
 272         let body = self.put(&format!("/crates/{}/{}/unyank", krate, version), &[])?;
 273         assert!(serde_json::from_str::<R>(&body)?.ok);
 274         Ok(())
 275     }
 276
 277     fn put(&mut self, path: &str, b: &[u8]) -> Result<String> {
 278         self.handle.put(true)?;
 279         self.req(path, Some(b), Auth::Authorized)
 280     }
 281
 282     fn get(&mut self, path: &str) -> Result<String> {
 283         self.handle.get(true)?;
 284         self.req(path, None, Auth::Authorized)
 285     }
 286
 287     fn delete(&mut self, path: &str, b: Option<&[u8]>) -> Result<String> {
 288         self.handle.custom_request("DELETE")?;
 289         self.req(path, b, Auth::Authorized)
 290     }
 291
 292     fn req(&mut self, path: &str, body: Option<&[u8]>, authorized: Auth) -> Result<String> {
 293         self.handle.url(&format!("{}/api/v1{}", self.host, path))?;
 294         let mut headers = List::new();
 295         headers.append("Accept: application/json")?;
 296         headers.append("Content-Type: application/json")?;
 297
 298         if authorized == Auth::Authorized {
 299             let token = match self.token.as_ref() {
 300                 Some(s) => s,
 301                 None => bail!("no upload token found, please run `cargo login`"),
 302             };
 303             headers.append(&format!("Authorization: {}", token))?;
 304         }
 305         self.handle.http_headers(headers)?;
 306         match body {
 307             Some(mut body) => {
 308                 self.handle.upload(true)?;
 309                 self.handle.in_filesize(body.len() as u64)?;
 310                 self.handle(&mut |buf| body.read(buf).unwrap_or(0))
 311             }
 312             None => self.handle(&mut |_| 0),
 313         }
 314     }
 315
 316     fn handle(&mut self, read: &mut dyn FnMut(&mut [u8]) -> usize) -> Result<String> {
 317         let mut headers = Vec::new();
 318         let mut body = Vec::new();
 319         let started;
 320         {
 321             let mut handle = self.handle.transfer();
 322             handle.read_function(|buf| Ok(read(buf)))?;
 323             handle.write_function(|data| {
 324                 body.extend_from_slice(data);
 325                 Ok(data.len())
 326             })?;
 327             handle.header_function(|data| {
 328                 headers.push(String::from_utf8_lossy(data).into_owned());
 329                 true
 330             })?;
 331             started = Instant::now();
 332             handle.perform()?;
 333         }
 334
 335         let body = match String::from_utf8(body) {
 336             Ok(body) => body,
 337             Err(..) => bail!("response body was not valid utf-8"),
 338         };
 339         let errors = serde_json::from_str::<ApiErrorList>(&body)
 340             .ok()
 341             .map(|s| s.errors.into_iter().map(|s| s.detail).collect::<Vec<_>>());
 342
 343         match (self.handle.response_code()?, errors) {
 344             (0, None) | (200, None) => {}
 345             (503, None) if started.elapsed().as_secs() >= 29 && self.host_is_crates_io() => bail!(
 346                 "Request timed out after 30 seconds. If you're trying to \
 347                  upload a crate it may be too large. If the crate is under \
 348                  10MB in size, you can email help@crates.io for assistance."
 349             ),
 350             (code, Some(errors)) => {
 351                 let reason = reason(code);
 352                 bail!(
 353                     "api errors (status {} {}): {}",
 354                     code,
 355                     reason,
 356                     errors.join(", ")
 357                 )
 358             }
 359             (code, None) => bail!(
 360                 "failed to get a 200 OK response, got {}\n\
 361                  headers:\n\
 362                  \t{}\n\
 363                  body:\n\
 364                  {}",
 365                 code,
 366                 headers.join("\n\t"),
 367                 body,
 368             ),
 369         }
 370
 371         Ok(body)
 372     }
 373 }
 374
 375 fn reason(code: u32) -> &'static str {
 376     // Taken from https://developer.mozilla.org/en-US/docs/Web/HTTP/Status
 377     match code {
 378         100 => "Continue",
 379         101 => "Switching Protocol",
 380         103 => "Early Hints",
 381         200 => "OK",
 382         201 => "Created",
 383         202 => "Accepted",
 384         203 => "Non-Authoritative Information",
 385         204 => "No Content",
 386         205 => "Reset Content",
 387         206 => "Partial Content",
 388         300 => "Multiple Choice",
 389         301 => "Moved Permanently",
 390         302 => "Found",
 391         303 => "See Other",
 392         304 => "Not Modified",
 393         307 => "Temporary Redirect",
 394         308 => "Permanent Redirect",
 395         400 => "Bad Request",
 396         401 => "Unauthorized",
 397         402 => "Payment Required",
 398         403 => "Forbidden",
 399         404 => "Not Found",
 400         405 => "Method Not Allowed",
 401         406 => "Not Acceptable",
 402         407 => "Proxy Authentication Required",
 403         408 => "Request Timeout",
 404         409 => "Conflict",
 405         410 => "Gone",
 406         411 => "Length Required",
 407         412 => "Precondition Failed",
 408         413 => "Payload Too Large",
 409         414 => "URI Too Long",
 410         415 => "Unsupported Media Type",
 411         416 => "Request Range Not Satisfiable",
 412         417 => "Expectation Failed",
 413         429 => "Too Many Requests",
 414         431 => "Request Header Fields Too Large",
 415         500 => "Internal Server Error",
 416         501 => "Not Implemented",
 417         502 => "Bad Gateway",
 418         503 => "Service Unavailable",
 419         504 => "Gateway Timeout",
 420         _ => "<unknown>",
 421     }
 422 }
 423
 424 /// Returns `true` if the host of the given URL is "crates.io".
 425 pub fn is_url_crates_io(url: &str) -> bool {
 426     Url::parse(url)
 427         .map(|u| u.host_str() == Some("crates.io"))
 428         .unwrap_or(false)
 429 }