use std::fmt::Write as _;
use std::fs::{self, File};
use std::io::{BufRead, BufReader, Write};
-use std::net::TcpListener;
+use std::net::{SocketAddr, TcpListener};
use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
use std::thread;
use tar::{Builder, Header};
use url::Url;
RegistryBuilder::new().alternative(true).build();
}
+pub struct RegistryServer {
+ done: Arc<AtomicBool>,
+ server: Option<thread::JoinHandle<()>>,
+ addr: SocketAddr,
+}
+
+impl RegistryServer {
+ pub fn addr(&self) -> SocketAddr {
+ self.addr
+ }
+}
+
+impl Drop for RegistryServer {
+ fn drop(&mut self) {
+ self.done.store(true, Ordering::SeqCst);
+ // NOTE: we can't actually await the server since it's blocked in accept()
+ let _ = self.server.take();
+ }
+}
+
+#[must_use]
+pub fn serve_registry(registry_path: PathBuf) -> RegistryServer {
+ let listener = TcpListener::bind("127.0.0.1:0").unwrap();
+ let addr = listener.local_addr().unwrap();
+ let done = Arc::new(AtomicBool::new(false));
+ let done2 = done.clone();
+
+ let t = thread::spawn(move || {
+ let mut line = String::new();
+ 'server: while !done2.load(Ordering::SeqCst) {
+ let (socket, _) = listener.accept().unwrap();
+ // Let's implement a very naive static file HTTP server.
+ let mut buf = BufReader::new(socket);
+
+ // First, the request line:
+ // GET /path HTTPVERSION
+ line.clear();
+ if buf.read_line(&mut line).unwrap() == 0 {
+ // Connection terminated.
+ continue;
+ }
+
+ assert!(line.starts_with("GET "), "got non-GET request: {}", line);
+ let path = PathBuf::from(
+ line.split_whitespace()
+ .skip(1)
+ .next()
+ .unwrap()
+ .trim_start_matches('/'),
+ );
+
+ let file = registry_path.join(path);
+ if file.exists() {
+ // Grab some other headers we may care about.
+ let mut if_modified_since = None;
+ let mut if_none_match = None;
+ loop {
+ line.clear();
+ if buf.read_line(&mut line).unwrap() == 0 {
+ continue 'server;
+ }
+
+ if line == "\r\n" {
+ // End of headers.
+ line.clear();
+ break;
+ }
+
+ let value = line
+ .splitn(2, ':')
+ .skip(1)
+ .next()
+ .map(|v| v.trim())
+ .unwrap();
+
+ if line.starts_with("If-Modified-Since:") {
+ if_modified_since = Some(value.to_owned());
+ } else if line.starts_with("If-None-Match:") {
+ if_none_match = Some(value.trim_matches('"').to_owned());
+ }
+ }
+
+ // Now grab info about the file.
+ let data = fs::read(&file).unwrap();
+ let etag = Sha256::new().update(&data).finish_hex();
+ let last_modified = format!("{:?}", file.metadata().unwrap().modified().unwrap());
+
+ // Start to construct our response:
+ let mut any_match = false;
+ let mut all_match = true;
+ if let Some(expected) = if_none_match {
+ if etag != expected {
+ all_match = false;
+ } else {
+ any_match = true;
+ }
+ }
+ if let Some(expected) = if_modified_since {
+ // NOTE: Equality comparison is good enough for tests.
+ if last_modified != expected {
+ all_match = false;
+ } else {
+ any_match = true;
+ }
+ }
+
+ // Write out the main response line.
+ if any_match && all_match {
+ buf.get_mut()
+ .write_all(b"HTTP/1.1 304 Not Modified\r\n")
+ .unwrap();
+ } else {
+ buf.get_mut().write_all(b"HTTP/1.1 200 OK\r\n").unwrap();
+ }
+ // TODO: Support 451 for crate index deletions.
+
+ // Write out other headers.
+ buf.get_mut()
+ .write_all(format!("Content-Length: {}\r\n", data.len()).as_bytes())
+ .unwrap();
+ buf.get_mut()
+ .write_all(format!("ETag: \"{}\"\r\n", etag).as_bytes())
+ .unwrap();
+ buf.get_mut()
+ .write_all(format!("Last-Modified: {}\r\n", last_modified).as_bytes())
+ .unwrap();
+
+ // And finally, write out the body.
+ buf.get_mut().write_all(b"\r\n").unwrap();
+ buf.get_mut().write_all(&data).unwrap();
+ } else {
+ loop {
+ line.clear();
+ if buf.read_line(&mut line).unwrap() == 0 {
+ // Connection terminated.
+ continue 'server;
+ }
+
+ if line == "\r\n" {
+ break;
+ }
+ }
+
+ buf.get_mut()
+ .write_all(b"HTTP/1.1 404 Not Found\r\n\r\n")
+ .unwrap();
+ buf.get_mut().write_all(b"\r\n").unwrap();
+ }
+ buf.get_mut().flush().unwrap();
+ }
+ });
+
+ RegistryServer {
+ addr,
+ server: Some(t),
+ done,
+ }
+}
+
/// Creates a new on-disk registry.
pub fn init_registry(registry_path: PathBuf, dl_url: String, api_url: Url, api_path: PathBuf) {
// Initialize a new registry.
no_index_update: bool = ("Do not update the registry index even if the cache is outdated"),
panic_abort_tests: bool = ("Enable support to run tests with -Cpanic=abort"),
host_config: bool = ("Enable the [host] section in the .cargo/config.toml file"),
+ http_registry: bool = ("Support HTTP-based crate registries"),
target_applies_to_host: bool = ("Enable the `target-applies-to-host` key in the .cargo/config.toml file"),
rustdoc_map: bool = ("Allow passing external documentation mappings to rustdoc"),
separate_nightlies: bool = (HIDDEN),
"multitarget" => self.multitarget = parse_empty(k, v)?,
"rustdoc-map" => self.rustdoc_map = parse_empty(k, v)?,
"terminal-width" => self.terminal_width = Some(parse_usize_opt(v)?),
+ "http-registry" => self.http_registry = parse_empty(k, v)?,
"namespaced-features" => stabilized_warn(k, "1.60", STABILISED_NAMESPACED_FEATURES),
"weak-dep-features" => stabilized_warn(k, "1.60", STABILIZED_WEAK_DEP_FEATURES),
"credential-process" => self.credential_process = parse_empty(k, v)?,
) -> CargoResult<PackageSet<'cfg>> {
// We've enabled the `http2` feature of `curl` in Cargo, so treat
// failures here as fatal as it would indicate a build-time problem.
- //
- // Note that the multiplexing support is pretty new so we're having it
- // off-by-default temporarily.
- //
- // Also note that pipelining is disabled as curl authors have indicated
- // that it's buggy, and we've empirically seen that it's buggy with HTTP
- // proxies.
let mut multi = Multi::new();
let multiplexing = config.http_config()?.multiplexing.unwrap_or(true);
multi
return Ok(Some(pkg));
}
- // Ask the original source fo this `PackageId` for the corresponding
+ // Ask the original source for this `PackageId` for the corresponding
// package. That may immediately come back and tell us that the package
// is ready, or it could tell us that it needs to be downloaded.
let mut sources = self.set.sources.borrow_mut();
// initiate dozens of connections to crates.io, but rather only one.
// Once the main one is opened we realized that pipelining is possible
// and multiplexing is possible with static.crates.io. All in all this
- // reduces the number of connections done to a more manageable state.
+ // reduces the number of connections down to a more manageable state.
try_old_curl!(handle.pipewait(true), "pipewait");
handle.write_function(move |buf| {
}
self.load(namespace, kind)?;
+
+ // This isn't strictly necessary since it will be called later.
+ // However it improves error messages for sources that issue errors
+ // in `block_until_ready` because the callers here have context about
+ // which deps are being resolved.
self.block_until_ready()?;
Ok(())
}
// First up we need to actually resolve each `deps` specification to
// precisely one summary. We're not using the `query` method below as it
// internally uses maps we're building up as part of this method
- // (`patches_available` and `patches). Instead we're going straight to
+ // (`patches_available` and `patches`). Instead we're going straight to
// the source to load information from it.
//
// Remember that each dependency listed in `[patch]` has to resolve to
Ok(SourceId::new(SourceKind::Registry, url, None)?
.with_precise(Some("locked".to_string())))
}
+ "sparse" => {
+ let url = string.into_url()?;
+ Ok(SourceId::new(SourceKind::Registry, url, None)?
+ .with_precise(Some("locked".to_string())))
+ }
"path" => {
let url = url.into_url()?;
SourceId::new(SourceKind::Path, url, None)
self,
yanked_whitelist,
config,
- ))),
+ )?)),
SourceKind::LocalRegistry => {
let path = match self.inner.url.to_file_path() {
Ok(p) => p,
}
let api_host = {
let _lock = config.acquire_package_cache_lock()?;
- let mut src = RegistrySource::remote(sid, &HashSet::new(), config);
+ let mut src = RegistrySource::remote(sid, &HashSet::new(), config)?;
// Only update the index if the config is not available or `force` is set.
if force_update {
src.invalidate_cache()
specified"
)
}
- if !config.network_allowed() {
- bail!("can't make HTTP request in the offline mode")
+ if config.offline() {
+ bail!(
+ "attempting to make an HTTP request, but --offline was \
+ specified"
+ )
}
// The timeout option for libcurl by default times out the entire transfer,
impl<'cfg> Source for PathSource<'cfg> {
fn query(&mut self, dep: &Dependency, f: &mut dyn FnMut(Summary)) -> Poll<CargoResult<()>> {
- if !self.updated {
- return Poll::Pending;
- }
+ self.update()?;
for s in self.packages.iter().map(|p| p.summary()) {
if dep.matches(s) {
f(s.clone())
_dep: &Dependency,
f: &mut dyn FnMut(Summary),
) -> Poll<CargoResult<()>> {
- if !self.updated {
- return Poll::Pending;
- }
+ self.update()?;
for s in self.packages.iter().map(|p| p.summary()) {
f(s.clone())
}
fn download(&mut self, id: PackageId) -> CargoResult<MaybePackage> {
trace!("getting packages; id={}", id);
-
+ self.update()?;
let pkg = self.packages.iter().find(|pkg| pkg.package_id() == id);
pkg.cloned()
.map(MaybePackage::Ready)
--- /dev/null
+use anyhow::Context;
+use cargo_util::Sha256;
+
+use crate::core::PackageId;
+use crate::sources::registry::make_dep_prefix;
+use crate::sources::registry::MaybeLock;
+use crate::sources::registry::{
+ RegistryConfig, CHECKSUM_TEMPLATE, CRATE_TEMPLATE, LOWER_PREFIX_TEMPLATE, PREFIX_TEMPLATE,
+ VERSION_TEMPLATE,
+};
+use crate::util::errors::CargoResult;
+use crate::util::{Config, Filesystem};
+use std::fmt::Write as FmtWrite;
+use std::fs::{self, File, OpenOptions};
+use std::io::prelude::*;
+use std::io::SeekFrom;
+use std::str;
+
+pub(super) fn filename(pkg: PackageId) -> String {
+ format!("{}-{}.crate", pkg.name(), pkg.version())
+}
+
+pub(super) fn download(
+ cache_path: &Filesystem,
+ config: &Config,
+ pkg: PackageId,
+ checksum: &str,
+ registry_config: RegistryConfig,
+) -> CargoResult<MaybeLock> {
+ let filename = filename(pkg);
+ let path = cache_path.join(&filename);
+ let path = config.assert_package_cache_locked(&path);
+
+ // Attempt to open a read-only copy first to avoid an exclusive write
+ // lock and also work with read-only filesystems. Note that we check the
+ // length of the file like below to handle interrupted downloads.
+ //
+ // If this fails then we fall through to the exclusive path where we may
+ // have to redownload the file.
+ if let Ok(dst) = File::open(path) {
+ let meta = dst.metadata()?;
+ if meta.len() > 0 {
+ return Ok(MaybeLock::Ready(dst));
+ }
+ }
+
+ let mut url = registry_config.dl;
+ if !url.contains(CRATE_TEMPLATE)
+ && !url.contains(VERSION_TEMPLATE)
+ && !url.contains(PREFIX_TEMPLATE)
+ && !url.contains(LOWER_PREFIX_TEMPLATE)
+ && !url.contains(CHECKSUM_TEMPLATE)
+ {
+ // Original format before customizing the download URL was supported.
+ write!(
+ url,
+ "/{}/{}/download",
+ pkg.name(),
+ pkg.version().to_string()
+ )
+ .unwrap();
+ } else {
+ let prefix = make_dep_prefix(&*pkg.name());
+ url = url
+ .replace(CRATE_TEMPLATE, &*pkg.name())
+ .replace(VERSION_TEMPLATE, &pkg.version().to_string())
+ .replace(PREFIX_TEMPLATE, &prefix)
+ .replace(LOWER_PREFIX_TEMPLATE, &prefix.to_lowercase())
+ .replace(CHECKSUM_TEMPLATE, checksum);
+ }
+
+ Ok(MaybeLock::Download {
+ url,
+ descriptor: pkg.to_string(),
+ })
+}
+
+pub(super) fn finish_download(
+ cache_path: &Filesystem,
+ config: &Config,
+ pkg: PackageId,
+ checksum: &str,
+ data: &[u8],
+) -> CargoResult<File> {
+ // Verify what we just downloaded
+ let actual = Sha256::new().update(data).finish_hex();
+ if actual != checksum {
+ anyhow::bail!("failed to verify the checksum of `{}`", pkg)
+ }
+
+ let filename = filename(pkg);
+ cache_path.create_dir()?;
+ let path = cache_path.join(&filename);
+ let path = config.assert_package_cache_locked(&path);
+ let mut dst = OpenOptions::new()
+ .create(true)
+ .read(true)
+ .write(true)
+ .open(&path)
+ .with_context(|| format!("failed to open `{}`", path.display()))?;
+ let meta = dst.metadata()?;
+ if meta.len() > 0 {
+ return Ok(dst);
+ }
+
+ dst.write_all(data)?;
+ dst.seek(SeekFrom::Start(0))?;
+ Ok(dst)
+}
+
+pub(super) fn is_crate_downloaded(
+ cache_path: &Filesystem,
+ config: &Config,
+ pkg: PackageId,
+) -> bool {
+ let path = cache_path.join(filename(pkg));
+ let path = config.assert_package_cache_locked(&path);
+ if let Ok(meta) = fs::metadata(path) {
+ return meta.len() > 0;
+ }
+ false
+}
--- /dev/null
+//! Access to a HTTP-based crate registry.
+//!
+//! See [`HttpRegistry`] for details.
+
+use crate::core::{PackageId, SourceId};
+use crate::ops;
+use crate::sources::registry::download;
+use crate::sources::registry::MaybeLock;
+use crate::sources::registry::{LoadResponse, RegistryConfig, RegistryData};
+use crate::util::errors::CargoResult;
+use crate::util::{Config, Filesystem, IntoUrl, Progress, ProgressStyle};
+use anyhow::Context;
+use cargo_util::paths;
+use curl::easy::{HttpVersion, List};
+use curl::multi::{EasyHandle, Multi};
+use log::{debug, trace};
+use std::cell::{Cell, RefCell};
+use std::collections::{HashMap, HashSet};
+use std::fs::{self, File};
+use std::path::{Path, PathBuf};
+use std::str;
+use std::task::Poll;
+use std::time::Duration;
+use url::Url;
+
+const ETAG: &'static str = "ETag";
+const LAST_MODIFIED: &'static str = "Last-Modified";
+const UNKNOWN: &'static str = "Unknown";
+
+/// A registry served by the HTTP-based registry API.
+///
+/// This type is primarily accessed through the [`RegistryData`] trait.
+///
+/// `HttpRegistry` implements the HTTP-based registry API outlined in [RFC 2789]. Read the RFC for
+/// the complete protocol, but _roughly_ the implementation loads each index file (e.g.,
+/// config.json or re/ge/regex) from an HTTP service rather than from a locally cloned git
+/// repository. The remote service can more or less be a static file server that simply serves the
+/// contents of the origin git repository.
+///
+/// Implemented naively, this leads to a significant amount of network traffic, as a lookup of any
+/// index file would need to check with the remote backend if the index file has changed. This
+/// cost is somewhat mitigated by the use of HTTP conditional fetches (`If-Modified-Since` and
+/// `If-None-Match` for `ETag`s) which can be efficiently handled by HTTP/2.
+///
+/// [RFC 2789]: https://github.com/rust-lang/rfcs/pull/2789
+pub struct HttpRegistry<'cfg> {
+ index_path: Filesystem,
+ cache_path: Filesystem,
+ source_id: SourceId,
+ config: &'cfg Config,
+
+ /// Store the server URL without the protocol prefix (sparse+)
+ url: Url,
+
+ /// HTTP multi-handle for asynchronous/parallel requests.
+ multi: Multi,
+
+ /// Has the client requested a cache update?
+ ///
+ /// Only if they have do we double-check the freshness of each locally-stored index file.
+ requested_update: bool,
+
+ /// State for currently pending index downloads.
+ downloads: Downloads<'cfg>,
+
+ /// Does the config say that we can use HTTP multiplexing?
+ multiplexing: bool,
+
+ /// What paths have we already fetched since the last index update?
+ ///
+ /// We do not need to double-check any of these index files since we have already done so.
+ fresh: HashSet<PathBuf>,
+
+ /// Have we started to download any index files?
+ fetch_started: bool,
+
+ /// Cached registry configuration.
+ registry_config: Option<RegistryConfig>,
+}
+
+/// Helper for downloading crates.
+pub struct Downloads<'cfg> {
+ /// When a download is started, it is added to this map. The key is a
+ /// "token" (see `Download::token`). It is removed once the download is
+ /// finished.
+ pending: HashMap<usize, (Download, EasyHandle)>,
+ /// Set of paths currently being downloaded, mapped to their tokens.
+ /// This should stay in sync with `pending`.
+ pending_ids: HashMap<PathBuf, usize>,
+ /// The final result of each download. A pair `(token, result)`. This is a
+ /// temporary holding area, needed because curl can report multiple
+ /// downloads at once, but the main loop (`wait`) is written to only
+ /// handle one at a time.
+ results: HashMap<PathBuf, Result<CompletedDownload, curl::Error>>,
+ /// The next ID to use for creating a token (see `Download::token`).
+ next: usize,
+ /// Progress bar.
+ progress: RefCell<Option<Progress<'cfg>>>,
+ /// Number of downloads that have successfully finished.
+ downloads_finished: usize,
+}
+
+struct Download {
+ /// The token for this download, used as the key of the `Downloads::pending` map
+ /// and stored in `EasyHandle` as well.
+ token: usize,
+
+ /// The path of the package that we're downloading.
+ path: PathBuf,
+
+ /// Actual downloaded data, updated throughout the lifetime of this download.
+ data: RefCell<Vec<u8>>,
+
+ /// ETag or Last-Modified header received from the server (if any).
+ index_version: RefCell<Option<String>>,
+
+ /// Statistics updated from the progress callback in libcurl.
+ total: Cell<u64>,
+ current: Cell<u64>,
+}
+
+struct CompletedDownload {
+ response_code: u32,
+ data: Vec<u8>,
+ index_version: String,
+}
+
+impl<'cfg> HttpRegistry<'cfg> {
+ pub fn new(source_id: SourceId, config: &'cfg Config, name: &str) -> HttpRegistry<'cfg> {
+ let url = source_id
+ .url()
+ .to_string()
+ .trim_start_matches("sparse+")
+ .trim_end_matches('/')
+ .into_url()
+ .expect("a url with the protocol stripped should still be valid");
+
+ HttpRegistry {
+ index_path: config.registry_index_path().join(name),
+ cache_path: config.registry_cache_path().join(name),
+ source_id,
+ config,
+ url,
+ multi: Multi::new(),
+ multiplexing: false,
+ downloads: Downloads {
+ next: 0,
+ pending: HashMap::new(),
+ pending_ids: HashMap::new(),
+ results: HashMap::new(),
+ progress: RefCell::new(Some(Progress::with_style(
+ "Fetching",
+ ProgressStyle::Ratio,
+ config,
+ ))),
+ downloads_finished: 0,
+ },
+ fresh: HashSet::new(),
+ requested_update: false,
+ fetch_started: false,
+ registry_config: None,
+ }
+ }
+
+ fn handle_http_header(buf: &[u8]) -> Option<(&str, &str)> {
+ if buf.is_empty() {
+ return None;
+ }
+ let buf = std::str::from_utf8(buf).ok()?.trim_end();
+ // Don't let server sneak extra lines anywhere.
+ if buf.contains('\n') {
+ return None;
+ }
+ let (tag, value) = buf.split_once(':')?;
+ let value = value.trim();
+ Some((tag, value))
+ }
+
+ fn start_fetch(&mut self) -> CargoResult<()> {
+ if self.fetch_started {
+ // We only need to run the setup code once.
+ return Ok(());
+ }
+ self.fetch_started = true;
+
+ // We've enabled the `http2` feature of `curl` in Cargo, so treat
+ // failures here as fatal as it would indicate a build-time problem.
+ self.multiplexing = self.config.http_config()?.multiplexing.unwrap_or(true);
+
+ self.multi
+ .pipelining(false, self.multiplexing)
+ .with_context(|| "failed to enable multiplexing/pipelining in curl")?;
+
+ // let's not flood the server with connections
+ self.multi.set_max_host_connections(2)?;
+
+ self.config
+ .shell()
+ .status("Updating", self.source_id.display_index())?;
+
+ Ok(())
+ }
+
+ fn handle_completed_downloads(&mut self) -> CargoResult<()> {
+ assert_eq!(
+ self.downloads.pending.len(),
+ self.downloads.pending_ids.len()
+ );
+
+ // Collect the results from the Multi handle.
+ let pending = &mut self.downloads.pending;
+ self.multi.messages(|msg| {
+ let token = msg.token().expect("failed to read token");
+ let (_, handle) = &pending[&token];
+ let result = match msg.result_for(handle) {
+ Some(result) => result,
+ None => return, // transfer is not yet complete.
+ };
+
+ let (download, mut handle) = pending.remove(&token).unwrap();
+ self.downloads.pending_ids.remove(&download.path).unwrap();
+
+ let result = match result {
+ Ok(()) => {
+ self.downloads.downloads_finished += 1;
+ match handle.response_code() {
+ Ok(code) => Ok(CompletedDownload {
+ response_code: code,
+ data: download.data.take(),
+ index_version: download
+ .index_version
+ .take()
+ .unwrap_or_else(|| UNKNOWN.to_string()),
+ }),
+ Err(e) => Err(e),
+ }
+ }
+ Err(e) => Err(e),
+ };
+ self.downloads.results.insert(download.path, result);
+ });
+ self.downloads.tick()?;
+
+ Ok(())
+ }
+
+ fn full_url(&self, path: &Path) -> String {
+ format!("{}/{}", self.url, path.display())
+ }
+
+ fn is_fresh(&self, path: &Path) -> bool {
+ if !self.requested_update {
+ trace!(
+ "using local {} as user did not request update",
+ path.display()
+ );
+ true
+ } else if self.config.cli_unstable().no_index_update {
+ trace!("using local {} in no_index_update mode", path.display());
+ true
+ } else if self.config.offline() {
+ trace!("using local {} in offline mode", path.display());
+ true
+ } else if self.fresh.contains(path) {
+ trace!("using local {} as it was already fetched", path.display());
+ true
+ } else {
+ debug!("checking freshness of {}", path.display());
+ false
+ }
+ }
+}
+
+impl<'cfg> RegistryData for HttpRegistry<'cfg> {
+ fn prepare(&self) -> CargoResult<()> {
+ Ok(())
+ }
+
+ fn index_path(&self) -> &Filesystem {
+ &self.index_path
+ }
+
+ fn assert_index_locked<'a>(&self, path: &'a Filesystem) -> &'a Path {
+ self.config.assert_package_cache_locked(path)
+ }
+
+ fn is_updated(&self) -> bool {
+ self.requested_update
+ }
+
+ fn load(
+ &mut self,
+ _root: &Path,
+ path: &Path,
+ index_version: Option<&str>,
+ ) -> Poll<CargoResult<LoadResponse>> {
+ trace!("load: {}", path.display());
+ if let Some(_token) = self.downloads.pending_ids.get(path) {
+ debug!("dependency is still pending: {}", path.display());
+ return Poll::Pending;
+ }
+
+ if let Some(index_version) = index_version {
+ trace!(
+ "local cache of {} is available at version `{}`",
+ path.display(),
+ index_version
+ );
+ if self.is_fresh(path) {
+ return Poll::Ready(Ok(LoadResponse::CacheValid));
+ }
+ } else if self.fresh.contains(path) {
+ debug!(
+ "cache did not contain previously downloaded file {}",
+ path.display()
+ );
+ }
+
+ if let Some(result) = self.downloads.results.remove(path) {
+ let result =
+ result.with_context(|| format!("download of {} failed", path.display()))?;
+ debug!(
+ "index file downloaded with status code {}",
+ result.response_code
+ );
+ trace!("index file version: {}", result.index_version);
+
+ if !self.fresh.insert(path.to_path_buf()) {
+ debug!("downloaded the index file `{}` twice", path.display())
+ }
+
+ match result.response_code {
+ 200 => {}
+ 304 => {
+ // Not Modified: the data in the cache is still the latest.
+ if index_version.is_none() {
+ return Poll::Ready(Err(anyhow::anyhow!(
+ "server said not modified (HTTP 304) when no local cache exists"
+ )));
+ }
+ return Poll::Ready(Ok(LoadResponse::CacheValid));
+ }
+ 404 | 410 | 451 => {
+ // The crate was not found or deleted from the registry.
+ return Poll::Ready(Ok(LoadResponse::NotFound));
+ }
+ code => {
+ return Err(anyhow::anyhow!(
+ "server returned unexpected HTTP status code {} for {}\nbody: {}",
+ code,
+ self.full_url(path),
+ str::from_utf8(&result.data).unwrap_or("<invalid utf8>"),
+ ))
+ .into();
+ }
+ }
+
+ return Poll::Ready(Ok(LoadResponse::Data {
+ raw_data: result.data,
+ index_version: Some(result.index_version),
+ }));
+ }
+
+ if self.config.offline() {
+ return Poll::Ready(Err(anyhow::anyhow!(
+ "can't download index file from '{}': you are in offline mode (--offline)",
+ self.url
+ )));
+ }
+
+ // Looks like we're going to have to do a network request.
+ self.start_fetch()?;
+
+ // Load the registry config.
+ if self.registry_config.is_none() && path != Path::new("config.json") {
+ match self.config()? {
+ Poll::Ready(_) => {}
+ Poll::Pending => return Poll::Pending,
+ }
+ }
+
+ let mut handle = ops::http_handle(self.config)?;
+ let full_url = self.full_url(path);
+ debug!("fetch {}", full_url);
+ handle.get(true)?;
+ handle.url(&full_url)?;
+ handle.follow_location(true)?;
+
+ // Enable HTTP/2 if possible.
+ if self.multiplexing {
+ handle.http_version(HttpVersion::V2)?;
+ } else {
+ handle.http_version(HttpVersion::V11)?;
+ }
+
+ // This is an option to `libcurl` which indicates that if there's a
+ // bunch of parallel requests to the same host they all wait until the
+ // pipelining status of the host is known. This means that we won't
+ // initiate dozens of connections to crates.io, but rather only one.
+ // Once the main one is opened we realized that pipelining is possible
+ // and multiplexing is possible with static.crates.io. All in all this
+ // reduces the number of connections done to a more manageable state.
+ handle.pipewait(true)?;
+
+ // Make sure we don't send data back if it's the same as we have in the index.
+ let mut headers = List::new();
+ if let Some(index_version) = index_version {
+ if let Some((key, value)) = index_version.split_once(':') {
+ match key {
+ ETAG => headers.append(&format!("If-None-Match: {}", value.trim()))?,
+ LAST_MODIFIED => {
+ headers.append(&format!("If-Modified-Since: {}", value.trim()))?
+ }
+ _ => debug!("unexpected index version: {}", index_version),
+ }
+ }
+ }
+ handle.http_headers(headers)?;
+
+ // We're going to have a bunch of downloads all happening "at the same time".
+ // So, we need some way to track what headers/data/responses are for which request.
+ // We do that through this token. Each request (and associated response) gets one.
+ let token = self.downloads.next;
+ self.downloads.next += 1;
+ debug!("downloading {} as {}", path.display(), token);
+ assert_eq!(
+ self.downloads.pending_ids.insert(path.to_path_buf(), token),
+ None,
+ "path queued for download more than once"
+ );
+
+ // Each write should go to self.downloads.pending[&token].data.
+ // Since the write function must be 'static, we access downloads through a thread-local.
+ // That thread-local is set up in `block_until_ready` when it calls self.multi.perform,
+ // which is what ultimately calls this method.
+ handle.write_function(move |buf| {
+ trace!("{} - {} bytes of data", token, buf.len());
+ tls::with(|downloads| {
+ if let Some(downloads) = downloads {
+ downloads.pending[&token]
+ .0
+ .data
+ .borrow_mut()
+ .extend_from_slice(buf);
+ }
+ });
+ Ok(buf.len())
+ })?;
+
+ // Same goes for the progress function -- it goes through thread-local storage.
+ handle.progress(true)?;
+ handle.progress_function(move |dl_total, dl_cur, _, _| {
+ tls::with(|downloads| match downloads {
+ Some(d) => d.progress(token, dl_total as u64, dl_cur as u64),
+ None => false,
+ })
+ })?;
+
+ // And ditto for the header function.
+ handle.header_function(move |buf| {
+ if let Some((tag, value)) = Self::handle_http_header(buf) {
+ let is_etag = tag.eq_ignore_ascii_case(ETAG);
+ let is_lm = tag.eq_ignore_ascii_case(LAST_MODIFIED);
+ if is_etag || is_lm {
+ tls::with(|downloads| {
+ if let Some(downloads) = downloads {
+ let mut index_version =
+ downloads.pending[&token].0.index_version.borrow_mut();
+ if is_etag {
+ *index_version = Some(format!("{}: {}", ETAG, value));
+ } else if index_version.is_none() && is_lm {
+ *index_version = Some(format!("{}: {}", LAST_MODIFIED, value));
+ };
+ }
+ })
+ }
+ }
+
+ true
+ })?;
+
+ let dl = Download {
+ token,
+ data: RefCell::new(Vec::new()),
+ path: path.to_path_buf(),
+ index_version: RefCell::new(None),
+ total: Cell::new(0),
+ current: Cell::new(0),
+ };
+
+ // Finally add the request we've lined up to the pool of requests that cURL manages.
+ let mut handle = self.multi.add(handle)?;
+ handle.set_token(token)?;
+ self.downloads.pending.insert(dl.token, (dl, handle));
+
+ Poll::Pending
+ }
+
+ fn config(&mut self) -> Poll<CargoResult<Option<RegistryConfig>>> {
+ if self.registry_config.is_some() {
+ return Poll::Ready(Ok(self.registry_config.clone()));
+ }
+ debug!("loading config");
+ let index_path = self.config.assert_package_cache_locked(&self.index_path);
+ let config_json_path = index_path.join("config.json");
+ if self.is_fresh(Path::new("config.json")) {
+ match fs::read(&config_json_path) {
+ Ok(raw_data) => match serde_json::from_slice(&raw_data) {
+ Ok(json) => {
+ self.registry_config = Some(json);
+ return Poll::Ready(Ok(self.registry_config.clone()));
+ }
+ Err(e) => log::debug!("failed to decode cached config.json: {}", e),
+ },
+ Err(e) => log::debug!("failed to read config.json cache: {}", e),
+ }
+ }
+
+ match self.load(Path::new(""), Path::new("config.json"), None)? {
+ Poll::Ready(LoadResponse::Data {
+ raw_data,
+ index_version: _,
+ }) => {
+ trace!("config loaded");
+ self.registry_config = Some(serde_json::from_slice(&raw_data)?);
+ if paths::create_dir_all(&config_json_path.parent().unwrap()).is_ok() {
+ if let Err(e) = fs::write(&config_json_path, &raw_data) {
+ log::debug!("failed to write config.json cache: {}", e);
+ }
+ }
+ Poll::Ready(Ok(self.registry_config.clone()))
+ }
+ Poll::Ready(LoadResponse::NotFound) => {
+ Poll::Ready(Err(anyhow::anyhow!("config.json not found in registry")))
+ }
+ Poll::Ready(LoadResponse::CacheValid) => {
+ panic!("config.json is not stored in the index cache")
+ }
+ Poll::Pending => Poll::Pending,
+ }
+ }
+
+ fn invalidate_cache(&mut self) {
+ // Actually updating the index is more or less a no-op for this implementation.
+ // All it does is ensure that a subsequent load will double-check files with the
+ // server rather than rely on a locally cached copy of the index files.
+ debug!("invalidated index cache");
+ self.requested_update = true;
+ }
+
+ fn download(&mut self, pkg: PackageId, checksum: &str) -> CargoResult<MaybeLock> {
+ let registry_config = loop {
+ match self.config()? {
+ Poll::Pending => self.block_until_ready()?,
+ Poll::Ready(cfg) => break cfg.unwrap(),
+ }
+ };
+ download::download(
+ &self.cache_path,
+ &self.config,
+ pkg,
+ checksum,
+ registry_config,
+ )
+ }
+
+ fn finish_download(
+ &mut self,
+ pkg: PackageId,
+ checksum: &str,
+ data: &[u8],
+ ) -> CargoResult<File> {
+ download::finish_download(&self.cache_path, &self.config, pkg, checksum, data)
+ }
+
+ fn is_crate_downloaded(&self, pkg: PackageId) -> bool {
+ download::is_crate_downloaded(&self.cache_path, &self.config, pkg)
+ }
+
+ fn block_until_ready(&mut self) -> CargoResult<()> {
+ let initial_pending_count = self.downloads.pending.len();
+ trace!(
+ "block_until_ready: {} transfers pending",
+ initial_pending_count
+ );
+
+ loop {
+ self.handle_completed_downloads()?;
+
+ let remaining_in_multi = tls::set(&self.downloads, || {
+ self.multi
+ .perform()
+ .with_context(|| "failed to perform http requests")
+ })?;
+ trace!("{} transfers remaining", remaining_in_multi);
+
+ if remaining_in_multi == 0 {
+ return Ok(());
+ }
+
+ // We have no more replies to provide the caller with,
+ // so we need to wait until cURL has something new for us.
+ let timeout = self
+ .multi
+ .get_timeout()?
+ .unwrap_or_else(|| Duration::new(5, 0));
+ self.multi
+ .wait(&mut [], timeout)
+ .with_context(|| "failed to wait on curl `Multi`")?;
+ }
+ }
+}
+
+impl<'cfg> Downloads<'cfg> {
+ fn progress(&self, token: usize, total: u64, cur: u64) -> bool {
+ let dl = &self.pending[&token].0;
+ dl.total.set(total);
+ dl.current.set(cur);
+ true
+ }
+
+ fn tick(&self) -> CargoResult<()> {
+ let mut progress = self.progress.borrow_mut();
+ let progress = progress.as_mut().unwrap();
+
+ progress.tick(
+ self.downloads_finished,
+ self.downloads_finished + self.pending.len(),
+ "",
+ )
+ }
+}
+
+mod tls {
+ use super::Downloads;
+ use std::cell::Cell;
+
+ thread_local!(static PTR: Cell<usize> = Cell::new(0));
+
+ pub(crate) fn with<R>(f: impl FnOnce(Option<&Downloads<'_>>) -> R) -> R {
+ let ptr = PTR.with(|p| p.get());
+ if ptr == 0 {
+ f(None)
+ } else {
+ // Safety: * `ptr` is only set by `set` below which ensures the type is correct.
+ let ptr = unsafe { &*(ptr as *const Downloads<'_>) };
+ f(Some(ptr))
+ }
+ }
+
+ pub(crate) fn set<R>(dl: &Downloads<'_>, f: impl FnOnce() -> R) -> R {
+ struct Reset<'a, T: Copy>(&'a Cell<T>, T);
+
+ impl<'a, T: Copy> Drop for Reset<'a, T> {
+ fn drop(&mut self) {
+ self.0.set(self.1);
+ }
+ }
+
+ PTR.with(|p| {
+ let _reset = Reset(p, p.get());
+ p.set(dl as *const Downloads<'_> as usize);
+ f()
+ })
+ }
+}
use std::collections::{HashMap, HashSet};
use std::convert::TryInto;
use std::fs;
+use std::io::ErrorKind;
use std::path::Path;
use std::str;
use std::task::Poll;
yanked_whitelist: &HashSet<PackageId>,
f: &mut dyn FnMut(Summary),
) -> Poll<CargoResult<()>> {
- if self.config.offline()
- && self.query_inner_with_online(dep, load, yanked_whitelist, f, false)?
- != Poll::Ready(0)
- {
- return Poll::Ready(Ok(()));
+ if self.config.offline() {
+ match self.query_inner_with_online(dep, load, yanked_whitelist, f, false)? {
+ Poll::Ready(0) => {}
+ Poll::Ready(_) => return Poll::Ready(Ok(())),
+ Poll::Pending => return Poll::Pending,
+ }
// If offline, and there are no matches, try again with online.
// This is necessary for dependencies that are not used (such as
// target-cfg or optional), but are not downloaded. Normally the
/// for `relative` from the underlying index (aka typically libgit2 with
/// crates.io) and then parse everything in there.
///
- /// * `index_version` - a version string to describe the current state of
- /// the index which for remote registries is the current git sha and
- /// for local registries is not available.
/// * `root` - this is the root argument passed to `load`
/// * `cache_root` - this is the root on the filesystem itself of where to
/// store cache files.
Err(e) => log::debug!("cache missing for {:?} error: {}", relative, e),
}
- let mut response = load.load(root, relative, index_version.as_deref())?;
- // In debug builds, perform a second load without the cache so that
- // we can validate that the cache is correct.
- if cfg!(debug_assertions) && matches!(response, Poll::Ready(LoadResponse::CacheValid)) {
- response = load.load(root, relative, None)?;
- }
- let response = match response {
+ let response = match load.load(root, relative, index_version.as_deref())? {
Poll::Pending => return Poll::Pending,
Poll::Ready(response) => response,
};
- let mut bytes_to_cache = None;
- let mut version_to_cache = None;
- let mut ret = Summaries::default();
match response {
LoadResponse::CacheValid => {
log::debug!("fast path for registry cache of {:?}", relative);
}
LoadResponse::NotFound => {
debug_assert!(cached_summaries.is_none());
+ if let Err(e) = fs::remove_file(cache_path) {
+ if e.kind() != ErrorKind::NotFound {
+ log::debug!("failed to remove from cache: {}", e);
+ }
+ }
return Poll::Ready(Ok(None));
}
LoadResponse::Data {
// to find the versions)
log::debug!("slow path for {:?}", relative);
let mut cache = SummariesCache::default();
+ let mut ret = Summaries::default();
ret.raw_data = raw_data;
for line in split(&ret.raw_data, b'\n') {
// Attempt forwards-compatibility on the index by ignoring
ret.versions.insert(version, summary.into());
}
if let Some(index_version) = index_version {
- bytes_to_cache = Some(cache.serialize(index_version.as_str()));
- version_to_cache = Some(index_version);
- }
- }
- }
-
- // If we've got debug assertions enabled and the cache was previously
- // present and considered fresh this is where the debug assertions
- // actually happens to verify that our cache is indeed fresh and
- // computes exactly the same value as before.
- let cache_contents = cached_summaries.as_ref().map(|s| &s.raw_data);
- if cfg!(debug_assertions)
- && index_version.as_deref() == version_to_cache.as_deref()
- && cached_summaries.is_some()
- && bytes_to_cache.as_ref() != cache_contents
- {
- panic!(
- "original cache contents:\n{:?}\n\
- does not equal new cache contents:\n{:?}\n",
- cache_contents.as_ref().map(|s| String::from_utf8_lossy(s)),
- bytes_to_cache.as_ref().map(|s| String::from_utf8_lossy(s)),
- );
- }
+ log::trace!("caching index_version {}", index_version);
+ let cache_bytes = cache.serialize(index_version.as_str());
+ // Once we have our `cache_bytes` which represents the `Summaries` we're
+ // about to return, write that back out to disk so future Cargo
+ // invocations can use it.
+ //
+ // This is opportunistic so we ignore failure here but are sure to log
+ // something in case of error.
+ if paths::create_dir_all(cache_path.parent().unwrap()).is_ok() {
+ let path = Filesystem::new(cache_path.clone());
+ config.assert_package_cache_locked(&path);
+ if let Err(e) = fs::write(cache_path, &cache_bytes) {
+ log::info!("failed to write cache: {}", e);
+ }
+ }
- // Once we have our `cache_bytes` which represents the `Summaries` we're
- // about to return, write that back out to disk so future Cargo
- // invocations can use it.
- //
- // This is opportunistic so we ignore failure here but are sure to log
- // something in case of error.
- if let Some(cache_bytes) = bytes_to_cache {
- if paths::create_dir_all(cache_path.parent().unwrap()).is_ok() {
- let path = Filesystem::new(cache_path.clone());
- config.assert_package_cache_locked(&path);
- if let Err(e) = fs::write(cache_path, cache_bytes) {
- log::info!("failed to write cache: {}", e);
+ // If we've got debug assertions enabled read back in the cached values
+ // and assert they match the expected result.
+ #[cfg(debug_assertions)]
+ {
+ let readback = SummariesCache::parse(&cache_bytes)
+ .expect("failed to parse cache we just wrote");
+ assert_eq!(
+ readback.index_version, index_version,
+ "index_version mismatch"
+ );
+ assert_eq!(readback.versions, cache.versions, "versions mismatch");
+ }
}
+ Poll::Ready(Ok(Some(ret)))
}
}
-
- Poll::Ready(Ok(Some(ret)))
}
/// Parses an open `File` which represents information previously cached by
}
fn load(
- &self,
+ &mut self,
root: &Path,
path: &Path,
_index_version: Option<&str>,
}
/// The `config.json` file stored in the index.
-#[derive(Deserialize)]
+#[derive(Deserialize, Debug, Clone)]
+#[serde(rename_all = "kebab-case")]
pub struct RegistryConfig {
/// Download endpoint for all crates.
///
/// * `path` is the relative path to the package to load (like `ca/rg/cargo`).
/// * `index_version` is the version of the requested crate data currently in cache.
fn load(
- &self,
+ &mut self,
root: &Path,
path: &Path,
index_version: Option<&str>,
Download { url: String, descriptor: String },
}
+mod download;
+mod http_remote;
mod index;
mod local;
mod remote;
source_id: SourceId,
yanked_whitelist: &HashSet<PackageId>,
config: &'cfg Config,
- ) -> RegistrySource<'cfg> {
+ ) -> CargoResult<RegistrySource<'cfg>> {
let name = short_name(source_id);
- let ops = remote::RemoteRegistry::new(source_id, config, &name);
- RegistrySource::new(source_id, config, &name, Box::new(ops), yanked_whitelist)
+ let ops = if source_id.url().scheme().starts_with("sparse+") {
+ if !config.cli_unstable().http_registry {
+ anyhow::bail!("Usage of HTTP-based registries requires `-Z http-registry`");
+ }
+ Box::new(http_remote::HttpRegistry::new(source_id, config, &name)) as Box<_>
+ } else {
+ Box::new(remote::RemoteRegistry::new(source_id, config, &name)) as Box<_>
+ };
+ Ok(RegistrySource::new(
+ source_id,
+ config,
+ &name,
+ ops,
+ yanked_whitelist,
+ ))
}
pub fn local(
}
fn download(&mut self, package: PackageId) -> CargoResult<MaybePackage> {
- let hash = self
- .index
- .hash(package, &mut *self.ops)?
- .expect("we got to downloading a dep while pending!?");
+ let hash = loop {
+ match self.index.hash(package, &mut *self.ops)? {
+ Poll::Pending => self.block_until_ready()?,
+ Poll::Ready(hash) => break hash,
+ }
+ };
match self.ops.download(package, hash)? {
MaybeLock::Ready(file) => self.get_pkg(package, &file).map(MaybePackage::Ready),
MaybeLock::Download { url, descriptor } => {
}
fn finish_download(&mut self, package: PackageId, data: Vec<u8>) -> CargoResult<Package> {
- let hash = self
- .index
- .hash(package, &mut *self.ops)?
- .expect("we got to downloading a dep while pending!?");
+ let hash = loop {
+ match self.index.hash(package, &mut *self.ops)? {
+ Poll::Pending => self.block_until_ready()?,
+ Poll::Ready(hash) => break hash,
+ }
+ };
let file = self.ops.finish_download(package, hash, &data)?;
self.get_pkg(package, &file)
}
self.ops.block_until_ready()
}
}
+
+fn make_dep_prefix(name: &str) -> String {
+ match name.len() {
+ 1 => String::from("1"),
+ 2 => String::from("2"),
+ 3 => format!("3/{}", &name[..1]),
+ _ => format!("{}/{}", &name[0..2], &name[2..4]),
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::make_dep_prefix;
+
+ #[test]
+ fn dep_prefix() {
+ assert_eq!(make_dep_prefix("a"), "1");
+ assert_eq!(make_dep_prefix("ab"), "2");
+ assert_eq!(make_dep_prefix("abc"), "3/a");
+ assert_eq!(make_dep_prefix("Abc"), "3/A");
+ assert_eq!(make_dep_prefix("AbCd"), "Ab/Cd");
+ assert_eq!(make_dep_prefix("aBcDe"), "aB/cD");
+ }
+}
use crate::core::{GitReference, PackageId, SourceId};
use crate::sources::git;
+use crate::sources::registry::download;
use crate::sources::registry::MaybeLock;
-use crate::sources::registry::{
- LoadResponse, RegistryConfig, RegistryData, CHECKSUM_TEMPLATE, CRATE_TEMPLATE,
- LOWER_PREFIX_TEMPLATE, PREFIX_TEMPLATE, VERSION_TEMPLATE,
-};
+use crate::sources::registry::{LoadResponse, RegistryConfig, RegistryData};
use crate::util::errors::CargoResult;
use crate::util::interning::InternedString;
use crate::util::{Config, Filesystem};
use anyhow::Context as _;
-use cargo_util::{paths, registry::make_dep_path, Sha256};
+use cargo_util::paths;
use lazycell::LazyCell;
use log::{debug, trace};
use std::cell::{Cell, Ref, RefCell};
-use std::fmt::Write as FmtWrite;
-use std::fs::{self, File, OpenOptions};
-use std::io::prelude::*;
-use std::io::SeekFrom;
+use std::fs::File;
use std::mem;
use std::path::Path;
use std::str;
repo: LazyCell<git2::Repository>,
head: Cell<Option<git2::Oid>>,
current_sha: Cell<Option<InternedString>>,
- needs_update: Cell<bool>, // Does this registry need to be updated?
- updated: bool, // Has this registry been updated this session?
+ needs_update: bool, // Does this registry need to be updated?
+ updated: bool, // Has this registry been updated this session?
}
impl<'cfg> RemoteRegistry<'cfg> {
repo: LazyCell::new(),
head: Cell::new(None),
current_sha: Cell::new(None),
- needs_update: Cell::new(false),
+ needs_update: false,
updated: false,
}
}
Ok(Ref::map(self.tree.borrow(), |s| s.as_ref().unwrap()))
}
- fn filename(&self, pkg: PackageId) -> String {
- format!("{}-{}.crate", pkg.name(), pkg.version())
- }
-
fn current_version(&self) -> Option<InternedString> {
if let Some(sha) = self.current_sha.get() {
return Some(sha);
}
fn load(
- &self,
+ &mut self,
_root: &Path,
path: &Path,
index_version: Option<&str>,
) -> Poll<CargoResult<LoadResponse>> {
- if self.needs_update.get() {
+ if self.needs_update {
return Poll::Pending;
}
// Check if the cache is valid.
Err(_) if !self.updated => {
// If git returns an error and we haven't updated the repo, return
// pending to allow an update to try again.
- self.needs_update.set(true);
+ self.needs_update = true;
Poll::Pending
}
Err(e)
}
fn block_until_ready(&mut self) -> CargoResult<()> {
- if !self.needs_update.get() {
+ if !self.needs_update {
return Ok(());
}
self.updated = true;
- self.needs_update.set(false);
+ self.needs_update = false;
if self.config.offline() {
return Ok(());
fn invalidate_cache(&mut self) {
if !self.updated {
- self.needs_update.set(true);
+ self.needs_update = true;
}
}
}
fn download(&mut self, pkg: PackageId, checksum: &str) -> CargoResult<MaybeLock> {
- let filename = self.filename(pkg);
-
- // Attempt to open an read-only copy first to avoid an exclusive write
- // lock and also work with read-only filesystems. Note that we check the
- // length of the file like below to handle interrupted downloads.
- //
- // If this fails then we fall through to the exclusive path where we may
- // have to redownload the file.
- let path = self.cache_path.join(&filename);
- let path = self.config.assert_package_cache_locked(&path);
- if let Ok(dst) = File::open(&path) {
- let meta = dst.metadata()?;
- if meta.len() > 0 {
- return Ok(MaybeLock::Ready(dst));
- }
- }
-
- let config = loop {
+ let registry_config = loop {
match self.config()? {
Poll::Pending => self.block_until_ready()?,
Poll::Ready(cfg) => break cfg.unwrap(),
}
};
- let mut url = config.dl;
- if !url.contains(CRATE_TEMPLATE)
- && !url.contains(VERSION_TEMPLATE)
- && !url.contains(PREFIX_TEMPLATE)
- && !url.contains(LOWER_PREFIX_TEMPLATE)
- && !url.contains(CHECKSUM_TEMPLATE)
- {
- write!(url, "/{}/{}/download", CRATE_TEMPLATE, VERSION_TEMPLATE).unwrap();
- }
- let prefix = make_dep_path(&*pkg.name(), true);
- let url = url
- .replace(CRATE_TEMPLATE, &*pkg.name())
- .replace(VERSION_TEMPLATE, &pkg.version().to_string())
- .replace(PREFIX_TEMPLATE, &prefix)
- .replace(LOWER_PREFIX_TEMPLATE, &prefix.to_lowercase())
- .replace(CHECKSUM_TEMPLATE, checksum);
-
- Ok(MaybeLock::Download {
- url,
- descriptor: pkg.to_string(),
- })
+ download::download(
+ &self.cache_path,
+ &self.config,
+ pkg,
+ checksum,
+ registry_config,
+ )
}
fn finish_download(
checksum: &str,
data: &[u8],
) -> CargoResult<File> {
- // Verify what we just downloaded
- let actual = Sha256::new().update(data).finish_hex();
- if actual != checksum {
- anyhow::bail!("failed to verify the checksum of `{}`", pkg)
- }
-
- let filename = self.filename(pkg);
- self.cache_path.create_dir()?;
- let path = self.cache_path.join(&filename);
- let path = self.config.assert_package_cache_locked(&path);
- let mut dst = OpenOptions::new()
- .create(true)
- .read(true)
- .write(true)
- .open(&path)
- .with_context(|| format!("failed to open `{}`", path.display()))?;
- let meta = dst.metadata()?;
- if meta.len() > 0 {
- return Ok(dst);
- }
-
- dst.write_all(data)?;
- dst.seek(SeekFrom::Start(0))?;
- Ok(dst)
+ download::finish_download(&self.cache_path, &self.config, pkg, checksum, data)
}
fn is_crate_downloaded(&self, pkg: PackageId) -> bool {
- let filename = format!("{}-{}.crate", pkg.name(), pkg.version());
- let path = Path::new(&filename);
-
- let path = self.cache_path.join(path);
- let path = self.config.assert_package_cache_locked(&path);
- if let Ok(meta) = fs::metadata(path) {
- return meta.len() > 0;
- }
- false
+ download::is_crate_downloaded(&self.cache_path, &self.config, pkg)
}
}
-use crate::util::errors::CargoResult;
+use crate::util::{errors::CargoResult, IntoUrl};
use std::hash::{self, Hash};
use url::Url;
url.path_segments_mut().unwrap().pop().push(&last);
}
+ // Ignore the protocol specifier (if any).
+ if url.scheme().starts_with("sparse+") {
+ // NOTE: it is illegal to use set_scheme to change sparse+http(s) to http(s).
+ url = url
+ .to_string()
+ .strip_prefix("sparse+")
+ .expect("we just found that prefix")
+ .into_url()
+ .expect("a valid url without a protocol specifier should still be valid");
+ }
+
Ok(CanonicalUrl(url))
}
* Registries
* [credential-process](#credential-process) — Adds support for fetching registry tokens from an external authentication program.
* [`cargo logout`](#cargo-logout) — Adds the `logout` command to remove the currently saved registry token.
+ * [http-registry](#http-registry) — Adds support for fetching from http registries (`sparse+`)
### allow-features
}
```
+### http-registry
+* Tracking Issue: [9069](https://github.com/rust-lang/cargo/issues/9069)
+* RFC: [#2789](https://github.com/rust-lang/rfcs/pull/2789)
+
+The `http-registry` feature allows cargo to interact with remote registries served
+over http rather than git. These registries can be identified by urls starting with
+`sparse+http://` or `sparse+https://`.
+
+When fetching index metadata over http, cargo only downloads the metadata for relevant
+crates, which can save significant time and bandwidth.
+
+The format of the http index is identical to a checkout of a git-based index.
### credential-process
* Tracking Issue: [#8933](https://github.com/rust-lang/cargo/issues/8933)
[ERROR] failed to download `opt_dep v1.0.0`
Caused by:
- can't make HTTP request in the offline mode
+ attempting to make an HTTP request, but --offline was specified
",
)
.with_status(101)
[ERROR] failed to download `bar v0.1.0`
Caused by:
- can't make HTTP request in the offline mode
+ attempting to make an HTTP request, but --offline was specified
",
)
.run();
use cargo::core::SourceId;
use cargo_test_support::paths::{self, CargoPathExt};
-use cargo_test_support::registry::{self, registry_path, Dependency, Package};
-use cargo_test_support::{basic_manifest, project};
+use cargo_test_support::registry::{
+ self, registry_path, serve_registry, Dependency, Package, RegistryServer,
+};
+use cargo_test_support::{basic_manifest, project, Execs, Project};
use cargo_test_support::{cargo_process, registry::registry_url};
use cargo_test_support::{git, install::cargo_home, t};
use cargo_util::paths::remove_dir_all;
use std::path::Path;
use std::process::Stdio;
+fn cargo_http(p: &Project, s: &str) -> Execs {
+ let mut e = p.cargo(s);
+ e.arg("-Zhttp-registry").masquerade_as_nightly_cargo();
+ e
+}
+
+fn cargo_stable(p: &Project, s: &str) -> Execs {
+ p.cargo(s)
+}
+
+fn setup_http() -> RegistryServer {
+ let server = serve_registry(registry_path());
+ configure_source_replacement_for_http(&server.addr().to_string());
+ server
+}
+
+fn configure_source_replacement_for_http(addr: &str) {
+ let root = paths::root();
+ t!(fs::create_dir(&root.join(".cargo")));
+ t!(fs::write(
+ root.join(".cargo/config"),
+ format!(
+ "
+ [source.crates-io]
+ replace-with = 'dummy-registry'
+
+ [source.dummy-registry]
+ registry = 'sparse+http://{}'
+ ",
+ addr
+ )
+ ));
+}
+
#[cargo_test]
-fn simple() {
+fn simple_http() {
+ let _server = setup_http();
+ simple(cargo_http);
+}
+
+#[cargo_test]
+fn simple_git() {
+ simple(cargo_stable);
+}
+
+fn simple(cargo: fn(&Project, &str) -> Execs) {
let p = project()
.file(
"Cargo.toml",
Package::new("bar", "0.0.1").publish();
- p.cargo("build")
+ cargo(&p, "build")
.with_stderr(
"\
[UPDATING] `dummy-registry` index
)
.run();
- p.cargo("clean").run();
+ cargo(&p, "clean").run();
// Don't download a second time
- p.cargo("build")
+ cargo(&p, "build")
.with_stderr(
"\
[COMPILING] bar v0.0.1
}
#[cargo_test]
-fn deps() {
+fn deps_http() {
+ let _server = setup_http();
+ deps(cargo_http);
+}
+
+#[cargo_test]
+fn deps_git() {
+ deps(cargo_stable);
+}
+
+fn deps(cargo: fn(&Project, &str) -> Execs) {
let p = project()
.file(
"Cargo.toml",
Package::new("baz", "0.0.1").publish();
Package::new("bar", "0.0.1").dep("baz", "*").publish();
- p.cargo("build")
+ cargo(&p, "build")
.with_stderr(
"\
[UPDATING] `dummy-registry` index
}
#[cargo_test]
-fn nonexistent() {
+fn nonexistent_http() {
+ let _server = setup_http();
+ nonexistent(cargo_http);
+}
+
+#[cargo_test]
+fn nonexistent_git() {
+ nonexistent(cargo_stable);
+}
+
+fn nonexistent(cargo: fn(&Project, &str) -> Execs) {
Package::new("init", "0.0.1").publish();
let p = project()
.file("src/main.rs", "fn main() {}")
.build();
- p.cargo("build")
+ cargo(&p, "build")
.with_status(101)
.with_stderr(
"\
}
#[cargo_test]
-fn wrong_case() {
+fn wrong_case_http() {
+ let _server = setup_http();
+ wrong_case(cargo_http);
+}
+
+#[cargo_test]
+fn wrong_case_git() {
+ wrong_case(cargo_stable);
+}
+
+fn wrong_case(cargo: fn(&Project, &str) -> Execs) {
Package::new("init", "0.0.1").publish();
let p = project()
.build();
// #5678 to make this work
- p.cargo("build")
+ cargo(&p, "build")
.with_status(101)
.with_stderr(
"\
}
#[cargo_test]
-fn mis_hyphenated() {
+fn mis_hyphenated_http() {
+ let _server = setup_http();
+ mis_hyphenated(cargo_http);
+}
+
+#[cargo_test]
+fn mis_hyphenated_git() {
+ mis_hyphenated(cargo_stable);
+}
+
+fn mis_hyphenated(cargo: fn(&Project, &str) -> Execs) {
Package::new("mis-hyphenated", "0.0.1").publish();
let p = project()
.build();
// #2775 to make this work
- p.cargo("build")
+ cargo(&p, "build")
.with_status(101)
.with_stderr(
"\
}
#[cargo_test]
-fn wrong_version() {
+fn wrong_version_http() {
+ let _server = setup_http();
+ wrong_version(cargo_http);
+}
+
+#[cargo_test]
+fn wrong_version_git() {
+ wrong_version(cargo_stable);
+}
+
+fn wrong_version(cargo: fn(&Project, &str) -> Execs) {
let p = project()
.file(
"Cargo.toml",
Package::new("foo", "0.0.1").publish();
Package::new("foo", "0.0.2").publish();
- p.cargo("build")
+ cargo(&p, "build")
.with_status(101)
.with_stderr_contains(
"\
Package::new("foo", "0.0.3").publish();
Package::new("foo", "0.0.4").publish();
- p.cargo("build")
+ cargo(&p, "build")
.with_status(101)
.with_stderr_contains(
"\
}
#[cargo_test]
-fn bad_cksum() {
+fn bad_cksum_http() {
+ let _server = setup_http();
+ bad_cksum(cargo_http);
+}
+
+#[cargo_test]
+fn bad_cksum_git() {
+ bad_cksum(cargo_stable);
+}
+
+fn bad_cksum(cargo: fn(&Project, &str) -> Execs) {
let p = project()
.file(
"Cargo.toml",
pkg.publish();
t!(File::create(&pkg.archive_dst()));
- p.cargo("build -v")
+ cargo(&p, "build -v")
.with_status(101)
.with_stderr(
"\
}
#[cargo_test]
-fn update_registry() {
+fn update_registry_http() {
+ let _server = setup_http();
+ update_registry(cargo_http);
+}
+
+#[cargo_test]
+fn update_registry_git() {
+ update_registry(cargo_stable);
+}
+
+fn update_registry(cargo: fn(&Project, &str) -> Execs) {
Package::new("init", "0.0.1").publish();
let p = project()
.file("src/main.rs", "fn main() {}")
.build();
- p.cargo("build")
+ cargo(&p, "build")
.with_status(101)
.with_stderr_contains(
"\
Package::new("notyet", "0.0.1").publish();
- p.cargo("build")
+ cargo(&p, "build")
.with_stderr(
"\
[UPDATING] `dummy-registry` index
}
#[cargo_test]
-fn package_with_path_deps() {
+fn package_with_path_deps_http() {
+ let _server = setup_http();
+ package_with_path_deps(cargo_http);
+}
+
+#[cargo_test]
+fn package_with_path_deps_git() {
+ package_with_path_deps(cargo_stable);
+}
+
+fn package_with_path_deps(cargo: fn(&Project, &str) -> Execs) {
Package::new("init", "0.0.1").publish();
let p = project()
.file("notyet/src/lib.rs", "")
.build();
- p.cargo("package")
+ cargo(&p, "package")
.with_status(101)
.with_stderr_contains(
"\
Package::new("notyet", "0.0.1").publish();
- p.cargo("package")
+ cargo(&p, "package")
.with_stderr(
"\
[PACKAGING] foo v0.0.1 ([CWD])
}
#[cargo_test]
-fn lockfile_locks() {
+fn lockfile_locks_http() {
+ let _server = setup_http();
+ lockfile_locks(cargo_http);
+}
+
+#[cargo_test]
+fn lockfile_locks_git() {
+ lockfile_locks(cargo_stable);
+}
+
+fn lockfile_locks(cargo: fn(&Project, &str) -> Execs) {
let p = project()
.file(
"Cargo.toml",
Package::new("bar", "0.0.1").publish();
- p.cargo("build")
+ cargo(&p, "build")
.with_stderr(
"\
[UPDATING] `[..]` index
p.root().move_into_the_past();
Package::new("bar", "0.0.2").publish();
- p.cargo("build").with_stdout("").run();
+ cargo(&p, "build").with_stdout("").run();
}
#[cargo_test]
-fn lockfile_locks_transitively() {
+fn lockfile_locks_transitively_http() {
+ let _server = setup_http();
+ lockfile_locks_transitively(cargo_http);
+}
+
+#[cargo_test]
+fn lockfile_locks_transitively_git() {
+ lockfile_locks_transitively(cargo_stable);
+}
+
+fn lockfile_locks_transitively(cargo: fn(&Project, &str) -> Execs) {
let p = project()
.file(
"Cargo.toml",
Package::new("baz", "0.0.1").publish();
Package::new("bar", "0.0.1").dep("baz", "*").publish();
- p.cargo("build")
+ cargo(&p, "build")
.with_stderr(
"\
[UPDATING] `[..]` index
Package::new("baz", "0.0.2").publish();
Package::new("bar", "0.0.2").dep("baz", "*").publish();
- p.cargo("build").with_stdout("").run();
+ cargo(&p, "build").with_stdout("").run();
}
#[cargo_test]
-fn yanks_are_not_used() {
+fn yanks_are_not_used_http() {
+ let _server = setup_http();
+ yanks_are_not_used(cargo_http);
+}
+
+#[cargo_test]
+fn yanks_are_not_used_git() {
+ yanks_are_not_used(cargo_stable);
+}
+
+fn yanks_are_not_used(cargo: fn(&Project, &str) -> Execs) {
let p = project()
.file(
"Cargo.toml",
.yanked(true)
.publish();
- p.cargo("build")
+ cargo(&p, "build")
.with_stderr(
"\
[UPDATING] `[..]` index
}
#[cargo_test]
-fn relying_on_a_yank_is_bad() {
+fn relying_on_a_yank_is_bad_http() {
+ let _server = setup_http();
+ relying_on_a_yank_is_bad(cargo_http);
+}
+
+#[cargo_test]
+fn relying_on_a_yank_is_bad_git() {
+ relying_on_a_yank_is_bad(cargo_stable);
+}
+
+fn relying_on_a_yank_is_bad(cargo: fn(&Project, &str) -> Execs) {
let p = project()
.file(
"Cargo.toml",
Package::new("baz", "0.0.2").yanked(true).publish();
Package::new("bar", "0.0.1").dep("baz", "=0.0.2").publish();
- p.cargo("build")
+ cargo(&p, "build")
.with_status(101)
.with_stderr_contains(
"\
}
#[cargo_test]
-fn yanks_in_lockfiles_are_ok() {
+fn yanks_in_lockfiles_are_ok_http() {
+ let _server = setup_http();
+ yanks_in_lockfiles_are_ok(cargo_http);
+}
+
+#[cargo_test]
+fn yanks_in_lockfiles_are_ok_git() {
+ yanks_in_lockfiles_are_ok(cargo_stable);
+}
+
+fn yanks_in_lockfiles_are_ok(cargo: fn(&Project, &str) -> Execs) {
let p = project()
.file(
"Cargo.toml",
Package::new("bar", "0.0.1").publish();
- p.cargo("build").run();
+ cargo(&p, "build").run();
registry_path().join("3").rm_rf();
Package::new("bar", "0.0.1").yanked(true).publish();
- p.cargo("build").with_stdout("").run();
+ cargo(&p, "build").with_stdout("").run();
- p.cargo("update")
+ cargo(&p, "update")
.with_status(101)
.with_stderr_contains(
"\
}
#[cargo_test]
-fn yanks_in_lockfiles_are_ok_for_other_update() {
+fn yanks_in_lockfiles_are_ok_for_other_update_http() {
+ let _server = setup_http();
+ yanks_in_lockfiles_are_ok_for_other_update(cargo_http);
+}
+
+#[cargo_test]
+fn yanks_in_lockfiles_are_ok_for_other_update_git() {
+ yanks_in_lockfiles_are_ok_for_other_update(cargo_stable);
+}
+
+fn yanks_in_lockfiles_are_ok_for_other_update(cargo: fn(&Project, &str) -> Execs) {
let p = project()
.file(
"Cargo.toml",
Package::new("bar", "0.0.1").publish();
Package::new("baz", "0.0.1").publish();
- p.cargo("build").run();
+ cargo(&p, "build").run();
registry_path().join("3").rm_rf();
Package::new("bar", "0.0.1").yanked(true).publish();
Package::new("baz", "0.0.1").publish();
- p.cargo("build").with_stdout("").run();
+ cargo(&p, "build").with_stdout("").run();
Package::new("baz", "0.0.2").publish();
- p.cargo("update")
+ cargo(&p, "update")
.with_status(101)
.with_stderr_contains(
"\
)
.run();
- p.cargo("update -p baz")
+ cargo(&p, "update -p baz")
.with_stderr_contains(
"\
[UPDATING] `[..]` index
}
#[cargo_test]
-fn yanks_in_lockfiles_are_ok_with_new_dep() {
+fn yanks_in_lockfiles_are_ok_with_new_dep_http() {
+ let _server = setup_http();
+ yanks_in_lockfiles_are_ok_with_new_dep(cargo_http);
+}
+
+#[cargo_test]
+fn yanks_in_lockfiles_are_ok_with_new_dep_git() {
+ yanks_in_lockfiles_are_ok_with_new_dep(cargo_stable);
+}
+
+fn yanks_in_lockfiles_are_ok_with_new_dep(cargo: fn(&Project, &str) -> Execs) {
let p = project()
.file(
"Cargo.toml",
Package::new("bar", "0.0.1").publish();
- p.cargo("build").run();
+ cargo(&p, "build").run();
registry_path().join("3").rm_rf();
"#,
);
- p.cargo("build").with_stdout("").run();
+ cargo(&p, "build").with_stdout("").run();
}
#[cargo_test]
-fn update_with_lockfile_if_packages_missing() {
+fn update_with_lockfile_if_packages_missing_http() {
+ let _server = setup_http();
+ update_with_lockfile_if_packages_missing(cargo_http);
+}
+
+#[cargo_test]
+fn update_with_lockfile_if_packages_missing_git() {
+ update_with_lockfile_if_packages_missing(cargo_stable);
+}
+
+fn update_with_lockfile_if_packages_missing(cargo: fn(&Project, &str) -> Execs) {
let p = project()
.file(
"Cargo.toml",
.build();
Package::new("bar", "0.0.1").publish();
- p.cargo("build").run();
+ cargo(&p, "build").run();
p.root().move_into_the_past();
paths::home().join(".cargo/registry").rm_rf();
- p.cargo("build")
+ cargo(&p, "build")
.with_stderr(
"\
[UPDATING] `[..]` index
}
#[cargo_test]
-fn update_lockfile() {
+fn update_lockfile_http() {
+ let _server = setup_http();
+ update_lockfile(cargo_http);
+}
+
+#[cargo_test]
+fn update_lockfile_git() {
+ update_lockfile(cargo_stable);
+}
+
+fn update_lockfile(cargo: fn(&Project, &str) -> Execs) {
let p = project()
.file(
"Cargo.toml",
println!("0.0.1");
Package::new("bar", "0.0.1").publish();
- p.cargo("build").run();
+ cargo(&p, "build").run();
Package::new("bar", "0.0.2").publish();
Package::new("bar", "0.0.3").publish();
paths::home().join(".cargo/registry").rm_rf();
println!("0.0.2 update");
- p.cargo("update -p bar --precise 0.0.2")
+ cargo(&p, "update -p bar --precise 0.0.2")
.with_stderr(
"\
[UPDATING] `[..]` index
.run();
println!("0.0.2 build");
- p.cargo("build")
+ cargo(&p, "build")
.with_stderr(
"\
[DOWNLOADING] crates ...
.run();
println!("0.0.3 update");
- p.cargo("update -p bar")
+ cargo(&p, "update -p bar")
.with_stderr(
"\
[UPDATING] `[..]` index
.run();
println!("0.0.3 build");
- p.cargo("build")
+ cargo(&p, "build")
.with_stderr(
"\
[DOWNLOADING] crates ...
println!("new dependencies update");
Package::new("bar", "0.0.4").dep("spam", "0.2.5").publish();
Package::new("spam", "0.2.5").publish();
- p.cargo("update -p bar")
+ cargo(&p, "update -p bar")
.with_stderr(
"\
[UPDATING] `[..]` index
println!("new dependencies update");
Package::new("bar", "0.0.5").publish();
- p.cargo("update -p bar")
+ cargo(&p, "update -p bar")
.with_stderr(
"\
[UPDATING] `[..]` index
}
#[cargo_test]
-fn dev_dependency_not_used() {
+fn dev_dependency_not_used_http() {
+ let _server = setup_http();
+ dev_dependency_not_used(cargo_http);
+}
+
+#[cargo_test]
+fn dev_dependency_not_used_git() {
+ dev_dependency_not_used(cargo_stable);
+}
+
+fn dev_dependency_not_used(cargo: fn(&Project, &str) -> Execs) {
let p = project()
.file(
"Cargo.toml",
Package::new("baz", "0.0.1").publish();
Package::new("bar", "0.0.1").dev_dep("baz", "*").publish();
- p.cargo("build")
+ cargo(&p, "build")
.with_stderr(
"\
[UPDATING] `[..]` index
}
#[cargo_test]
-fn bad_license_file() {
+fn bad_license_file_http() {
+ let _server = setup_http();
+ bad_license_file(cargo_http);
+}
+
+#[cargo_test]
+fn bad_license_file_git() {
+ bad_license_file(cargo_stable);
+}
+
+fn bad_license_file(cargo: fn(&Project, &str) -> Execs) {
Package::new("foo", "1.0.0").publish();
let p = project()
.file(
)
.file("src/main.rs", "fn main() {}")
.build();
- p.cargo("publish -v --token sekrit")
+ cargo(&p, "publish -v --token sekrit")
.with_status(101)
.with_stderr_contains("[ERROR] the license file `foo` does not exist")
.run();
}
#[cargo_test]
-fn updating_a_dep() {
+fn updating_a_dep_http() {
+ let _server = setup_http();
+ updating_a_dep(cargo_http);
+}
+
+#[cargo_test]
+fn updating_a_dep_git() {
+ updating_a_dep(cargo_stable);
+}
+
+fn updating_a_dep(cargo: fn(&Project, &str) -> Execs) {
let p = project()
.file(
"Cargo.toml",
Package::new("bar", "0.0.1").publish();
- p.cargo("build")
+ cargo(&p, "build")
.with_stderr(
"\
[UPDATING] `[..]` index
Package::new("bar", "0.1.0").publish();
println!("second");
- p.cargo("build")
+ cargo(&p, "build")
.with_stderr(
"\
[UPDATING] `[..]` index
}
#[cargo_test]
-fn git_and_registry_dep() {
+fn git_and_registry_dep_http() {
+ let _server = setup_http();
+ git_and_registry_dep(cargo_http);
+}
+
+#[cargo_test]
+fn git_and_registry_dep_git() {
+ git_and_registry_dep(cargo_stable);
+}
+
+fn git_and_registry_dep(cargo: fn(&Project, &str) -> Execs) {
let b = git::repo(&paths::root().join("b"))
.file(
"Cargo.toml",
Package::new("a", "0.0.1").publish();
p.root().move_into_the_past();
- p.cargo("build")
+ cargo(&p, "build")
.with_stderr(
"\
[UPDATING] [..]
p.root().move_into_the_past();
println!("second");
- p.cargo("build").with_stdout("").run();
+ cargo(&p, "build").with_stdout("").run();
}
#[cargo_test]
-fn update_publish_then_update() {
+fn update_publish_then_update_http() {
+ let _server = setup_http();
+ update_publish_then_update(cargo_http);
+}
+
+#[cargo_test]
+fn update_publish_then_update_git() {
+ update_publish_then_update(cargo_stable);
+}
+
+fn update_publish_then_update(cargo: fn(&Project, &str) -> Execs) {
// First generate a Cargo.lock and a clone of the registry index at the
// "head" of the current registry.
let p = project()
.file("src/main.rs", "fn main() {}")
.build();
Package::new("a", "0.1.0").publish();
- p.cargo("build").run();
+ cargo(&p, "build").run();
// Next, publish a new package and back up the copy of the registry we just
// created.
)
.file("src/main.rs", "fn main() {}")
.build();
- p2.cargo("build").run();
+ cargo(&p2, "build").run();
registry.rm_rf();
t!(fs::rename(&backup, ®istry));
t!(fs::rename(
// Finally, build the first project again (with our newer Cargo.lock) which
// should force an update of the old registry, download the new crate, and
// then build everything again.
- p.cargo("build")
+ cargo(&p, "build")
.with_stderr(
"\
[UPDATING] [..]
}
#[cargo_test]
-fn fetch_downloads() {
+fn fetch_downloads_http() {
+ let _server = setup_http();
+ fetch_downloads(cargo_http);
+}
+
+#[cargo_test]
+fn fetch_downloads_git() {
+ fetch_downloads(cargo_stable);
+}
+
+fn fetch_downloads(cargo: fn(&Project, &str) -> Execs) {
let p = project()
.file(
"Cargo.toml",
Package::new("a", "0.1.0").publish();
- p.cargo("fetch")
+ cargo(&p, "fetch")
.with_stderr(
"\
[UPDATING] `[..]` index
}
#[cargo_test]
-fn update_transitive_dependency() {
+fn update_transitive_dependency_http() {
+ let _server = setup_http();
+ update_transitive_dependency(cargo_http);
+}
+
+#[cargo_test]
+fn update_transitive_dependency_git() {
+ update_transitive_dependency(cargo_stable);
+}
+
+fn update_transitive_dependency(cargo: fn(&Project, &str) -> Execs) {
let p = project()
.file(
"Cargo.toml",
Package::new("a", "0.1.0").dep("b", "*").publish();
Package::new("b", "0.1.0").publish();
- p.cargo("fetch").run();
+ cargo(&p, "fetch").run();
Package::new("b", "0.1.1").publish();
- p.cargo("update -pb")
+ cargo(&p, "update -pb")
.with_stderr(
"\
[UPDATING] `[..]` index
)
.run();
- p.cargo("build")
+ cargo(&p, "build")
.with_stderr(
"\
[DOWNLOADING] crates ...
}
#[cargo_test]
-fn update_backtracking_ok() {
+fn update_backtracking_ok_http() {
+ let _server = setup_http();
+ update_backtracking_ok(cargo_http);
+}
+
+#[cargo_test]
+fn update_backtracking_ok_git() {
+ update_backtracking_ok(cargo_stable);
+}
+
+fn update_backtracking_ok(cargo: fn(&Project, &str) -> Execs) {
let p = project()
.file(
"Cargo.toml",
.publish();
Package::new("openssl", "0.1.0").publish();
- p.cargo("generate-lockfile").run();
+ cargo(&p, "generate-lockfile").run();
Package::new("openssl", "0.1.1").publish();
Package::new("hyper", "0.6.6")
.dep("cookie", "0.1.0")
.publish();
- p.cargo("update -p hyper")
+ cargo(&p, "update -p hyper")
.with_stderr(
"\
[UPDATING] `[..]` index
}
#[cargo_test]
-fn update_multiple_packages() {
+fn update_multiple_packages_http() {
+ let _server = setup_http();
+ update_multiple_packages(cargo_http);
+}
+
+#[cargo_test]
+fn update_multiple_packages_git() {
+ update_multiple_packages(cargo_stable);
+}
+
+fn update_multiple_packages(cargo: fn(&Project, &str) -> Execs) {
let p = project()
.file(
"Cargo.toml",
Package::new("b", "0.1.0").publish();
Package::new("c", "0.1.0").publish();
- p.cargo("fetch").run();
+ cargo(&p, "fetch").run();
Package::new("a", "0.1.1").publish();
Package::new("b", "0.1.1").publish();
Package::new("c", "0.1.1").publish();
- p.cargo("update -pa -pb")
+ cargo(&p, "update -pa -pb")
.with_stderr(
"\
[UPDATING] `[..]` index
)
.run();
- p.cargo("update -pb -pc")
+ cargo(&p, "update -pb -pc")
.with_stderr(
"\
[UPDATING] `[..]` index
)
.run();
- p.cargo("build")
+ cargo(&p, "build")
.with_stderr_contains("[DOWNLOADED] a v0.1.1 (registry `dummy-registry`)")
.with_stderr_contains("[DOWNLOADED] b v0.1.1 (registry `dummy-registry`)")
.with_stderr_contains("[DOWNLOADED] c v0.1.1 (registry `dummy-registry`)")
}
#[cargo_test]
-fn bundled_crate_in_registry() {
+fn bundled_crate_in_registry_http() {
+ let _server = setup_http();
+ bundled_crate_in_registry(cargo_http);
+}
+
+#[cargo_test]
+fn bundled_crate_in_registry_git() {
+ bundled_crate_in_registry(cargo_stable);
+}
+
+fn bundled_crate_in_registry(cargo: fn(&Project, &str) -> Execs) {
let p = project()
.file(
"Cargo.toml",
.file("bar/src/lib.rs", "")
.publish();
- p.cargo("run").run();
+ cargo(&p, "run").run();
+}
+
+#[cargo_test]
+fn update_same_prefix_oh_my_how_was_this_a_bug_http() {
+ let _server = setup_http();
+ update_same_prefix_oh_my_how_was_this_a_bug(cargo_http);
}
#[cargo_test]
-fn update_same_prefix_oh_my_how_was_this_a_bug() {
+fn update_same_prefix_oh_my_how_was_this_a_bug_git() {
+ update_same_prefix_oh_my_how_was_this_a_bug(cargo_stable);
+}
+
+fn update_same_prefix_oh_my_how_was_this_a_bug(cargo: fn(&Project, &str) -> Execs) {
let p = project()
.file(
"Cargo.toml",
.dep("foobar", "0.2.0")
.publish();
- p.cargo("generate-lockfile").run();
- p.cargo("update -pfoobar --precise=0.2.0").run();
+ cargo(&p, "generate-lockfile").run();
+ cargo(&p, "update -pfoobar --precise=0.2.0").run();
+}
+
+#[cargo_test]
+fn use_semver_http() {
+ let _server = setup_http();
+ use_semver(cargo_http);
}
#[cargo_test]
-fn use_semver() {
+fn use_semver_git() {
+ use_semver(cargo_stable);
+}
+
+fn use_semver(cargo: fn(&Project, &str) -> Execs) {
let p = project()
.file(
"Cargo.toml",
Package::new("foo", "1.2.3-alpha.0").publish();
- p.cargo("build").run();
+ cargo(&p, "build").run();
}
#[cargo_test]
-fn use_semver_package_incorrectly() {
+fn use_semver_package_incorrectly_http() {
+ let _server = setup_http();
+ use_semver_package_incorrectly(cargo_http);
+}
+
+#[cargo_test]
+fn use_semver_package_incorrectly_git() {
+ use_semver_package_incorrectly(cargo_stable);
+}
+
+fn use_semver_package_incorrectly(cargo: fn(&Project, &str) -> Execs) {
let p = project()
.file(
"Cargo.toml",
.file("b/src/main.rs", "fn main() {}")
.build();
- p.cargo("build")
+ cargo(&p, "build")
.with_status(101)
.with_stderr(
"\
}
#[cargo_test]
-fn only_download_relevant() {
+fn only_download_relevant_http() {
+ let _server = setup_http();
+ only_download_relevant(cargo_http);
+}
+
+#[cargo_test]
+fn only_download_relevant_git() {
+ only_download_relevant(cargo_stable);
+}
+
+fn only_download_relevant(cargo: fn(&Project, &str) -> Execs) {
let p = project()
.file(
"Cargo.toml",
Package::new("bar", "0.1.0").publish();
Package::new("baz", "0.1.0").publish();
- p.cargo("build")
+ cargo(&p, "build")
.with_stderr(
"\
[UPDATING] `[..]` index
}
#[cargo_test]
-fn resolve_and_backtracking() {
+fn resolve_and_backtracking_http() {
+ let _server = setup_http();
+ resolve_and_backtracking(cargo_http);
+}
+
+#[cargo_test]
+fn resolve_and_backtracking_git() {
+ resolve_and_backtracking(cargo_stable);
+}
+
+fn resolve_and_backtracking(cargo: fn(&Project, &str) -> Execs) {
let p = project()
.file(
"Cargo.toml",
.publish();
Package::new("foo", "0.1.0").publish();
- p.cargo("build").run();
+ cargo(&p, "build").run();
+}
+
+#[cargo_test]
+fn upstream_warnings_on_extra_verbose_http() {
+ let _server = setup_http();
+ upstream_warnings_on_extra_verbose(cargo_http);
}
#[cargo_test]
-fn upstream_warnings_on_extra_verbose() {
+fn upstream_warnings_on_extra_verbose_git() {
+ upstream_warnings_on_extra_verbose(cargo_stable);
+}
+
+fn upstream_warnings_on_extra_verbose(cargo: fn(&Project, &str) -> Execs) {
let p = project()
.file(
"Cargo.toml",
.file("src/lib.rs", "fn unused() {}")
.publish();
- p.cargo("build -vv")
+ cargo(&p, "build -vv")
.with_stderr_contains("[..]warning: function is never used[..]")
.run();
}
#[cargo_test]
-fn disallow_network() {
+fn disallow_network_http() {
+ let _server = setup_http();
+ let p = project()
+ .file(
+ "Cargo.toml",
+ r#"
+ [project]
+ name = "bar"
+ version = "0.5.0"
+ authors = []
+
+ [dependencies]
+ foo = "*"
+ "#,
+ )
+ .file("src/main.rs", "fn main() {}")
+ .build();
+
+ cargo_http(&p, "build --frozen")
+ .with_status(101)
+ .with_stderr(
+ "\
+[UPDATING] [..]
+[ERROR] failed to get `foo` as a dependency of package `bar v0.5.0 ([..])`
+
+Caused by:
+ failed to query replaced source registry `crates-io`
+
+Caused by:
+ attempting to make an HTTP request, but --frozen was specified
+",
+ )
+ .run();
+}
+
+#[cargo_test]
+fn disallow_network_git() {
let p = project()
.file(
"Cargo.toml",
.file("src/main.rs", "fn main() {}")
.build();
- p.cargo("build --frozen")
+ cargo_stable(&p, "build --frozen")
.with_status(101)
.with_stderr(
"\
}
#[cargo_test]
-fn add_dep_dont_update_registry() {
+fn add_dep_dont_update_registry_http() {
+ let _server = setup_http();
+ add_dep_dont_update_registry(cargo_http);
+}
+
+#[cargo_test]
+fn add_dep_dont_update_registry_git() {
+ add_dep_dont_update_registry(cargo_stable);
+}
+
+fn add_dep_dont_update_registry(cargo: fn(&Project, &str) -> Execs) {
let p = project()
.file(
"Cargo.toml",
Package::new("remote", "0.3.4").publish();
- p.cargo("build").run();
+ cargo(&p, "build").run();
p.change_file(
"Cargo.toml",
"#,
);
- p.cargo("build")
+ cargo(&p, "build")
.with_stderr(
"\
[COMPILING] bar v0.5.0 ([..])
}
#[cargo_test]
-fn bump_version_dont_update_registry() {
+fn bump_version_dont_update_registry_http() {
+ let _server = setup_http();
+ bump_version_dont_update_registry(cargo_http);
+}
+
+#[cargo_test]
+fn bump_version_dont_update_registry_git() {
+ bump_version_dont_update_registry(cargo_stable);
+}
+
+fn bump_version_dont_update_registry(cargo: fn(&Project, &str) -> Execs) {
let p = project()
.file(
"Cargo.toml",
Package::new("remote", "0.3.4").publish();
- p.cargo("build").run();
+ cargo(&p, "build").run();
p.change_file(
"Cargo.toml",
"#,
);
- p.cargo("build")
+ cargo(&p, "build")
.with_stderr(
"\
[COMPILING] bar v0.6.0 ([..])
}
#[cargo_test]
-fn toml_lies_but_index_is_truth() {
+fn toml_lies_but_index_is_truth_http() {
+ let _server = setup_http();
+ toml_lies_but_index_is_truth(cargo_http);
+}
+
+#[cargo_test]
+fn toml_lies_but_index_is_truth_git() {
+ toml_lies_but_index_is_truth(cargo_stable);
+}
+
+fn toml_lies_but_index_is_truth(cargo: fn(&Project, &str) -> Execs) {
Package::new("foo", "0.2.0").publish();
Package::new("bar", "0.3.0")
.dep("foo", "0.2.0")
.file("src/main.rs", "fn main() {}")
.build();
- p.cargo("build -v").run();
+ cargo(&p, "build -v").run();
+}
+
+#[cargo_test]
+fn vv_prints_warnings_http() {
+ let _server = setup_http();
+ vv_prints_warnings(cargo_http);
}
#[cargo_test]
-fn vv_prints_warnings() {
+fn vv_prints_warnings_git() {
+ vv_prints_warnings(cargo_stable);
+}
+
+fn vv_prints_warnings(cargo: fn(&Project, &str) -> Execs) {
Package::new("foo", "0.2.0")
.file(
"src/lib.rs",
.file("src/main.rs", "fn main() {}")
.build();
- p.cargo("build -vv").run();
+ cargo(&p, "build -vv").run();
+}
+
+#[cargo_test]
+fn bad_and_or_malicious_packages_rejected_http() {
+ let _server = setup_http();
+ bad_and_or_malicious_packages_rejected(cargo_http);
}
#[cargo_test]
-fn bad_and_or_malicious_packages_rejected() {
+fn bad_and_or_malicious_packages_rejected_git() {
+ bad_and_or_malicious_packages_rejected(cargo_stable);
+}
+
+fn bad_and_or_malicious_packages_rejected(cargo: fn(&Project, &str) -> Execs) {
Package::new("foo", "0.2.0")
.extra_file("foo-0.1.0/src/lib.rs", "")
.publish();
.file("src/main.rs", "fn main() {}")
.build();
- p.cargo("build -vv")
+ cargo(&p, "build -vv")
.with_status(101)
.with_stderr(
"\
}
#[cargo_test]
-fn git_init_templatedir_missing() {
+fn git_init_templatedir_missing_http() {
+ let _server = setup_http();
+ git_init_templatedir_missing(cargo_http);
+}
+
+#[cargo_test]
+fn git_init_templatedir_missing_git() {
+ git_init_templatedir_missing(cargo_stable);
+}
+
+fn git_init_templatedir_missing(cargo: fn(&Project, &str) -> Execs) {
Package::new("foo", "0.2.0").dep("bar", "*").publish();
Package::new("bar", "0.2.0").publish();
.file("src/main.rs", "fn main() {}")
.build();
- p.cargo("build").run();
+ cargo(&p, "build").run();
remove_dir_all(paths::home().join(".cargo/registry")).unwrap();
fs::write(
)
.unwrap();
- p.cargo("build").run();
- p.cargo("build").run();
+ cargo(&p, "build").run();
+ cargo(&p, "build").run();
}
#[cargo_test]
-fn rename_deps_and_features() {
+fn rename_deps_and_features_http() {
+ let _server = setup_http();
+ rename_deps_and_features(cargo_http);
+}
+
+#[cargo_test]
+fn rename_deps_and_features_git() {
+ rename_deps_and_features(cargo_stable);
+}
+
+fn rename_deps_and_features(cargo: fn(&Project, &str) -> Execs) {
Package::new("foo", "0.1.0")
.file("src/lib.rs", "pub fn f1() {}")
.publish();
)
.build();
- p.cargo("build").run();
- p.cargo("build --features bar/foo01").run();
- p.cargo("build --features bar/another").run();
+ cargo(&p, "build").run();
+ cargo(&p, "build --features bar/foo01").run();
+ cargo(&p, "build --features bar/another").run();
+}
+
+#[cargo_test]
+fn ignore_invalid_json_lines_http() {
+ let _server = setup_http();
+ ignore_invalid_json_lines(cargo_http);
}
#[cargo_test]
-fn ignore_invalid_json_lines() {
+fn ignore_invalid_json_lines_git() {
+ ignore_invalid_json_lines(cargo_stable);
+}
+
+fn ignore_invalid_json_lines(cargo: fn(&Project, &str) -> Execs) {
Package::new("foo", "0.1.0").publish();
Package::new("foo", "0.1.1").invalid_json(true).publish();
Package::new("foo", "0.2.0").publish();
.file("src/lib.rs", "")
.build();
- p.cargo("build").run();
+ cargo(&p, "build").run();
+}
+
+#[cargo_test]
+fn readonly_registry_still_works_http() {
+ let _server = setup_http();
+ readonly_registry_still_works(cargo_http);
}
#[cargo_test]
-fn readonly_registry_still_works() {
+fn readonly_registry_still_works_git() {
+ readonly_registry_still_works(cargo_stable);
+}
+
+fn readonly_registry_still_works(cargo: fn(&Project, &str) -> Execs) {
Package::new("foo", "0.1.0").publish();
let p = project()
.file("src/lib.rs", "")
.build();
- p.cargo("generate-lockfile").run();
- p.cargo("fetch --locked").run();
+ cargo(&p, "generate-lockfile").run();
+ cargo(&p, "fetch --locked").run();
chmod_readonly(&paths::home(), true);
- p.cargo("build").run();
+ cargo(&p, "build").run();
// make sure we un-readonly the files afterwards so "cargo clean" can remove them (#6934)
chmod_readonly(&paths::home(), false);
}
#[cargo_test]
-fn registry_index_rejected() {
+fn registry_index_rejected_http() {
+ let _server = setup_http();
+ registry_index_rejected(cargo_http);
+}
+
+#[cargo_test]
+fn registry_index_rejected_git() {
+ registry_index_rejected(cargo_stable);
+}
+
+fn registry_index_rejected(cargo: fn(&Project, &str) -> Execs) {
Package::new("dep", "0.1.0").publish();
let p = project()
.file("src/lib.rs", "")
.build();
- p.cargo("check")
+ cargo(&p, "check")
.with_status(101)
.with_stderr(
"\
)
.run();
- p.cargo("login")
+ cargo(&p, "login")
.with_status(101)
.with_stderr(
"\
}
#[cargo_test]
-fn ignores_unknown_index_version() {
+fn ignores_unknown_index_version_http() {
+ let _server = setup_http();
+ ignores_unknown_index_version(cargo_http);
+}
+
+#[cargo_test]
+fn ignores_unknown_index_version_git() {
+ ignores_unknown_index_version(cargo_stable);
+}
+
+fn ignores_unknown_index_version(cargo: fn(&Project, &str) -> Execs) {
// If the version field is not understood, it is ignored.
Package::new("bar", "1.0.0").publish();
Package::new("bar", "1.0.1").schema_version(9999).publish();
.file("src/lib.rs", "")
.build();
- p.cargo("tree")
+ cargo(&p, "tree")
.with_stdout(
"foo v0.1.0 [..]\n\
└── bar v1.0.0\n\
)
.run();
}
+
+#[cargo_test]
+fn http_requires_z_flag() {
+ let _server = setup_http();
+ let p = project()
+ .file(
+ "Cargo.toml",
+ r#"
+ [project]
+ name = "foo"
+ version = "0.0.1"
+ authors = []
+
+ [dependencies]
+ bar = ">= 0.0.0"
+ "#,
+ )
+ .file("src/main.rs", "fn main() {}")
+ .build();
+
+ p.cargo("build")
+ .with_status(101)
+ .with_stderr_contains(" Usage of HTTP-based registries requires `-Z http-registry`")
+ .run();
+}
paths::home().join(".cargo"),
);
let lock = cfg.acquire_package_cache_lock().unwrap();
- let mut regsrc = RegistrySource::remote(sid, &HashSet::new(), &cfg);
+ let mut regsrc = RegistrySource::remote(sid, &HashSet::new(), &cfg).unwrap();
regsrc.invalidate_cache();
regsrc.block_until_ready().unwrap();
drop(lock);
projects / cargo.git / commitdiff