ceph/src/Beast/examples/http_crawl.cpp

   1 //
   2 // Copyright (c) 2013-2017 Vinnie Falco (vinnie dot falco at gmail dot com)
   3 //
   4 // Distributed under the Boost Software License, Version 1.0. (See accompanying
   5 // file LICENSE_1_0.txt or copy at http://www.boost.org/LICENSE_1_0.txt)
   6 //
   7
   8 #include "urls_large_data.hpp"
   9
  10 #include <beast/core/streambuf.hpp>
  11 #include <beast/http.hpp>
  12 #include <boost/asio.hpp>
  13 #include <boost/lexical_cast.hpp>
  14 #include <iostream>
  15
  16 using namespace beast::http;
  17 using namespace boost::asio;
  18
  19 template<class String>
  20 void
  21 err(beast::error_code const& ec, String const& what)
  22 {
  23     std::cerr << what << ": " << ec.message() << std::endl;
  24 }
  25
  26 int main(int, char const*[])
  27 {
  28     io_service ios;
  29     for(auto const& host : urls_large_data())
  30     {
  31         try
  32         {
  33             ip::tcp::resolver r(ios);
  34             auto it = r.resolve(
  35                 ip::tcp::resolver::query{host, "http"});
  36             ip::tcp::socket sock(ios);
  37             connect(sock, it);
  38             auto ep = sock.remote_endpoint();
  39             request<string_body> req;
  40             req.method = "GET";
  41             req.url = "/";
  42             req.version = 11;
  43             req.fields.insert("Host", host + std::string(":") +
  44                 boost::lexical_cast<std::string>(ep.port()));
  45             req.fields.insert("User-Agent", "beast/http");
  46             prepare(req);
  47             write(sock, req);
  48             response<string_body> res;
  49             streambuf sb;
  50             beast::http::read(sock, sb, res);
  51             std::cout << res;
  52         }
  53         catch(beast::system_error const& ec)
  54         {
  55             std::cerr << host << ": " << ec.what();
  56         }
  57         catch(...)
  58         {
  59             std::cerr << host << ": unknown exception" << std::endl;
  60         }
  61     }
  62 }