ceph/src/boost/tools/quickbook/src/syntax_highlight.cpp

   1 /*=============================================================================
   2     Copyright (c) 2002 2004 2006 Joel de Guzman
   3     Copyright (c) 2004 Eric Niebler
   4     http://spirit.sourceforge.net/
   5
   6     Use, modification and distribution is subject to the Boost Software
   7     License, Version 1.0. (See accompanying file LICENSE_1_0.txt or copy at
   8     http://www.boost.org/LICENSE_1_0.txt)
   9 =============================================================================*/
  10 #include <boost/spirit/include/classic_core.hpp>
  11 #include <boost/spirit/include/classic_confix.hpp>
  12 #include <boost/spirit/include/classic_chset.hpp>
  13 #include <boost/spirit/include/classic_symbols.hpp>
  14 #include <boost/spirit/include/classic_loops.hpp>
  15 #include "grammar.hpp"
  16 #include "state.hpp"
  17 #include "actions.hpp"
  18 #include "syntax_highlight.hpp"
  19 #include "utils.hpp"
  20 #include "files.hpp"
  21 #include "stream.hpp"
  22 #include "phrase_tags.hpp"
  23
  24 namespace quickbook
  25 {
  26     namespace cl = boost::spirit::classic;
  27
  28     // Syntax Highlight Actions
  29
  30     struct syntax_highlight_actions
  31     {
  32         quickbook::state& state;
  33         do_macro_action do_macro_impl;
  34
  35         // State
  36         bool support_callouts;
  37         quickbook::string_view marked_text;
  38
  39         syntax_highlight_actions(quickbook::state& state_, bool is_block_) :
  40             state(state_),
  41             do_macro_impl(state_),
  42             support_callouts(is_block_ && (qbk_version_n >= 107u ||
  43                 state.current_file->is_code_snippets)),
  44             marked_text()
  45         {}
  46
  47         void span(parse_iterator, parse_iterator, char const*);
  48         void span_start(parse_iterator, parse_iterator, char const*);
  49         void span_end(parse_iterator, parse_iterator);
  50         void unexpected_char(parse_iterator, parse_iterator);
  51         void plain_char(parse_iterator, parse_iterator);
  52         void pre_escape_back(parse_iterator, parse_iterator);
  53         void post_escape_back(parse_iterator, parse_iterator);
  54         void do_macro(std::string const&);
  55
  56         void mark_text(parse_iterator, parse_iterator);
  57         void callout(parse_iterator, parse_iterator);
  58     };
  59
  60     void syntax_highlight_actions::span(parse_iterator first,
  61             parse_iterator last, char const* name)
  62     {
  63         state.phrase << "<phrase role=\"" << name << "\">";
  64         while (first != last)
  65             detail::print_char(*first++, state.phrase.get());
  66         state.phrase << "</phrase>";
  67     }
  68
  69     void syntax_highlight_actions::span_start(parse_iterator first,
  70             parse_iterator last, char const* name)
  71     {
  72         state.phrase << "<phrase role=\"" << name << "\">";
  73         while (first != last)
  74             detail::print_char(*first++, state.phrase.get());
  75     }
  76
  77     void syntax_highlight_actions::span_end(parse_iterator first,
  78             parse_iterator last)
  79     {
  80         while (first != last)
  81             detail::print_char(*first++, state.phrase.get());
  82         state.phrase << "</phrase>";
  83     }
  84
  85     void syntax_highlight_actions::unexpected_char(parse_iterator first,
  86             parse_iterator last)
  87     {
  88         file_position const pos = state.current_file->position_of(first.base());
  89
  90         detail::outwarn(state.current_file->path, pos.line)
  91             << "in column:" << pos.column
  92             << ", unexpected character: " << std::string(first.base(), last.base())
  93             << "\n";
  94
  95         // print out an unexpected character
  96         state.phrase << "<phrase role=\"error\">";
  97         while (first != last)
  98             detail::print_char(*first++, state.phrase.get());
  99         state.phrase << "</phrase>";
 100     }
 101
 102     void syntax_highlight_actions::plain_char(parse_iterator first,
 103             parse_iterator last)
 104     {
 105         while (first != last)
 106             detail::print_char(*first++, state.phrase.get());
 107     }
 108
 109     void syntax_highlight_actions::pre_escape_back(parse_iterator,
 110             parse_iterator)
 111     {
 112         state.push_output(); // save the stream
 113     }
 114
 115     void syntax_highlight_actions::post_escape_back(parse_iterator,
 116             parse_iterator)
 117     {
 118         std::string tmp;
 119         state.phrase.swap(tmp);
 120         state.pop_output(); // restore the stream
 121         state.phrase << tmp;
 122     }
 123
 124     void syntax_highlight_actions::do_macro(std::string const& v)
 125     {
 126         do_macro_impl(v);
 127     }
 128
 129     void syntax_highlight_actions::mark_text(parse_iterator first,
 130             parse_iterator last)
 131     {
 132         marked_text = quickbook::string_view(first.base(), last.base() - first.base());
 133     }
 134
 135     void syntax_highlight_actions::callout(parse_iterator, parse_iterator)
 136     {
 137         state.phrase << state.add_callout(qbk_value(state.current_file,
 138             marked_text.begin(), marked_text.end()));
 139         marked_text.clear();
 140     }
 141
 142     // Syntax
 143
 144     struct keywords_holder
 145     {
 146         cl::symbols<> cpp, python;
 147
 148         keywords_holder()
 149         {
 150             cpp
 151                     =   "alignas", "alignof", "and_eq", "and", "asm", "auto",
 152                         "bitand", "bitor", "bool", "break", "case", "catch",
 153                         "char", "char16_t", "char32_t", "class", "compl",
 154                         "const", "const_cast", "constexpr", "continue",
 155                         "decltype", "default", "delete", "do", "double",
 156                         "dynamic_cast",  "else", "enum", "explicit", "export",
 157                         "extern", "false", "float", "for", "friend", "goto",
 158                         "if", "inline", "int", "long", "mutable", "namespace",
 159                         "new", "noexcept", "not_eq", "not", "nullptr",
 160                         "operator", "or_eq", "or", "private", "protected",
 161                         "public", "register", "reinterpret_cast", "return",
 162                         "short", "signed", "sizeof", "static", "static_assert",
 163                         "static_cast", "struct", "switch", "template", "this",
 164                         "thread_local", "throw", "true", "try", "typedef",
 165                         "typeid", "typename", "union", "unsigned", "using",
 166                         "virtual", "void", "volatile", "wchar_t", "while",
 167                         "xor_eq", "xor"
 168                     ;
 169
 170             python
 171                     =
 172                     "and",       "del",       "for",       "is",        "raise",
 173                     "assert",    "elif",      "from",      "lambda",    "return",
 174                     "break",     "else",      "global",    "not",       "try",
 175                     "class",     "except",    "if",        "or",        "while",
 176                     "continue",  "exec",      "import",    "pass",      "yield",
 177                     "def",       "finally",   "in",        "print",
 178
 179                     // Technically "as" and "None" are not yet keywords (at Python
 180                     // 2.4). They are destined to become keywords, and we treat them
 181                     // as such for syntax highlighting purposes.
 182
 183                     "as", "None"
 184                     ;
 185         }
 186     };
 187
 188     namespace {
 189         keywords_holder keywords;
 190     }
 191
 192     // Grammar for C++ highlighting
 193     struct cpp_highlight : public cl::grammar<cpp_highlight>
 194     {
 195         explicit cpp_highlight(syntax_highlight_actions& actions_)
 196             : actions(actions_) {}
 197
 198         template <typename Scanner>
 199         struct definition
 200         {
 201             definition(cpp_highlight const& self)
 202                 : g(self.actions.state.grammar())
 203             {
 204                 member_action1<syntax_highlight_actions, char const*>
 205                     span(self.actions, &syntax_highlight_actions::span),
 206                     span_start(self.actions, &syntax_highlight_actions::span_start);
 207                 member_action<syntax_highlight_actions>
 208                     span_end(self.actions, &syntax_highlight_actions::span_end),
 209                     unexpected_char(self.actions, &syntax_highlight_actions::unexpected_char),
 210                     plain_char(self.actions, &syntax_highlight_actions::plain_char),
 211                     pre_escape_back(self.actions, &syntax_highlight_actions::pre_escape_back),
 212                     post_escape_back(self.actions, &syntax_highlight_actions::post_escape_back),
 213                     mark_text(self.actions, &syntax_highlight_actions::mark_text),
 214                     callout(self.actions, &syntax_highlight_actions::callout);
 215                 member_action_value<syntax_highlight_actions, std::string const&>
 216                     do_macro(self.actions, &syntax_highlight_actions::do_macro);
 217                 error_action error(self.actions.state);
 218
 219                 program =
 220                     *(  (*cl::space_p)                  [plain_char]
 221                     >>  (line_start | rest_of_line)
 222                     >>  *rest_of_line
 223                     )
 224                     ;
 225
 226                 line_start =
 227                         preprocessor                    [span("preprocessor")]
 228                     ;
 229
 230                 rest_of_line =
 231                         (+cl::blank_p)                  [plain_char]
 232                     |   macro
 233                     |   escape
 234                     |   cl::eps_p(ph::var(self.actions.support_callouts))
 235                     >>  (   line_callout                [callout]
 236                         |   inline_callout              [callout]
 237                         )
 238                     |   comment
 239                     |   keyword                         [span("keyword")]
 240                     |   identifier                      [span("identifier")]
 241                     |   special                         [span("special")]
 242                     |   string_                         [span("string")]
 243                     |   char_                           [span("char")]
 244                     |   number                          [span("number")]
 245                     |   ~cl::eps_p(cl::eol_p)
 246                     >>  u8_codepoint_p                  [unexpected_char]
 247                     ;
 248
 249                 macro =
 250                     // must not be followed by alpha or underscore
 251                     cl::eps_p(self.actions.state.macro
 252                         >> (cl::eps_p - (cl::alpha_p | '_')))
 253                     >> self.actions.state.macro
 254                                                         [do_macro]
 255                     ;
 256
 257                 escape =
 258                     cl::str_p("``")                     [pre_escape_back]
 259                     >>
 260                     (
 261                         (
 262                             (
 263                                 (+(cl::anychar_p - "``") >> cl::eps_p("``"))
 264                                 & g.phrase_start
 265                             )
 266                             >>  cl::str_p("``")
 267                         )
 268                         |
 269                         (
 270                             cl::eps_p                   [error]
 271                             >> *cl::anychar_p
 272                         )
 273                     )                                   [post_escape_back]
 274                     ;
 275
 276                 preprocessor
 277                     =   '#' >> *cl::space_p >> ((cl::alpha_p | '_') >> *(cl::alnum_p | '_'))
 278                     ;
 279
 280                 inline_callout
 281                     =   cl::confix_p(
 282                             "/*<" >> *cl::space_p,
 283                             (*cl::anychar_p)            [mark_text],
 284                             ">*/"
 285                         )
 286                         ;
 287
 288                 line_callout
 289                     =   cl::confix_p(
 290                             "/*<<" >> *cl::space_p,
 291                             (*cl::anychar_p)            [mark_text],
 292                             ">>*/"
 293                         )
 294                     >>  *cl::space_p
 295                     ;
 296
 297                 comment
 298                     =   cl::str_p("//")                 [span_start("comment")]
 299                     >>  *(  escape
 300                         |   (+(cl::anychar_p - (cl::eol_p | "``")))
 301                                                         [plain_char]
 302                         )
 303                     >>  cl::eps_p                       [span_end]
 304                     |   cl::str_p("/*")                 [span_start("comment")]
 305                     >>  *(  escape
 306                         |   (+(cl::anychar_p - (cl::str_p("*/") | "``")))
 307                                                         [plain_char]
 308                         )
 309                     >>  (!cl::str_p("*/"))              [span_end]
 310                     ;
 311
 312                 keyword
 313                     =   keywords.cpp >> (cl::eps_p - (cl::alnum_p | '_'))
 314                     ;   // make sure we recognize whole words only
 315
 316                 special
 317                     =   +cl::chset_p("~!%^&*()+={[}]:;,<.>?/|\\#-")
 318                     ;
 319
 320                 string_char = ('\\' >> u8_codepoint_p) | (cl::anychar_p - '\\');
 321
 322                 string_
 323                     =   !cl::as_lower_d['l'] >> cl::confix_p('"', *string_char, '"')
 324                     ;
 325
 326                 char_
 327                     =   !cl::as_lower_d['l'] >> cl::confix_p('\'', *string_char, '\'')
 328                     ;
 329
 330                 number
 331                     =   (
 332                             cl::as_lower_d["0x"] >> cl::hex_p
 333                         |   '0' >> cl::oct_p
 334                         |   cl::real_p
 335                         )
 336                         >>  *cl::as_lower_d[cl::chset_p("ldfu")]
 337                     ;
 338
 339                 identifier
 340                     =   (cl::alpha_p | '_') >> *(cl::alnum_p | '_')
 341                     ;
 342             }
 343
 344             cl::rule<Scanner>
 345                             program, line_start, rest_of_line, macro, preprocessor,
 346                             inline_callout, line_callout, comment,
 347                             special, string_,
 348                             char_, number, identifier, keyword, escape,
 349                             string_char;
 350
 351             quickbook_grammar& g;
 352
 353             cl::rule<Scanner> const&
 354             start() const { return program; }
 355         };
 356
 357         syntax_highlight_actions& actions;
 358     };
 359
 360     // Grammar for Python highlighting
 361     // See also: The Python Reference Manual
 362     // http://docs.python.org/ref/ref.html
 363     struct python_highlight : public cl::grammar<python_highlight>
 364     {
 365         explicit python_highlight(syntax_highlight_actions& actions_)
 366             : actions(actions_) {}
 367
 368         template <typename Scanner>
 369         struct definition
 370         {
 371             definition(python_highlight const& self)
 372                 : g(self.actions.state.grammar())
 373             {
 374                 member_action1<syntax_highlight_actions, char const*>
 375                     span(self.actions, &syntax_highlight_actions::span),
 376                     span_start(self.actions, &syntax_highlight_actions::span_start);
 377                 member_action<syntax_highlight_actions>
 378                     span_end(self.actions, &syntax_highlight_actions::span_end),
 379                     unexpected_char(self.actions, &syntax_highlight_actions::unexpected_char),
 380                     plain_char(self.actions, &syntax_highlight_actions::plain_char),
 381                     pre_escape_back(self.actions, &syntax_highlight_actions::pre_escape_back),
 382                     post_escape_back(self.actions, &syntax_highlight_actions::post_escape_back),
 383                     mark_text(self.actions, &syntax_highlight_actions::mark_text),
 384                     callout(self.actions, &syntax_highlight_actions::callout);
 385                 member_action_value<syntax_highlight_actions, std::string const&>
 386                     do_macro(self.actions, &syntax_highlight_actions::do_macro);
 387                 error_action error(self.actions.state);
 388
 389                 program
 390                     =
 391                     *(  (+cl::space_p)                  [plain_char]
 392                     |   macro
 393                     |   escape
 394                     |   cl::eps_p(ph::var(self.actions.support_callouts))
 395                     >>  (   line_callout                [callout]
 396                         |   inline_callout              [callout]
 397                         )
 398                     |   comment
 399                     |   keyword                         [span("keyword")]
 400                     |   identifier                      [span("identifier")]
 401                     |   special                         [span("special")]
 402                     |   string_                         [span("string")]
 403                     |   number                          [span("number")]
 404                     |   u8_codepoint_p                  [unexpected_char]
 405                     )
 406                     ;
 407
 408                 macro =
 409                     // must not be followed by alpha or underscore
 410                     cl::eps_p(self.actions.state.macro
 411                         >> (cl::eps_p - (cl::alpha_p | '_')))
 412                     >> self.actions.state.macro
 413                                                         [do_macro]
 414                     ;
 415
 416                 escape =
 417                     cl::str_p("``")                     [pre_escape_back]
 418                     >>
 419                     (
 420                         (
 421                             (
 422                                 (+(cl::anychar_p - "``") >> cl::eps_p("``"))
 423                                 & g.phrase_start
 424                             )
 425                             >>  cl::str_p("``")
 426                         )
 427                         |
 428                         (
 429                             cl::eps_p                   [error]
 430                             >> *cl::anychar_p
 431                         )
 432                     )                                   [post_escape_back]
 433                     ;
 434
 435                 inline_callout
 436                     =   "#<" >> *cl::space_p >>
 437                         (*(cl::anychar_p - cl::eol_p))  [mark_text]
 438                     ;
 439
 440                 line_callout
 441                     =   cl::confix_p(
 442                             "#<<" >> *cl::space_p,
 443                             (*cl::anychar_p)            [mark_text],
 444                             (cl::eol_p | cl::end_p)
 445                         )
 446                     ;
 447
 448                 comment
 449                     =   cl::str_p("#")                  [span_start("comment")]
 450                     >>  *(  escape
 451                         |   (+(cl::anychar_p - (cl::eol_p | "``")))
 452                                                         [plain_char]
 453                         )
 454                     >>  cl::eps_p                       [span_end]
 455                     ;
 456
 457                 keyword
 458                     =   keywords.python >> (cl::eps_p - (cl::alnum_p | '_'))
 459                     ;   // make sure we recognize whole words only
 460
 461                 special
 462                     =   +cl::chset_p("~!%^&*()+={[}]:;,<.>/|\\-")
 463                     ;
 464
 465                 string_prefix
 466                     =    cl::as_lower_d[cl::str_p("u") >> ! cl::str_p("r")]
 467                     ;
 468
 469                 string_
 470                     =   ! string_prefix >> (long_string | short_string)
 471                     ;
 472
 473                 string_char = ('\\' >> u8_codepoint_p) | (cl::anychar_p - '\\');
 474
 475                 short_string
 476                     =   cl::confix_p('\'', * string_char, '\'') |
 477                         cl::confix_p('"', * string_char, '"')
 478                     ;
 479
 480                 long_string
 481                     // Note: the "cl::str_p" on the next two lines work around
 482                     // an INTERNAL COMPILER ERROR when using VC7.1
 483                     =   cl::confix_p(cl::str_p("'''"), * string_char, "'''") |
 484                         cl::confix_p(cl::str_p("\"\"\""), * string_char, "\"\"\"")
 485                     ;
 486
 487                 number
 488                     =   (
 489                             cl::as_lower_d["0x"] >> cl::hex_p
 490                         |   '0' >> cl::oct_p
 491                         |   cl::real_p
 492                         )
 493                         >>  *cl::as_lower_d[cl::chset_p("lj")]
 494                     ;
 495
 496                 identifier
 497                     =   (cl::alpha_p | '_') >> *(cl::alnum_p | '_')
 498                     ;
 499             }
 500
 501             cl::rule<Scanner>
 502                             program, macro, inline_callout, line_callout,
 503                             comment, special, string_, string_prefix,
 504                             short_string, long_string, number, identifier, keyword,
 505                             escape, string_char;
 506
 507             quickbook_grammar& g;
 508
 509             cl::rule<Scanner> const&
 510             start() const { return program; }
 511         };
 512
 513         syntax_highlight_actions& actions;
 514     };
 515
 516     // Grammar for plain text (no actual highlighting)
 517     struct teletype_highlight : public cl::grammar<teletype_highlight>
 518     {
 519         teletype_highlight(syntax_highlight_actions& actions_)
 520             : actions(actions_) {}
 521
 522         template <typename Scanner>
 523         struct definition
 524         {
 525             definition(teletype_highlight const& self)
 526                 : g(self.actions.state.grammar())
 527             {
 528                 member_action<syntax_highlight_actions>
 529                     plain_char(self.actions, &syntax_highlight_actions::plain_char),
 530                     pre_escape_back(self.actions, &syntax_highlight_actions::pre_escape_back),
 531                     post_escape_back(self.actions, &syntax_highlight_actions::post_escape_back);
 532                 member_action_value<syntax_highlight_actions, std::string const&>
 533                     do_macro(self.actions, &syntax_highlight_actions::do_macro);
 534                 error_action error(self.actions.state);
 535
 536                 program
 537                     =
 538                     *(  macro
 539                     |   escape
 540                     |   u8_codepoint_p                  [plain_char]
 541                     )
 542                     ;
 543
 544                 macro =
 545                     // must not be followed by alpha or underscore
 546                     cl::eps_p(self.actions.state.macro
 547                         >> (cl::eps_p - (cl::alpha_p | '_')))
 548                     >> self.actions.state.macro
 549                                                         [do_macro]
 550                     ;
 551
 552                 escape =
 553                     cl::str_p("``")                     [pre_escape_back]
 554                     >>
 555                     (
 556                         (
 557                             (
 558                                 (+(cl::anychar_p - "``") >> cl::eps_p("``"))
 559                                 & g.phrase_start
 560                             )
 561                             >>  cl::str_p("``")
 562                         )
 563                         |
 564                         (
 565                             cl::eps_p                   [error]
 566                             >> *cl::anychar_p
 567                         )
 568                     )                                   [post_escape_back]
 569                     ;
 570             }
 571
 572             cl::rule<Scanner> program, macro, escape;
 573
 574             quickbook_grammar& g;
 575
 576             cl::rule<Scanner> const&
 577             start() const { return program; }
 578         };
 579
 580         syntax_highlight_actions& actions;
 581     };
 582
 583     void syntax_highlight(
 584         parse_iterator first,
 585         parse_iterator last,
 586         quickbook::state& state,
 587         source_mode_type source_mode,
 588         bool is_block)
 589     {
 590         syntax_highlight_actions syn_actions(state, is_block);
 591
 592         // print the code with syntax coloring
 593         switch(source_mode)
 594         {
 595             case source_mode_tags::cpp: {
 596                 cpp_highlight cpp_p(syn_actions);
 597                 boost::spirit::classic::parse(first, last, cpp_p);
 598                 break;
 599             }
 600             case source_mode_tags::python: {
 601                 python_highlight python_p(syn_actions);
 602                 boost::spirit::classic::parse(first, last, python_p);
 603                 break;
 604             }
 605             case source_mode_tags::teletype: {
 606                 teletype_highlight teletype_p(syn_actions);
 607                 boost::spirit::classic::parse(first, last, teletype_p);
 608                 break;
 609             }
 610             default:
 611                 BOOST_ASSERT(0);
 612         }
 613     }
 614 }