2 * Licensed to the Apache Software Foundation (ASF) under one
3 * or more contributor license agreements. See the NOTICE file
4 * distributed with this work for additional information
5 * regarding copyright ownership. The ASF licenses this file
6 * to you under the Apache License, Version 2.0 (the
7 * "License"); you may not use this file except in compliance
8 * with the License. You may obtain a copy of the License at
10 * http://www.apache.org/licenses/LICENSE-2.0
12 * Unless required by applicable law or agreed to in writing,
13 * software distributed under the License is distributed on an
14 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15 * KIND, either express or implied. See the License for the
16 * specific language governing permissions and limitations
19 module thrift.protocol.json;
26 import std.string : format;
27 import std.traits : isIntegral;
28 import std.typetuple : allSatisfy, TypeTuple;
29 import std.utf : toUTF8;
30 import thrift.protocol.base;
31 import thrift.transport.base;
33 alias Base64Impl!('+', '/', Base64.NoPadding) Base64NoPad;
36 * Implementation of the Thrift JSON protocol.
38 final class TJsonProtocol(Transport = TTransport) if (
39 isTTransport!Transport
42 * Constructs a new instance.
45 * trans = The transport to use.
46 * containerSizeLimit = If positive, the container size is limited to the
47 * given number of items.
48 * stringSizeLimit = If positive, the string length is limited to the
49 * given number of bytes.
51 this(Transport trans, int containerSizeLimit = 0, int stringSizeLimit = 0) {
53 this.containerSizeLimit = containerSizeLimit;
54 this.stringSizeLimit = stringSizeLimit;
56 context_ = new Context();
57 reader_ = new LookaheadReader(trans);
60 Transport transport() @property {
65 destroy(contextStack_);
66 context_ = new Context();
67 reader_ = new LookaheadReader(trans_);
71 * If positive, limits the number of items of deserialized containers to the
74 * This is useful to avoid allocating excessive amounts of memory when broken
75 * data is received. If the limit is exceeded, a SIZE_LIMIT-type
76 * TProtocolException is thrown.
78 * Defaults to zero (no limit).
80 int containerSizeLimit;
83 * If positive, limits the length of deserialized strings/binary data to the
84 * given number of bytes.
86 * This is useful to avoid allocating excessive amounts of memory when broken
87 * data is received. If the limit is exceeded, a SIZE_LIMIT-type
88 * TProtocolException is thrown.
90 * Note: For binary data, the limit applies to the length of the
91 * Base64-encoded string data, not the resulting byte array.
93 * Defaults to zero (no limit).
101 void writeBool(bool b) {
102 writeJsonInteger(b ? 1 : 0);
105 void writeByte(byte b) {
109 void writeI16(short i16) {
110 writeJsonInteger(i16);
113 void writeI32(int i32) {
114 writeJsonInteger(i32);
117 void writeI64(long i64) {
118 writeJsonInteger(i64);
121 void writeDouble(double dub) {
122 context_.write(trans_);
125 if (dub is double.nan) {
127 } else if (dub is double.infinity) {
128 value = INFINITY_STRING;
129 } else if (dub is -double.infinity) {
130 value = NEG_INFINITY_STRING;
133 bool escapeNum = value !is null || context_.escapeNum;
136 /* precision is 17 */
137 value = format("%.17g", dub);
140 if (escapeNum) trans_.write(STRING_DELIMITER);
141 trans_.write(cast(ubyte[])value);
142 if (escapeNum) trans_.write(STRING_DELIMITER);
145 void writeString(string str) {
146 context_.write(trans_);
147 trans_.write(STRING_DELIMITER);
151 trans_.write(STRING_DELIMITER);
154 void writeBinary(ubyte[] buf) {
155 context_.write(trans_);
157 trans_.write(STRING_DELIMITER);
160 auto toWrite = take(buf, 3);
161 Base64NoPad.encode(toWrite, b[]);
162 trans_.write(b[0 .. toWrite.length + 1]);
163 buf.popFrontN(toWrite.length);
165 trans_.write(STRING_DELIMITER);
168 void writeMessageBegin(TMessage msg) {
169 writeJsonArrayBegin();
170 writeJsonInteger(THRIFT_JSON_VERSION);
171 writeString(msg.name);
172 writeJsonInteger(cast(byte)msg.type);
173 writeJsonInteger(msg.seqid);
176 void writeMessageEnd() {
180 void writeStructBegin(TStruct tstruct) {
181 writeJsonObjectBegin();
184 void writeStructEnd() {
185 writeJsonObjectEnd();
188 void writeFieldBegin(TField field) {
189 writeJsonInteger(field.id);
190 writeJsonObjectBegin();
191 writeString(getNameFromTType(field.type));
194 void writeFieldEnd() {
195 writeJsonObjectEnd();
198 void writeFieldStop() {}
200 void writeListBegin(TList list) {
201 writeJsonArrayBegin();
202 writeString(getNameFromTType(list.elemType));
203 writeJsonInteger(list.size);
206 void writeListEnd() {
210 void writeMapBegin(TMap map) {
211 writeJsonArrayBegin();
212 writeString(getNameFromTType(map.keyType));
213 writeString(getNameFromTType(map.valueType));
214 writeJsonInteger(map.size);
215 writeJsonObjectBegin();
219 writeJsonObjectEnd();
223 void writeSetBegin(TSet set) {
224 writeJsonArrayBegin();
225 writeString(getNameFromTType(set.elemType));
226 writeJsonInteger(set.size);
239 return readJsonInteger!byte() ? true : false;
243 return readJsonInteger!byte();
247 return readJsonInteger!short();
251 return readJsonInteger!int();
255 return readJsonInteger!long();
258 double readDouble() {
259 context_.read(reader_);
261 if (reader_.peek() == STRING_DELIMITER) {
262 auto str = readJsonString(true);
263 if (str == NAN_STRING) {
266 if (str == INFINITY_STRING) {
267 return double.infinity;
269 if (str == NEG_INFINITY_STRING) {
270 return -double.infinity;
273 if (!context_.escapeNum) {
274 // Throw exception -- we should not be in a string in this case
275 throw new TProtocolException("Numeric data unexpectedly quoted",
276 TProtocolException.Type.INVALID_DATA);
279 return to!double(str);
280 } catch (ConvException e) {
281 throw new TProtocolException(`Expected numeric value; got "` ~ str ~
282 `".`, TProtocolException.Type.INVALID_DATA);
286 if (context_.escapeNum) {
287 // This will throw - we should have had a quote if escapeNum == true
288 readJsonSyntaxChar(STRING_DELIMITER);
291 auto str = readJsonNumericChars();
293 return to!double(str);
294 } catch (ConvException e) {
295 throw new TProtocolException(`Expected numeric value; got "` ~ str ~
296 `".`, TProtocolException.Type.INVALID_DATA);
301 string readString() {
302 return readJsonString(false);
305 ubyte[] readBinary() {
306 return Base64NoPad.decode(readString());
309 TMessage readMessageBegin() {
312 readJsonArrayBegin();
314 auto ver = readJsonInteger!short();
315 if (ver != THRIFT_JSON_VERSION) {
316 throw new TProtocolException("Message contained bad version.",
317 TProtocolException.Type.BAD_VERSION);
320 msg.name = readString();
321 msg.type = cast(TMessageType)readJsonInteger!byte();
322 msg.seqid = readJsonInteger!short();
327 void readMessageEnd() {
331 TStruct readStructBegin() {
332 readJsonObjectBegin();
336 void readStructEnd() {
340 TField readFieldBegin() {
344 auto ch = reader_.peek();
345 if (ch == OBJECT_END) {
348 f.id = readJsonInteger!short();
349 readJsonObjectBegin();
350 f.type = getTTypeFromName(readString());
356 void readFieldEnd() {
360 TList readListBegin() {
361 readJsonArrayBegin();
362 auto type = getTTypeFromName(readString());
363 auto size = readContainerSize();
364 return TList(type, size);
371 TMap readMapBegin() {
372 readJsonArrayBegin();
373 auto keyType = getTTypeFromName(readString());
374 auto valueType = getTTypeFromName(readString());
375 auto size = readContainerSize();
376 readJsonObjectBegin();
377 return TMap(keyType, valueType, size);
385 TSet readSetBegin() {
386 readJsonArrayBegin();
387 auto type = getTTypeFromName(readString());
388 auto size = readContainerSize();
389 return TSet(type, size);
397 void pushContext(Context c) {
398 contextStack_ ~= context_;
403 context_ = contextStack_.back;
404 contextStack_.popBack();
405 contextStack_.assumeSafeAppend();
412 // Write the character ch as a Json escape sequence ("\u00xx")
413 void writeJsonEscapeChar(ubyte ch) {
414 trans_.write(ESCAPE_PREFIX);
415 trans_.write(ESCAPE_PREFIX);
416 auto outCh = hexChar(cast(ubyte)(ch >> 4));
417 trans_.write((&outCh)[0 .. 1]);
419 trans_.write((&outCh)[0 .. 1]);
422 // Write the character ch as part of a Json string, escaping as appropriate.
423 void writeJsonChar(ubyte ch) {
425 if (ch == '\\') { // Only special character >= 0x30 is '\'
426 trans_.write(BACKSLASH);
427 trans_.write(BACKSLASH);
429 trans_.write((&ch)[0 .. 1]);
433 auto outCh = kJsonCharTable[ch];
434 // Check if regular character, backslash escaped, or Json escaped
436 trans_.write((&ch)[0 .. 1]);
437 } else if (outCh > 1) {
438 trans_.write(BACKSLASH);
439 trans_.write((&outCh)[0 .. 1]);
441 writeJsonEscapeChar(ch);
446 // Convert the given integer type to a Json number, or a string
447 // if the context requires it (eg: key in a map pair).
448 void writeJsonInteger(T)(T num) if (isIntegral!T) {
449 context_.write(trans_);
451 auto escapeNum = context_.escapeNum();
452 if (escapeNum) trans_.write(STRING_DELIMITER);
453 trans_.write(cast(ubyte[])to!string(num));
454 if (escapeNum) trans_.write(STRING_DELIMITER);
457 void writeJsonObjectBegin() {
458 context_.write(trans_);
459 trans_.write(OBJECT_BEGIN);
460 pushContext(new PairContext());
463 void writeJsonObjectEnd() {
465 trans_.write(OBJECT_END);
468 void writeJsonArrayBegin() {
469 context_.write(trans_);
470 trans_.write(ARRAY_BEGIN);
471 pushContext(new ListContext());
474 void writeJsonArrayEnd() {
476 trans_.write(ARRAY_END);
483 int readContainerSize() {
484 auto size = readJsonInteger!int();
486 throw new TProtocolException(TProtocolException.Type.NEGATIVE_SIZE);
487 } else if (containerSizeLimit > 0 && size > containerSizeLimit) {
488 throw new TProtocolException(TProtocolException.Type.SIZE_LIMIT);
493 void readJsonSyntaxChar(ubyte[1] ch) {
494 return readSyntaxChar(reader_, ch);
497 wchar readJsonEscapeChar() {
498 auto a = reader_.read();
499 auto b = reader_.read();
500 auto c = reader_.read();
501 auto d = reader_.read();
503 (hexVal(a[0]) << 12) + (hexVal(b[0]) << 8) +
504 (hexVal(c[0]) << 4) + hexVal(d[0])
508 string readJsonString(bool skipContext = false) {
509 if (!skipContext) context_.read(reader_);
511 readJsonSyntaxChar(STRING_DELIMITER);
512 auto buffer = appender!string();
517 auto ch = reader_.read();
518 if (ch == STRING_DELIMITER) {
523 if (stringSizeLimit > 0 && bytesRead > stringSizeLimit) {
524 throw new TProtocolException(TProtocolException.Type.SIZE_LIMIT);
527 if (ch == BACKSLASH) {
529 if (ch == ESCAPE_CHAR) {
530 auto wch = readJsonEscapeChar();
531 if (wch >= 0xD800 && wch <= 0xDBFF) {
533 } else if (wch >= 0xDC00 && wch <= 0xDFFF && wchs.length == 0) {
534 throw new TProtocolException("Missing UTF-16 high surrogate.",
535 TProtocolException.Type.INVALID_DATA);
538 buffer.put(wchs.toUTF8);
543 auto pos = countUntil(kEscapeChars[], ch[0]);
545 throw new TProtocolException("Expected control char, got '" ~
546 cast(char)ch[0] ~ "'.", TProtocolException.Type.INVALID_DATA);
548 ch = kEscapeCharVals[pos];
551 if (wchs.length != 0) {
552 throw new TProtocolException("Missing UTF-16 low surrogate.",
553 TProtocolException.Type.INVALID_DATA);
558 if (wchs.length != 0) {
559 throw new TProtocolException("Missing UTF-16 low surrogate.",
560 TProtocolException.Type.INVALID_DATA);
565 // Reads a sequence of characters, stopping at the first one that is not
566 // a valid Json numeric character.
567 string readJsonNumericChars() {
570 auto ch = reader_.peek();
571 if (!isJsonNumeric(ch[0])) {
580 // Reads a sequence of characters and assembles them into a number,
581 // returning them via num
582 T readJsonInteger(T)() if (isIntegral!T) {
583 context_.read(reader_);
584 if (context_.escapeNum()) {
585 readJsonSyntaxChar(STRING_DELIMITER);
587 auto str = readJsonNumericChars();
591 } catch (ConvException e) {
592 throw new TProtocolException(`Expected numeric value, got "` ~ str ~ `".`,
593 TProtocolException.Type.INVALID_DATA);
595 if (context_.escapeNum()) {
596 readJsonSyntaxChar(STRING_DELIMITER);
601 void readJsonObjectBegin() {
602 context_.read(reader_);
603 readJsonSyntaxChar(OBJECT_BEGIN);
604 pushContext(new PairContext());
607 void readJsonObjectEnd() {
608 readJsonSyntaxChar(OBJECT_END);
612 void readJsonArrayBegin() {
613 context_.read(reader_);
614 readJsonSyntaxChar(ARRAY_BEGIN);
615 pushContext(new ListContext());
618 void readJsonArrayEnd() {
619 readJsonSyntaxChar(ARRAY_END);
624 final class LookaheadReader {
625 this(Transport trans) {
633 trans_.readAll(data_);
640 trans_.readAll(data_);
653 * Class to serve as base Json context and as base class for other context
658 * Write context data to the transport. Default is to do nothing.
660 void write(Transport trans) {}
663 * Read context data from the transport. Default is to do nothing.
665 void read(LookaheadReader reader) {}
668 * Return true if numbers need to be escaped as strings in this context.
669 * Default behavior is to return false.
671 bool escapeNum() @property {
676 // Context class for object member key-value pairs
677 class PairContext : Context {
683 override void write(Transport trans) {
688 trans.write(colon_ ? PAIR_SEP : ELEM_SEP);
693 override void read(LookaheadReader reader) {
698 auto ch = (colon_ ? PAIR_SEP : ELEM_SEP);
700 return readSyntaxChar(reader, ch);
704 // Numbers must be turned into strings if they are the key part of a pair
705 override bool escapeNum() @property {
714 class ListContext : Context {
719 override void write(Transport trans) {
723 trans.write(ELEM_SEP);
727 override void read(LookaheadReader reader) {
731 readSyntaxChar(reader, ELEM_SEP);
739 // Read 1 character from the transport trans and verify that it is the
740 // expected character ch.
741 // Throw a protocol exception if it is not.
742 void readSyntaxChar(LookaheadReader reader, ubyte[1] ch) {
743 auto ch2 = reader.read();
745 throw new TProtocolException("Expected '" ~ cast(char)ch[0] ~ "', got '" ~
746 cast(char)ch2[0] ~ "'.", TProtocolException.Type.INVALID_DATA);
751 // Probably need to implement a better stack at some point.
752 Context[] contextStack_;
756 LookaheadReader reader_;
760 * TJsonProtocol construction helper to avoid having to explicitly specify
761 * the transport type, i.e. to allow the constructor being called using IFTI
762 * (see $(LINK2 http://d.puremagic.com/issues/show_bug.cgi?id=6082, D Bugzilla
763 * enhancement requet 6082)).
765 TJsonProtocol!Transport tJsonProtocol(Transport)(Transport trans,
766 int containerSizeLimit = 0, int stringSizeLimit = 0
767 ) if (isTTransport!Transport) {
768 return new TJsonProtocol!Transport(trans, containerSizeLimit, stringSizeLimit);
772 import std.exception;
773 import thrift.transport.memory;
775 // Check the message header format.
776 auto buf = new TMemoryBuffer;
777 auto json = tJsonProtocol(buf);
778 json.writeMessageBegin(TMessage("foo", TMessageType.CALL, 0));
779 json.writeMessageEnd();
781 auto header = new ubyte[13];
783 enforce(cast(char[])header == `[1,"foo",1,0]`);
787 import std.exception;
788 import thrift.transport.memory;
790 // Check that short binary data is read correctly (the Thrift JSON format
791 // does not include padding chars in the Base64 encoded data).
792 auto buf = new TMemoryBuffer;
793 auto json = tJsonProtocol(buf);
794 json.writeBinary([1, 2]);
796 enforce(json.readBinary() == [1, 2]);
800 import std.exception;
801 import thrift.transport.memory;
803 auto buf = new TMemoryBuffer(cast(ubyte[])"\"\\u0e01 \\ud835\\udd3e\"");
804 auto json = tJsonProtocol(buf);
805 auto str = json.readString();
806 enforce(str == "ก 𝔾");
810 // Thrown if low surrogate is missing.
811 import std.exception;
812 import thrift.transport.memory;
814 auto buf = new TMemoryBuffer(cast(ubyte[])"\"\\u0e01 \\ud835\"");
815 auto json = tJsonProtocol(buf);
816 assertThrown!TProtocolException(json.readString());
820 // Thrown if high surrogate is missing.
821 import std.exception;
822 import thrift.transport.memory;
824 auto buf = new TMemoryBuffer(cast(ubyte[])"\"\\u0e01 \\udd3e\"");
825 auto json = tJsonProtocol(buf);
826 assertThrown!TProtocolException(json.readString());
830 import thrift.internal.test.protocol;
831 testContainerSizeLimit!(TJsonProtocol!())();
832 testStringSizeLimit!(TJsonProtocol!())();
836 * TProtocolFactory creating a TJsonProtocol instance for passed in
839 * The optional Transports template tuple parameter can be used to specify
840 * one or more TTransport implementations to specifically instantiate
841 * TJsonProtocol for. If the actual transport types encountered at
842 * runtime match one of the transports in the list, a specialized protocol
843 * instance is created. Otherwise, a generic TTransport version is used.
845 class TJsonProtocolFactory(Transports...) if (
846 allSatisfy!(isTTransport, Transports)
847 ) : TProtocolFactory {
848 TProtocol getProtocol(TTransport trans) const {
849 foreach (Transport; TypeTuple!(Transports, TTransport)) {
850 auto concreteTrans = cast(Transport)trans;
852 auto p = new TJsonProtocol!Transport(concreteTrans);
856 throw new TProtocolException(
857 "Passed null transport to TJsonProtocolFactoy.");
862 immutable ubyte[1] OBJECT_BEGIN = '{';
863 immutable ubyte[1] OBJECT_END = '}';
864 immutable ubyte[1] ARRAY_BEGIN = '[';
865 immutable ubyte[1] ARRAY_END = ']';
866 immutable ubyte[1] NEWLINE = '\n';
867 immutable ubyte[1] PAIR_SEP = ':';
868 immutable ubyte[1] ELEM_SEP = ',';
869 immutable ubyte[1] BACKSLASH = '\\';
870 immutable ubyte[1] STRING_DELIMITER = '"';
871 immutable ubyte[1] ZERO_CHAR = '0';
872 immutable ubyte[1] ESCAPE_CHAR = 'u';
873 immutable ubyte[4] ESCAPE_PREFIX = cast(ubyte[4])r"\u00";
875 enum THRIFT_JSON_VERSION = 1;
877 immutable NAN_STRING = "NaN";
878 immutable INFINITY_STRING = "Infinity";
879 immutable NEG_INFINITY_STRING = "-Infinity";
881 string getNameFromTType(TType typeID) {
882 final switch (typeID) {
905 case TType.STOP: goto case;
907 assert(false, "Invalid type passed.");
911 TType getTTypeFromName(string name) {
913 if (name.length > 1) {
916 result = TType.DOUBLE;
943 result = TType.STRUCT;
946 if (name[1] == 't') {
947 result = TType.STRING;
949 else if (name[1] == 'e') {
960 if (result == TType.STOP) {
961 throw new TProtocolException("Unrecognized type",
962 TProtocolException.Type.NOT_IMPLEMENTED);
967 // This table describes the handling for the first 0x30 characters
968 // 0 : escape using "\u00xx" notation
969 // 1 : just output index
970 // <other> : escape using "\<other>" notation
971 immutable ubyte[0x30] kJsonCharTable = [
972 // 0 1 2 3 4 5 6 7 8 9 A B C D E F
973 0, 0, 0, 0, 0, 0, 0, 0,'b','t','n', 0,'f','r', 0, 0, // 0
974 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1
975 1, 1,'"', 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 2
978 // This string's characters must match up with the elements in kEscapeCharVals.
979 // I don't have '/' on this list even though it appears on www.json.org --
980 // it is not in the RFC
981 immutable kEscapeChars = cast(ubyte[7]) `"\\bfnrt`;
983 // The elements of this array must match up with the sequence of characters in
985 immutable ubyte[7] kEscapeCharVals = [
986 '"', '\\', '\b', '\f', '\n', '\r', '\t',
989 // Return the integer value of a hex character ch.
990 // Throw a protocol exception if the character is not [0-9a-f].
991 ubyte hexVal(ubyte ch) {
992 if ((ch >= '0') && (ch <= '9')) {
993 return cast(ubyte)(ch - '0');
994 } else if ((ch >= 'a') && (ch <= 'f')) {
995 return cast(ubyte)(ch - 'a' + 10);
998 throw new TProtocolException("Expected hex val ([0-9a-f]), got '" ~
999 ch ~ "'.", TProtocolException.Type.INVALID_DATA);
1003 // Return the hex character representing the integer val. The value is masked
1004 // to make sure it is in the correct range.
1005 ubyte hexChar(ubyte val) {
1008 return cast(ubyte)(val + '0');
1010 return cast(ubyte)(val - 10 + 'a');
1014 // Return true if the character ch is in [-+0-9.Ee]; false otherwise
1015 bool isJsonNumeric(ubyte ch) {