ceph/src/arrow/csharp/src/Apache.Arrow/Ipc/MessageSerializer.cs

   1 // Licensed to the Apache Software Foundation (ASF) under one or more
   2 // contributor license agreements. See the NOTICE file distributed with
   3 // this work for additional information regarding copyright ownership.
   4 // The ASF licenses this file to You under the Apache License, Version 2.0
   5 // (the "License"); you may not use this file except in compliance with
   6 // the License.  You may obtain a copy of the License at
   7 //
   8 //     http://www.apache.org/licenses/LICENSE-2.0
   9 //
  10 // Unless required by applicable law or agreed to in writing, software
  11 // distributed under the License is distributed on an "AS IS" BASIS,
  12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 // See the License for the specific language governing permissions and
  14 // limitations under the License.
  15
  16 using System;
  17 using System.Collections.Generic;
  18 using System.Diagnostics;
  19 using System.IO;
  20 using Apache.Arrow.Types;
  21
  22 namespace Apache.Arrow.Ipc
  23 {
  24     internal class MessageSerializer
  25     {
  26         public const int IpcContinuationToken = -1;
  27
  28         public static Types.NumberType GetNumberType(int bitWidth, bool signed)
  29         {
  30             if (signed)
  31             {
  32                 if (bitWidth == 8)
  33                     return Types.Int8Type.Default;
  34                 if (bitWidth == 16)
  35                     return Types.Int16Type.Default;
  36                 if (bitWidth == 32)
  37                     return Types.Int32Type.Default;
  38                 if (bitWidth == 64)
  39                     return Types.Int64Type.Default;
  40             }
  41             else
  42             {
  43                 if (bitWidth == 8)
  44                     return Types.UInt8Type.Default;
  45                 if (bitWidth == 16)
  46                     return Types.UInt16Type.Default;
  47                 if (bitWidth == 32)
  48                     return Types.UInt32Type.Default;
  49                 if (bitWidth == 64)
  50                     return Types.UInt64Type.Default;
  51             }
  52             throw new Exception($"Unexpected bit width of {bitWidth} for " +
  53                                 $"{(signed ? "signed " : "unsigned")} integer.");
  54         }
  55
  56         internal static Schema GetSchema(Flatbuf.Schema schema, ref DictionaryMemo dictionaryMemo)
  57         {
  58             List<Field> fields = new List<Field>();
  59             for (int i = 0; i < schema.FieldsLength; i++)
  60             {
  61                 Flatbuf.Field field = schema.Fields(i).GetValueOrDefault();
  62                 fields.Add(FieldFromFlatbuffer(field, ref dictionaryMemo));
  63             }
  64
  65             Dictionary<string, string> metadata = schema.CustomMetadataLength > 0 ? new Dictionary<string, string>() : null;
  66             for (int i = 0; i < schema.CustomMetadataLength; i++)
  67             {
  68                 Flatbuf.KeyValue keyValue = schema.CustomMetadata(i).GetValueOrDefault();
  69
  70                 metadata[keyValue.Key] = keyValue.Value;
  71             }
  72
  73             return new Schema(fields, metadata, copyCollections: false);
  74         }
  75
  76         private static Field FieldFromFlatbuffer(Flatbuf.Field flatbufField, ref DictionaryMemo dictionaryMemo)
  77         {
  78             Field[] childFields = flatbufField.ChildrenLength > 0 ? new Field[flatbufField.ChildrenLength] : null;
  79             for (int i = 0; i < flatbufField.ChildrenLength; i++)
  80             {
  81                 Flatbuf.Field? childFlatbufField = flatbufField.Children(i);
  82                 childFields[i] = FieldFromFlatbuffer(childFlatbufField.Value, ref dictionaryMemo);
  83             }
  84
  85             Flatbuf.DictionaryEncoding? dictionaryEncoding = flatbufField.Dictionary;
  86             IArrowType type = GetFieldArrowType(flatbufField, childFields);
  87
  88             if (dictionaryEncoding.HasValue)
  89             {
  90                 Flatbuf.Int? indexTypeAsInt = dictionaryEncoding.Value.IndexType;
  91                 IArrowType indexType = indexTypeAsInt.HasValue ?
  92                     GetNumberType(indexTypeAsInt.Value.BitWidth, indexTypeAsInt.Value.IsSigned) :
  93                     GetNumberType(Int32Type.Default.BitWidth, Int32Type.Default.IsSigned);
  94
  95                 type = new DictionaryType(indexType, type, dictionaryEncoding.Value.IsOrdered);
  96             }
  97
  98             Dictionary<string, string> metadata = flatbufField.CustomMetadataLength > 0 ? new Dictionary<string, string>() : null;
  99             for (int i = 0; i < flatbufField.CustomMetadataLength; i++)
 100             {
 101                 Flatbuf.KeyValue keyValue = flatbufField.CustomMetadata(i).GetValueOrDefault();
 102
 103                 metadata[keyValue.Key] = keyValue.Value;
 104             }
 105
 106             var arrowField = new Field(flatbufField.Name, type, flatbufField.Nullable, metadata, copyCollections: false);
 107
 108             if (dictionaryEncoding.HasValue)
 109             {
 110                 dictionaryMemo ??= new DictionaryMemo();
 111                 dictionaryMemo.AddField(dictionaryEncoding.Value.Id, arrowField);
 112             }
 113
 114             return arrowField;
 115         }
 116
 117         private static Types.IArrowType GetFieldArrowType(Flatbuf.Field field, Field[] childFields = null)
 118         {
 119             switch (field.TypeType)
 120             {
 121                 case Flatbuf.Type.Int:
 122                     Flatbuf.Int intMetaData = field.Type<Flatbuf.Int>().Value;
 123                     return MessageSerializer.GetNumberType(intMetaData.BitWidth, intMetaData.IsSigned);
 124                 case Flatbuf.Type.FloatingPoint:
 125                     Flatbuf.FloatingPoint floatingPointTypeMetadata = field.Type<Flatbuf.FloatingPoint>().Value;
 126                     switch (floatingPointTypeMetadata.Precision)
 127                     {
 128                         case Flatbuf.Precision.SINGLE:
 129                             return Types.FloatType.Default;
 130                         case Flatbuf.Precision.DOUBLE:
 131                             return Types.DoubleType.Default;
 132                         case Flatbuf.Precision.HALF:
 133                             return Types.HalfFloatType.Default;
 134                         default:
 135                             throw new InvalidDataException("Unsupported floating point precision");
 136                     }
 137                 case Flatbuf.Type.Bool:
 138                     return new Types.BooleanType();
 139                 case Flatbuf.Type.Decimal:
 140                     Flatbuf.Decimal decMeta = field.Type<Flatbuf.Decimal>().Value;
 141                     switch (decMeta.BitWidth)
 142                     {
 143                         case 128:
 144                             return new Types.Decimal128Type(decMeta.Precision, decMeta.Scale);
 145                         case 256:
 146                             return new Types.Decimal256Type(decMeta.Precision, decMeta.Scale);
 147                         default:
 148                             throw new InvalidDataException("Unsupported decimal bit width " + decMeta.BitWidth);
 149                     }
 150                 case Flatbuf.Type.Date:
 151                     Flatbuf.Date dateMeta = field.Type<Flatbuf.Date>().Value;
 152                     switch (dateMeta.Unit)
 153                     {
 154                         case Flatbuf.DateUnit.DAY:
 155                             return Types.Date32Type.Default;
 156                         case Flatbuf.DateUnit.MILLISECOND:
 157                             return Types.Date64Type.Default;
 158                         default:
 159                             throw new InvalidDataException("Unsupported date unit");
 160                     }
 161                 case Flatbuf.Type.Time:
 162                     Flatbuf.Time timeMeta = field.Type<Flatbuf.Time>().Value;
 163                     switch (timeMeta.BitWidth)
 164                     {
 165                         case 32:
 166                             return new Types.Time32Type(timeMeta.Unit.ToArrow());
 167                         case 64:
 168                             return new Types.Time64Type(timeMeta.Unit.ToArrow());
 169                         default:
 170                             throw new InvalidDataException("Unsupported time bit width");
 171                     }
 172                 case Flatbuf.Type.Timestamp:
 173                     Flatbuf.Timestamp timestampTypeMetadata = field.Type<Flatbuf.Timestamp>().Value;
 174                     Types.TimeUnit unit = timestampTypeMetadata.Unit.ToArrow();
 175                     string timezone = timestampTypeMetadata.Timezone;
 176                     return new Types.TimestampType(unit, timezone);
 177                 case Flatbuf.Type.Interval:
 178                     Flatbuf.Interval intervalMetadata = field.Type<Flatbuf.Interval>().Value;
 179                     return new Types.IntervalType(intervalMetadata.Unit.ToArrow());
 180                 case Flatbuf.Type.Utf8:
 181                     return new Types.StringType();
 182                 case Flatbuf.Type.FixedSizeBinary:
 183                     Flatbuf.FixedSizeBinary fixedSizeBinaryMetadata = field.Type<Flatbuf.FixedSizeBinary>().Value;
 184                     return new Types.FixedSizeBinaryType(fixedSizeBinaryMetadata.ByteWidth);
 185                 case Flatbuf.Type.Binary:
 186                     return Types.BinaryType.Default;
 187                 case Flatbuf.Type.List:
 188                     if (childFields == null || childFields.Length != 1)
 189                     {
 190                         throw new InvalidDataException($"List type must have exactly one child.");
 191                     }
 192                     return new Types.ListType(childFields[0]);
 193                 case Flatbuf.Type.Struct_:
 194                     Debug.Assert(childFields != null);
 195                     return new Types.StructType(childFields);
 196                 default:
 197                     throw new InvalidDataException($"Arrow primitive '{field.TypeType}' is unsupported.");
 198             }
 199         }
 200     }
 201 }