Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Directory.Packages.props
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@
<PackageVersion Include="Grpc.Tools" Version="2.83.0" PrivateAssets="All" />
<PackageVersion Include="K4os.Compression.LZ4.Streams" Version="1.3.8" />
<PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="8.0.0" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="6.0.0" />
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="18.11.0" />
<PackageVersion Include="Microsoft.SourceLink.GitHub" Version="10.0.403" />
<PackageVersion Include="pythonnet" Version="3.2.1" />
Expand Down
14 changes: 14 additions & 0 deletions src/Apache.Arrow.Flight.Sql/FlightSqlServer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,20 @@ public static Schema GetTableSchema(bool includeTableSchemaField)
}

protected FlightSqlServer(ILoggerFactory? factory = null)
: this(context: null, factory)
{
}

/// <summary>
/// Creates a server whose record-batch readers and writers use <paramref name="context"/>.
/// </summary>
/// <param name="context">
/// Supplies the allocator, compression codec factory and extension type registry; <c>null</c>
/// uses the defaults.
/// </param>
/// <param name="factory">Creates the logger for the server; <c>null</c> disables logging.</param>
protected FlightSqlServer(ArrowContext? context, ILoggerFactory? factory)
: base(context)
{
Logger = factory?.CreateLogger(typeof(FlightSqlServer));
}
Expand Down
2 changes: 2 additions & 0 deletions src/Apache.Arrow.Flight/Apache.Arrow.Flight.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@
<PackageReference Include="Google.Protobuf" />
<PackageReference Include="Grpc.Net.Client" />
<PackageReference Include="Grpc.Tools" PrivateAssets="All" />
<!-- For [ActivatorUtilitiesConstructor] on FlightClient; see the comment there. -->
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" />
<PackageReference Include="System.Memory" />
</ItemGroup>

Expand Down
45 changes: 34 additions & 11 deletions src/Apache.Arrow.Flight/Client/FlightClient.cs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
using Apache.Arrow.Flight.Protocol;
using Apache.Arrow.Flight.Server.Internal;
using Grpc.Core;
using Microsoft.Extensions.DependencyInjection;

namespace Apache.Arrow.Flight.Client
{
Expand All @@ -30,22 +31,45 @@ public class FlightClient
private readonly ArrowContext _context;

public FlightClient(ChannelBase grpcChannel)
: this(grpcChannel, null)
: this(grpcChannel, context: null)
{
}

/// <summary>
/// Creates a client whose record-batch readers and writers use <paramref name="context"/>.
/// </summary>
/// <param name="grpcChannel">The gRPC channel to the Flight server.</param>
/// <param name="context">
/// Supplies the allocator, compression codec factory and extension type registry; <c>null</c>
/// uses the defaults.
/// </param>
public FlightClient(ChannelBase grpcChannel, ArrowContext context)
{
_client = new FlightService.FlightServiceClient(grpcChannel);
_context = context;
}

public FlightClient(CallInvoker callInvoker)
: this(callInvoker, null)
: this(callInvoker, context: null)
{
}

private FlightClient(CallInvoker callInvoker, ArrowContext context)
/// <summary>
/// Creates a client whose record-batch readers and writers use <paramref name="context"/>.
/// </summary>
/// <remarks>
/// Grpc.Net.ClientFactory creates clients through <c>ActivatorUtilities</c>, passing a
/// <see cref="CallInvoker"/>. With two constructors accepting one it could not choose, so
/// this one is marked as the constructor to use. A client created that way gets the
/// <see cref="ArrowContext"/> registered for dependency injection, if there is one.
/// </remarks>
/// <param name="callInvoker">The gRPC call invoker for the Flight server.</param>
/// <param name="context">
/// Supplies the allocator, compression codec factory and extension type registry; <c>null</c>
/// uses the defaults.
/// </param>
[ActivatorUtilitiesConstructor]
public FlightClient(CallInvoker callInvoker, ArrowContext context = null)
{
_client = new FlightService.FlightServiceClient(callInvoker);
_context = context;
Expand All @@ -55,8 +79,7 @@ private FlightClient(CallInvoker callInvoker, ArrowContext context)
/// Creates a <see cref="FlightClient"/> from a <see cref="CallInvoker"/> with an <see cref="ArrowContext"/>.
/// </summary>
/// <remarks>
/// A factory method is used instead of a constructor overload so that dependency injection
/// (e.g. Grpc.Net.ClientFactory) can still resolve the single-argument constructor unambiguously.
/// Equivalent to <see cref="FlightClient(CallInvoker, ArrowContext)"/>.
/// </remarks>
/// <param name="callInvoker">The <see cref="CallInvoker"/> to use for gRPC calls</param>
/// <param name="context">The <see cref="ArrowContext"/> carrying the compression codec factory and other reader configuration</param>
Expand All @@ -78,7 +101,7 @@ public AsyncServerStreamingCall<FlightInfo> ListFlights(FlightCriteria criteria,
}

var response = _client.ListFlights(criteria.ToProtocol(), headers, deadline, cancellationToken);
var convertStream = new StreamReader<Protocol.FlightInfo, FlightInfo>(response.ResponseStream, inFlight => new FlightInfo(inFlight));
var convertStream = new StreamReader<Protocol.FlightInfo, FlightInfo>(response.ResponseStream, inFlight => new FlightInfo(inFlight, _context?.ExtensionRegistry));

return new AsyncServerStreamingCall<FlightInfo>(convertStream, response.ResponseHeadersAsync, response.GetStatus, response.GetTrailers, response.Dispose);
}
Expand Down Expand Up @@ -119,7 +142,7 @@ public AsyncUnaryCall<FlightInfo> GetInfo(FlightDescriptor flightDescriptor, Met

var flightInfo = flightInfoResult
.ResponseAsync
.ContinueWith(async flightInfo => new FlightInfo(await flightInfo.ConfigureAwait(false)))
.ContinueWith(async flightInfo => new FlightInfo(await flightInfo.ConfigureAwait(false), _context?.ExtensionRegistry))
.Unwrap();

return new AsyncUnaryCall<FlightInfo>(
Expand Down Expand Up @@ -166,7 +189,7 @@ public Task<FlightRecordBatchDuplexStreamingCall> StartPut(FlightDescriptor flig
public FlightRecordBatchDuplexStreamingCall StartPut(FlightDescriptor flightDescriptor, Metadata headers, System.DateTime? deadline, CancellationToken cancellationToken = default)
{
var channels = _client.DoPut(headers, deadline, cancellationToken);
var requestStream = new FlightClientRecordBatchStreamWriter(channels.RequestStream, flightDescriptor);
var requestStream = new FlightClientRecordBatchStreamWriter(channels.RequestStream, flightDescriptor, _context);
var readStream = new StreamReader<Protocol.PutResult, FlightPutResult>(channels.ResponseStream, putResult => new FlightPutResult(putResult));
return new FlightRecordBatchDuplexStreamingCall(
requestStream,
Expand All @@ -191,7 +214,7 @@ public FlightRecordBatchDuplexStreamingCall StartPut(FlightDescriptor flightDesc
public async Task<FlightRecordBatchDuplexStreamingCall> StartPut(FlightDescriptor flightDescriptor, Schema schema, Metadata headers, System.DateTime? deadline, CancellationToken cancellationToken = default)
{
var channels = _client.DoPut(headers, deadline, cancellationToken);
var requestStream = new FlightClientRecordBatchStreamWriter(channels.RequestStream, flightDescriptor);
var requestStream = new FlightClientRecordBatchStreamWriter(channels.RequestStream, flightDescriptor, _context);
var readStream = new StreamReader<Protocol.PutResult, FlightPutResult>(channels.ResponseStream, putResult => new FlightPutResult(putResult));
var streamingCall = new FlightRecordBatchDuplexStreamingCall(
requestStream,
Expand Down Expand Up @@ -233,7 +256,7 @@ public FlightRecordBatchExchangeCall DoExchange(FlightDescriptor flightDescripto
public FlightRecordBatchExchangeCall DoExchange(FlightDescriptor flightDescriptor, Metadata headers, System.DateTime? deadline, CancellationToken cancellationToken = default)
{
var channel = _client.DoExchange(headers, deadline, cancellationToken);
var requestStream = new FlightClientRecordBatchStreamWriter(channel.RequestStream, flightDescriptor);
var requestStream = new FlightClientRecordBatchStreamWriter(channel.RequestStream, flightDescriptor, _context);
var responseStream = new FlightClientRecordBatchStreamReader(channel.ResponseStream, _context);
var call = new FlightRecordBatchExchangeCall(
requestStream,
Expand Down Expand Up @@ -269,7 +292,7 @@ public AsyncUnaryCall<Schema> GetSchema(FlightDescriptor flightDescriptor, Metad

var schema = schemaResult
.ResponseAsync
.ContinueWith(async schema => FlightMessageSerializer.DecodeSchema((await schemaResult.ResponseAsync.ConfigureAwait(false)).Schema.Memory))
.ContinueWith(async schema => FlightMessageSerializer.DecodeSchema((await schemaResult.ResponseAsync.ConfigureAwait(false)).Schema.Memory, _context?.ExtensionRegistry))
.Unwrap();

return new AsyncUnaryCall<Schema>(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ public class FlightClientRecordBatchStreamWriter : FlightRecordBatchStreamWriter
{
private readonly IClientStreamWriter<Protocol.FlightData> _clientStreamWriter;
private bool _completed = false;
internal FlightClientRecordBatchStreamWriter(IClientStreamWriter<Protocol.FlightData> clientStreamWriter, FlightDescriptor flightDescriptor) : base(clientStreamWriter, flightDescriptor)
internal FlightClientRecordBatchStreamWriter(IClientStreamWriter<Protocol.FlightData> clientStreamWriter, FlightDescriptor flightDescriptor, ArrowContext context) : base(clientStreamWriter, flightDescriptor, context)
{
_clientStreamWriter = clientStreamWriter;
}
Expand Down
4 changes: 2 additions & 2 deletions src/Apache.Arrow.Flight/FlightInfo.cs
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,9 @@ namespace Apache.Arrow.Flight
{
public class FlightInfo
{
internal FlightInfo(Protocol.FlightInfo flightInfo)
internal FlightInfo(Protocol.FlightInfo flightInfo, ExtensionTypeRegistry extensionRegistry = null)
{
Schema = flightInfo.Schema?.Length > 0 ? FlightMessageSerializer.DecodeSchema(flightInfo.Schema.Memory) : null;
Schema = flightInfo.Schema?.Length > 0 ? FlightMessageSerializer.DecodeSchema(flightInfo.Schema.Memory, extensionRegistry) : null;
Descriptor = new FlightDescriptor(flightInfo.FlightDescriptor);

var endpoints = new List<FlightEndpoint>();
Expand Down
6 changes: 4 additions & 2 deletions src/Apache.Arrow.Flight/FlightRecordBatchStreamWriter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,13 @@ public abstract class FlightRecordBatchStreamWriter : IAsyncStreamWriter<RecordB
private FlightDataStream _flightDataStream;
private readonly IAsyncStreamWriter<Protocol.FlightData> _clientStreamWriter;
private readonly FlightDescriptor _flightDescriptor;
private readonly ArrowContext _context;

private bool _disposed;

private protected FlightRecordBatchStreamWriter(IAsyncStreamWriter<Protocol.FlightData> clientStreamWriter, FlightDescriptor flightDescriptor)
private protected FlightRecordBatchStreamWriter(IAsyncStreamWriter<Protocol.FlightData> clientStreamWriter, FlightDescriptor flightDescriptor, ArrowContext context)
{
_context = context;
_clientStreamWriter = clientStreamWriter;
_flightDescriptor = flightDescriptor;
}
Expand All @@ -52,7 +54,7 @@ public async Task SetupStream(Schema schema)
{
throw new InvalidOperationException("Flight data stream is already set");
}
_flightDataStream = new FlightDataStream(_clientStreamWriter, _flightDescriptor, schema);
_flightDataStream = new FlightDataStream(_clientStreamWriter, _flightDescriptor, schema, _context);
await _flightDataStream.SendSchema().ConfigureAwait(false);
}

Expand Down
4 changes: 2 additions & 2 deletions src/Apache.Arrow.Flight/Internal/FlightDataStream.cs
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,8 @@ internal class FlightDataStream : ArrowStreamWriter
private readonly IAsyncStreamWriter<Protocol.FlightData> _clientStreamWriter;
private Protocol.FlightData _currentFlightData;

public FlightDataStream(IAsyncStreamWriter<Protocol.FlightData> clientStreamWriter, FlightDescriptor flightDescriptor, Schema schema)
: base(new MemoryStream(), schema)
public FlightDataStream(IAsyncStreamWriter<Protocol.FlightData> clientStreamWriter, FlightDescriptor flightDescriptor, Schema schema, ArrowContext context)
: base(new MemoryStream(), schema, leaveOpen: false, options: null, context?.Allocator, context?.CompressionCodecFactory)
Comment thread
CurtHagenlocher marked this conversation as resolved.
Resolved
{
_clientStreamWriter = clientStreamWriter;
_flightDescriptor = flightDescriptor;
Expand Down
8 changes: 4 additions & 4 deletions src/Apache.Arrow.Flight/Internal/FlightMessageSerializer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ namespace Apache.Arrow.Flight
{
internal static class FlightMessageSerializer
{
public static Schema DecodeSchema(ReadOnlyMemory<byte> buffer)
public static Schema DecodeSchema(ReadOnlyMemory<byte> buffer, ExtensionTypeRegistry extensionRegistry = null)
{
if (buffer.IsEmpty) return null;
int bufferPosition = 0;
Expand All @@ -47,15 +47,15 @@ public static Schema DecodeSchema(ReadOnlyMemory<byte> buffer)
ByteBuffer schemaBuffer = ArrowReaderImplementation.CreateByteBuffer(buffer.Slice(bufferPosition));
//DictionaryBatch not supported for now
DictionaryMemo dictionaryMemo = null;
var schema = MessageSerializer.GetSchema(ArrowReaderImplementation.ReadMessage<Flatbuf.Schema>(schemaBuffer), ref dictionaryMemo);
var schema = MessageSerializer.GetSchema(ArrowReaderImplementation.ReadMessage<Flatbuf.Schema>(schemaBuffer), ref dictionaryMemo, extensionRegistry ?? ExtensionTypeRegistry.Default);
return schema;
}

internal static Schema DecodeSchema(ByteBuffer schemaBuffer)
internal static Schema DecodeSchema(ByteBuffer schemaBuffer, ExtensionTypeRegistry extensionRegistry = null)
{
//DictionaryBatch not supported for now
DictionaryMemo dictionaryMemo = null;
var schema = MessageSerializer.GetSchema(ArrowReaderImplementation.ReadMessage<Flatbuf.Schema>(schemaBuffer), ref dictionaryMemo);
var schema = MessageSerializer.GetSchema(ArrowReaderImplementation.ReadMessage<Flatbuf.Schema>(schemaBuffer), ref dictionaryMemo, extensionRegistry ?? ExtensionTypeRegistry.Default);
return schema;
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ public override async ValueTask<Schema> ReadSchemaAsync(CancellationToken cancel
switch (message.HeaderType)
{
case MessageHeader.Schema:
_schema = FlightMessageSerializer.DecodeSchema(message.ByteBuffer);
_schema = FlightMessageSerializer.DecodeSchema(message.ByteBuffer, _extensionRegistry);
break;
default:
throw new Exception($"Expected schema as the first message, but got: {message.HeaderType.ToString()}");
Expand Down
27 changes: 27 additions & 0 deletions src/Apache.Arrow.Flight/Server/FlightServer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,33 @@ namespace Apache.Arrow.Flight.Server
{
public abstract class FlightServer
{
protected FlightServer()
: this(context: null)
{
}

/// <summary>
/// Creates a server whose record-batch readers and writers use <paramref name="context"/>.
/// </summary>
/// <remarks>
/// This applies however the server is hosted, including when it is created by dependency
/// injection: the derived server receives the context in its own constructor and passes it on.
/// </remarks>
/// <param name="context">
/// Supplies the allocator, compression codec factory and extension type registry; <c>null</c>
/// uses the defaults.
/// </param>
protected FlightServer(ArrowContext context)
{
Context = context;
}

/// <summary>
/// The context used for the record batches this server reads and writes, or <c>null</c>
/// for the defaults.
/// </summary>
public ArrowContext Context { get; }

public virtual Task DoPut(FlightServerRecordBatchStreamReader requestStream, IAsyncStreamWriter<FlightPutResult> responseStream, ServerCallContext context)
{
throw new NotImplementedException();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,20 @@ namespace Apache.Arrow.Flight.Server
public class FlightServerRecordBatchStreamReader : FlightRecordBatchStreamReader
{
public FlightServerRecordBatchStreamReader(IAsyncStreamReader<FlightData> flightDataStream)
: this(flightDataStream, null)
: this(flightDataStream, context: null)
{
}

public FlightServerRecordBatchStreamReader(IAsyncStreamReader<FlightData> flightDataStream, ArrowContext context) : base(new StreamReader<FlightData, Protocol.FlightData>(flightDataStream, data => data.ToProtocol()), context)
/// <summary>
/// Creates a reader that decodes record batches using <paramref name="context"/>.
/// </summary>
/// <param name="flightDataStream">The stream of Flight data to read.</param>
/// <param name="context">
/// Supplies the allocator, compression codec factory and extension type registry; <c>null</c>
/// uses the defaults.
/// </param>
public FlightServerRecordBatchStreamReader(IAsyncStreamReader<FlightData> flightDataStream, ArrowContext context)
: base(new StreamReader<FlightData, Protocol.FlightData>(flightDataStream, data => data.ToProtocol()), context)
{
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,25 @@ namespace Apache.Arrow.Flight.Server
{
public class FlightServerRecordBatchStreamWriter : FlightRecordBatchStreamWriter, IServerStreamWriter<RecordBatch>
{
public FlightServerRecordBatchStreamWriter(IServerStreamWriter<FlightData> clientStreamWriter) : base(new StreamWriter<Protocol.FlightData, FlightData>(clientStreamWriter, data => new FlightData(data)), null)
public FlightServerRecordBatchStreamWriter(IServerStreamWriter<FlightData> clientStreamWriter)
: this(clientStreamWriter, context: null)
{
}

internal FlightServerRecordBatchStreamWriter(IServerStreamWriter<Protocol.FlightData> clientStreamWriter) : base(clientStreamWriter, null)
/// <summary>
/// Creates a writer that encodes record batches using <paramref name="context"/>.
/// </summary>
/// <param name="clientStreamWriter">The stream to write Flight data to.</param>
/// <param name="context">
/// Supplies the allocator, compression codec factory and extension type registry; <c>null</c>
/// uses the defaults.
/// </param>
public FlightServerRecordBatchStreamWriter(IServerStreamWriter<FlightData> clientStreamWriter, ArrowContext context)
: base(new StreamWriter<Protocol.FlightData, FlightData>(clientStreamWriter, data => new FlightData(data)), null, context)
{
}

internal FlightServerRecordBatchStreamWriter(IServerStreamWriter<Protocol.FlightData> clientStreamWriter, ArrowContext context) : base(clientStreamWriter, null, context)
{
}
}
Expand Down
Loading
Loading