From 5926da3a08cd9607b110844d948ab29973e077c6 Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Mon, 1 Jun 2026 15:38:18 -0700 Subject: [PATCH 1/3] HBASE-30194 [thirdparty] Onboard libthrift to hbase-thirdparty Co-Authored-by: Claude --- hbase-shaded-thrift/pom.xml | 228 ++++ .../main/appended-resources/META-INF/LICENSE | 19 + .../main/appended-resources/META-INF/NOTICE | 8 + .../src/main/java/META-INF/MANIFEST.MF | 2 + .../apache/thrift/AsyncProcessFunction.java | 64 ++ .../java/org/apache/thrift/EncodingUtils.java | 148 +++ .../main/java/org/apache/thrift/Option.java | 143 +++ .../org/apache/thrift/ProcessFunction.java | 96 ++ .../apache/thrift/TApplicationException.java | 147 +++ .../org/apache/thrift/TAsyncProcessor.java | 33 + .../main/java/org/apache/thrift/TBase.java | 74 ++ .../apache/thrift/TBaseAsyncProcessor.java | 115 ++ .../java/org/apache/thrift/TBaseHelper.java | 298 +++++ .../org/apache/thrift/TBaseProcessor.java | 44 + .../apache/thrift/TByteArrayOutputStream.java | 57 + .../org/apache/thrift/TConfiguration.java | 102 ++ .../java/org/apache/thrift/TDeserializer.java | 730 ++++++++++++ .../main/java/org/apache/thrift/TEnum.java | 24 + .../java/org/apache/thrift/TEnumHelper.java | 47 + .../java/org/apache/thrift/TException.java | 42 + .../java/org/apache/thrift/TFieldIdEnum.java | 36 + .../apache/thrift/TFieldRequirementType.java | 31 + .../thrift/THttpClientResponseHandler.java | 74 ++ .../apache/thrift/TMultiplexedProcessor.java | 168 +++ .../thrift/TNonblockingMultiFetchClient.java | 374 ++++++ .../thrift/TNonblockingMultiFetchStats.java | 132 +++ .../java/org/apache/thrift/TProcessor.java | 30 + .../org/apache/thrift/TProcessorFactory.java | 40 + .../java/org/apache/thrift/TSerializable.java | 42 + .../java/org/apache/thrift/TSerializer.java | 83 ++ .../org/apache/thrift/TServiceClient.java | 96 ++ .../apache/thrift/TServiceClientFactory.java | 48 + .../main/java/org/apache/thrift/TUnion.java | 293 +++++ .../apache/thrift/annotation/Nullable.java | 30 + .../thrift/async/AsyncMethodCallback.java | 47 + .../async/AsyncMethodFutureAdapter.java | 35 + .../org/apache/thrift/async/TAsyncClient.java | 113 ++ .../thrift/async/TAsyncClientFactory.java | 25 + .../thrift/async/TAsyncClientManager.java | 209 ++++ .../apache/thrift/async/TAsyncMethodCall.java | 294 +++++ .../apache/thrift/meta_data/EnumMetaData.java | 31 + .../thrift/meta_data/FieldMetaData.java | 104 ++ .../thrift/meta_data/FieldValueMetaData.java | 71 ++ .../apache/thrift/meta_data/ListMetaData.java | 29 + .../apache/thrift/meta_data/MapMetaData.java | 31 + .../apache/thrift/meta_data/SetMetaData.java | 29 + .../thrift/meta_data/StructMetaData.java | 31 + .../org/apache/thrift/partial/EnumCache.java | 88 ++ .../thrift/partial/PartialThriftComparer.java | 342 ++++++ .../java/org/apache/thrift/partial/README.md | 112 ++ .../org/apache/thrift/partial/TFieldData.java | 44 + .../apache/thrift/partial/ThriftField.java | 193 ++++ .../partial/ThriftFieldValueProcessor.java | 93 ++ .../apache/thrift/partial/ThriftMetadata.java | 552 +++++++++ .../thrift/partial/ThriftStructProcessor.java | 181 +++ .../org/apache/thrift/partial/Validate.java | 233 ++++ .../apache/thrift/protocol/ShortStack.java | 77 ++ .../apache/thrift/protocol/TBase64Utils.java | 118 ++ .../thrift/protocol/TBinaryProtocol.java | 572 ++++++++++ .../thrift/protocol/TCompactProtocol.java | 968 ++++++++++++++++ .../org/apache/thrift/protocol/TField.java | 63 ++ .../apache/thrift/protocol/TJSONProtocol.java | 1004 +++++++++++++++++ .../TLegacyUuidProtocolDecorator.java | 98 ++ .../org/apache/thrift/protocol/TList.java | 43 + .../java/org/apache/thrift/protocol/TMap.java | 49 + .../org/apache/thrift/protocol/TMessage.java | 78 ++ .../apache/thrift/protocol/TMessageType.java | 28 + .../thrift/protocol/TMultiplexedProtocol.java | 88 ++ .../org/apache/thrift/protocol/TProtocol.java | 577 ++++++++++ .../thrift/protocol/TProtocolDecorator.java | 320 ++++++ .../thrift/protocol/TProtocolException.java | 77 ++ .../thrift/protocol/TProtocolFactory.java | 28 + .../apache/thrift/protocol/TProtocolUtil.java | 214 ++++ .../apache/thrift/protocol/TReadProtocol.java | 50 + .../java/org/apache/thrift/protocol/TSet.java | 47 + .../thrift/protocol/TSimpleJSONProtocol.java | 509 +++++++++ .../org/apache/thrift/protocol/TStruct.java | 37 + .../thrift/protocol/TTupleProtocol.java | 122 ++ .../org/apache/thrift/protocol/TType.java | 41 + .../thrift/protocol/TWriteProtocol.java | 52 + .../org/apache/thrift/scheme/IScheme.java | 30 + .../apache/thrift/scheme/SchemeFactory.java | 24 + .../apache/thrift/scheme/StandardScheme.java | 23 + .../org/apache/thrift/scheme/TupleScheme.java | 23 + .../server/AbstractNonblockingServer.java | 626 ++++++++++ .../org/apache/thrift/server/Invocation.java | 21 + .../apache/thrift/server/ServerContext.java | 55 + .../thrift/server/TExtensibleServlet.java | 168 +++ .../org/apache/thrift/server/THsHaServer.java | 194 ++++ .../thrift/server/TNonblockingServer.java | 234 ++++ .../thrift/server/TSaslNonblockingServer.java | 488 ++++++++ .../org/apache/thrift/server/TServer.java | 160 +++ .../thrift/server/TServerEventHandler.java | 49 + .../org/apache/thrift/server/TServlet.java | 120 ++ .../apache/thrift/server/TSimpleServer.java | 120 ++ .../thrift/server/TThreadPoolServer.java | 316 ++++++ .../server/TThreadedSelectorServer.java | 730 ++++++++++++ .../thrift/transport/AutoExpandingBuffer.java | 49 + .../AutoExpandingBufferReadTransport.java | 89 ++ .../AutoExpandingBufferWriteTransport.java | 93 ++ .../transport/SocketAddressProvider.java | 30 + .../apache/thrift/transport/TByteBuffer.java | 111 ++ .../thrift/transport/TEOFException.java | 28 + .../thrift/transport/TEndpointTransport.java | 120 ++ .../thrift/transport/TFileProcessor.java | 122 ++ .../thrift/transport/TFileTransport.java | 612 ++++++++++ .../apache/thrift/transport/THttpClient.java | 329 ++++++ .../thrift/transport/TIOStreamTransport.java | 209 ++++ .../thrift/transport/TMemoryBuffer.java | 120 ++ .../transport/TMemoryInputTransport.java | 125 ++ .../thrift/transport/TMemoryTransport.java | 88 ++ .../transport/TNonblockingSSLSocket.java | 263 +++++ .../transport/TNonblockingServerSocket.java | 186 +++ .../TNonblockingServerTransport.java | 35 + .../thrift/transport/TNonblockingSocket.java | 214 ++++ .../transport/TNonblockingTransport.java | 50 + .../transport/TSSLTransportFactory.java | 528 +++++++++ .../transport/TSaslClientTransport.java | 109 ++ .../transport/TSaslServerTransport.java | 224 ++++ .../thrift/transport/TSaslTransport.java | 510 +++++++++ .../thrift/transport/TSeekableFile.java | 37 + .../thrift/transport/TServerSocket.java | 154 +++ .../thrift/transport/TServerTransport.java | 89 ++ .../transport/TSimpleFileTransport.java | 225 ++++ .../org/apache/thrift/transport/TSocket.java | 261 +++++ .../thrift/transport/TStandardFile.java | 60 + .../apache/thrift/transport/TTransport.java | 202 ++++ .../thrift/transport/TTransportException.java | 78 ++ .../thrift/transport/TTransportFactory.java | 38 + .../thrift/transport/TZlibTransport.java | 154 +++ .../layered/TFastFramedTransport.java | 201 ++++ .../transport/layered/TFramedTransport.java | 186 +++ .../transport/layered/TLayeredTransport.java | 53 + .../transport/sasl/DataFrameHeaderReader.java | 45 + .../transport/sasl/DataFrameReader.java | 28 + .../transport/sasl/DataFrameWriter.java | 69 ++ .../transport/sasl/FixedSizeHeaderReader.java | 73 ++ .../transport/sasl/FrameHeaderReader.java | 61 + .../thrift/transport/sasl/FrameReader.java | 155 +++ .../thrift/transport/sasl/FrameWriter.java | 127 +++ .../transport/sasl/NegotiationStatus.java | 59 + .../sasl/NonblockingSaslHandler.java | 537 +++++++++ .../sasl/SaslNegotiationFrameReader.java | 28 + .../sasl/SaslNegotiationFrameWriter.java | 63 ++ .../sasl/SaslNegotiationHeaderReader.java | 58 + .../thrift/transport/sasl/SaslPeer.java | 96 ++ .../thrift/transport/sasl/ServerSaslPeer.java | 104 ++ .../sasl/TBaseSaslProcessorFactory.java | 36 + .../sasl/TInvalidSaslFrameException.java | 28 + .../sasl/TSaslNegotiationException.java | 76 ++ .../transport/sasl/TSaslProcessorFactory.java | 32 + .../transport/sasl/TSaslServerDefinition.java | 45 + .../transport/sasl/TSaslServerFactory.java | 70 ++ .../org/apache/thrift/utils/StringUtils.java | 77 ++ .../HBASE-30194-01-servlet-javax.patch | 39 + .../patches/HBASE-30194-02-httpclient4.patch | 129 +++ .../src/main/resources/META-INF/LICENSE.txt | 315 ++++++ .../src/main/resources/META-INF/NOTICE.txt | 5 + pom.xml | 5 + 159 files changed, 23768 insertions(+) create mode 100644 hbase-shaded-thrift/pom.xml create mode 100644 hbase-shaded-thrift/src/main/appended-resources/META-INF/LICENSE create mode 100644 hbase-shaded-thrift/src/main/appended-resources/META-INF/NOTICE create mode 100644 hbase-shaded-thrift/src/main/java/META-INF/MANIFEST.MF create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/AsyncProcessFunction.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/EncodingUtils.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/Option.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/ProcessFunction.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TApplicationException.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TAsyncProcessor.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TBase.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TBaseAsyncProcessor.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TBaseHelper.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TBaseProcessor.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TByteArrayOutputStream.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TConfiguration.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TDeserializer.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TEnum.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TEnumHelper.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TException.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TFieldIdEnum.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TFieldRequirementType.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/THttpClientResponseHandler.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TMultiplexedProcessor.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TNonblockingMultiFetchClient.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TNonblockingMultiFetchStats.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TProcessor.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TProcessorFactory.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TSerializable.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TSerializer.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TServiceClient.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TServiceClientFactory.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TUnion.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/annotation/Nullable.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/async/AsyncMethodCallback.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/async/AsyncMethodFutureAdapter.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/async/TAsyncClient.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/async/TAsyncClientFactory.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/async/TAsyncClientManager.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/async/TAsyncMethodCall.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/meta_data/EnumMetaData.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/meta_data/FieldMetaData.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/meta_data/FieldValueMetaData.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/meta_data/ListMetaData.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/meta_data/MapMetaData.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/meta_data/SetMetaData.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/meta_data/StructMetaData.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/EnumCache.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/PartialThriftComparer.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/README.md create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/TFieldData.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/ThriftField.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/ThriftFieldValueProcessor.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/ThriftMetadata.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/ThriftStructProcessor.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/Validate.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/ShortStack.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TBase64Utils.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TBinaryProtocol.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TCompactProtocol.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TField.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TJSONProtocol.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TLegacyUuidProtocolDecorator.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TList.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TMap.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TMessage.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TMessageType.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TMultiplexedProtocol.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TProtocol.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TProtocolDecorator.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TProtocolException.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TProtocolFactory.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TProtocolUtil.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TReadProtocol.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TSet.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TSimpleJSONProtocol.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TStruct.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TTupleProtocol.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TType.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TWriteProtocol.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/scheme/IScheme.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/scheme/SchemeFactory.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/scheme/StandardScheme.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/scheme/TupleScheme.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/server/AbstractNonblockingServer.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/server/Invocation.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/server/ServerContext.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TExtensibleServlet.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/server/THsHaServer.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TNonblockingServer.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TSaslNonblockingServer.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TServer.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TServerEventHandler.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TServlet.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TSimpleServer.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TThreadPoolServer.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TThreadedSelectorServer.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/AutoExpandingBuffer.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/AutoExpandingBufferReadTransport.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/AutoExpandingBufferWriteTransport.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/SocketAddressProvider.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TByteBuffer.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TEOFException.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TEndpointTransport.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TFileProcessor.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TFileTransport.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/THttpClient.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TIOStreamTransport.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TMemoryBuffer.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TMemoryInputTransport.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TMemoryTransport.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TNonblockingSSLSocket.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TNonblockingServerSocket.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TNonblockingServerTransport.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TNonblockingSocket.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TNonblockingTransport.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TSSLTransportFactory.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TSaslClientTransport.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TSaslServerTransport.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TSaslTransport.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TSeekableFile.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TServerSocket.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TServerTransport.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TSimpleFileTransport.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TSocket.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TStandardFile.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TTransport.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TTransportException.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TTransportFactory.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TZlibTransport.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/layered/TFastFramedTransport.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/layered/TFramedTransport.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/layered/TLayeredTransport.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/DataFrameHeaderReader.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/DataFrameReader.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/DataFrameWriter.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/FixedSizeHeaderReader.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/FrameHeaderReader.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/FrameReader.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/FrameWriter.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/NegotiationStatus.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/NonblockingSaslHandler.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/SaslNegotiationFrameReader.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/SaslNegotiationFrameWriter.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/SaslNegotiationHeaderReader.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/SaslPeer.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/ServerSaslPeer.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/TBaseSaslProcessorFactory.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/TInvalidSaslFrameException.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/TSaslNegotiationException.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/TSaslProcessorFactory.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/TSaslServerDefinition.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/TSaslServerFactory.java create mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/utils/StringUtils.java create mode 100644 hbase-shaded-thrift/src/main/patches/HBASE-30194-01-servlet-javax.patch create mode 100644 hbase-shaded-thrift/src/main/patches/HBASE-30194-02-httpclient4.patch create mode 100644 hbase-shaded-thrift/src/main/resources/META-INF/LICENSE.txt create mode 100644 hbase-shaded-thrift/src/main/resources/META-INF/NOTICE.txt diff --git a/hbase-shaded-thrift/pom.xml b/hbase-shaded-thrift/pom.xml new file mode 100644 index 0000000..b4e7de2 --- /dev/null +++ b/hbase-shaded-thrift/pom.xml @@ -0,0 +1,228 @@ + + + + 4.0.0 + + org.apache.hbase.thirdparty + hbase-thirdparty + ${revision} + .. + + hbase-shaded-thrift + Apache HBase Patched and Relocated (Shaded) Thrift + Pulls down libthrift, patches it, compiles, and then relocates/shades. + + + ${hbase.unsafe.and.protobuf.java.version} + ${java.release.version} + ${java.release.version} + 0.23.0 + + + + + org.slf4j + slf4j-api + 1.7.36 + provided + + + javax.servlet + javax.servlet-api + 3.1.0 + provided + + + org.apache.httpcomponents + httpclient + 4.5.13 + provided + + + org.apache.httpcomponents + httpcore + 4.4.13 + provided + + + org.apache.commons + commons-lang3 + 3.12.0 + provided + + + + + + + maven-clean-plugin + + + + ${basedir}/src/main/java + + **/** + + false + + + ${basedir}/src/main/resources + + **/** + + false + + + + + + pre-generate-sources + + clean + + generate-sources + + + + + + org.apache.maven.plugins + maven-dependency-plugin + + + unpack + + unpack + + generate-sources + + + + + org.apache.thrift + libthrift + ${thrift.source.version} + sources + jar + true + ${basedir}/src/main/java + **/** + + + + org.apache.thrift + libthrift + ${thrift.source.version} + jar + true + ${basedir}/src/main/resources + META-INF/LICENSE.txt,META-INF/NOTICE.txt + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + org.apache.maven.plugins + maven-patch-plugin + + ${basedir} + false + + + + patch + + apply + + process-sources + + 1 + ${basedir}/src/main/patches + ${project.build.directory}/patches-applied.txt + true + + + + + + org.apache.maven.plugins + maven-source-plugin + + + attach-sources + + jar-no-fork + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + + shade + + package + + true + true + + + org.apache.thrift + ${rename.offset}.org.apache.thrift + + + + + + + + + diff --git a/hbase-shaded-thrift/src/main/appended-resources/META-INF/LICENSE b/hbase-shaded-thrift/src/main/appended-resources/META-INF/LICENSE new file mode 100644 index 0000000..8c1cee5 --- /dev/null +++ b/hbase-shaded-thrift/src/main/appended-resources/META-INF/LICENSE @@ -0,0 +1,19 @@ + +--- + +This project incorporates portions of the 'Apache Thrift' project (libthrift), +which is published under the Apache License, Version 2.0. + + Copyright (C) 2006 - 2024, The Apache Software Foundation + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/hbase-shaded-thrift/src/main/appended-resources/META-INF/NOTICE b/hbase-shaded-thrift/src/main/appended-resources/META-INF/NOTICE new file mode 100644 index 0000000..11ae5f0 --- /dev/null +++ b/hbase-shaded-thrift/src/main/appended-resources/META-INF/NOTICE @@ -0,0 +1,8 @@ + +--- + +Apache Thrift +Copyright (C) 2006 - 2024, The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). diff --git a/hbase-shaded-thrift/src/main/java/META-INF/MANIFEST.MF b/hbase-shaded-thrift/src/main/java/META-INF/MANIFEST.MF new file mode 100644 index 0000000..58630c0 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/META-INF/MANIFEST.MF @@ -0,0 +1,2 @@ +Manifest-Version: 1.0 + diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/AsyncProcessFunction.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/AsyncProcessFunction.java new file mode 100644 index 0000000..4e65ae6 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/AsyncProcessFunction.java @@ -0,0 +1,64 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.thrift; + +import org.apache.thrift.async.AsyncMethodCallback; +import org.apache.thrift.protocol.TMessage; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.server.AbstractNonblockingServer; + +public abstract class AsyncProcessFunction { + final String methodName; + + public AsyncProcessFunction(String methodName) { + this.methodName = methodName; + } + + public abstract boolean isOneway(); + + public abstract void start(I iface, T args, AsyncMethodCallback resultHandler) + throws TException; + + public abstract T getEmptyArgsInstance(); + + public abstract A getEmptyResultInstance(); + + public abstract AsyncMethodCallback getResultHandler( + final AbstractNonblockingServer.AsyncFrameBuffer fb, int seqid); + + public String getMethodName() { + return methodName; + } + + public void sendResponse( + final AbstractNonblockingServer.AsyncFrameBuffer fb, + final TSerializable result, + final byte type, + final int seqid) + throws TException { + TProtocol oprot = fb.getOutputProtocol(); + + oprot.writeMessageBegin(new TMessage(getMethodName(), type, seqid)); + result.write(oprot); + oprot.writeMessageEnd(); + oprot.getTransport().flush(); + + fb.responseReady(); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/EncodingUtils.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/EncodingUtils.java new file mode 100644 index 0000000..d4bc7ac --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/EncodingUtils.java @@ -0,0 +1,148 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift; + +/** Utility methods for use when encoding/decoding raw data as byte arrays. */ +public class EncodingUtils { + + /** + * Encode integer as a series of 4 bytes into buf starting at position 0 + * within that buffer. + * + * @param integer The integer to encode. + * @param buf The buffer to write to. + */ + public static final void encodeBigEndian(final int integer, final byte[] buf) { + encodeBigEndian(integer, buf, 0); + } + + /** + * Encode integer as a series of 4 bytes into buf starting at position + * offset. + * + * @param integer The integer to encode. + * @param buf The buffer to write to. + * @param offset The offset within buf to start the encoding. + */ + public static final void encodeBigEndian(final int integer, final byte[] buf, int offset) { + buf[offset] = (byte) (0xff & (integer >> 24)); + buf[offset + 1] = (byte) (0xff & (integer >> 16)); + buf[offset + 2] = (byte) (0xff & (integer >> 8)); + buf[offset + 3] = (byte) (0xff & (integer)); + } + + /** + * Decode a series of 4 bytes from buf, starting at position 0, and interpret them as + * an integer. + * + * @param buf The buffer to read from. + * @return An integer, as read from the buffer. + */ + public static final int decodeBigEndian(final byte[] buf) { + return decodeBigEndian(buf, 0); + } + + /** + * Decode a series of 4 bytes from buf, start at offset, and interpret + * them as an integer. + * + * @param buf The buffer to read from. + * @param offset The offset with buf to start the decoding. + * @return An integer, as read from the buffer. + */ + public static final int decodeBigEndian(final byte[] buf, int offset) { + return ((buf[offset] & 0xff) << 24) + | ((buf[offset + 1] & 0xff) << 16) + | ((buf[offset + 2] & 0xff) << 8) + | ((buf[offset + 3] & 0xff)); + } + + /** + * Bitfield utilities. Returns true if the bit at position is set in v. + * + * @param v the value whose bit is to be checked. + * @param position the 0 based bit number indicating the bit to check. + * @return true if the bit at position is set in v. + */ + public static final boolean testBit(byte v, int position) { + return testBit((int) v, position); + } + + public static final boolean testBit(short v, int position) { + return testBit((int) v, position); + } + + public static final boolean testBit(int v, int position) { + return (v & (1 << position)) != 0; + } + + public static final boolean testBit(long v, int position) { + return (v & (1L << position)) != 0L; + } + + /** + * Returns v, with the bit at position set to zero. + * + * @param v the value whose bit is to be cleared. + * @param position the 0 based bit number indicating the bit to clear. + * @return v, with the bit at position set to zero. + */ + public static final byte clearBit(byte v, int position) { + return (byte) clearBit((int) v, position); + } + + public static final short clearBit(short v, int position) { + return (short) clearBit((int) v, position); + } + + public static final int clearBit(int v, int position) { + return v & ~(1 << position); + } + + public static final long clearBit(long v, int position) { + return v & ~(1L << position); + } + + /** + * Returns v, with the bit at position set to 1 or 0 depending on value. + * + * @param v the value whose bit is to be set. + * @param position the 0 based bit number indicating the bit to set. + * @param value if true, the given bit is set to 1; otherwise it is set to 0. + * @return v, with the bit at position set to 0 (if value is false) or 1 (if value is true). + */ + public static final byte setBit(byte v, int position, boolean value) { + return (byte) setBit((int) v, position, value); + } + + public static final short setBit(short v, int position, boolean value) { + return (short) setBit((int) v, position, value); + } + + public static final int setBit(int v, int position, boolean value) { + if (value) return v | (1 << position); + else return clearBit(v, position); + } + + public static final long setBit(long v, int position, boolean value) { + if (value) return v | (1L << position); + else return clearBit(v, position); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/Option.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/Option.java new file mode 100644 index 0000000..601bcc1 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/Option.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift; + +import java.util.Optional; + +/** Implementation of the Option type pattern */ +public abstract class Option { + + @SuppressWarnings("rawtypes") + private static final Option NONE = new None(); + + /** + * Whether the Option is defined or not + * + * @return true if the Option is defined (of type Some) false if the Option is not defined (of + * type None) + */ + public abstract boolean isDefined(); + + /** + * Get the value of the Option (if it is defined) + * + * @return the value + * @throws IllegalStateException if called on a None + */ + public abstract T get(); + + /** + * Get the contained value (if defined) or else return a default value + * + * @param other what to return if the value is not defined (a None) + * @return either the value, or other if the value is not defined + */ + public T or(T other) { + if (isDefined()) { + return get(); + } else { + return other; + } + } + + /** + * Turn this Option into Java 8 Optional type + * + * @return Java 8+ Optional Type + */ + public Optional toOptional() { + if (isDefined()) { + return Optional.of(get()); + } else { + return Optional.empty(); + } + } + + /** The None type, representing an absent value (instead of "null") */ + public static class None extends Option { + public boolean isDefined() { + return false; + } + + public T get() { + throw new IllegalStateException("Cannot call get() on None"); + } + + public String toString() { + return "None"; + } + } + + /** + * The Some type, representing an existence of some value + * + * @param The type of value + */ + public static class Some extends Option { + private final T value; + + public Some(T value) { + this.value = value; + } + + public boolean isDefined() { + return true; + } + + public T get() { + return value; + } + + public String toString() { + return "Some(" + value + ")"; + } + } + + /** + * Wraps value in an Option type, depending on whether or not value is null + * + * @param value the value to wrap in Option + * @param the type of value + * @return Some(value) if value is not null, None if value is null + */ + public static Option fromNullable(T value) { + if (value != null) { + return some(value); + } else { + return none(); + } + } + + /** + * Wrap value in a Some type (NB! value must not be null!) + * + * @param value the value to wrap. + * @param the type of value + * @return a new Some(value) + */ + public static Some some(T value) { + return new Some(value); + } + + @SuppressWarnings("unchecked") + public static None none() { + return (None) NONE; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/ProcessFunction.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/ProcessFunction.java new file mode 100644 index 0000000..1f406cc --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/ProcessFunction.java @@ -0,0 +1,96 @@ +package org.apache.thrift; + +import org.apache.thrift.protocol.TMessage; +import org.apache.thrift.protocol.TMessageType; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.protocol.TProtocolException; +import org.apache.thrift.transport.TTransportException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class ProcessFunction { + private final String methodName; + + private static final Logger LOGGER = LoggerFactory.getLogger(ProcessFunction.class.getName()); + + public ProcessFunction(String methodName) { + this.methodName = methodName; + } + + public final void process(int seqid, TProtocol iprot, TProtocol oprot, I iface) + throws TException { + T args = getEmptyArgsInstance(); + try { + args.read(iprot); + } catch (TProtocolException e) { + iprot.readMessageEnd(); + TApplicationException x = + new TApplicationException(TApplicationException.PROTOCOL_ERROR, e.getMessage()); + oprot.writeMessageBegin(new TMessage(getMethodName(), TMessageType.EXCEPTION, seqid)); + x.write(oprot); + oprot.writeMessageEnd(); + oprot.getTransport().flush(); + return; + } + iprot.readMessageEnd(); + TSerializable result = null; + byte msgType = TMessageType.REPLY; + + try { + result = getResult(iface, args); + } catch (TTransportException ex) { + LOGGER.error("Transport error while processing " + getMethodName(), ex); + throw ex; + } catch (TApplicationException ex) { + LOGGER.error("Internal application error processing " + getMethodName(), ex); + result = ex; + msgType = TMessageType.EXCEPTION; + } catch (Exception ex) { + LOGGER.error("Internal error processing " + getMethodName(), ex); + if (rethrowUnhandledExceptions()) throw new RuntimeException(ex.getMessage(), ex); + if (!isOneway()) { + result = + new TApplicationException( + TApplicationException.INTERNAL_ERROR, + "Internal error processing " + getMethodName()); + msgType = TMessageType.EXCEPTION; + } + } + + if (!isOneway()) { + oprot.writeMessageBegin(new TMessage(getMethodName(), msgType, seqid)); + result.write(oprot); + oprot.writeMessageEnd(); + oprot.getTransport().flush(); + } + } + + private void handleException(int seqid, TProtocol oprot) throws TException { + if (!isOneway()) { + TApplicationException x = + new TApplicationException( + TApplicationException.INTERNAL_ERROR, "Internal error processing " + getMethodName()); + oprot.writeMessageBegin(new TMessage(getMethodName(), TMessageType.EXCEPTION, seqid)); + x.write(oprot); + oprot.writeMessageEnd(); + oprot.getTransport().flush(); + } + } + + protected boolean rethrowUnhandledExceptions() { + return false; + } + + public abstract boolean isOneway(); + + public abstract A getResult(I iface, T args) throws TException; + + public abstract T getEmptyArgsInstance(); + + /** Returns null when this is a oneWay function. */ + public abstract A getEmptyResultInstance(); + + public String getMethodName() { + return methodName; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/TApplicationException.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TApplicationException.java new file mode 100644 index 0000000..daf1072 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TApplicationException.java @@ -0,0 +1,147 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift; + +import org.apache.thrift.protocol.TField; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.protocol.TProtocolUtil; +import org.apache.thrift.protocol.TStruct; +import org.apache.thrift.protocol.TType; + +/** Application level exception */ +public class TApplicationException extends TException implements TSerializable { + + private static final TStruct TAPPLICATION_EXCEPTION_STRUCT = new TStruct("TApplicationException"); + private static final TField MESSAGE_FIELD = new TField("message", TType.STRING, (short) 1); + private static final TField TYPE_FIELD = new TField("type", TType.I32, (short) 2); + + private static final long serialVersionUID = 1L; + + public static final int UNKNOWN = 0; + public static final int UNKNOWN_METHOD = 1; + public static final int INVALID_MESSAGE_TYPE = 2; + public static final int WRONG_METHOD_NAME = 3; + public static final int BAD_SEQUENCE_ID = 4; + public static final int MISSING_RESULT = 5; + public static final int INTERNAL_ERROR = 6; + public static final int PROTOCOL_ERROR = 7; + public static final int INVALID_TRANSFORM = 8; + public static final int INVALID_PROTOCOL = 9; + public static final int UNSUPPORTED_CLIENT_TYPE = 10; + + protected int type_ = UNKNOWN; + private String message_ = null; + + public TApplicationException() { + super(); + } + + public TApplicationException(int type) { + super(); + type_ = type; + } + + public TApplicationException(int type, String message) { + super(message); + type_ = type; + } + + public TApplicationException(String message) { + super(message); + } + + public int getType() { + return type_; + } + + @Override + public String getMessage() { + if (message_ == null) { + return super.getMessage(); + } else { + return message_; + } + } + + public void read(TProtocol iprot) throws TException { + TField field; + iprot.readStructBegin(); + + String message = null; + int type = UNKNOWN; + + while (true) { + field = iprot.readFieldBegin(); + if (field.type == TType.STOP) { + break; + } + switch (field.id) { + case 1: + if (field.type == TType.STRING) { + message = iprot.readString(); + } else { + TProtocolUtil.skip(iprot, field.type); + } + break; + case 2: + if (field.type == TType.I32) { + type = iprot.readI32(); + } else { + TProtocolUtil.skip(iprot, field.type); + } + break; + default: + TProtocolUtil.skip(iprot, field.type); + break; + } + iprot.readFieldEnd(); + } + iprot.readStructEnd(); + type_ = type; + message_ = message; + } + + /** + * Convenience factory method for constructing a TApplicationException given a TProtocol input + * + * @param iprot protocol from which an instance of TApplicationException is read. + * @return an instance of TApplicationException read from iprot. + * @throws TException if there is an error reading from iprot. + */ + public static TApplicationException readFrom(TProtocol iprot) throws TException { + TApplicationException result = new TApplicationException(); + result.read(iprot); + return result; + } + + public void write(TProtocol oprot) throws TException { + oprot.writeStructBegin(TAPPLICATION_EXCEPTION_STRUCT); + if (getMessage() != null) { + oprot.writeFieldBegin(MESSAGE_FIELD); + oprot.writeString(getMessage()); + oprot.writeFieldEnd(); + } + oprot.writeFieldBegin(TYPE_FIELD); + oprot.writeI32(type_); + oprot.writeFieldEnd(); + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/TAsyncProcessor.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TAsyncProcessor.java new file mode 100644 index 0000000..3d488ba --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TAsyncProcessor.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.thrift; + +import org.apache.thrift.server.AbstractNonblockingServer.AsyncFrameBuffer; + +public interface TAsyncProcessor { + /** + * Process a single frame. + * + *

Note: Implementations must call fb.responseReady() once processing is complete + * + * @param fb the frame buffer to process. + * @throws TException if the frame cannot be processed + */ + void process(final AsyncFrameBuffer fb) throws TException; +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/TBase.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TBase.java new file mode 100644 index 0000000..67c6ee8 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TBase.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift; + +import java.io.Serializable; + +/** Generic base interface for generated Thrift objects. */ +public interface TBase, F extends TFieldIdEnum> + extends Comparable, TSerializable, Serializable { + + /** + * Get the F instance that corresponds to fieldId. + * + * @param fieldId the ID of the requested field. + * @return F instance that corresponds to fieldId. + */ + F fieldForId(int fieldId); + + /** + * Check if a field is currently set or unset. + * + * @param field the field to check. + * @return true if the field is set, false otherwise. + */ + boolean isSet(F field); + + /** + * Get a field's value by field variable. Primitive types will be wrapped in the appropriate + * "boxed" types. + * + * @param field the field whose value is requested. + * @return the value of the requested field. + */ + Object getFieldValue(F field); + + /** + * Set a field's value by field variable. Primitive types must be "boxed" in the appropriate + * object wrapper type. + * + * @param field the field whose value is to be set. + * @param value the value to be assigned to field. + */ + void setFieldValue(F field, Object value); + + /** + * Performs a deep copy of this instance and returns the copy. + * + * @return a deep copy of this instance. + */ + T deepCopy(); + + /** + * Return to the state of having just been initialized, as though you had just called the default + * constructor. + */ + void clear(); +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/TBaseAsyncProcessor.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TBaseAsyncProcessor.java new file mode 100644 index 0000000..eedb8cb --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TBaseAsyncProcessor.java @@ -0,0 +1,115 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.thrift; + +import java.util.Collections; +import java.util.Map; +import org.apache.thrift.async.AsyncMethodCallback; +import org.apache.thrift.protocol.*; +import org.apache.thrift.server.AbstractNonblockingServer.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TBaseAsyncProcessor implements TAsyncProcessor, TProcessor { + protected final Logger LOGGER = LoggerFactory.getLogger(getClass().getName()); + + final I iface; + final Map> processMap; + + public TBaseAsyncProcessor( + I iface, + Map> processMap) { + this.iface = iface; + this.processMap = processMap; + } + + public Map> + getProcessMapView() { + return Collections.unmodifiableMap(processMap); + } + + public void process(final AsyncFrameBuffer fb) throws TException { + + final TProtocol in = fb.getInputProtocol(); + final TProtocol out = fb.getOutputProtocol(); + + // Find processing function + final TMessage msg = in.readMessageBegin(); + AsyncProcessFunction fn = processMap.get(msg.name); + if (fn == null) { + TProtocolUtil.skip(in, TType.STRUCT); + in.readMessageEnd(); + + TApplicationException x = + new TApplicationException( + TApplicationException.UNKNOWN_METHOD, "Invalid method name: '" + msg.name + "'"); + LOGGER.debug("Invalid method name", x); + + // this means it is a two-way request, so we can send a reply + if (msg.type == TMessageType.CALL) { + out.writeMessageBegin(new TMessage(msg.name, TMessageType.EXCEPTION, msg.seqid)); + x.write(out); + out.writeMessageEnd(); + out.getTransport().flush(); + } + fb.responseReady(); + return; + } + + // Get Args + TBase args = fn.getEmptyArgsInstance(); + + try { + args.read(in); + } catch (TProtocolException e) { + in.readMessageEnd(); + + TApplicationException x = + new TApplicationException(TApplicationException.PROTOCOL_ERROR, e.getMessage()); + LOGGER.debug("Could not retrieve function arguments", x); + + if (!fn.isOneway()) { + out.writeMessageBegin(new TMessage(msg.name, TMessageType.EXCEPTION, msg.seqid)); + x.write(out); + out.writeMessageEnd(); + out.getTransport().flush(); + } + fb.responseReady(); + return; + } + in.readMessageEnd(); + + if (fn.isOneway()) { + fb.responseReady(); + } + + // start off processing function + AsyncMethodCallback resultHandler = fn.getResultHandler(fb, msg.seqid); + try { + fn.start(iface, args, resultHandler); + } catch (Exception e) { + LOGGER.debug("Exception handling function", e); + resultHandler.onError(e); + } + return; + } + + @Override + public void process(TProtocol in, TProtocol out) throws TException {} +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/TBaseHelper.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TBaseHelper.java new file mode 100644 index 0000000..be7a9f3 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TBaseHelper.java @@ -0,0 +1,298 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.thrift; + +import java.io.Serializable; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; + +public final class TBaseHelper { + + private TBaseHelper() {} + + private static final Comparator comparator = new NestedStructureComparator(); + + public static int compareTo(Object o1, Object o2) { + if (o1 instanceof Comparable) { + return compareTo((Comparable) o1, (Comparable) o2); + } else if (o1 instanceof List) { + return compareTo((List) o1, (List) o2); + } else if (o1 instanceof Set) { + return compareTo((Set) o1, (Set) o2); + } else if (o1 instanceof Map) { + return compareTo((Map) o1, (Map) o2); + } else if (o1 instanceof byte[]) { + return compareTo((byte[]) o1, (byte[]) o2); + } else { + throw new IllegalArgumentException("Cannot compare objects of type " + o1.getClass()); + } + } + + public static int compareTo(boolean a, boolean b) { + return Boolean.compare(a, b); + } + + public static int compareTo(byte a, byte b) { + return Byte.compare(a, b); + } + + public static int compareTo(short a, short b) { + return Short.compare(a, b); + } + + public static int compareTo(int a, int b) { + return Integer.compare(a, b); + } + + public static int compareTo(long a, long b) { + return Long.compare(a, b); + } + + public static int compareTo(double a, double b) { + return Double.compare(a, b); + } + + public static int compareTo(String a, String b) { + return a.compareTo(b); + } + + public static int compareTo(byte[] a, byte[] b) { + int compare = compareTo(a.length, b.length); + if (compare == 0) { + for (int i = 0; i < a.length; i++) { + compare = compareTo(a[i], b[i]); + if (compare != 0) { + break; + } + } + } + return compare; + } + + public static int compareTo(Comparable a, Comparable b) { + return a.compareTo(b); + } + + public static int compareTo(List a, List b) { + int compare = compareTo(a.size(), b.size()); + if (compare == 0) { + for (int i = 0; i < a.size(); i++) { + compare = comparator.compare(a.get(i), b.get(i)); + if (compare != 0) { + break; + } + } + } + return compare; + } + + public static int compareTo(Set a, Set b) { + int compare = compareTo(a.size(), b.size()); + if (compare == 0) { + ArrayList sortedA = new ArrayList(a); + ArrayList sortedB = new ArrayList(b); + + Collections.sort(sortedA, comparator); + Collections.sort(sortedB, comparator); + + Iterator iterA = sortedA.iterator(); + Iterator iterB = sortedB.iterator(); + + // Compare each item. + while (iterA.hasNext() && iterB.hasNext()) { + compare = comparator.compare(iterA.next(), iterB.next()); + if (compare != 0) { + break; + } + } + } + return compare; + } + + public static int compareTo(Map a, Map b) { + int lastComparison = compareTo(a.size(), b.size()); + if (lastComparison != 0) { + return lastComparison; + } + + // Sort a and b so we can compare them. + SortedMap sortedA = new TreeMap(comparator); + sortedA.putAll(a); + Iterator iterA = sortedA.entrySet().iterator(); + SortedMap sortedB = new TreeMap(comparator); + sortedB.putAll(b); + Iterator iterB = sortedB.entrySet().iterator(); + + // Compare each item. + while (iterA.hasNext() && iterB.hasNext()) { + Map.Entry entryA = iterA.next(); + Map.Entry entryB = iterB.next(); + lastComparison = comparator.compare(entryA.getKey(), entryB.getKey()); + if (lastComparison != 0) { + return lastComparison; + } + lastComparison = comparator.compare(entryA.getValue(), entryB.getValue()); + if (lastComparison != 0) { + return lastComparison; + } + } + + return 0; + } + + /** Comparator to compare items inside a structure (e.g. a list, set, or map). */ + private static class NestedStructureComparator implements Comparator, Serializable { + public int compare(Object oA, Object oB) { + if (oA == null && oB == null) { + return 0; + } else if (oA == null) { + return -1; + } else if (oB == null) { + return 1; + } else if (oA instanceof List) { + return compareTo((List) oA, (List) oB); + } else if (oA instanceof Set) { + return compareTo((Set) oA, (Set) oB); + } else if (oA instanceof Map) { + return compareTo((Map) oA, (Map) oB); + } else if (oA instanceof byte[]) { + return compareTo((byte[]) oA, (byte[]) oB); + } else { + return compareTo((Comparable) oA, (Comparable) oB); + } + } + } + + public static void toString(Collection bbs, StringBuilder sb) { + Iterator it = bbs.iterator(); + if (!it.hasNext()) { + sb.append("[]"); + } else { + sb.append("["); + while (true) { + ByteBuffer bb = it.next(); + org.apache.thrift.TBaseHelper.toString(bb, sb); + if (!it.hasNext()) { + sb.append("]"); + return; + } else { + sb.append(", "); + } + } + } + } + + public static void toString(ByteBuffer bb, StringBuilder sb) { + byte[] buf = bb.array(); + + int arrayOffset = bb.arrayOffset(); + int offset = arrayOffset + bb.position(); + int origLimit = arrayOffset + bb.limit(); + int limit = (origLimit - offset > 128) ? offset + 128 : origLimit; + + for (int i = offset; i < limit; i++) { + if (i > offset) { + sb.append(" "); + } + sb.append(paddedByteString(buf[i])); + } + if (origLimit != limit) { + sb.append("..."); + } + } + + public static String paddedByteString(byte b) { + int extended = (b | 0x100) & 0x1ff; + return Integer.toHexString(extended).toUpperCase().substring(1); + } + + public static byte[] byteBufferToByteArray(ByteBuffer byteBuffer) { + if (wrapsFullArray(byteBuffer)) { + return byteBuffer.array(); + } + byte[] target = new byte[byteBuffer.remaining()]; + byteBufferToByteArray(byteBuffer, target, 0); + return target; + } + + public static boolean wrapsFullArray(ByteBuffer byteBuffer) { + return byteBuffer.hasArray() + && byteBuffer.position() == 0 + && byteBuffer.arrayOffset() == 0 + && byteBuffer.remaining() == byteBuffer.capacity(); + } + + public static int byteBufferToByteArray(ByteBuffer byteBuffer, byte[] target, int offset) { + int remaining = byteBuffer.remaining(); + System.arraycopy( + byteBuffer.array(), + byteBuffer.arrayOffset() + byteBuffer.position(), + target, + offset, + remaining); + return remaining; + } + + public static ByteBuffer rightSize(ByteBuffer in) { + if (in == null) { + return null; + } + if (wrapsFullArray(in)) { + return in; + } + return ByteBuffer.wrap(byteBufferToByteArray(in)); + } + + public static ByteBuffer copyBinary(final ByteBuffer orig) { + if (orig == null) { + return null; + } + ByteBuffer copy = ByteBuffer.wrap(new byte[orig.remaining()]); + if (orig.hasArray()) { + System.arraycopy( + orig.array(), orig.arrayOffset() + orig.position(), copy.array(), 0, orig.remaining()); + } else { + orig.slice().get(copy.array()); + } + + return copy; + } + + public static byte[] copyBinary(final byte[] orig) { + return (orig == null) ? null : Arrays.copyOf(orig, orig.length); + } + + public static int hashCode(long value) { + return Long.hashCode(value); + } + + public static int hashCode(double value) { + return Double.hashCode(value); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/TBaseProcessor.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TBaseProcessor.java new file mode 100644 index 0000000..2cd805f --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TBaseProcessor.java @@ -0,0 +1,44 @@ +package org.apache.thrift; + +import java.util.Collections; +import java.util.Map; +import org.apache.thrift.protocol.TMessage; +import org.apache.thrift.protocol.TMessageType; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.protocol.TProtocolUtil; +import org.apache.thrift.protocol.TType; + +public abstract class TBaseProcessor implements TProcessor { + private final I iface; + private final Map> processMap; + + protected TBaseProcessor( + I iface, + Map> processFunctionMap) { + this.iface = iface; + this.processMap = processFunctionMap; + } + + public Map> getProcessMapView() { + return Collections.unmodifiableMap(processMap); + } + + @Override + public void process(TProtocol in, TProtocol out) throws TException { + TMessage msg = in.readMessageBegin(); + ProcessFunction fn = processMap.get(msg.name); + if (fn == null) { + TProtocolUtil.skip(in, TType.STRUCT); + in.readMessageEnd(); + TApplicationException x = + new TApplicationException( + TApplicationException.UNKNOWN_METHOD, "Invalid method name: '" + msg.name + "'"); + out.writeMessageBegin(new TMessage(msg.name, TMessageType.EXCEPTION, msg.seqid)); + x.write(out); + out.writeMessageEnd(); + out.getTransport().flush(); + } else { + fn.process(msg.seqid, in, out, iface); + } + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/TByteArrayOutputStream.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TByteArrayOutputStream.java new file mode 100644 index 0000000..69bb5ba --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TByteArrayOutputStream.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift; + +import java.io.ByteArrayOutputStream; +import java.nio.charset.Charset; + +/** Class that allows access to the underlying buf without doing deep copies on it. */ +public class TByteArrayOutputStream extends ByteArrayOutputStream { + + private final int initialSize; + + public TByteArrayOutputStream(int size) { + super(size); + this.initialSize = size; + } + + public TByteArrayOutputStream() { + this(32); + } + + public byte[] get() { + return buf; + } + + public void reset() { + super.reset(); + if (buf.length > initialSize) { + buf = new byte[initialSize]; + } + } + + public int len() { + return count; + } + + public String toString(Charset charset) { + return new String(buf, 0, count, charset); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/TConfiguration.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TConfiguration.java new file mode 100644 index 0000000..0391d5b --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TConfiguration.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.thrift; + +public class TConfiguration { + public static final int DEFAULT_MAX_MESSAGE_SIZE = 100 * 1024 * 1024; + public static final int DEFAULT_MAX_FRAME_SIZE = + 16384000; // this value is used consistently across all Thrift libraries + public static final int DEFAULT_RECURSION_DEPTH = 64; + + private int maxMessageSize; + private int maxFrameSize; + private int recursionLimit; + + public TConfiguration() { + this(DEFAULT_MAX_MESSAGE_SIZE, DEFAULT_MAX_FRAME_SIZE, DEFAULT_RECURSION_DEPTH); + } + + public TConfiguration(int maxMessageSize, int maxFrameSize, int recursionLimit) { + this.maxFrameSize = maxFrameSize; + this.maxMessageSize = maxMessageSize; + this.recursionLimit = recursionLimit; + } + + public int getMaxMessageSize() { + return maxMessageSize; + } + + public int getMaxFrameSize() { + return maxFrameSize; + } + + public int getRecursionLimit() { + return recursionLimit; + } + + public void setMaxMessageSize(int maxMessageSize) { + this.maxMessageSize = maxMessageSize; + } + + public void setMaxFrameSize(int maxFrameSize) { + this.maxFrameSize = maxFrameSize; + } + + public void setRecursionLimit(int recursionLimit) { + this.recursionLimit = recursionLimit; + } + + public static final TConfiguration DEFAULT = new Builder().build(); + + public static TConfiguration.Builder custom() { + return new Builder(); + } + + public static class Builder { + private int maxMessageSize; + private int maxFrameSize; + private int recursionLimit; + + Builder() { + super(); + this.maxFrameSize = DEFAULT_MAX_FRAME_SIZE; + this.maxMessageSize = DEFAULT_MAX_MESSAGE_SIZE; + this.recursionLimit = DEFAULT_RECURSION_DEPTH; + } + + public Builder setMaxMessageSize(int maxMessageSize) { + this.maxMessageSize = maxMessageSize; + return this; + } + + public Builder setMaxFrameSize(int maxFrameSize) { + this.maxFrameSize = maxFrameSize; + return this; + } + + public Builder setRecursionLimit(int recursionLimit) { + this.recursionLimit = recursionLimit; + return this; + } + + public TConfiguration build() { + return new TConfiguration(maxMessageSize, maxFrameSize, recursionLimit); + } + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/TDeserializer.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TDeserializer.java new file mode 100644 index 0000000..1cdc548 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TDeserializer.java @@ -0,0 +1,730 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift; + +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.util.Collection; +import org.apache.thrift.meta_data.EnumMetaData; +import org.apache.thrift.meta_data.StructMetaData; +import org.apache.thrift.partial.TFieldData; +import org.apache.thrift.partial.ThriftFieldValueProcessor; +import org.apache.thrift.partial.ThriftMetadata; +import org.apache.thrift.partial.ThriftStructProcessor; +import org.apache.thrift.partial.Validate; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.protocol.TField; +import org.apache.thrift.protocol.TList; +import org.apache.thrift.protocol.TMap; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.protocol.TProtocolFactory; +import org.apache.thrift.protocol.TProtocolUtil; +import org.apache.thrift.protocol.TSet; +import org.apache.thrift.protocol.TType; +import org.apache.thrift.transport.TMemoryInputTransport; +import org.apache.thrift.transport.TTransportException; + +/** Generic utility for easily deserializing objects from a byte array or Java String. */ +public class TDeserializer { + private final TProtocol protocol_; + private final TMemoryInputTransport trans_; + + // Metadata that describes fields to deserialize during partial deserialization. + private ThriftMetadata.ThriftStruct metadata_ = null; + + // Processor that handles deserialized field values during partial deserialization. + private ThriftFieldValueProcessor processor_ = null; + + /** + * Create a new TDeserializer that uses the TBinaryProtocol by default. + * + * @throws TTransportException if there an error initializing the underlying transport. + */ + public TDeserializer() throws TTransportException { + this(new TBinaryProtocol.Factory()); + } + + /** + * Create a new TDeserializer. It will use the TProtocol specified by the factory that is passed + * in. + * + * @param protocolFactory Factory to create a protocol + * @throws TTransportException if there an error initializing the underlying transport. + */ + public TDeserializer(TProtocolFactory protocolFactory) throws TTransportException { + trans_ = new TMemoryInputTransport(new TConfiguration()); + protocol_ = protocolFactory.getProtocol(trans_); + } + + /** + * Construct a new TDeserializer that supports partial deserialization that outputs instances of + * type controlled by the given {@code processor}. + * + * @param thriftClass a TBase derived class. + * @param fieldNames list of fields to deserialize. + * @param processor the Processor that handles deserialized field values. + * @param protocolFactory the Factory to create a protocol. + */ + public TDeserializer( + Class thriftClass, + Collection fieldNames, + ThriftFieldValueProcessor processor, + TProtocolFactory protocolFactory) + throws TTransportException { + this(protocolFactory); + + Validate.checkNotNull(thriftClass, "thriftClass"); + Validate.checkNotNull(fieldNames, "fieldNames"); + Validate.checkNotNull(processor, "processor"); + + metadata_ = ThriftMetadata.ThriftStruct.fromFieldNames(thriftClass, fieldNames); + processor_ = processor; + } + + /** + * Construct a new TDeserializer that supports partial deserialization that outputs {@code TBase} + * instances. + * + * @param thriftClass a TBase derived class. + * @param fieldNames list of fields to deserialize. + * @param protocolFactory the Factory to create a protocol. + */ + public TDeserializer( + Class thriftClass, + Collection fieldNames, + TProtocolFactory protocolFactory) + throws TTransportException { + this(thriftClass, fieldNames, new ThriftStructProcessor(), protocolFactory); + } + + /** + * Gets the metadata used for partial deserialization. + * + * @return the metadata used for partial deserialization. + */ + public ThriftMetadata.ThriftStruct getMetadata() { + return metadata_; + } + + /** + * Deserialize the Thrift object from a byte array. + * + * @param base The object to read into + * @param bytes The array to read from + * @throws TException if an error is encountered during deserialization. + */ + public void deserialize(TBase base, byte[] bytes) throws TException { + deserialize(base, bytes, 0, bytes.length); + } + + /** + * Deserialize the Thrift object from a byte array. + * + * @param base The object to read into + * @param bytes The array to read from + * @param offset The offset into {@code bytes} + * @param length The length to read from {@code bytes} + * @throws TException if an error is encountered during deserialization. + */ + public void deserialize(TBase base, byte[] bytes, int offset, int length) throws TException { + if (this.isPartialDeserializationMode()) { + this.partialDeserializeThriftObject(base, bytes, offset, length); + } else { + try { + trans_.reset(bytes, offset, length); + base.read(protocol_); + } finally { + trans_.clear(); + protocol_.reset(); + } + } + } + + /** + * Deserialize the Thrift object from a Java string, using a specified character set for decoding. + * + * @param base The object to read into + * @param data The string to read from + * @param charset Valid JVM charset + * @throws TException if an error is encountered during deserialization. + */ + public void deserialize(TBase base, String data, String charset) throws TException { + try { + deserialize(base, data.getBytes(charset)); + } catch (UnsupportedEncodingException uex) { + throw new TException("JVM DOES NOT SUPPORT ENCODING: " + charset); + } finally { + protocol_.reset(); + } + } + + /** + * Deserialize only a single Thrift object (addressed by recursively using field id) from a byte + * record. + * + * @param tb The object to read into + * @param bytes The serialized object to read from + * @param fieldIdPathFirst First of the FieldId's that define a path tb + * @param fieldIdPathRest The rest FieldId's that define a path tb + * @throws TException if an error is encountered during deserialization. + */ + public void partialDeserialize( + TBase tb, byte[] bytes, TFieldIdEnum fieldIdPathFirst, TFieldIdEnum... fieldIdPathRest) + throws TException { + try { + if (locateField(bytes, fieldIdPathFirst, fieldIdPathRest) != null) { + // if this line is reached, iprot will be positioned at the start of tb. + tb.read(protocol_); + } + } catch (Exception e) { + throw new TException(e); + } finally { + trans_.clear(); + protocol_.reset(); + } + } + + /** + * Deserialize only a boolean field (addressed by recursively using field id) from a byte record. + * + * @param bytes The serialized object to read from + * @param fieldIdPathFirst First of the FieldId's that define a path to a boolean field + * @param fieldIdPathRest The rest FieldId's that define a path to a boolean field + * @return the deserialized value. + * @throws TException if an error is encountered during deserialization. + */ + public Boolean partialDeserializeBool( + byte[] bytes, TFieldIdEnum fieldIdPathFirst, TFieldIdEnum... fieldIdPathRest) + throws TException { + return (Boolean) partialDeserializeField(TType.BOOL, bytes, fieldIdPathFirst, fieldIdPathRest); + } + + /** + * Deserialize only a byte field (addressed by recursively using field id) from a byte record. + * + * @param bytes The serialized object to read from + * @param fieldIdPathFirst First of the FieldId's that define a path to a byte field + * @param fieldIdPathRest The rest FieldId's that define a path to a byte field + * @return the deserialized value. + * @throws TException if an error is encountered during deserialization. + */ + public Byte partialDeserializeByte( + byte[] bytes, TFieldIdEnum fieldIdPathFirst, TFieldIdEnum... fieldIdPathRest) + throws TException { + return (Byte) partialDeserializeField(TType.BYTE, bytes, fieldIdPathFirst, fieldIdPathRest); + } + + /** + * Deserialize only a double field (addressed by recursively using field id) from a byte record. + * + * @param bytes The serialized object to read from + * @param fieldIdPathFirst First of the FieldId's that define a path to a double field + * @param fieldIdPathRest The rest FieldId's that define a path to a double field + * @return the deserialized value. + * @throws TException if an error is encountered during deserialization. + */ + public Double partialDeserializeDouble( + byte[] bytes, TFieldIdEnum fieldIdPathFirst, TFieldIdEnum... fieldIdPathRest) + throws TException { + return (Double) partialDeserializeField(TType.DOUBLE, bytes, fieldIdPathFirst, fieldIdPathRest); + } + + /** + * Deserialize only an i16 field (addressed by recursively using field id) from a byte record. + * + * @param bytes The serialized object to read from + * @param fieldIdPathFirst First of the FieldId's that define a path to an i16 field + * @param fieldIdPathRest The rest FieldId's that define a path to an i16 field + * @return the deserialized value. + * @throws TException if an error is encountered during deserialization. + */ + public Short partialDeserializeI16( + byte[] bytes, TFieldIdEnum fieldIdPathFirst, TFieldIdEnum... fieldIdPathRest) + throws TException { + return (Short) partialDeserializeField(TType.I16, bytes, fieldIdPathFirst, fieldIdPathRest); + } + + /** + * Deserialize only an i32 field (addressed by recursively using field id) from a byte record. + * + * @param bytes The serialized object to read from + * @param fieldIdPathFirst First of the FieldId's that define a path to an i32 field + * @param fieldIdPathRest The rest FieldId's that define a path to an i32 field + * @return the deserialized value. + * @throws TException if an error is encountered during deserialization. + */ + public Integer partialDeserializeI32( + byte[] bytes, TFieldIdEnum fieldIdPathFirst, TFieldIdEnum... fieldIdPathRest) + throws TException { + return (Integer) partialDeserializeField(TType.I32, bytes, fieldIdPathFirst, fieldIdPathRest); + } + + /** + * Deserialize only an i64 field (addressed by recursively using field id) from a byte record. + * + * @param bytes The serialized object to read from + * @param fieldIdPathFirst First of the FieldId's that define a path to an i64 field + * @param fieldIdPathRest The rest FieldId's that define a path to an i64 field + * @return the deserialized value. + * @throws TException if an error is encountered during deserialization. + */ + public Long partialDeserializeI64( + byte[] bytes, TFieldIdEnum fieldIdPathFirst, TFieldIdEnum... fieldIdPathRest) + throws TException { + return (Long) partialDeserializeField(TType.I64, bytes, fieldIdPathFirst, fieldIdPathRest); + } + + /** + * Deserialize only a string field (addressed by recursively using field id) from a byte record. + * + * @param bytes The serialized object to read from + * @param fieldIdPathFirst First of the FieldId's that define a path to a string field + * @param fieldIdPathRest The rest FieldId's that define a path to a string field + * @return the deserialized value. + * @throws TException if an error is encountered during deserialization. + */ + public String partialDeserializeString( + byte[] bytes, TFieldIdEnum fieldIdPathFirst, TFieldIdEnum... fieldIdPathRest) + throws TException { + return (String) partialDeserializeField(TType.STRING, bytes, fieldIdPathFirst, fieldIdPathRest); + } + + /** + * Deserialize only a binary field (addressed by recursively using field id) from a byte record. + * + * @param bytes The serialized object to read from + * @param fieldIdPathFirst First of the FieldId's that define a path to a binary field + * @param fieldIdPathRest The rest FieldId's that define a path to a binary field + * @return the deserialized value. + * @throws TException if an error is encountered during deserialization. + */ + public ByteBuffer partialDeserializeByteArray( + byte[] bytes, TFieldIdEnum fieldIdPathFirst, TFieldIdEnum... fieldIdPathRest) + throws TException { + // TType does not have binary, so we use the arbitrary num 100 + return (ByteBuffer) + partialDeserializeField((byte) 100, bytes, fieldIdPathFirst, fieldIdPathRest); + } + + /** + * Deserialize only the id of the field set in a TUnion (addressed by recursively using field id) + * from a byte record. + * + * @param bytes The serialized object to read from + * @param fieldIdPathFirst First of the FieldId's that define a path to a TUnion + * @param fieldIdPathRest The rest FieldId's that define a path to a TUnion + * @return the deserialized value. + * @throws TException if an error is encountered during deserialization. + */ + public Short partialDeserializeSetFieldIdInUnion( + byte[] bytes, TFieldIdEnum fieldIdPathFirst, TFieldIdEnum... fieldIdPathRest) + throws TException { + try { + TField field = locateField(bytes, fieldIdPathFirst, fieldIdPathRest); + if (field != null) { + protocol_.readStructBegin(); // The Union + return protocol_.readFieldBegin().id; // The field set in the union + } + return null; + } catch (Exception e) { + throw new TException(e); + } finally { + trans_.clear(); + protocol_.reset(); + } + } + + private Object partialDeserializeField( + byte ttype, byte[] bytes, TFieldIdEnum fieldIdPathFirst, TFieldIdEnum... fieldIdPathRest) + throws TException { + try { + TField field = locateField(bytes, fieldIdPathFirst, fieldIdPathRest); + if (field != null) { + if (ttype == field.type) { + // if this point is reached, iprot will be positioned at the start of + // the field + switch (ttype) { + case TType.BOOL: + return protocol_.readBool(); + case TType.BYTE: + return protocol_.readByte(); + case TType.DOUBLE: + return protocol_.readDouble(); + case TType.I16: + return protocol_.readI16(); + case TType.I32: + return protocol_.readI32(); + case TType.I64: + return protocol_.readI64(); + case TType.STRING: + return protocol_.readString(); + default: + return null; + } + } + // hack to differentiate between string and binary + if (ttype == 100 && field.type == TType.STRING) { + return protocol_.readBinary(); + } + } + return null; + } catch (Exception e) { + throw new TException(e); + } finally { + trans_.clear(); + protocol_.reset(); + } + } + + private TField locateField( + byte[] bytes, TFieldIdEnum fieldIdPathFirst, TFieldIdEnum... fieldIdPathRest) + throws TException { + trans_.reset(bytes); + + TFieldIdEnum[] fieldIdPath = new TFieldIdEnum[fieldIdPathRest.length + 1]; + fieldIdPath[0] = fieldIdPathFirst; + System.arraycopy(fieldIdPathRest, 0, fieldIdPath, 1, fieldIdPathRest.length); + + // index into field ID path being currently searched for + int curPathIndex = 0; + + // this will be the located field, or null if it is not located + TField field = null; + + protocol_.readStructBegin(); + + while (curPathIndex < fieldIdPath.length) { + field = protocol_.readFieldBegin(); + // we can stop searching if we either see a stop or we go past the field + // id we're looking for (since fields should now be serialized in asc + // order). + if (field.type == TType.STOP || field.id > fieldIdPath[curPathIndex].getThriftFieldId()) { + return null; + } + + if (field.id != fieldIdPath[curPathIndex].getThriftFieldId()) { + // Not the field we're looking for. Skip field. + TProtocolUtil.skip(protocol_, field.type); + protocol_.readFieldEnd(); + } else { + // This field is the next step in the path. Step into field. + curPathIndex++; + if (curPathIndex < fieldIdPath.length) { + protocol_.readStructBegin(); + } + } + } + return field; + } + + /** + * Deserialize the Thrift object from a Java string, using the default JVM charset encoding. + * + * @param base The object to read into + * @param data The string to read from + * @throws TException if an error is encountered during deserialization. + */ + public void fromString(TBase base, String data) throws TException { + deserialize(base, data.getBytes()); + } + + // ---------------------------------------------------------------------- + // Methods related to partial deserialization. + + /** + * Partially deserializes the given serialized blob. + * + * @param bytes the serialized blob. + * @return deserialized instance. + * @throws TException if an error is encountered during deserialization. + */ + public Object partialDeserializeObject(byte[] bytes) throws TException { + return this.partialDeserializeObject(bytes, 0, bytes.length); + } + + /** + * Partially deserializes the given serialized blob into the given {@code TBase} instance. + * + * @param base the instance into which the given blob is deserialized. + * @param bytes the serialized blob. + * @param offset the blob is read starting at this offset. + * @param length the size of blob read (in number of bytes). + * @return deserialized instance. + * @throws TException if an error is encountered during deserialization. + */ + public Object partialDeserializeThriftObject(TBase base, byte[] bytes, int offset, int length) + throws TException { + ensurePartialThriftDeserializationMode(); + + return this.partialDeserializeObject(base, bytes, offset, length); + } + + /** + * Partially deserializes the given serialized blob. + * + * @param bytes the serialized blob. + * @param offset the blob is read starting at this offset. + * @param length the size of blob read (in number of bytes). + * @return deserialized instance. + * @throws TException if an error is encountered during deserialization. + */ + public Object partialDeserializeObject(byte[] bytes, int offset, int length) throws TException { + ensurePartialDeserializationMode(); + + return this.partialDeserializeObject(null, bytes, offset, length); + } + + /** + * Partially deserializes the given serialized blob. + * + * @param instance the instance into which the given blob is deserialized. + * @param bytes the serialized blob. + * @param offset the blob is read starting at this offset. + * @param length the size of blob read (in number of bytes). + * @return deserialized instance. + * @throws TException if an error is encountered during deserialization. + */ + private Object partialDeserializeObject(Object instance, byte[] bytes, int offset, int length) + throws TException { + ensurePartialDeserializationMode(); + + this.trans_.reset(bytes, offset, length); + this.protocol_.reset(); + return this.deserializeStruct(instance, this.metadata_); + } + + private Object deserialize(ThriftMetadata.ThriftObject data) throws TException { + + Object value; + byte fieldType = data.data.valueMetaData.type; + switch (fieldType) { + case TType.STRUCT: + return this.deserializeStruct(null, (ThriftMetadata.ThriftStruct) data); + + case TType.LIST: + return this.deserializeList((ThriftMetadata.ThriftList) data); + + case TType.MAP: + return this.deserializeMap((ThriftMetadata.ThriftMap) data); + + case TType.SET: + return this.deserializeSet((ThriftMetadata.ThriftSet) data); + + case TType.ENUM: + return this.deserializeEnum((ThriftMetadata.ThriftEnum) data); + + case TType.BOOL: + return this.protocol_.readBool(); + + case TType.BYTE: + return this.protocol_.readByte(); + + case TType.I16: + return this.protocol_.readI16(); + + case TType.I32: + return this.protocol_.readI32(); + + case TType.I64: + return this.protocol_.readI64(); + + case TType.DOUBLE: + return this.protocol_.readDouble(); + + case TType.STRING: + if (((ThriftMetadata.ThriftPrimitive) data).isBinary()) { + return this.processor_.prepareBinary(this.protocol_.readBinary()); + } else { + return this.processor_.prepareString(this.protocol_.readBinary()); + } + + default: + throw unsupportedFieldTypeException(fieldType); + } + } + + private Object deserializeStruct(Object instance, ThriftMetadata.ThriftStruct data) + throws TException { + + if (instance == null) { + instance = this.processor_.createNewStruct(data); + } + + this.protocol_.readStructBegin(); + while (true) { + int tfieldData = this.protocol_.readFieldBeginData(); + byte tfieldType = TFieldData.getType(tfieldData); + if (tfieldType == TType.STOP) { + break; + } + + Integer id = (int) TFieldData.getId(tfieldData); + ThriftMetadata.ThriftObject field = (ThriftMetadata.ThriftObject) data.fields.get(id); + + if (field != null) { + this.deserializeStructField(instance, field.fieldId, field); + } else { + this.protocol_.skip(tfieldType); + } + this.protocol_.readFieldEnd(); + } + this.protocol_.readStructEnd(); + + return this.processor_.prepareStruct(instance); + } + + private void deserializeStructField( + Object instance, TFieldIdEnum fieldId, ThriftMetadata.ThriftObject data) throws TException { + + byte fieldType = data.data.valueMetaData.type; + Object value; + + switch (fieldType) { + case TType.BOOL: + this.processor_.setBool(instance, fieldId, this.protocol_.readBool()); + break; + + case TType.BYTE: + this.processor_.setByte(instance, fieldId, this.protocol_.readByte()); + break; + + case TType.I16: + this.processor_.setInt16(instance, fieldId, this.protocol_.readI16()); + break; + + case TType.I32: + this.processor_.setInt32(instance, fieldId, this.protocol_.readI32()); + break; + + case TType.I64: + this.processor_.setInt64(instance, fieldId, this.protocol_.readI64()); + break; + + case TType.DOUBLE: + this.processor_.setDouble(instance, fieldId, this.protocol_.readDouble()); + break; + + case TType.STRING: + if (((ThriftMetadata.ThriftPrimitive) data).isBinary()) { + this.processor_.setBinary(instance, fieldId, this.protocol_.readBinary()); + } else { + this.processor_.setString(instance, fieldId, this.protocol_.readBinary()); + } + break; + + case TType.STRUCT: + value = this.deserializeStruct(null, (ThriftMetadata.ThriftStruct) data); + this.processor_.setStructField(instance, fieldId, value); + break; + + case TType.LIST: + value = this.deserializeList((ThriftMetadata.ThriftList) data); + this.processor_.setListField(instance, fieldId, value); + break; + + case TType.MAP: + value = this.deserializeMap((ThriftMetadata.ThriftMap) data); + this.processor_.setMapField(instance, fieldId, value); + break; + + case TType.SET: + value = this.deserializeSet((ThriftMetadata.ThriftSet) data); + this.processor_.setSetField(instance, fieldId, value); + break; + + case TType.ENUM: + value = this.deserializeEnum((ThriftMetadata.ThriftEnum) data); + this.processor_.setEnumField(instance, fieldId, value); + break; + + default: + throw new RuntimeException("Unsupported field type: " + fieldId.toString()); + } + } + + private Object deserializeList(ThriftMetadata.ThriftList data) throws TException { + + TList tlist = this.protocol_.readListBegin(); + Object instance = this.processor_.createNewList(tlist.size); + for (int i = 0; i < tlist.size; i++) { + Object value = this.deserialize(data.elementData); + this.processor_.setListElement(instance, i, value); + } + this.protocol_.readListEnd(); + return this.processor_.prepareList(instance); + } + + private Object deserializeMap(ThriftMetadata.ThriftMap data) throws TException { + TMap tmap = this.protocol_.readMapBegin(); + Object instance = this.processor_.createNewMap(tmap.size); + for (int i = 0; i < tmap.size; i++) { + Object key = this.deserialize(data.keyData); + Object val = this.deserialize(data.valueData); + this.processor_.setMapElement(instance, i, key, val); + } + this.protocol_.readMapEnd(); + return this.processor_.prepareMap(instance); + } + + private Object deserializeSet(ThriftMetadata.ThriftSet data) throws TException { + TSet tset = this.protocol_.readSetBegin(); + Object instance = this.processor_.createNewSet(tset.size); + for (int i = 0; i < tset.size; i++) { + Object eltValue = this.deserialize(data.elementData); + this.processor_.setSetElement(instance, i, eltValue); + } + this.protocol_.readSetEnd(); + return this.processor_.prepareSet(instance); + } + + private Object deserializeEnum(ThriftMetadata.ThriftEnum data) throws TException { + int ordinal = this.protocol_.readI32(); + Class enumClass = ((EnumMetaData) data.data.valueMetaData).enumClass; + return this.processor_.prepareEnum(enumClass, ordinal); + } + + private Class getStructClass(ThriftMetadata.ThriftStruct data) { + return (Class) ((StructMetaData) data.data.valueMetaData).structClass; + } + + private static UnsupportedOperationException unsupportedFieldTypeException(byte fieldType) { + return new UnsupportedOperationException("field type not supported: " + fieldType); + } + + private boolean isPartialDeserializationMode() { + return (this.metadata_ != null) && (this.processor_ != null); + } + + private void ensurePartialDeserializationMode() throws IllegalStateException { + if (!this.isPartialDeserializationMode()) { + throw new IllegalStateException( + "Members metadata and processor must be correctly initialized in order to use this method"); + } + } + + private void ensurePartialThriftDeserializationMode() throws IllegalStateException { + this.ensurePartialDeserializationMode(); + + if (!(this.processor_ instanceof ThriftStructProcessor)) { + throw new IllegalStateException( + "processor must be an instance of ThriftStructProcessor to use this method"); + } + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/TEnum.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TEnum.java new file mode 100644 index 0000000..300496d --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TEnum.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift; + +public interface TEnum { + int getValue(); +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/TEnumHelper.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TEnumHelper.java new file mode 100644 index 0000000..b003346 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TEnumHelper.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift; + +import java.lang.reflect.Method; + +/** Utility class with static methods for interacting with TEnum */ +public class TEnumHelper { + + /* no instantiation */ + private TEnumHelper() {} + + /** + * Given a TEnum class and integer value, this method will return the associated constant from the + * given TEnum class. This method MUST be modified should the name of the 'findByValue' method + * change. + * + * @param enumClass TEnum from which to return a matching constant. + * @param value Value for which to return the constant. + * @return The constant in 'enumClass' whose value is 'value' or null if something went wrong. + */ + public static TEnum getByValue(Class enumClass, int value) { + try { + Method method = enumClass.getMethod("findByValue", int.class); + return (TEnum) method.invoke(null, value); + } catch (ReflectiveOperationException nsme) { + return null; + } + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/TException.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TException.java new file mode 100644 index 0000000..b2273d9 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TException.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift; + +/** Generic exception class for Thrift. */ +public class TException extends Exception { + + private static final long serialVersionUID = 1L; + + public TException() { + super(); + } + + public TException(String message) { + super(message); + } + + public TException(Throwable cause) { + super(cause); + } + + public TException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/TFieldIdEnum.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TFieldIdEnum.java new file mode 100644 index 0000000..417cd84 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TFieldIdEnum.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.thrift; + +/** Interface for all generated struct Fields objects. */ +public interface TFieldIdEnum { + /** + * Gets the Thrift field id for the named field. + * + * @return the Thrift field id for the named field. + */ + short getThriftFieldId(); + + /** + * Gets the field's name, exactly as in the IDL. + * + * @return the field's name, exactly as in the IDL. + */ + String getFieldName(); +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/TFieldRequirementType.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TFieldRequirementType.java new file mode 100644 index 0000000..8769dbb --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TFieldRequirementType.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift; + +/** Requirement type constants. */ +public final class TFieldRequirementType { + + /* no instantiation */ + private TFieldRequirementType() {} + + public static final byte REQUIRED = 1; + public static final byte OPTIONAL = 2; + public static final byte DEFAULT = 3; +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/THttpClientResponseHandler.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/THttpClientResponseHandler.java new file mode 100644 index 0000000..de2f150 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/THttpClientResponseHandler.java @@ -0,0 +1,74 @@ +package org.apache.thrift; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.client.ResponseHandler; +import org.apache.http.client.ClientProtocolException; + +public class THttpClientResponseHandler implements ResponseHandler { + @Override + public InputStream handleResponse(HttpResponse response) + throws ClientProtocolException, IOException { + try (InputStream is = response.getEntity().getContent()) { + int responseCode = response.getStatusLine().getStatusCode(); + if (responseCode != HttpStatus.SC_OK) { + throw new IOException("HTTP Response code: " + responseCode); + } + byte[] readByteArray = readIntoByteArray(is); + try { + // Indicate we're done with the content. + consume(response.getEntity()); + } catch (IOException ioe) { + // We ignore this exception, it might only mean the server has no + // keep-alive capability. + } + return new ByteArrayInputStream(readByteArray); + } catch (IOException ioe) { + throw ioe; + } + } + + /** + * Read the responses into a byte array so we can release the connection early. This implies that + * the whole content will have to be read in memory, and that momentarily we might use up twice + * the memory (while the thrift struct is being read up the chain). Proceeding differently might + * lead to exhaustion of connections and thus to app failure. + * + * @param is input stream + * @return read bytes + * @throws IOException when exception during read + */ + private static byte[] readIntoByteArray(InputStream is) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + byte[] buf = new byte[1024]; + int len; + do { + len = is.read(buf); + if (len > 0) { + baos.write(buf, 0, len); + } + } while (-1 != len); + return baos.toByteArray(); + } + + /** + * copy from org.apache.http.util.EntityUtils#consume. Android has it's own httpcore that doesn't + * have a consume. + */ + private static void consume(final HttpEntity entity) throws IOException { + if (entity == null) { + return; + } + if (entity.isStreaming()) { + InputStream instream = entity.getContent(); + if (instream != null) { + instream.close(); + } + } + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/TMultiplexedProcessor.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TMultiplexedProcessor.java new file mode 100644 index 0000000..60c7944 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TMultiplexedProcessor.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift; + +import java.util.HashMap; +import java.util.Map; +import org.apache.thrift.protocol.*; + +/** + * TMultiplexedProcessor is a TProcessor allowing a single TServer + * to provide multiple services. + * + *

To do so, you instantiate the processor and then register additional processors with it, as + * shown in the following example: + * + *

+ * + * + * TMultiplexedProcessor processor = new TMultiplexedProcessor(); + * + * processor.registerProcessor( + * "Calculator", + * new Calculator.Processor(new CalculatorHandler())); + * + * processor.registerProcessor( + * "WeatherReport", + * new WeatherReport.Processor(new WeatherReportHandler())); + * + * TServerTransport t = new TServerSocket(9090); + * TSimpleServer server = new TSimpleServer(processor, t); + * + * server.serve(); + * + * + *
+ */ +public class TMultiplexedProcessor implements TProcessor { + + private final Map SERVICE_PROCESSOR_MAP = new HashMap(); + private TProcessor defaultProcessor; + + /** + * 'Register' a service with this TMultiplexedProcessor. This allows us to broker + * requests to individual services by using the service name to select them at request time. + * + * @param serviceName Name of a service, has to be identical to the name declared in the Thrift + * IDL, e.g. "WeatherReport". + * @param processor Implementation of a service, usually referred to as "handlers", e.g. + * WeatherReportHandler implementing WeatherReport.Iface. + */ + public void registerProcessor(String serviceName, TProcessor processor) { + SERVICE_PROCESSOR_MAP.put(serviceName, processor); + } + + /** + * Register a service to be called to process queries without service name + * + * @param processor the service to be called. + */ + public void registerDefault(TProcessor processor) { + defaultProcessor = processor; + } + + /** + * This implementation of process performs the following steps: + * + *
    + *
  1. Read the beginning of the message. + *
  2. Extract the service name from the message. + *
  3. Using the service name to locate the appropriate processor. + *
  4. Dispatch to the processor, with a decorated instance of TProtocol that allows + * readMessageBegin() to return the original TMessage. + *
+ * + * @throws TProtocolException If the message type is not CALL or ONEWAY, if the service name was + * not found in the message, or if the service name was not found in the service map. You + * called {@link #registerProcessor(String, TProcessor) registerProcessor} during + * initialization, right? :) + */ + @Override + public void process(TProtocol iprot, TProtocol oprot) throws TException { + /* + Use the actual underlying protocol (e.g. TBinaryProtocol) to read the + message header. This pulls the message "off the wire", which we'll + deal with at the end of this method. + */ + TMessage message = iprot.readMessageBegin(); + + if (message.type != TMessageType.CALL && message.type != TMessageType.ONEWAY) { + throw new TProtocolException( + TProtocolException.NOT_IMPLEMENTED, "This should not have happened!?"); + } + + // Extract the service name + int index = message.name.indexOf(TMultiplexedProtocol.SEPARATOR); + if (index < 0) { + if (defaultProcessor != null) { + // Dispatch processing to the stored processor + defaultProcessor.process(new StoredMessageProtocol(iprot, message), oprot); + return; + } + throw new TProtocolException( + TProtocolException.NOT_IMPLEMENTED, + "Service name not found in message name: " + + message.name + + ". Did you " + + "forget to use a TMultiplexProtocol in your client?"); + } + + // Create a new TMessage, something that can be consumed by any TProtocol + String serviceName = message.name.substring(0, index); + TProcessor actualProcessor = SERVICE_PROCESSOR_MAP.get(serviceName); + if (actualProcessor == null) { + throw new TProtocolException( + TProtocolException.NOT_IMPLEMENTED, + "Service name not found: " + + serviceName + + ". Did you forget " + + "to call registerProcessor()?"); + } + + // Create a new TMessage, removing the service name + TMessage standardMessage = + new TMessage( + message.name.substring(serviceName.length() + TMultiplexedProtocol.SEPARATOR.length()), + message.type, + message.seqid); + + // Dispatch processing to the stored processor + actualProcessor.process(new StoredMessageProtocol(iprot, standardMessage), oprot); + } + + /** + * Our goal was to work with any protocol. In order to do that, we needed to allow them to call + * readMessageBegin() and get a TMessage in exactly the standard format, without the service name + * prepended to TMessage.name. + */ + private static class StoredMessageProtocol extends TProtocolDecorator { + TMessage messageBegin; + + public StoredMessageProtocol(TProtocol protocol, TMessage messageBegin) { + super(protocol); + this.messageBegin = messageBegin; + } + + @Override + public TMessage readMessageBegin() throws TException { + return messageBegin; + } + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/TNonblockingMultiFetchClient.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TNonblockingMultiFetchClient.java new file mode 100644 index 0000000..26ea6d0 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TNonblockingMultiFetchClient.java @@ -0,0 +1,374 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.thrift; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.SelectionKey; +import java.nio.channels.Selector; +import java.nio.channels.SocketChannel; +import java.util.Collections; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.FutureTask; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class uses a single thread to set up non-blocking sockets to a set of remote servers + * (hostname and port pairs), and sends a same request to all these servers. It then fetches + * responses from servers. + * + *

Parameters: int maxRecvBufBytesPerServer - an upper limit for receive buffer size per server + * (in byte). If a response from a server exceeds this limit, the client will not allocate memory or + * read response data for it. + * + *

int fetchTimeoutSeconds - time limit for fetching responses from all servers (in second). + * After the timeout, the fetch job is stopped and available responses are returned. + * + *

ByteBuffer requestBuf - request message that is sent to all servers. + * + *

Output: Responses are stored in an array of ByteBuffers. Index of elements in this array + * corresponds to index of servers in the server list. Content in a ByteBuffer may be in one of the + * following forms: 1. First 4 bytes form an integer indicating length of following data, then + * followed by the data. 2. First 4 bytes form an integer indicating length of following data, then + * followed by nothing - this happens when the response data size exceeds maxRecvBufBytesPerServer, + * and the client will not read any response data. 3. No data in the ByteBuffer - this happens when + * the server does not return any response within fetchTimeoutSeconds. + * + *

In some special cases (no servers are given, fetchTimeoutSeconds less than or equal to 0, + * requestBuf is null), the return is null. + * + *

Note: It assumes all remote servers are TNonblockingServers and use TFramedTransport. + */ +public class TNonblockingMultiFetchClient { + + private static final Logger LOGGER = LoggerFactory.getLogger(TNonblockingMultiFetchClient.class); + + // if the size of the response msg exceeds this limit (in byte), we will + // not read the msg + private final int maxRecvBufBytesPerServer; + + // time limit for fetching data from all servers (in second) + private final int fetchTimeoutSeconds; + + // store request that will be sent to servers + private final ByteBuffer requestBuf; + private ByteBuffer requestBufDuplication; + + // a list of remote servers + private final List servers; + + // store fetch results + private final TNonblockingMultiFetchStats stats; + private ByteBuffer[] recvBuf; + + public TNonblockingMultiFetchClient( + int maxRecvBufBytesPerServer, + int fetchTimeoutSeconds, + ByteBuffer requestBuf, + List servers) { + this.maxRecvBufBytesPerServer = maxRecvBufBytesPerServer; + this.fetchTimeoutSeconds = fetchTimeoutSeconds; + this.requestBuf = requestBuf; + this.servers = servers; + + stats = new TNonblockingMultiFetchStats(); + recvBuf = null; + } + + public synchronized int getMaxRecvBufBytesPerServer() { + return maxRecvBufBytesPerServer; + } + + public synchronized int getFetchTimeoutSeconds() { + return fetchTimeoutSeconds; + } + + /** + * Returns a copy of requestBuf, so that requestBuf will not be modified by others. + * + * @return a copy of requestBuf. + */ + public synchronized ByteBuffer getRequestBuf() { + if (requestBuf == null) { + return null; + } else { + if (requestBufDuplication == null) { + requestBufDuplication = requestBuf.duplicate(); + } + return requestBufDuplication; + } + } + + public synchronized List getServerList() { + if (servers == null) { + return null; + } + return Collections.unmodifiableList(servers); + } + + public synchronized TNonblockingMultiFetchStats getFetchStats() { + return stats; + } + + /** + * Main entry function for fetching from servers. + * + * @return The fetched data. + */ + public synchronized ByteBuffer[] fetch() { + // clear previous results + recvBuf = null; + stats.clear(); + + if (servers == null || servers.size() == 0 || requestBuf == null || fetchTimeoutSeconds <= 0) { + return recvBuf; + } + + ExecutorService executor = Executors.newSingleThreadExecutor(); + MultiFetch multiFetch = new MultiFetch(); + FutureTask task = new FutureTask(multiFetch, null); + executor.execute(task); + try { + task.get(fetchTimeoutSeconds, TimeUnit.SECONDS); + } catch (InterruptedException ie) { + // attempt to cancel execution of the task. + task.cancel(true); + LOGGER.error("Interrupted during fetch", ie); + } catch (ExecutionException ee) { + // attempt to cancel execution of the task. + task.cancel(true); + LOGGER.error("Exception during fetch", ee); + } catch (TimeoutException te) { + // attempt to cancel execution of the task. + task.cancel(true); + LOGGER.error("Timeout for fetch", te); + } + + executor.shutdownNow(); + multiFetch.close(); + return recvBuf; + } + + /** + * Private class that does real fetch job. Users are not allowed to directly use this class, as + * its run() function may run forever. + */ + private class MultiFetch implements Runnable { + private Selector selector; + + /** + * main entry function for fetching. + * + *

Server responses are stored in TNonblocingMultiFetchClient.recvBuf, and fetch statistics + * is in TNonblockingMultiFetchClient.stats. + * + *

Sanity check for parameters has been done in TNonblockingMultiFetchClient before calling + * this function. + */ + public void run() { + long t1 = System.currentTimeMillis(); + + int numTotalServers = servers.size(); + stats.setNumTotalServers(numTotalServers); + + // buffer for receiving response from servers + recvBuf = new ByteBuffer[numTotalServers]; + // buffer for sending request + ByteBuffer[] sendBuf = new ByteBuffer[numTotalServers]; + long[] numBytesRead = new long[numTotalServers]; + int[] frameSize = new int[numTotalServers]; + boolean[] hasReadFrameSize = new boolean[numTotalServers]; + + try { + selector = Selector.open(); + } catch (IOException ioe) { + LOGGER.error("Selector opens error", ioe); + return; + } + + for (int i = 0; i < numTotalServers; i++) { + // create buffer to send request to server. + sendBuf[i] = requestBuf.duplicate(); + // create buffer to read response's frame size from server + recvBuf[i] = ByteBuffer.allocate(4); + stats.incTotalRecvBufBytes(4); + + InetSocketAddress server = servers.get(i); + SocketChannel s = null; + SelectionKey key = null; + try { + s = SocketChannel.open(); + s.configureBlocking(false); + // now this method is non-blocking + s.connect(server); + key = s.register(selector, s.validOps()); + // attach index of the key + key.attach(i); + } catch (Exception e) { + stats.incNumConnectErrorServers(); + LOGGER.error("Set up socket to server {} error", server, e); + + // free resource + if (s != null) { + try { + s.close(); + } catch (Exception ex) { + LOGGER.error("failed to free up socket", ex); + } + } + if (key != null) { + key.cancel(); + } + } + } + + // wait for events + while (stats.getNumReadCompletedServers() + stats.getNumConnectErrorServers() + < stats.getNumTotalServers()) { + // if the thread is interrupted (e.g., task is cancelled) + if (Thread.currentThread().isInterrupted()) { + return; + } + + try { + selector.select(); + } catch (Exception e) { + LOGGER.error("Selector selects error", e); + continue; + } + + Iterator it = selector.selectedKeys().iterator(); + while (it.hasNext()) { + SelectionKey selKey = it.next(); + it.remove(); + + // get previously attached index + int index = (Integer) selKey.attachment(); + + if (selKey.isValid() && selKey.isConnectable()) { + // if this socket throws an exception (e.g., connection refused), + // print error msg and skip it. + try { + SocketChannel sChannel = (SocketChannel) selKey.channel(); + sChannel.finishConnect(); + } catch (Exception e) { + stats.incNumConnectErrorServers(); + LOGGER.error("Socket {} connects to server {} error", index, servers.get(index), e); + } + } + + if (selKey.isValid() && selKey.isWritable() && sendBuf[index].hasRemaining()) { + // if this socket throws an exception, print error msg and + // skip it. + try { + SocketChannel sChannel = (SocketChannel) selKey.channel(); + sChannel.write(sendBuf[index]); + } catch (Exception e) { + LOGGER.error("Socket {} writes to server {} error", index, servers.get(index), e); + } + } + + if (selKey.isValid() && selKey.isReadable()) { + // if this socket throws an exception, print error msg and + // skip it. + try { + SocketChannel sChannel = (SocketChannel) selKey.channel(); + int bytesRead = sChannel.read(recvBuf[index]); + + if (bytesRead > 0) { + numBytesRead[index] += bytesRead; + + if (!hasReadFrameSize[index] && recvBuf[index].remaining() == 0) { + // if the frame size has been read completely, then prepare + // to read the actual frame. + frameSize[index] = recvBuf[index].getInt(0); + + if (frameSize[index] <= 0) { + stats.incNumInvalidFrameSize(); + LOGGER.error( + "Read an invalid frame size {} from {}. Does the server use TFramedTransport?", + frameSize[index], + servers.get(index)); + sChannel.close(); + continue; + } + + if (frameSize[index] + 4 > stats.getMaxResponseBytes()) { + stats.setMaxResponseBytes(frameSize[index] + 4); + } + + if (frameSize[index] + 4 > maxRecvBufBytesPerServer) { + stats.incNumOverflowedRecvBuf(); + LOGGER.error( + "Read frame size {} from {}, total buffer size would exceed limit {}", + frameSize[index], + servers.get(index), + maxRecvBufBytesPerServer); + sChannel.close(); + continue; + } + + // reallocate buffer for actual frame data + recvBuf[index] = ByteBuffer.allocate(frameSize[index] + 4); + recvBuf[index].putInt(frameSize[index]); + + stats.incTotalRecvBufBytes(frameSize[index]); + hasReadFrameSize[index] = true; + } + + if (hasReadFrameSize[index] && numBytesRead[index] >= frameSize[index] + 4) { + // has read all data + sChannel.close(); + stats.incNumReadCompletedServers(); + long t2 = System.currentTimeMillis(); + stats.setReadTime(t2 - t1); + } + } + } catch (Exception e) { + LOGGER.error("Socket {} reads from server {} error", index, servers.get(index), e); + } + } + } + } + } + + /** dispose any resource allocated */ + public void close() { + try { + if (selector.isOpen()) { + for (SelectionKey selKey : selector.keys()) { + SocketChannel sChannel = (SocketChannel) selKey.channel(); + sChannel.close(); + } + selector.close(); + } + } catch (IOException e) { + LOGGER.error("Free resource error", e); + } + } + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/TNonblockingMultiFetchStats.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TNonblockingMultiFetchStats.java new file mode 100644 index 0000000..304dc09 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TNonblockingMultiFetchStats.java @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.thrift; + +/** This class keeps track of statistics for TNonblockinMultiFetchClient. */ +public class TNonblockingMultiFetchStats { + private int numTotalServers; + private int numReadCompletedServers; + private int numConnectErrorServers; + private int totalRecvBufBytes; + private int maxResponseBytes; + private int numOverflowedRecvBuf; + private int numInvalidFrameSize; + // time from the beginning of fetch() function to the reading finish + // time of the last socket (in millisecond) + private long readTime; + + public TNonblockingMultiFetchStats() { + clear(); + } + + public void clear() { + numTotalServers = 0; + numReadCompletedServers = 0; + numConnectErrorServers = 0; + totalRecvBufBytes = 0; + maxResponseBytes = 0; + numOverflowedRecvBuf = 0; + numInvalidFrameSize = 0; + readTime = 0; + } + + public String toString() { + String stats = + String.format( + "numTotalServers=%d, " + + "numReadCompletedServers=%d, numConnectErrorServers=%d, " + + "numUnresponsiveServers=%d, totalRecvBufBytes=%fM, " + + "maxResponseBytes=%d, numOverflowedRecvBuf=%d, " + + "numInvalidFrameSize=%d, readTime=%dms", + numTotalServers, + numReadCompletedServers, + numConnectErrorServers, + (numTotalServers - numReadCompletedServers - numConnectErrorServers), + totalRecvBufBytes / 1024.0 / 1024, + maxResponseBytes, + numOverflowedRecvBuf, + numInvalidFrameSize, + readTime); + return stats; + } + + public void setNumTotalServers(int val) { + numTotalServers = val; + } + + public void setMaxResponseBytes(int val) { + maxResponseBytes = val; + } + + public void setReadTime(long val) { + readTime = val; + } + + public void incNumReadCompletedServers() { + numReadCompletedServers++; + } + + public void incNumConnectErrorServers() { + numConnectErrorServers++; + } + + public void incNumOverflowedRecvBuf() { + numOverflowedRecvBuf++; + } + + public void incTotalRecvBufBytes(int val) { + totalRecvBufBytes += val; + } + + public void incNumInvalidFrameSize() { + numInvalidFrameSize++; + } + + public int getMaxResponseBytes() { + return maxResponseBytes; + } + + public int getNumReadCompletedServers() { + return numReadCompletedServers; + } + + public int getNumConnectErrorServers() { + return numConnectErrorServers; + } + + public int getNumTotalServers() { + return numTotalServers; + } + + public int getNumOverflowedRecvBuf() { + return numOverflowedRecvBuf; + } + + public int getTotalRecvBufBytes() { + return totalRecvBufBytes; + } + + public int getNumInvalidFrameSize() { + return numInvalidFrameSize; + } + + public long getReadTime() { + return readTime; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/TProcessor.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TProcessor.java new file mode 100644 index 0000000..96b02a4 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TProcessor.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift; + +import org.apache.thrift.protocol.TProtocol; + +/** + * A processor is a generic object which operates upon an input stream and writes to some output + * stream. + */ +public interface TProcessor { + void process(TProtocol in, TProtocol out) throws TException; +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/TProcessorFactory.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TProcessorFactory.java new file mode 100644 index 0000000..fab4005 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TProcessorFactory.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift; + +import org.apache.thrift.transport.TTransport; + +/** The default processor factory just returns a singleton instance. */ +public class TProcessorFactory { + + private final TProcessor processor_; + + public TProcessorFactory(TProcessor processor) { + processor_ = processor; + } + + public TProcessor getProcessor(TTransport trans) { + return processor_; + } + + public boolean isAsyncProcessor() { + return processor_ instanceof TAsyncProcessor; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/TSerializable.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TSerializable.java new file mode 100644 index 0000000..7333d41 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TSerializable.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift; + +import org.apache.thrift.protocol.TProtocol; + +/** Generic base interface for generated Thrift objects. */ +public interface TSerializable { + + /** + * Reads the TObject from the given input protocol. + * + * @param iprot Input protocol + * @throws TException if there is an error reading from iprot + */ + void read(TProtocol iprot) throws TException; + + /** + * Writes the objects out to the protocol + * + * @param oprot Output protocol + * @throws TException if there is an error writing to oprot + */ + void write(TProtocol oprot) throws TException; +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/TSerializer.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TSerializer.java new file mode 100644 index 0000000..8bcd494 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TSerializer.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift; + +import java.io.ByteArrayOutputStream; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.protocol.TProtocolFactory; +import org.apache.thrift.transport.TIOStreamTransport; +import org.apache.thrift.transport.TTransportException; + +/** Generic utility for easily serializing objects into a byte array or Java String. */ +public class TSerializer { + + /** This is the byte array that data is actually serialized into */ + private final ByteArrayOutputStream baos_ = new ByteArrayOutputStream(); + + /** Internal protocol used for serializing objects. */ + private final TProtocol protocol_; + + /** + * Create a new TSerializer that uses the TBinaryProtocol by default. + * + * @throws TTransportException if there an error initializing the underlying transport. + */ + public TSerializer() throws TTransportException { + this(new TBinaryProtocol.Factory()); + } + + /** + * Create a new TSerializer. It will use the TProtocol specified by the factory that is passed in. + * + * @param protocolFactory Factory to create a protocol + * @throws TTransportException if there is an error initializing the underlying transport. + */ + public TSerializer(TProtocolFactory protocolFactory) throws TTransportException { + /* This transport wraps that byte array */ + TIOStreamTransport transport_ = new TIOStreamTransport(new TConfiguration(), baos_); + protocol_ = protocolFactory.getProtocol(transport_); + } + + /** + * Serialize the Thrift object into a byte array. The process is simple, just clear the byte array + * output, write the object into it, and grab the raw bytes. + * + * @param base The object to serialize + * @return Serialized object in byte[] format + * @throws TException if an error is encountered during serialization. + */ + public byte[] serialize(TBase base) throws TException { + baos_.reset(); + base.write(protocol_); + return baos_.toByteArray(); + } + + /** + * Serialize the Thrift object into a Java string, using the default JVM charset encoding. + * + * @param base The object to serialize + * @return Serialized object as a String + * @throws TException if an error is encountered during serialization. + */ + public String toString(TBase base) throws TException { + return new String(serialize(base)); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/TServiceClient.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TServiceClient.java new file mode 100644 index 0000000..6c37596 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TServiceClient.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift; + +import org.apache.thrift.protocol.TMessage; +import org.apache.thrift.protocol.TMessageType; +import org.apache.thrift.protocol.TProtocol; + +/** + * A TServiceClient is used to communicate with a TService implementation across protocols and + * transports. + */ +public abstract class TServiceClient { + public TServiceClient(TProtocol prot) { + this(prot, prot); + } + + public TServiceClient(TProtocol iprot, TProtocol oprot) { + iprot_ = iprot; + oprot_ = oprot; + } + + protected TProtocol iprot_; + protected TProtocol oprot_; + + protected int seqid_; + + /** + * Get the TProtocol being used as the input (read) protocol. + * + * @return the TProtocol being used as the input (read) protocol. + */ + public TProtocol getInputProtocol() { + return this.iprot_; + } + + /** + * Get the TProtocol being used as the output (write) protocol. + * + * @return the TProtocol being used as the output (write) protocol. + */ + public TProtocol getOutputProtocol() { + return this.oprot_; + } + + protected void sendBase(String methodName, TBase args) throws TException { + sendBase(methodName, args, TMessageType.CALL); + } + + protected void sendBaseOneway(String methodName, TBase args) throws TException { + sendBase(methodName, args, TMessageType.ONEWAY); + } + + private void sendBase(String methodName, TBase args, byte type) throws TException { + oprot_.writeMessageBegin(new TMessage(methodName, type, ++seqid_)); + args.write(oprot_); + oprot_.writeMessageEnd(); + oprot_.getTransport().flush(); + } + + protected void receiveBase(TBase result, String methodName) throws TException { + TMessage msg = iprot_.readMessageBegin(); + if (msg.type == TMessageType.EXCEPTION) { + TApplicationException x = new TApplicationException(); + x.read(iprot_); + iprot_.readMessageEnd(); + throw x; + } + if (msg.seqid != seqid_) { + throw new TApplicationException( + TApplicationException.BAD_SEQUENCE_ID, + String.format( + "%s failed: out of sequence response: expected %d but got %d", + methodName, seqid_, msg.seqid)); + } + result.read(iprot_); + iprot_.readMessageEnd(); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/TServiceClientFactory.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TServiceClientFactory.java new file mode 100644 index 0000000..54dd967 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TServiceClientFactory.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift; + +import org.apache.thrift.protocol.TProtocol; + +/** + * A TServiceClientFactory provides a general way to get a TServiceClient connected to a remote + * TService via a protocol. + * + * @param the type of TServiceClient to get. + */ +public interface TServiceClientFactory { + /** + * Get a brand-new T using prot as both the input and output protocol. + * + * @param prot The protocol to use for getting T. + * @return A brand-new T using prot as both the input and output protocol. + */ + T getClient(TProtocol prot); + + /** + * Get a brand new T using the specified input and output protocols. The input and output + * protocols may be the same instance. + * + * @param iprot The input protocol to use for getting T. + * @param oprot The output protocol to use for getting T. + * @return a brand new T using the specified input and output protocols + */ + T getClient(TProtocol iprot, TProtocol oprot); +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/TUnion.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TUnion.java new file mode 100644 index 0000000..65c92dc --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/TUnion.java @@ -0,0 +1,293 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.thrift; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.thrift.protocol.TField; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.protocol.TProtocolException; +import org.apache.thrift.protocol.TStruct; +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.SchemeFactory; +import org.apache.thrift.scheme.StandardScheme; +import org.apache.thrift.scheme.TupleScheme; + +public abstract class TUnion, F extends TFieldIdEnum> + implements TBase { + + protected Object value_; + protected F setField_; + + protected TUnion() { + setField_ = null; + value_ = null; + } + + private static final Map, SchemeFactory> schemes = + new HashMap, SchemeFactory>(); + + static { + schemes.put(StandardScheme.class, new TUnionStandardSchemeFactory()); + schemes.put(TupleScheme.class, new TUnionTupleSchemeFactory()); + } + + protected TUnion(F setField, Object value) { + setFieldValue(setField, value); + } + + protected TUnion(TUnion other) { + if (!other.getClass().equals(this.getClass())) { + throw new ClassCastException(); + } + setField_ = other.setField_; + value_ = deepCopyObject(other.value_); + } + + private static Object deepCopyObject(Object o) { + if (o instanceof TBase) { + return ((TBase) o).deepCopy(); + } else if (o instanceof ByteBuffer) { + return TBaseHelper.copyBinary((ByteBuffer) o); + } else if (o instanceof List) { + return deepCopyList((List) o); + } else if (o instanceof Set) { + return deepCopySet((Set) o); + } else if (o instanceof Map) { + return deepCopyMap((Map) o); + } else { + return o; + } + } + + private static Map deepCopyMap(Map map) { + Map copy = new HashMap(map.size()); + for (Map.Entry entry : map.entrySet()) { + copy.put(deepCopyObject(entry.getKey()), deepCopyObject(entry.getValue())); + } + return copy; + } + + private static Set deepCopySet(Set set) { + Set copy = new HashSet(set.size()); + for (Object o : set) { + copy.add(deepCopyObject(o)); + } + return copy; + } + + private static List deepCopyList(List list) { + List copy = new ArrayList(list.size()); + for (Object o : list) { + copy.add(deepCopyObject(o)); + } + return copy; + } + + public F getSetField() { + return setField_; + } + + public Object getFieldValue() { + return value_; + } + + public Object getFieldValue(F fieldId) { + if (fieldId != setField_) { + throw new IllegalArgumentException( + "Cannot get the value of field " + + fieldId + + " because union's set field is " + + setField_); + } + + return getFieldValue(); + } + + public Object getFieldValue(int fieldId) { + return getFieldValue(enumForId((short) fieldId)); + } + + public boolean isSet() { + return setField_ != null; + } + + public boolean isSet(F fieldId) { + return setField_ == fieldId; + } + + public boolean isSet(int fieldId) { + return isSet(enumForId((short) fieldId)); + } + + public void read(TProtocol iprot) throws TException { + schemes.get(iprot.getScheme()).getScheme().read(iprot, this); + } + + public void setFieldValue(F fieldId, Object value) { + checkType(fieldId, value); + setField_ = fieldId; + value_ = value; + } + + public void setFieldValue(int fieldId, Object value) { + setFieldValue(enumForId((short) fieldId), value); + } + + public void write(TProtocol oprot) throws TException { + schemes.get(oprot.getScheme()).getScheme().write(oprot, this); + } + + /** + * Implementation should be generated so that we can efficiently type check various values. + * + * @param setField the field to assign value to. + * @param value the value to be assigned to setField. + * @throws ClassCastException if the type of value is incompatible with the type of setField. + */ + protected abstract void checkType(F setField, Object value) throws ClassCastException; + + /** + * Implementation should be generated to read the right stuff from the wire based on the field + * header. + * + * @param iprot input protocol from which to read a value. + * @param field the field whose value is to be read from iprot. + * @return read Object based on the field header, as specified by the argument. + * @throws TException on error during read. + */ + protected abstract Object standardSchemeReadValue(TProtocol iprot, TField field) + throws TException; + + protected abstract void standardSchemeWriteValue(TProtocol oprot) throws TException; + + protected abstract Object tupleSchemeReadValue(TProtocol iprot, short fieldID) throws TException; + + protected abstract void tupleSchemeWriteValue(TProtocol oprot) throws TException; + + protected abstract TStruct getStructDesc(); + + protected abstract TField getFieldDesc(F setField); + + protected abstract F enumForId(short id); + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("<"); + sb.append(this.getClass().getSimpleName()); + sb.append(" "); + + if (getSetField() != null) { + Object v = getFieldValue(); + sb.append(getFieldDesc(getSetField()).name); + sb.append(":"); + if (v instanceof ByteBuffer) { + TBaseHelper.toString((ByteBuffer) v, sb); + } else { + sb.append(v.toString()); + } + } + sb.append(">"); + return sb.toString(); + } + + public final void clear() { + this.setField_ = null; + this.value_ = null; + } + + private static class TUnionStandardSchemeFactory implements SchemeFactory { + public TUnionStandardScheme getScheme() { + return new TUnionStandardScheme(); + } + } + + private static class TUnionStandardScheme extends StandardScheme { + + @Override + public void read(TProtocol iprot, TUnion struct) throws TException { + struct.setField_ = null; + struct.value_ = null; + + iprot.readStructBegin(); + + TField field = iprot.readFieldBegin(); + + struct.value_ = struct.standardSchemeReadValue(iprot, field); + if (struct.value_ != null) { + struct.setField_ = struct.enumForId(field.id); + } + + iprot.readFieldEnd(); + // this is so that we will eat the stop byte. we could put a check here to + // make sure that it actually *is* the stop byte, but it's faster to do it + // this way. + iprot.readFieldBegin(); + iprot.readStructEnd(); + } + + @Override + public void write(TProtocol oprot, TUnion struct) throws TException { + if (struct.getSetField() == null || struct.getFieldValue() == null) { + throw new TProtocolException("Cannot write a TUnion with no set value!"); + } + oprot.writeStructBegin(struct.getStructDesc()); + oprot.writeFieldBegin(struct.getFieldDesc(struct.setField_)); + struct.standardSchemeWriteValue(oprot); + oprot.writeFieldEnd(); + oprot.writeFieldStop(); + oprot.writeStructEnd(); + } + } + + private static class TUnionTupleSchemeFactory implements SchemeFactory { + public TUnionTupleScheme getScheme() { + return new TUnionTupleScheme(); + } + } + + private static class TUnionTupleScheme extends TupleScheme { + + @Override + public void read(TProtocol iprot, TUnion struct) throws TException { + struct.setField_ = null; + struct.value_ = null; + short fieldID = iprot.readI16(); + struct.value_ = struct.tupleSchemeReadValue(iprot, fieldID); + if (struct.value_ != null) { + struct.setField_ = struct.enumForId(fieldID); + } + } + + @Override + public void write(TProtocol oprot, TUnion struct) throws TException { + if (struct.getSetField() == null || struct.getFieldValue() == null) { + throw new TProtocolException("Cannot write a TUnion with no set value!"); + } + oprot.writeI16(struct.setField_.getThriftFieldId()); + struct.tupleSchemeWriteValue(oprot); + } + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/annotation/Nullable.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/annotation/Nullable.java new file mode 100644 index 0000000..55d4608 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/annotation/Nullable.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.annotation; + +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; + +/** + * Annotation indicating a field, method return, or method parameter may be {@code null}. We package + * our own annotation to avoid a mandatory third-party dependency. + */ +@Retention(RetentionPolicy.CLASS) +public @interface Nullable {} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/async/AsyncMethodCallback.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/async/AsyncMethodCallback.java new file mode 100644 index 0000000..3939bc2 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/async/AsyncMethodCallback.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.thrift.async; + +/** + * A handler interface asynchronous clients can implement to receive future notice of the results of + * an asynchronous method call. + * + * @param The return type of the asynchronously invoked method. + */ +public interface AsyncMethodCallback { + /** + * This method will be called when the remote side has completed invoking your method call and the + * result is fully read. For {@code oneway} method calls, this method will be called as soon as we + * have completed writing out the request. + * + * @param response The return value of the asynchronously invoked method; {@code null} for void + * methods which includes {@code oneway} methods. + */ + void onComplete(T response); + + /** + * This method will be called when there is either an unexpected client-side exception like an + * IOException or else when the remote method raises an exception, either declared in the IDL or + * due to an unexpected server-side error. + * + * @param exception The exception encountered processing the the asynchronous method call, may be + * a local exception or an unmarshalled remote exception. + */ + void onError(Exception exception); +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/async/AsyncMethodFutureAdapter.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/async/AsyncMethodFutureAdapter.java new file mode 100644 index 0000000..202af7b --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/async/AsyncMethodFutureAdapter.java @@ -0,0 +1,35 @@ +package org.apache.thrift.async; + +import java.util.concurrent.CompletableFuture; + +/** + * A simple adapter that bridges {@link AsyncMethodCallback} with {@link + * CompletableFuture}-returning style clients. Compiler generated code will invoke this adapter to + * implement {@code FutureClient}s. + * + * @param return type (can be {@link Void}). + */ +public final class AsyncMethodFutureAdapter implements AsyncMethodCallback { + + private AsyncMethodFutureAdapter() {} + + public static AsyncMethodFutureAdapter create() { + return new AsyncMethodFutureAdapter<>(); + } + + private final CompletableFuture future = new CompletableFuture<>(); + + public CompletableFuture getFuture() { + return future; + } + + @Override + public void onComplete(T response) { + future.complete(response); + } + + @Override + public void onError(Exception exception) { + future.completeExceptionally(exception); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/async/TAsyncClient.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/async/TAsyncClient.java new file mode 100644 index 0000000..0bfe1c9 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/async/TAsyncClient.java @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.thrift.async; + +import org.apache.thrift.protocol.TProtocolFactory; +import org.apache.thrift.transport.TNonblockingTransport; + +public abstract class TAsyncClient { + protected final TProtocolFactory ___protocolFactory; + protected final TNonblockingTransport ___transport; + protected final TAsyncClientManager ___manager; + protected TAsyncMethodCall ___currentMethod; + private Exception ___error; + private long ___timeout; + + public TAsyncClient( + TProtocolFactory protocolFactory, + TAsyncClientManager manager, + TNonblockingTransport transport) { + this(protocolFactory, manager, transport, 0); + } + + public TAsyncClient( + TProtocolFactory protocolFactory, + TAsyncClientManager manager, + TNonblockingTransport transport, + long timeout) { + this.___protocolFactory = protocolFactory; + this.___manager = manager; + this.___transport = transport; + this.___timeout = timeout; + } + + public TProtocolFactory getProtocolFactory() { + return ___protocolFactory; + } + + public long getTimeout() { + return ___timeout; + } + + public boolean hasTimeout() { + return ___timeout > 0; + } + + public void setTimeout(long timeout) { + this.___timeout = timeout; + } + + /** + * Is the client in an error state? + * + * @return If client in an error state? + */ + public boolean hasError() { + return ___error != null; + } + + /** + * Get the client's error - returns null if no error + * + * @return Get the client's error. + *

returns null if no error + */ + public Exception getError() { + return ___error; + } + + protected void checkReady() { + // Ensure we are not currently executing a method + if (___currentMethod != null) { + throw new IllegalStateException( + "Client is currently executing another method: " + ___currentMethod.getClass().getName()); + } + + // Ensure we're not in an error state + if (___error != null) { + throw new IllegalStateException("Client has an error!", ___error); + } + } + + /** Called by delegate method when finished */ + protected void onComplete() { + ___currentMethod = null; + } + + /** + * Called by delegate method on error. + * + * @param exception the exception indicating the current error condition. + */ + protected void onError(Exception exception) { + ___transport.close(); + ___currentMethod = null; + ___error = exception; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/async/TAsyncClientFactory.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/async/TAsyncClientFactory.java new file mode 100644 index 0000000..31ca288 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/async/TAsyncClientFactory.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.thrift.async; + +import org.apache.thrift.transport.TNonblockingTransport; + +public interface TAsyncClientFactory { + T getAsyncClient(TNonblockingTransport transport); +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/async/TAsyncClientManager.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/async/TAsyncClientManager.java new file mode 100644 index 0000000..256ab26 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/async/TAsyncClientManager.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.thrift.async; + +import java.io.IOException; +import java.io.Serializable; +import java.nio.channels.ClosedSelectorException; +import java.nio.channels.SelectionKey; +import java.nio.channels.Selector; +import java.nio.channels.spi.SelectorProvider; +import java.util.Comparator; +import java.util.Iterator; +import java.util.TreeSet; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.TimeoutException; +import org.apache.thrift.TException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Contains selector thread which transitions method call objects */ +public class TAsyncClientManager { + private static final Logger LOGGER = LoggerFactory.getLogger(TAsyncClientManager.class.getName()); + + private final SelectThread selectThread; + private final ConcurrentLinkedQueue> pendingCalls = + new ConcurrentLinkedQueue<>(); + + public TAsyncClientManager() throws IOException { + this.selectThread = new SelectThread(); + selectThread.start(); + } + + public void call(TAsyncMethodCall method) throws TException { + if (!isRunning()) { + throw new TException("SelectThread is not running"); + } + method.prepareMethodCall(); + pendingCalls.add(method); + selectThread.getSelector().wakeup(); + } + + public void stop() { + selectThread.finish(); + } + + public boolean isRunning() { + return selectThread.isAlive(); + } + + private class SelectThread extends Thread { + private final Selector selector; + private volatile boolean running; + private final TreeSet> timeoutWatchSet = + new TreeSet<>(new TAsyncMethodCallTimeoutComparator()); + + public SelectThread() throws IOException { + this.selector = SelectorProvider.provider().openSelector(); + this.running = true; + this.setName("TAsyncClientManager#SelectorThread " + this.getId()); + + // We don't want to hold up the JVM when shutting down + setDaemon(true); + } + + public Selector getSelector() { + return selector; + } + + public void finish() { + running = false; + selector.wakeup(); + } + + public void run() { + while (running) { + try { + try { + if (timeoutWatchSet.size() == 0) { + // No timeouts, so select indefinitely + selector.select(); + } else { + // We have a timeout pending, so calculate the time until then and select + // appropriately + long nextTimeout = timeoutWatchSet.first().getTimeoutTimestamp(); + long selectTime = nextTimeout - System.currentTimeMillis(); + if (selectTime > 0) { + // Next timeout is in the future, select and wake up then + selector.select(selectTime); + } else { + // Next timeout is now or in the past, select immediately, so we can time out + selector.selectNow(); + } + } + } catch (IOException e) { + LOGGER.error("Caught IOException in TAsyncClientManager!", e); + } + transitionMethods(); + timeoutMethods(); + startPendingMethods(); + } catch (Exception exception) { + LOGGER.error("Ignoring uncaught exception in SelectThread", exception); + } + } + + try { + selector.close(); + } catch (IOException ex) { + LOGGER.warn("Could not close selector. This may result in leaked resources!", ex); + } + } + + // Transition methods for ready keys + private void transitionMethods() { + try { + Iterator keys = selector.selectedKeys().iterator(); + while (keys.hasNext()) { + SelectionKey key = keys.next(); + keys.remove(); + if (!key.isValid()) { + // this can happen if the method call experienced an error and the + // key was cancelled. can also happen if we time out a method, which + // results in a channel close. + // just skip + continue; + } + TAsyncMethodCall methodCall = (TAsyncMethodCall) key.attachment(); + methodCall.transition(key); + + // If done or error occurred, remove from timeout watch set + if (methodCall.isFinished() || methodCall.getClient().hasError()) { + timeoutWatchSet.remove(methodCall); + } + } + } catch (ClosedSelectorException e) { + LOGGER.error("Caught ClosedSelectorException in TAsyncClientManager!", e); + } + } + + // Timeout any existing method calls + private void timeoutMethods() { + Iterator> iterator = timeoutWatchSet.iterator(); + long currentTime = System.currentTimeMillis(); + while (iterator.hasNext()) { + TAsyncMethodCall methodCall = iterator.next(); + if (currentTime >= methodCall.getTimeoutTimestamp()) { + iterator.remove(); + methodCall.onError( + new TimeoutException( + "Operation " + + methodCall.getClass() + + " timed out after " + + (currentTime - methodCall.getStartTime()) + + " ms.")); + } else { + break; + } + } + } + + // Start any new calls + private void startPendingMethods() { + TAsyncMethodCall methodCall; + while ((methodCall = pendingCalls.poll()) != null) { + // Catch registration errors. method will catch transition errors and cleanup. + try { + methodCall.start(selector); + + // If timeout specified and first transition went smoothly, add to timeout watch set + TAsyncClient client = methodCall.getClient(); + if (client.hasTimeout() && !client.hasError()) { + timeoutWatchSet.add(methodCall); + } + } catch (Exception exception) { + LOGGER.warn("Caught exception in TAsyncClientManager!", exception); + methodCall.onError(exception); + } + } + } + } + + /** Comparator used in TreeSet */ + private static class TAsyncMethodCallTimeoutComparator + implements Comparator>, Serializable { + @Override + public int compare(TAsyncMethodCall left, TAsyncMethodCall right) { + if (left.getTimeoutTimestamp() == right.getTimeoutTimestamp()) { + return (int) (left.getSequenceId() - right.getSequenceId()); + } else { + return (int) (left.getTimeoutTimestamp() - right.getTimeoutTimestamp()); + } + } + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/async/TAsyncMethodCall.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/async/TAsyncMethodCall.java new file mode 100644 index 0000000..9bcc8a9 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/async/TAsyncMethodCall.java @@ -0,0 +1,294 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.thrift.async; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.SelectionKey; +import java.nio.channels.Selector; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.protocol.TProtocolFactory; +import org.apache.thrift.transport.TMemoryBuffer; +import org.apache.thrift.transport.TNonblockingTransport; +import org.apache.thrift.transport.TTransportException; +import org.apache.thrift.transport.layered.TFramedTransport; + +/** + * Encapsulates an async method call. + * + *

Need to generate: + * + *

    + *
  • protected abstract void write_args(TProtocol protocol) + *
  • protected abstract T getResult() throws <Exception_1>, <Exception_2>, ... + *
+ * + * @param The return type of the encapsulated method call. + */ +public abstract class TAsyncMethodCall { + + private static final int INITIAL_MEMORY_BUFFER_SIZE = 128; + private static final AtomicLong sequenceIdCounter = new AtomicLong(0); + + public enum State { + CONNECTING, + WRITING_REQUEST_SIZE, + WRITING_REQUEST_BODY, + READING_RESPONSE_SIZE, + READING_RESPONSE_BODY, + RESPONSE_READ, + ERROR; + } + + /** Next step in the call, initialized by start() */ + private State state = null; + + protected final TNonblockingTransport transport; + private final TProtocolFactory protocolFactory; + protected final TAsyncClient client; + private final AsyncMethodCallback callback; + private final boolean isOneway; + private final long sequenceId; + private final long timeout; + + private ByteBuffer sizeBuffer; + private final byte[] sizeBufferArray = new byte[4]; + private ByteBuffer frameBuffer; + + private final long startTime = System.currentTimeMillis(); + + protected TAsyncMethodCall( + TAsyncClient client, + TProtocolFactory protocolFactory, + TNonblockingTransport transport, + AsyncMethodCallback callback, + boolean isOneway) { + this.transport = transport; + this.callback = callback; + this.protocolFactory = protocolFactory; + this.client = client; + this.isOneway = isOneway; + this.sequenceId = TAsyncMethodCall.sequenceIdCounter.getAndIncrement(); + this.timeout = client.getTimeout(); + } + + protected State getState() { + return state; + } + + protected boolean isFinished() { + return state == State.RESPONSE_READ; + } + + protected long getStartTime() { + return startTime; + } + + protected long getSequenceId() { + return sequenceId; + } + + public TAsyncClient getClient() { + return client; + } + + public boolean hasTimeout() { + return timeout > 0; + } + + public long getTimeoutTimestamp() { + return timeout + startTime; + } + + protected abstract void write_args(TProtocol protocol) throws TException; + + protected abstract T getResult() throws Exception; + + /** + * Initialize buffers. + * + * @throws TException if buffer initialization fails + */ + protected void prepareMethodCall() throws TException { + TMemoryBuffer memoryBuffer = new TMemoryBuffer(INITIAL_MEMORY_BUFFER_SIZE); + TProtocol protocol = protocolFactory.getProtocol(memoryBuffer); + write_args(protocol); + + int length = memoryBuffer.length(); + frameBuffer = ByteBuffer.wrap(memoryBuffer.getArray(), 0, length); + + TFramedTransport.encodeFrameSize(length, sizeBufferArray); + sizeBuffer = ByteBuffer.wrap(sizeBufferArray); + } + + /** + * Register with selector and start first state, which could be either connecting or writing. + * + * @throws IOException if register or starting fails + */ + void start(Selector sel) throws IOException { + SelectionKey key; + if (transport.isOpen()) { + state = State.WRITING_REQUEST_SIZE; + key = transport.registerSelector(sel, SelectionKey.OP_WRITE); + } else { + state = State.CONNECTING; + key = transport.registerSelector(sel, SelectionKey.OP_CONNECT); + + // non-blocking connect can complete immediately, + // in which case we should not expect the OP_CONNECT + if (transport.startConnect()) { + registerForFirstWrite(key); + } + } + + key.attach(this); + } + + void registerForFirstWrite(SelectionKey key) throws IOException { + state = State.WRITING_REQUEST_SIZE; + key.interestOps(SelectionKey.OP_WRITE); + } + + protected ByteBuffer getFrameBuffer() { + return frameBuffer; + } + + /** + * Transition to next state, doing whatever work is required. Since this method is only called by + * the selector thread, we can make changes to our select interests without worrying about + * concurrency. + * + * @param key selection key + */ + void transition(SelectionKey key) { + // Ensure key is valid + if (!key.isValid()) { + key.cancel(); + Exception e = new TTransportException("Selection key not valid!"); + onError(e); + return; + } + + // Transition function + try { + switch (state) { + case CONNECTING: + doConnecting(key); + break; + case WRITING_REQUEST_SIZE: + doWritingRequestSize(); + break; + case WRITING_REQUEST_BODY: + doWritingRequestBody(key); + break; + case READING_RESPONSE_SIZE: + doReadingResponseSize(); + break; + case READING_RESPONSE_BODY: + doReadingResponseBody(key); + break; + default: // RESPONSE_READ, ERROR, or bug + throw new IllegalStateException( + "Method call in state " + + state + + " but selector called transition method. Seems like a bug..."); + } + } catch (Exception e) { + key.cancel(); + key.attach(null); + onError(e); + } + } + + protected void onError(Exception e) { + client.onError(e); + callback.onError(e); + state = State.ERROR; + } + + private void doReadingResponseBody(SelectionKey key) throws TTransportException { + if (transport.read(frameBuffer) < 0) { + throw new TTransportException(TTransportException.END_OF_FILE, "Read call frame failed"); + } + if (frameBuffer.remaining() == 0) { + cleanUpAndFireCallback(key); + } + } + + private void cleanUpAndFireCallback(SelectionKey key) { + state = State.RESPONSE_READ; + key.interestOps(0); + // this ensures that the TAsyncMethod instance doesn't hang around + key.attach(null); + try { + T result = this.getResult(); + client.onComplete(); + callback.onComplete(result); + } catch (Exception e) { + key.cancel(); + onError(e); + } + } + + private void doReadingResponseSize() throws TTransportException { + if (transport.read(sizeBuffer) < 0) { + throw new TTransportException(TTransportException.END_OF_FILE, "Read call frame size failed"); + } + if (sizeBuffer.remaining() == 0) { + state = State.READING_RESPONSE_BODY; + frameBuffer = ByteBuffer.allocate(TFramedTransport.decodeFrameSize(sizeBufferArray)); + } + } + + private void doWritingRequestBody(SelectionKey key) throws TTransportException { + if (transport.write(frameBuffer) < 0) { + throw new TTransportException(TTransportException.END_OF_FILE, "Write call frame failed"); + } + if (frameBuffer.remaining() == 0) { + if (isOneway) { + cleanUpAndFireCallback(key); + } else { + state = State.READING_RESPONSE_SIZE; + sizeBuffer.rewind(); // Prepare to read incoming frame size + key.interestOps(SelectionKey.OP_READ); + } + } + } + + private void doWritingRequestSize() throws TTransportException { + if (transport.write(sizeBuffer) < 0) { + throw new TTransportException( + TTransportException.END_OF_FILE, "Write call frame size failed"); + } + if (sizeBuffer.remaining() == 0) { + state = State.WRITING_REQUEST_BODY; + } + } + + private void doConnecting(SelectionKey key) throws IOException { + if (!key.isConnectable() || !transport.finishConnect()) { + throw new IOException( + "not connectable or finishConnect returned false after we got an OP_CONNECT"); + } + registerForFirstWrite(key); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/meta_data/EnumMetaData.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/meta_data/EnumMetaData.java new file mode 100644 index 0000000..b6fc113 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/meta_data/EnumMetaData.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.meta_data; + +import org.apache.thrift.TEnum; + +public class EnumMetaData extends FieldValueMetaData { + public final Class enumClass; + + public EnumMetaData(byte type, Class sClass) { + super(type); + this.enumClass = sClass; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/meta_data/FieldMetaData.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/meta_data/FieldMetaData.java new file mode 100644 index 0000000..3a7c1e2 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/meta_data/FieldMetaData.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.meta_data; + +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import org.apache.thrift.TBase; +import org.apache.thrift.TFieldIdEnum; + +/** + * This class is used to store meta data about thrift fields. Every field in a a struct should have + * a corresponding instance of this class describing it. + * + *

The meta data is registered by ALL Thrift struct classes via a static {...} initializer block + * in the generated Thrift code. + * + *

Since different threads could be initializing different Thrift classes, calls to the public + * static methods of this class could be racy. + * + *

All methods of this class should be made thread safe. + */ +public class FieldMetaData implements java.io.Serializable { + public final String fieldName; + public final byte requirementType; + public final FieldValueMetaData valueMetaData; + private final Map fieldAnnotations; + private static final ConcurrentMap< + Class, Map> + structMap = new ConcurrentHashMap<>(); + + public FieldMetaData(String name, byte req, FieldValueMetaData vMetaData) { + this(name, req, vMetaData, Collections.emptyMap()); + } + + public FieldMetaData( + String fieldName, + byte requirementType, + FieldValueMetaData valueMetaData, + Map fieldAnnotations) { + this.fieldName = fieldName; + this.requirementType = requirementType; + this.valueMetaData = valueMetaData; + this.fieldAnnotations = fieldAnnotations; + } + + /** + * @return an unmodifiable view of the annotations for this field, empty if no annotations present + * or code gen param is not turned on + */ + public Map getFieldAnnotations() { + return Collections.unmodifiableMap(fieldAnnotations); + } + + public static , F extends TFieldIdEnum> void addStructMetaDataMap( + Class sClass, Map map) { + structMap.put(sClass, map); + } + + /** + * Returns a map with metadata (i.e. instances of FieldMetaData) that describe the fields of the + * given class. + * + * @param sClass The TBase class for which the metadata map is requested. It is not guaranteed + * that sClass will have been statically initialized before this method is called. A racy call + * to {@link FieldMetaData#addStructMetaDataMap(Class, Map)} from a different thread during + * static initialization of the Thrift class is possible. + */ + public static , F extends TFieldIdEnum> + Map getStructMetaDataMap(Class sClass) { + // Note: Do not use synchronized on this method declaration - it leads to a deadlock. + // Similarly, do not trigger sClass.newInstance() while holding a lock on structMap, + // it will lead to the same deadlock. + // See: https://issues.apache.org/jira/browse/THRIFT-5430 for details. + if (!structMap.containsKey(sClass)) { // Load class if it hasn't been loaded + try { + sClass.getDeclaredConstructor().newInstance(); + } catch (ReflectiveOperationException e) { + throw new RuntimeException( + e.getClass().getSimpleName() + " for TBase class: " + sClass.getName(), e); + } + } + //noinspection unchecked + return (Map) structMap.get(sClass); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/meta_data/FieldValueMetaData.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/meta_data/FieldValueMetaData.java new file mode 100644 index 0000000..d022e11 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/meta_data/FieldValueMetaData.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.meta_data; + +import org.apache.thrift.protocol.TType; + +/** + * FieldValueMetaData and collection of subclasses to store metadata about the value(s) of a field + */ +public class FieldValueMetaData implements java.io.Serializable { + public final byte type; + + private final boolean isTypedefType; + private final String typedefName; + private final boolean isBinary; + + public FieldValueMetaData(byte type, boolean binary) { + this.type = type; + this.isTypedefType = false; + this.typedefName = null; + this.isBinary = binary; + } + + public FieldValueMetaData(byte type) { + this(type, false); + } + + public FieldValueMetaData(byte type, String typedefName) { + this.type = type; + this.isTypedefType = true; + this.typedefName = typedefName; + this.isBinary = false; + } + + public boolean isTypedef() { + return isTypedefType; + } + + public String getTypedefName() { + return typedefName; + } + + public boolean isStruct() { + return type == TType.STRUCT; + } + + public boolean isContainer() { + return type == TType.LIST || type == TType.MAP || type == TType.SET; + } + + public boolean isBinary() { + return isBinary; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/meta_data/ListMetaData.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/meta_data/ListMetaData.java new file mode 100644 index 0000000..e87e97a --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/meta_data/ListMetaData.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.meta_data; + +public class ListMetaData extends FieldValueMetaData { + public final FieldValueMetaData elemMetaData; + + public ListMetaData(byte type, FieldValueMetaData eMetaData) { + super(type); + this.elemMetaData = eMetaData; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/meta_data/MapMetaData.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/meta_data/MapMetaData.java new file mode 100644 index 0000000..692a372 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/meta_data/MapMetaData.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.meta_data; + +public class MapMetaData extends FieldValueMetaData { + public final FieldValueMetaData keyMetaData; + public final FieldValueMetaData valueMetaData; + + public MapMetaData(byte type, FieldValueMetaData kMetaData, FieldValueMetaData vMetaData) { + super(type); + this.keyMetaData = kMetaData; + this.valueMetaData = vMetaData; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/meta_data/SetMetaData.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/meta_data/SetMetaData.java new file mode 100644 index 0000000..3f2b1e2 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/meta_data/SetMetaData.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.meta_data; + +public class SetMetaData extends FieldValueMetaData { + public final FieldValueMetaData elemMetaData; + + public SetMetaData(byte type, FieldValueMetaData eMetaData) { + super(type); + this.elemMetaData = eMetaData; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/meta_data/StructMetaData.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/meta_data/StructMetaData.java new file mode 100644 index 0000000..98d7341 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/meta_data/StructMetaData.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.meta_data; + +import org.apache.thrift.TBase; + +public class StructMetaData extends FieldValueMetaData { + public final Class structClass; + + public StructMetaData(byte type, Class sClass) { + super(type); + this.structClass = sClass; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/EnumCache.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/EnumCache.java new file mode 100644 index 0000000..4fc7562 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/EnumCache.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.partial; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.util.HashMap; +import java.util.Map; +import org.apache.thrift.TEnum; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Provides a memoized way to lookup an enum by its value. + * + *

This class is used internally by {@code TDeserializer}. It is not intended to be used + * separately on its own. + */ +public class EnumCache { + private static final Logger LOG = LoggerFactory.getLogger(EnumCache.class); + + private Map, Map> classMap; + + public EnumCache() { + this.classMap = new HashMap<>(); + } + + /** + * Gets an instance of the enum type {@code enumClass} corresponding to the given {@code value}. + * + * @param enumClass class of the enum to be returned. + * @param value value returned by {@code getValue()}. + */ + public TEnum get(Class enumClass, int value) { + Validate.checkNotNull(enumClass, "enumClass"); + + Map valueMap = classMap.get(enumClass); + if (valueMap == null) { + valueMap = addClass(enumClass); + if (valueMap == null) { + return null; + } + } + + return valueMap.get(value); + } + + private Map addClass(Class enumClass) { + try { + Method valuesMethod = enumClass.getMethod("values"); + TEnum[] enumValues = (TEnum[]) valuesMethod.invoke(null); + Map valueMap = new HashMap<>(); + + for (TEnum enumValue : enumValues) { + valueMap.put(enumValue.getValue(), enumValue); + } + + classMap.put(enumClass, valueMap); + return valueMap; + } catch (NoSuchMethodException e) { + LOG.error("enum class does not have values() method", e); + return null; + } catch (IllegalAccessException e) { + LOG.error("Enum.values() method should be public!", e); + return null; + } catch (InvocationTargetException e) { + LOG.error("Enum.values() threw exception", e); + return null; + } + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/PartialThriftComparer.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/PartialThriftComparer.java new file mode 100644 index 0000000..e635dc6 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/PartialThriftComparer.java @@ -0,0 +1,342 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.partial; + +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.thrift.TBase; +import org.apache.thrift.protocol.TType; + +/** + * Enables comparison of two TBase instances such that the comparison is limited to the subset of + * fields defined by the supplied metadata. + * + *

This comparer is useful when comparing two instances where: -- one is generated by full + * deserialization. -- the other is generated by partial deserialization. + * + *

The typical use case is to establish correctness of partial deserialization. + */ +public class PartialThriftComparer { + + private enum ComparisonResult { + UNKNOWN, + EQUAL, + NOT_EQUAL + } + + // Metadata that defines the scope of comparison. + private ThriftMetadata.ThriftStruct metadata; + + /** + * Constructs an instance of {@link PartialThriftComparer}. + * + * @param metadata defines the scope of comparison. + */ + public PartialThriftComparer(ThriftMetadata.ThriftStruct metadata) { + this.metadata = metadata; + } + + /** + * Compares thrift objects {@code t1} and {@code t2} and returns true if they are equal false + * otherwise. The comparison is limited to the scope defined by {@code metadata}. + * + *

If the objects are not equal then it optionally records their differences if {@code sb} is + * supplied. + * + *

+ * + * @param t1 the first object. + * @param t2 the second object. + * @param sb if non-null, results of the comparison are returned in it. + * @return true if objects are equivalent, false otherwise. + */ + public boolean areEqual(T t1, T t2, StringBuilder sb) { + return this.areEqual(this.metadata, t1, t2, sb); + } + + private boolean areEqual( + ThriftMetadata.ThriftObject data, Object o1, Object o2, StringBuilder sb) { + + byte fieldType = data.data.valueMetaData.type; + switch (fieldType) { + case TType.STRUCT: + return this.areEqual((ThriftMetadata.ThriftStruct) data, o1, o2, sb); + + case TType.LIST: + return this.areEqual((ThriftMetadata.ThriftList) data, o1, o2, sb); + + case TType.MAP: + return this.areEqual((ThriftMetadata.ThriftMap) data, o1, o2, sb); + + case TType.SET: + return this.areEqual((ThriftMetadata.ThriftSet) data, o1, o2, sb); + + case TType.ENUM: + return this.areEqual((ThriftMetadata.ThriftEnum) data, o1, o2, sb); + + case TType.BOOL: + case TType.BYTE: + case TType.I16: + case TType.I32: + case TType.I64: + case TType.DOUBLE: + case TType.STRING: + return this.areEqual((ThriftMetadata.ThriftPrimitive) data, o1, o2, sb); + + default: + throw unsupportedFieldTypeException(fieldType); + } + } + + private boolean areEqual( + ThriftMetadata.ThriftStruct data, Object o1, Object o2, StringBuilder sb) { + ComparisonResult result = checkNullEquality(data, o1, o2, sb); + if (result != ComparisonResult.UNKNOWN) { + return result == ComparisonResult.EQUAL; + } + + TBase t1 = (TBase) o1; + TBase t2 = (TBase) o2; + + if (data.fields.size() == 0) { + if (t1.equals(t2)) { + return true; + } else { + appendNotEqual(data, sb, t1, t2, "struct1", "struct2"); + return false; + } + } else { + + boolean overallResult = true; + + for (Object o : data.fields.values()) { + ThriftMetadata.ThriftObject field = (ThriftMetadata.ThriftObject) o; + Object f1 = t1.getFieldValue(field.fieldId); + Object f2 = t2.getFieldValue(field.fieldId); + overallResult = overallResult && this.areEqual(field, f1, f2, sb); + } + + return overallResult; + } + } + + private boolean areEqual( + ThriftMetadata.ThriftPrimitive data, Object o1, Object o2, StringBuilder sb) { + + ComparisonResult result = checkNullEquality(data, o1, o2, sb); + if (result != ComparisonResult.UNKNOWN) { + return result == ComparisonResult.EQUAL; + } + + if (data.isBinary()) { + if (areBinaryFieldsEqual(o1, o2)) { + return true; + } + } else if (o1.equals(o2)) { + return true; + } + + appendNotEqual(data, sb, o1, o2, "o1", "o2"); + return false; + } + + private boolean areEqual(ThriftMetadata.ThriftEnum data, Object o1, Object o2, StringBuilder sb) { + + ComparisonResult result = checkNullEquality(data, o1, o2, sb); + if (result != ComparisonResult.UNKNOWN) { + return result == ComparisonResult.EQUAL; + } + + if (o1.equals(o2)) { + return true; + } + + appendNotEqual(data, sb, o1, o2, "o1", "o2"); + return false; + } + + private boolean areEqual(ThriftMetadata.ThriftList data, Object o1, Object o2, StringBuilder sb) { + + List l1 = (List) o1; + List l2 = (List) o2; + + ComparisonResult result = checkNullEquality(data, o1, o2, sb); + if (result != ComparisonResult.UNKNOWN) { + return result == ComparisonResult.EQUAL; + } + + if (!checkSizeEquality(data, l1, l2, sb, "list")) { + return false; + } + + for (int i = 0; i < l1.size(); i++) { + Object e1 = l1.get(i); + Object e2 = l2.get(i); + if (!this.areEqual(data.elementData, e1, e2, sb)) { + return false; + } + } + + return true; + } + + private boolean areEqual(ThriftMetadata.ThriftSet data, Object o1, Object o2, StringBuilder sb) { + + Set s1 = (Set) o1; + Set s2 = (Set) o2; + + ComparisonResult result = checkNullEquality(data, o1, o2, sb); + if (result != ComparisonResult.UNKNOWN) { + return result == ComparisonResult.EQUAL; + } + + if (!checkSizeEquality(data, s1, s2, sb, "set")) { + return false; + } + + for (Object e1 : s1) { + if (!s2.contains(e1)) { + appendResult(data, sb, "Element %s in s1 not found in s2", e1); + return false; + } + } + + return true; + } + + private boolean areEqual(ThriftMetadata.ThriftMap data, Object o1, Object o2, StringBuilder sb) { + + Map m1 = (Map) o1; + Map m2 = (Map) o2; + + ComparisonResult result = checkNullEquality(data, o1, o2, sb); + if (result != ComparisonResult.UNKNOWN) { + return result == ComparisonResult.EQUAL; + } + + if (!checkSizeEquality(data, m1.keySet(), m2.keySet(), sb, "map.keySet")) { + return false; + } + + for (Map.Entry e1 : m1.entrySet()) { + Object k1 = e1.getKey(); + + if (!m2.containsKey(k1)) { + appendResult(data, sb, "Key %s in m1 not found in m2", k1); + return false; + } + + Object v1 = e1.getValue(); + Object v2 = m2.get(k1); + if (!this.areEqual(data.valueData, v1, v2, sb)) { + return false; + } + } + + return true; + } + + private boolean areBinaryFieldsEqual(Object o1, Object o2) { + if (o1 instanceof byte[]) { + if (Arrays.equals((byte[]) o1, (byte[]) o2)) { + return true; + } + } else if (o1 instanceof ByteBuffer) { + if (((ByteBuffer) o1).compareTo((ByteBuffer) o2) == 0) { + return true; + } + } else { + throw new UnsupportedOperationException( + String.format("Unsupported binary field type: %s", o1.getClass().getName())); + } + + return false; + } + + private void appendResult( + ThriftMetadata.ThriftObject data, StringBuilder sb, String format, Object... args) { + if (sb != null) { + String msg = String.format(format, args); + sb.append(data.fieldId.getFieldName()); + sb.append(" : "); + sb.append(msg); + } + } + + private void appendNotEqual( + ThriftMetadata.ThriftObject data, + StringBuilder sb, + Object o1, + Object o2, + String o1name, + String o2name) { + + String o1s = o1.toString(); + String o2s = o2.toString(); + + if ((o1s.length() + o2s.length()) < 100) { + appendResult(data, sb, "%s (%s) != %s (%s)", o1name, o1s, o2name, o2s); + } else { + appendResult( + data, sb, "%s != %s\n%s =\n%s\n%s =\n%s\n", o1name, o2name, o1name, o1s, o2name, o2s); + } + } + + private ComparisonResult checkNullEquality( + ThriftMetadata.ThriftObject data, Object o1, Object o2, StringBuilder sb) { + if ((o1 == null) && (o2 == null)) { + return ComparisonResult.EQUAL; + } + + if (o1 == null) { + appendResult(data, sb, "o1 (null) != o2"); + } + + if (o2 == null) { + appendResult(data, sb, "o1 != o2 (null)"); + } + + return ComparisonResult.UNKNOWN; + } + + private boolean checkSizeEquality( + ThriftMetadata.ThriftObject data, + Collection c1, + Collection c2, + StringBuilder sb, + String typeName) { + + if (c1.size() != c2.size()) { + appendResult( + data, sb, "%s1.size(%d) != %s2.size(%d)", typeName, c1.size(), typeName, c2.size()); + return false; + } + + return true; + } + + static UnsupportedOperationException unsupportedFieldTypeException(byte fieldType) { + return new UnsupportedOperationException("field type not supported: '" + fieldType + "'"); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/README.md b/hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/README.md new file mode 100644 index 0000000..107a54e --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/README.md @@ -0,0 +1,112 @@ +# Partial Thrift Deserialization + +## Overview +This document describes how partial deserialization of Thrift works. There are two main goals of this documentation: +1. Make it easier to understand the current Java implementation in this folder. +1. Be useful in implementing partial deserialization support in additional languages. + +This document is divided into two high level areas. The first part explains important concepts relevant to partial deserialization. The second part describes components involved in the Java implementation in this folder. + +Moreover, this blog provides some performance numbers and addtional information: https://medium.com/pinterest-engineering/improving-data-processing-efficiency-using-partial-deserialization-of-thrift-16bc3a4a38b4 + +## Basic Concepts + +### Motivation + +The main motivation behind implementing this feature is to improve performance when we need to access only a subset of fields in any Thrift object. This situation arises often when big data is stored in Thrift encoded format (for example, SequenceFile with serialized Thrift values). Many data processing jobs may access this data. However, not every job needs to access every field of each object. In such cases, if we have prior knowledge of the fields needed for a given job, we can deserialize only that subset of fields and avoid the cost deserializing the rest of the fields. There are two benefits of this approach: we save cpu cycles by not deserializing unnecessary fields and we end up reducing gc pressure. Both of the savings quickly add up when processing billions of instances in a data processing job. + +### Partial deserialization + +Partial deserialization involves deserializing only a subset of the fields of a serialized Thrift object while efficiently skipping over the rest. One very important benefit of partial deserialization is that the output of the deserialization process is not limited to a `TBase` derived object. It can deserialize a serialized blob into any type by using an appropriate `ThriftFieldValueProcessor`. + +### Defining the subset of fields to deserialize + +The subset of fields to deserialize is defined using a list of fully qualified field names. For example, consider the Thrift `struct` definition below: + +```Thrift +struct SmallStruct { + 1: optional string stringValue; + 2: optional i16 i16Value; +} + +struct TestStruct { + 1: optional i16 i16Field; + 2: optional list structList; + 3: optional set structSet; + 4: optional map structMap; + 5: optional SmallStruct structField; +} +``` + +For the Thrift `struct`, each of the following line shows a fully qualified field definition. Partial deserialization uses a non-empty set of such field definitions to identify the subset of fields to deserialize. + +``` +- i16Field +- structList.stringValue +- structSet.i16Value +- structMap.stringValue +- structField.i16Value +``` + +Note that the syntax of denoting paths involving map fields do not support a way to define sub-fields of the key type. However, that limitation can be addressed in future by amending the syntax in a backword compatible way. + +For example, the field path `structMap.stringValue` shown above has leaf segment `stringValue` which is a field in map values. + +## Components + +The process of partial deserialization involves the following major components. We have listed names of the Java file(s) implementing each component for easier mapping to the source code. + +### Thrift Metadata + +Source files: +- ThriftField.java +- ThriftMetadata.java +- TDeserializer.java + +We saw in the previous section how we can identify the subset of fields to deserialize. As the first step, we need to compile the collection of field definitions into an internal efficient data structure that we can traverse at runtime. The compilation takes place internally when one creates an instance of TDeserializer using a constructor that accepts a list of field names. + +```Java +// First, create a collection of fully qualified field names. +List fieldNames = Arrays.asList("i16Field", "structField.i16Value"); + +// Create an instance of TDeserializer that supports partial deserialization. +TDeserializer deserializer = + new TDeserializer(TestStruct.class, fieldNames, new TBinaryProtocol.Factory()); +``` + +At this point, we have an efficient internal representation of the fields that need to get deserialized. + +### Partial Thrift Protocol + +Source files: +- TProtocol.java +- TBinaryProtocol.java +- TCompactProtocol.java + +This component implements efficient skipping over fields that need not be deserialized. The functionality to skip over fields has been added to the above protocols by addition of `skip*()` methods. The default implementation of each such method simply calls the corresponding `read*()` method in `TProtocol.java`. A derived protocol (for example, `TBinaryProtocol`) provides a more efficient implementation of each `skip*()` method. + +For example, `TBinaryProtocol` skips a field by incrementing internal offset into the transport buffer. + +### Partial Thrift Deserializer + +Source files: +- TDeserializer.java + +This component, traverses a serialized blob sequentially one field at a time. At the beginning of each field, it consults the informations stored in the compiled `ThriftMetadata` to see if that field needs to be deserialized. If yes, then the field is deserialized into a value as would normally take place during regular deserialization process. If that field is not in the target subset then the deserializer efficiently skips over that field. + +### Field Value Processor + +Source files: +- ThriftFieldValueProcessor.java +- ThriftStructProcessor.java + +One very important benefit of partial deserialization is that the output of the deserialization process is not limited to a `TBase` derived object. It can deserialize a serialized blob into any type by using an appropriate `ThriftFieldValueProcessor`. + +When the partial Thrift deserializer deserializes a field, it passes its value to a `ThriftFieldValueProcessor`. The processor gets to decide whether the value is stored as-is or is stored in some intermediate form. The default implementation of this interface is `ThriftStructProcessor`. This implementation outputs a `TBase` derived object. There are other implementations that exist (not included in this drop at present). For example, one implementation enables deserializing a Thrift blob directly into an `InternalRow` used by `Spark`. That has yielded orders of magnitude performance improvement over a `Spark` engine that consumes `Thrift` data using its default deserializer. + +### Miscellanious Helpers + +Files: +- TFieldData.java : Holds the type and id members of a TField into a single int. This encoding scheme obviates the need to instantiate TField during the partial deserialization process. +- EnumCache.java : Provides a memoized way to lookup an enum by its value. +- PartialThriftComparer.java : Enables comparison of two TBase instances such that the comparison is limited to the subset of fields defined by the supplied metadata. diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/TFieldData.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/TFieldData.java new file mode 100644 index 0000000..5119259 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/TFieldData.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.partial; + +/** + * Holds the type and id members of a {@link org.apache.thrift.protocol.TField} into a single int. + * + *

This encoding scheme obviates the need to instantiate TField during the partial + * deserialization process. + */ +public class TFieldData { + public static int encode(byte type) { + return type & 0xff; + } + + public static int encode(byte type, short id) { + return (type & 0xff) | (((int) id) << 8); + } + + public static byte getType(int data) { + return (byte) (0xff & data); + } + + public static short getId(int data) { + return (short) ((0xffff00 & data) >> 8); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/ThriftField.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/ThriftField.java new file mode 100644 index 0000000..a0fbda6 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/ThriftField.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.partial; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +/** + * Holds name of a thrift field and of its sub-fields recursively. + * + *

This class is meant to be used in conjunction with {@code TDeserializer}. + */ +public class ThriftField { + + /** Name of this field as it appears in a thrift file. Case sensitive. */ + public final String name; + + /** + * List of sub-fields of this field. + * + *

This list should have only those sub-fields that need to be deserialized by the {@code + * TDeserializer}. + */ + public final List fields; + + /** + * Constructs a {@link ThriftField}. + * + * @param name the name of this field as it appears in a thrift file. Case sensitive. + * @param fields List of sub-fields of this field. + */ + ThriftField(String name, List fields) { + Validate.checkNotNullAndNotEmpty(name, "name"); + Validate.checkNotNull(fields, "fields"); + + this.name = name; + this.fields = Collections.unmodifiableList(fields); + } + + /** Constructs a {@link ThriftField} that does not have any sub-fields. */ + ThriftField(String name) { + this(name, Collections.emptyList()); + } + + // Internal-only constructor that does not mark fields as read-only. + // That allows fromNames() to construct fields from names. + // The actual value of allowFieldAdds is ignored. + // It is used only for generating a different function signature. + ThriftField(String name, List fields, boolean allowFieldAdds) { + Validate.checkNotNullAndNotEmpty(name, "name"); + Validate.checkNotNull(fields, "fields"); + + this.name = name; + this.fields = fields; + } + + private int hashcode = 0; + + @Override + public int hashCode() { + if (this.hashcode == 0) { + int hc = this.name.toLowerCase().hashCode(); + for (ThriftField subField : this.fields) { + hc ^= subField.hashCode(); + } + + this.hashcode = hc; + } + + return this.hashcode; + } + + @Override + public boolean equals(Object o) { + if (o == null) { + return false; + } + + if (!(o instanceof ThriftField)) { + return false; + } + + ThriftField other = (ThriftField) o; + + if (!this.name.equalsIgnoreCase(other.name)) { + return false; + } + + if (this.fields.size() != other.fields.size()) { + return false; + } + + for (int i = 0; i < this.fields.size(); i++) { + if (!this.fields.get(i).equals(other.fields.get(i))) { + return false; + } + } + + return true; + } + + @Override + public String toString() { + return String.join(", ", this.getFieldNames()); + } + + public List getFieldNames() { + List fieldsList = new ArrayList<>(); + if (this.fields.size() == 0) { + fieldsList.add(this.name); + } else { + for (ThriftField f : this.fields) { + for (String subF : f.getFieldNames()) { + fieldsList.add(this.name + "." + subF); + } + } + } + + return fieldsList; + } + + /** + * Generates and returns n-ary tree of fields and their sub-fields. + * + *

+ * + * @param fieldNames collection of fully qualified field names. + *

for example, In case of PinJoin thrift struct, the following are valid field names -- + * signature -- pins.user.userId -- textSignal.termSignal.termDataMap + * @return n-ary tree of fields and their sub-fields. + */ + public static List fromNames(Collection fieldNames) { + Validate.checkNotNullAndNotEmpty(fieldNames, "fieldNames"); + + List fieldNamesList = new ArrayList<>(fieldNames); + Collections.sort(fieldNamesList, String.CASE_INSENSITIVE_ORDER); + + List fields = new ArrayList<>(); + + for (String fieldName : fieldNamesList) { + List tfields = fields; + String[] tokens = fieldName.split("\\."); + + for (String token : tokens) { + ThriftField field = findField(token, tfields); + if (field == null) { + field = new ThriftField(token, new ArrayList<>(), true); + tfields.add(field); + } + tfields = field.fields; + } + } + + return makeReadOnly(fields); + } + + private static ThriftField findField(String name, List fields) { + for (ThriftField field : fields) { + if (field.name.equalsIgnoreCase(name)) { + return field; + } + } + return null; + } + + private static List makeReadOnly(List fields) { + List result = new ArrayList<>(fields.size()); + for (ThriftField field : fields) { + ThriftField copy = new ThriftField(field.name, makeReadOnly(field.fields)); + result.add(copy); + } + return Collections.unmodifiableList(result); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/ThriftFieldValueProcessor.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/ThriftFieldValueProcessor.java new file mode 100644 index 0000000..edb106c --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/ThriftFieldValueProcessor.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.partial; + +import java.nio.ByteBuffer; +import org.apache.thrift.TEnum; +import org.apache.thrift.TFieldIdEnum; + +/** + * Provides an abstraction to process deserialized field values and place them into the collection + * that holds them. This abstraction allows different types of collections to be output from partial + * deserialization. + * + *

In case of the usual Thrift deserialization, the collection that holds field values is simply + * an instance of TBase. + */ +public interface ThriftFieldValueProcessor { + + // Struct related methods; + Object createNewStruct(ThriftMetadata.ThriftStruct metadata); + + V prepareStruct(Object instance); + + void setBool(V valueCollection, TFieldIdEnum fieldId, boolean value); + + void setByte(V valueCollection, TFieldIdEnum fieldId, byte value); + + void setInt16(V valueCollection, TFieldIdEnum fieldId, short value); + + void setInt32(V valueCollection, TFieldIdEnum fieldId, int value); + + void setInt64(V valueCollection, TFieldIdEnum fieldId, long value); + + void setDouble(V valueCollection, TFieldIdEnum fieldId, double value); + + void setBinary(V valueCollection, TFieldIdEnum fieldId, ByteBuffer value); + + void setString(V valueCollection, TFieldIdEnum fieldId, ByteBuffer buffer); + + void setEnumField(V valueCollection, TFieldIdEnum fieldId, Object value); + + void setListField(V valueCollection, TFieldIdEnum fieldId, Object value); + + void setMapField(V valueCollection, TFieldIdEnum fieldId, Object value); + + void setSetField(V valueCollection, TFieldIdEnum fieldId, Object value); + + void setStructField(V valueCollection, TFieldIdEnum fieldId, Object value); + + Object prepareEnum(Class enumClass, int ordinal); + + Object prepareString(ByteBuffer buffer); + + Object prepareBinary(ByteBuffer buffer); + + // List field related methods. + Object createNewList(int expectedSize); + + void setListElement(Object instance, int index, Object value); + + Object prepareList(Object instance); + + // Map field related methods. + Object createNewMap(int expectedSize); + + void setMapElement(Object instance, int index, Object key, Object value); + + Object prepareMap(Object instance); + + // Set field related methods. + Object createNewSet(int expectedSize); + + void setSetElement(Object instance, int index, Object value); + + Object prepareSet(Object instance); +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/ThriftMetadata.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/ThriftMetadata.java new file mode 100644 index 0000000..69f76cf --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/ThriftMetadata.java @@ -0,0 +1,552 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.partial; + +import java.io.Serializable; +import java.lang.reflect.Constructor; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; +import org.apache.thrift.TBase; +import org.apache.thrift.TFieldIdEnum; +import org.apache.thrift.TFieldRequirementType; +import org.apache.thrift.TUnion; +import org.apache.thrift.meta_data.FieldMetaData; +import org.apache.thrift.meta_data.ListMetaData; +import org.apache.thrift.meta_data.MapMetaData; +import org.apache.thrift.meta_data.SetMetaData; +import org.apache.thrift.meta_data.StructMetaData; +import org.apache.thrift.protocol.TType; + +/** + * Container for Thrift metadata classes such as {@link ThriftPrimitive}, {@link ThriftList}, etc. + * + *

This class is mainly used by {@code TDeserializer}. + */ +public class ThriftMetadata { + + enum FieldTypeEnum implements TFieldIdEnum { + ROOT((short) 0, "root"), + ENUM((short) 1, "enum"), + LIST_ELEMENT((short) 2, "listElement"), + MAP_KEY((short) 3, "mapKey"), + MAP_VALUE((short) 4, "mapValue"), + SET_ELEMENT((short) 5, "setElement"); + + private final short id; + private final String name; + + FieldTypeEnum(short id, String name) { + this.id = id; + this.name = name; + } + + @Override + public short getThriftFieldId() { + return id; + } + + @Override + public String getFieldName() { + return name; + } + } + + private enum ComparisonResult { + UNKNOWN, + EQUAL, + NOT_EQUAL + } + + /** + * Base class of field types that can be partially deserialized. + * + *

Holds metadata necessary for partial deserialization. The metadata is internally computed + * and used; therefore it is not visible to the users of {@code TDeserializer}. + */ + public abstract static class ThriftObject implements Serializable { + public final ThriftObject parent; + public final TFieldIdEnum fieldId; + public final FieldMetaData data; + + // Placeholder to attach additional data. This class or its descendents + // do not try to access or interpret this field. + public Object additionalData; + + ThriftObject(ThriftObject parent, TFieldIdEnum fieldId, FieldMetaData data) { + this.parent = parent; + this.fieldId = fieldId; + this.data = data; + } + + /** + * Converts this instance to formatted and indented string representation. + * + * @param sb the {@code StringBuilder} to add formatted strings to. + * @param level the current indent level. + */ + protected abstract void toPrettyString(StringBuilder sb, int level); + + /** Gets a space string whose length is proportional to the given indent level. */ + protected String getIndent(int level) { + return StringUtils.repeat(" ", level * 4); + } + + /** Helper method to append a formatted string to the given {@code StringBuilder}. */ + protected void append(StringBuilder sb, String format, Object... args) { + sb.append(String.format(format, args)); + } + + /** Gets the name of this field. */ + protected String getName() { + return this.fieldId.getFieldName(); + } + + protected List noFields = Collections.emptyList(); + + protected String getSubElementName(TFieldIdEnum fieldId) { + return getSubElementName(fieldId, "element"); + } + + protected String getSubElementName(TFieldIdEnum fieldId, String suffix) { + return String.format("%s_%s", fieldId.getFieldName(), suffix); + } + + private static class Factory { + + static ThriftObject createNew( + ThriftObject parent, TFieldIdEnum fieldId, FieldMetaData data, List fields) { + + byte fieldType = data.valueMetaData.type; + switch (fieldType) { + case TType.STRUCT: + return ThriftStructBase.create(parent, fieldId, data, fields); + + case TType.LIST: + return new ThriftList(parent, fieldId, data, fields); + + case TType.MAP: + return new ThriftMap(parent, fieldId, data, fields); + + case TType.SET: + return new ThriftSet(parent, fieldId, data, fields); + + case TType.ENUM: + return new ThriftEnum(parent, fieldId, data); + + case TType.BOOL: + case TType.BYTE: + case TType.I16: + case TType.I32: + case TType.I64: + case TType.DOUBLE: + case TType.STRING: + return new ThriftPrimitive(parent, fieldId, data); + + default: + throw unsupportedFieldTypeException(fieldType); + } + } + } + } + + /** Metadata about primitive types. */ + public static class ThriftPrimitive extends ThriftObject { + ThriftPrimitive(ThriftObject parent, TFieldIdEnum fieldId, FieldMetaData data) { + super(parent, fieldId, data); + } + + public boolean isBinary() { + return this.data.valueMetaData.isBinary(); + } + + @Override + protected void toPrettyString(StringBuilder sb, int level) { + String fieldType = this.getTypeName(); + this.append(sb, "%s%s %s;\n", this.getIndent(level), fieldType, this.getName()); + } + + private String getTypeName() { + byte fieldType = this.data.valueMetaData.type; + switch (fieldType) { + case TType.BOOL: + return "bool"; + + case TType.BYTE: + return "byte"; + + case TType.I16: + return "i16"; + + case TType.I32: + return "i32"; + + case TType.I64: + return "i64"; + + case TType.DOUBLE: + return "double"; + + case TType.STRING: + if (this.isBinary()) { + return "binary"; + } else { + return "string"; + } + + default: + throw unsupportedFieldTypeException(fieldType); + } + } + } + + public static class ThriftEnum extends ThriftObject { + private static EnumCache enums = new EnumCache(); + + ThriftEnum(ThriftObject parent, TFieldIdEnum fieldId, FieldMetaData data) { + super(parent, fieldId, data); + } + + @Override + protected void toPrettyString(StringBuilder sb, int level) { + this.append(sb, "%senum %s;\n", this.getIndent(level), this.getName()); + } + } + + /** Metadata of container like objects: list, set, map */ + public abstract static class ThriftContainer extends ThriftObject { + + public ThriftContainer(ThriftObject parent, TFieldIdEnum fieldId, FieldMetaData data) { + super(parent, fieldId, data); + } + + public abstract boolean hasUnion(); + } + + public static class ThriftList extends ThriftContainer { + public final ThriftObject elementData; + + ThriftList( + ThriftObject parent, TFieldIdEnum fieldId, FieldMetaData data, List fields) { + super(parent, fieldId, data); + + this.elementData = + ThriftObject.Factory.createNew( + this, + FieldTypeEnum.LIST_ELEMENT, + new FieldMetaData( + getSubElementName(fieldId), + TFieldRequirementType.REQUIRED, + ((ListMetaData) data.valueMetaData).elemMetaData), + fields); + } + + @Override + public boolean hasUnion() { + return this.elementData instanceof ThriftUnion; + } + + @Override + protected void toPrettyString(StringBuilder sb, int level) { + this.append(sb, "%slist<\n", this.getIndent(level)); + this.elementData.toPrettyString(sb, level + 1); + this.append(sb, "%s> %s;\n", this.getIndent(level), this.getName()); + } + } + + public static class ThriftSet extends ThriftContainer { + public final ThriftObject elementData; + + ThriftSet( + ThriftObject parent, TFieldIdEnum fieldId, FieldMetaData data, List fields) { + super(parent, fieldId, data); + + this.elementData = + ThriftObject.Factory.createNew( + this, + FieldTypeEnum.SET_ELEMENT, + new FieldMetaData( + getSubElementName(fieldId), + TFieldRequirementType.REQUIRED, + ((SetMetaData) data.valueMetaData).elemMetaData), + fields); + } + + @Override + public boolean hasUnion() { + return this.elementData instanceof ThriftUnion; + } + + @Override + protected void toPrettyString(StringBuilder sb, int level) { + this.append(sb, "%sset<\n", this.getIndent(level)); + this.elementData.toPrettyString(sb, level + 1); + this.append(sb, "%s> %s;\n", this.getIndent(level), this.getName()); + } + } + + public static class ThriftMap extends ThriftContainer { + public final ThriftObject keyData; + public final ThriftObject valueData; + + ThriftMap( + ThriftObject parent, TFieldIdEnum fieldId, FieldMetaData data, List fields) { + super(parent, fieldId, data); + + this.keyData = + ThriftObject.Factory.createNew( + this, + FieldTypeEnum.MAP_KEY, + new FieldMetaData( + getSubElementName(fieldId, "key"), + TFieldRequirementType.REQUIRED, + ((MapMetaData) data.valueMetaData).keyMetaData), + Collections.emptyList()); + + this.valueData = + ThriftObject.Factory.createNew( + this, + FieldTypeEnum.MAP_VALUE, + new FieldMetaData( + getSubElementName(fieldId, "value"), + TFieldRequirementType.REQUIRED, + ((MapMetaData) data.valueMetaData).valueMetaData), + fields); + } + + @Override + public boolean hasUnion() { + return (this.keyData instanceof ThriftUnion) || (this.valueData instanceof ThriftUnion); + } + + @Override + protected void toPrettyString(StringBuilder sb, int level) { + this.append(sb, "%smap<\n", this.getIndent(level)); + this.append(sb, "%skey = {\n", this.getIndent(level + 1)); + this.keyData.toPrettyString(sb, level + 2); + this.append(sb, "%s},\n", this.getIndent(level + 1)); + this.append(sb, "%svalue = {\n", this.getIndent(level + 1)); + this.valueData.toPrettyString(sb, level + 2); + this.append(sb, "%s}\n", this.getIndent(level + 1)); + this.append(sb, "%s> %s;\n", this.getIndent(level), this.getName()); + } + } + + /** + * Base class for metadata of ThriftStruct and ThriftUnion. Holds functionality that is common to + * both. + */ + public abstract static class ThriftStructBase extends ThriftObject { + public ThriftStructBase(ThriftObject parent, TFieldIdEnum fieldId, FieldMetaData data) { + super(parent, fieldId, data); + } + + public Class getStructClass() { + return getStructClass(this.data); + } + + public static Class getStructClass(FieldMetaData data) { + return (Class) ((StructMetaData) data.valueMetaData).structClass; + } + + public boolean isUnion() { + return isUnion(this.data); + } + + public static boolean isUnion(FieldMetaData data) { + return TUnion.class.isAssignableFrom(getStructClass(data)); + } + + public static ThriftStructBase create( + ThriftObject parent, + TFieldIdEnum fieldId, + FieldMetaData data, + Iterable fieldsData) { + + if (isUnion(data)) { + return new ThriftUnion<>(parent, fieldId, data, fieldsData); + } else { + return new ThriftStruct<>(parent, fieldId, data, fieldsData); + } + } + } + + /** Metadata of a Thrift union. Currently not adequately supported. */ + public static class ThriftUnion extends ThriftStructBase { + public ThriftUnion( + ThriftObject parent, + TFieldIdEnum fieldId, + FieldMetaData data, + Iterable fieldsData) { + super(parent, fieldId, data); + } + + @Override + protected void toPrettyString(StringBuilder sb, int level) { + String indent = this.getIndent(level); + String indent2 = this.getIndent(level + 1); + this.append(sb, "%sunion %s {\n", indent, this.getName()); + this.append(sb, "%s// unions not adequately supported at present.\n", indent2); + this.append(sb, "%s}\n", indent); + } + } + + /** Metadata of a Thrift struct. */ + public static class ThriftStruct extends ThriftStructBase { + public final Map fields; + + ThriftStruct( + ThriftObject parent, + TFieldIdEnum fieldId, + FieldMetaData data, + Iterable fieldsData) { + super(parent, fieldId, data); + + Class clasz = getStructClass(data); + this.fields = getFields(this, clasz, fieldsData); + } + + public T createNewStruct() { + try { + Class structClass = getStructClass(this.data); + Constructor declaredConstructor = structClass.getDeclaredConstructor(); + return declaredConstructor.newInstance(); + } catch (ReflectiveOperationException e) { + throw new RuntimeException(e); + } + } + + public static ThriftStruct of(Class clasz) { + return ThriftStruct.fromFields(clasz, Collections.emptyList()); + } + + public static ThriftStruct fromFieldNames( + Class clasz, Collection fieldNames) { + return fromFields(clasz, ThriftField.fromNames(fieldNames)); + } + + public static ThriftStruct fromFields( + Class clasz, Iterable fields) { + + Validate.checkNotNull(clasz, "clasz"); + Validate.checkNotNull(fields, "fields"); + + return new ThriftStruct<>( + null, + FieldTypeEnum.ROOT, + new FieldMetaData( + FieldTypeEnum.ROOT.getFieldName(), + TFieldRequirementType.REQUIRED, + new StructMetaData(TType.STRUCT, clasz)), + fields); + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + this.toPrettyString(sb, 0); + return sb.toString(); + } + + @Override + protected void toPrettyString(StringBuilder sb, int level) { + String indent = this.getIndent(level); + String indent2 = this.getIndent(level + 1); + this.append(sb, "%sstruct %s {\n", indent, this.getName()); + if (this.fields.size() == 0) { + this.append(sb, "%s*;", indent2); + } else { + List ids = new ArrayList<>(this.fields.keySet()); + Collections.sort(ids); + for (Integer id : ids) { + this.fields.get(id).toPrettyString(sb, level + 1); + } + } + this.append(sb, "%s}\n", indent); + } + + private static Map getFields( + ThriftStruct parent, Class clasz, Iterable fieldsData) { + + Map fieldsMetaData = + FieldMetaData.getStructMetaDataMap(clasz); + Map fields = new HashMap<>(); + boolean getAllFields = !fieldsData.iterator().hasNext(); + + if (getAllFields) { + for (Map.Entry entry : fieldsMetaData.entrySet()) { + TFieldIdEnum fieldId = entry.getKey(); + FieldMetaData fieldMetaData = entry.getValue(); + ThriftObject field = + ThriftObject.Factory.createNew( + parent, fieldId, fieldMetaData, Collections.emptyList()); + fields.put((int) fieldId.getThriftFieldId(), field); + } + } else { + for (ThriftField fieldData : fieldsData) { + String fieldName = fieldData.name; + FieldMetaData fieldMetaData = findFieldMetaData(fieldsMetaData, fieldName); + TFieldIdEnum fieldId = findFieldId(fieldsMetaData, fieldName); + ThriftObject field = + ThriftObject.Factory.createNew(parent, fieldId, fieldMetaData, fieldData.fields); + fields.put((int) fieldId.getThriftFieldId(), field); + } + } + + return fields; + } + + private static FieldMetaData findFieldMetaData( + Map fieldsMetaData, String fieldName) { + + for (FieldMetaData fieldData : fieldsMetaData.values()) { + if (fieldData.fieldName.equals(fieldName)) { + return fieldData; + } + } + + throw fieldNotFoundException(fieldName); + } + + private static TFieldIdEnum findFieldId( + Map fieldsMetaData, String fieldName) { + + for (TFieldIdEnum fieldId : fieldsMetaData.keySet()) { + if (fieldId.getFieldName().equals(fieldName)) { + return fieldId; + } + } + + throw fieldNotFoundException(fieldName); + } + } + + static IllegalArgumentException fieldNotFoundException(String fieldName) { + return new IllegalArgumentException("field not found: '" + fieldName + "'"); + } + + static UnsupportedOperationException unsupportedFieldTypeException(byte fieldType) { + return new UnsupportedOperationException("field type not supported: '" + fieldType + "'"); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/ThriftStructProcessor.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/ThriftStructProcessor.java new file mode 100644 index 0000000..249e6a2 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/ThriftStructProcessor.java @@ -0,0 +1,181 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.partial; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import org.apache.thrift.TBase; +import org.apache.thrift.TEnum; +import org.apache.thrift.TFieldIdEnum; + +/** + * Provides a way to create and initialize an instance of TBase during partial deserialization. + * + *

This class is supposed to be used as a helper class for {@code PartialThriftDeserializer}. + */ +public class ThriftStructProcessor implements ThriftFieldValueProcessor { + + private static final EnumCache enums = new EnumCache(); + + @Override + public Object createNewStruct(ThriftMetadata.ThriftStruct metadata) { + return metadata.createNewStruct(); + } + + @Override + public TBase prepareStruct(Object instance) { + return (TBase) instance; + } + + @Override + public Object createNewList(int expectedSize) { + return new Object[expectedSize]; + } + + @Override + public void setListElement(Object instance, int index, Object value) { + ((Object[]) instance)[index] = value; + } + + @Override + public Object prepareList(Object instance) { + return Arrays.asList((Object[]) instance); + } + + @Override + public Object createNewMap(int expectedSize) { + return new HashMap(expectedSize); + } + + @Override + public void setMapElement(Object instance, int index, Object key, Object value) { + ((HashMap) instance).put(key, value); + } + + @Override + public Object prepareMap(Object instance) { + return instance; + } + + @Override + public Object createNewSet(int expectedSize) { + return new HashSet(expectedSize); + } + + @Override + public void setSetElement(Object instance, int index, Object value) { + ((HashSet) instance).add(value); + } + + @Override + public Object prepareSet(Object instance) { + return instance; + } + + @Override + public Object prepareEnum(Class enumClass, int ordinal) { + return enums.get(enumClass, ordinal); + } + + @Override + public Object prepareString(ByteBuffer buffer) { + return byteBufferToString(buffer); + } + + @Override + public Object prepareBinary(ByteBuffer buffer) { + return buffer; + } + + @Override + public void setBool(TBase valueCollection, TFieldIdEnum fieldId, boolean value) { + valueCollection.setFieldValue(fieldId, value); + } + + @Override + public void setByte(TBase valueCollection, TFieldIdEnum fieldId, byte value) { + valueCollection.setFieldValue(fieldId, value); + } + + @Override + public void setInt16(TBase valueCollection, TFieldIdEnum fieldId, short value) { + valueCollection.setFieldValue(fieldId, value); + } + + @Override + public void setInt32(TBase valueCollection, TFieldIdEnum fieldId, int value) { + valueCollection.setFieldValue(fieldId, value); + } + + @Override + public void setInt64(TBase valueCollection, TFieldIdEnum fieldId, long value) { + valueCollection.setFieldValue(fieldId, value); + } + + @Override + public void setDouble(TBase valueCollection, TFieldIdEnum fieldId, double value) { + valueCollection.setFieldValue(fieldId, value); + } + + @Override + public void setBinary(TBase valueCollection, TFieldIdEnum fieldId, ByteBuffer value) { + valueCollection.setFieldValue(fieldId, value); + } + + @Override + public void setString(TBase valueCollection, TFieldIdEnum fieldId, ByteBuffer buffer) { + String value = byteBufferToString(buffer); + valueCollection.setFieldValue(fieldId, value); + } + + @Override + public void setEnumField(TBase valueCollection, TFieldIdEnum fieldId, Object value) { + valueCollection.setFieldValue(fieldId, value); + } + + @Override + public void setListField(TBase valueCollection, TFieldIdEnum fieldId, Object value) { + valueCollection.setFieldValue(fieldId, value); + } + + @Override + public void setMapField(TBase valueCollection, TFieldIdEnum fieldId, Object value) { + valueCollection.setFieldValue(fieldId, value); + } + + @Override + public void setSetField(TBase valueCollection, TFieldIdEnum fieldId, Object value) { + valueCollection.setFieldValue(fieldId, value); + } + + @Override + public void setStructField(TBase valueCollection, TFieldIdEnum fieldId, Object value) { + valueCollection.setFieldValue(fieldId, value); + } + + private static String byteBufferToString(ByteBuffer buffer) { + byte[] bytes = buffer.array(); + int pos = buffer.position(); + return new String(bytes, pos, buffer.limit() - pos, StandardCharsets.UTF_8); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/Validate.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/Validate.java new file mode 100644 index 0000000..f4bbd9a --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/Validate.java @@ -0,0 +1,233 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.partial; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collection; + +/** + * A superset of Validate class in Apache commons lang3. + * + *

It provides consistent message strings for frequently encountered checks. That simplifies + * callers because they have to supply only the name of the argument that failed a check instead of + * having to supply the entire message. + */ +public final class Validate { + private Validate() {} + + /** Validates that the given reference argument is not null. */ + public static void checkNotNull(Object obj, String argName) { + checkArgument(obj != null, "'%s' must not be null.", argName); + } + + /** Validates that the given integer argument is not zero or negative. */ + public static void checkPositiveInteger(long value, String argName) { + checkArgument(value > 0, "'%s' must be a positive integer.", argName); + } + + /** Validates that the given integer argument is not negative. */ + public static void checkNotNegative(long value, String argName) { + checkArgument(value >= 0, "'%s' must not be negative.", argName); + } + + /* + * Validates that the expression (that checks a required field is present) is true. + */ + public static void checkRequired(boolean isPresent, String argName) { + checkArgument(isPresent, "'%s' is required.", argName); + } + + /** Validates that the expression (that checks a field is valid) is true. */ + public static void checkValid(boolean isValid, String argName) { + checkArgument(isValid, "'%s' is invalid.", argName); + } + + /** Validates that the expression (that checks a field is valid) is true. */ + public static void checkValid(boolean isValid, String argName, String validValues) { + checkArgument(isValid, "'%s' is invalid. Valid values are: %s.", argName, validValues); + } + + /** Validates that the given string is not null and has non-zero length. */ + public static void checkNotNullAndNotEmpty(String arg, String argName) { + Validate.checkNotNull(arg, argName); + Validate.checkArgument(arg.length() > 0, "'%s' must not be empty.", argName); + } + + /** Validates that the given array is not null and has at least one element. */ + public static void checkNotNullAndNotEmpty(T[] array, String argName) { + Validate.checkNotNull(array, argName); + checkNotEmpty(array.length, argName); + } + + /** Validates that the given array is not null and has at least one element. */ + public static void checkNotNullAndNotEmpty(byte[] array, String argName) { + Validate.checkNotNull(array, argName); + checkNotEmpty(array.length, argName); + } + + /** Validates that the given array is not null and has at least one element. */ + public static void checkNotNullAndNotEmpty(short[] array, String argName) { + Validate.checkNotNull(array, argName); + checkNotEmpty(array.length, argName); + } + + /** Validates that the given array is not null and has at least one element. */ + public static void checkNotNullAndNotEmpty(int[] array, String argName) { + Validate.checkNotNull(array, argName); + checkNotEmpty(array.length, argName); + } + + /** Validates that the given array is not null and has at least one element. */ + public static void checkNotNullAndNotEmpty(long[] array, String argName) { + Validate.checkNotNull(array, argName); + checkNotEmpty(array.length, argName); + } + + /** Validates that the given buffer is not null and has non-zero capacity. */ + public static void checkNotNullAndNotEmpty(Iterable iter, String argName) { + Validate.checkNotNull(iter, argName); + int minNumElements = iter.iterator().hasNext() ? 1 : 0; + checkNotEmpty(minNumElements, argName); + } + + /** Validates that the given set is not null and has an exact number of items. */ + public static void checkNotNullAndNumberOfElements( + Collection collection, int numElements, String argName) { + Validate.checkNotNull(collection, argName); + checkArgument( + collection.size() == numElements, + "Number of elements in '%s' must be exactly %s, %s given.", + argName, + numElements, + collection.size()); + } + + /** Validates that the given two values are equal. */ + public static void checkValuesEqual( + long value1, String value1Name, long value2, String value2Name) { + checkArgument( + value1 == value2, + "'%s' (%s) must equal '%s' (%s).", + value1Name, + value1, + value2Name, + value2); + } + + /** Validates that the first value is an integer multiple of the second value. */ + public static void checkIntegerMultiple( + long value1, String value1Name, long value2, String value2Name) { + checkArgument( + (value1 % value2) == 0, + "'%s' (%s) must be an integer multiple of '%s' (%s).", + value1Name, + value1, + value2Name, + value2); + } + + /** Validates that the first value is greater than the second value. */ + public static void checkGreater(long value1, String value1Name, long value2, String value2Name) { + checkArgument( + value1 > value2, + "'%s' (%s) must be greater than '%s' (%s).", + value1Name, + value1, + value2Name, + value2); + } + + /** Validates that the first value is greater than or equal to the second value. */ + public static void checkGreaterOrEqual( + long value1, String value1Name, long value2, String value2Name) { + checkArgument( + value1 >= value2, + "'%s' (%s) must be greater than or equal to '%s' (%s).", + value1Name, + value1, + value2Name, + value2); + } + + /** Validates that the first value is less than or equal to the second value. */ + public static void checkLessOrEqual( + long value1, String value1Name, long value2, String value2Name) { + checkArgument( + value1 <= value2, + "'%s' (%s) must be less than or equal to '%s' (%s).", + value1Name, + value1, + value2Name, + value2); + } + + /** Validates that the given value is within the given range of values. */ + public static void checkWithinRange( + long value, String valueName, long minValueInclusive, long maxValueInclusive) { + checkArgument( + (value >= minValueInclusive) && (value <= maxValueInclusive), + "'%s' (%s) must be within the range [%s, %s].", + valueName, + value, + minValueInclusive, + maxValueInclusive); + } + + /** Validates that the given value is within the given range of values. */ + public static void checkWithinRange( + double value, String valueName, double minValueInclusive, double maxValueInclusive) { + checkArgument( + (value >= minValueInclusive) && (value <= maxValueInclusive), + "'%s' (%s) must be within the range [%s, %s].", + valueName, + value, + minValueInclusive, + maxValueInclusive); + } + + public static void checkPathExists(Path path, String argName) { + checkNotNull(path, argName); + checkArgument(Files.exists(path), "Path %s (%s) does not exist.", argName, path); + } + + public static void checkPathExistsAsDir(Path path, String argName) { + checkPathExists(path, argName); + checkArgument( + Files.isDirectory(path), "Path %s (%s) must point to a directory.", argName, path); + } + + public static void checkPathExistsAsFile(Path path, String argName) { + checkPathExists(path, argName); + checkArgument(Files.isRegularFile(path), "Path %s (%s) must point to a file.", argName, path); + } + + public static void checkArgument(boolean expression, String format, Object... args) { + org.apache.commons.lang3.Validate.isTrue(expression, format, args); + } + + public static void checkState(boolean expression, String format, Object... args) { + org.apache.commons.lang3.Validate.validState(expression, format, args); + } + + private static void checkNotEmpty(int arraySize, String argName) { + Validate.checkArgument(arraySize > 0, "'%s' must have at least one element.", argName); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/ShortStack.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/ShortStack.java new file mode 100644 index 0000000..64afb60 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/ShortStack.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.thrift.protocol; + +import java.util.Arrays; + +/** + * ShortStack is a short-specific Stack implementation written for the express purpose of very fast + * operations on TCompactProtocol's field id stack. This implementation performs at least 10x faster + * than java.util.Stack. + */ +class ShortStack { + + private short[] vector; + + /** Always points to the next location */ + private int top = 0; + + public ShortStack(int initialCapacity) { + vector = new short[initialCapacity]; + } + + public short pop() { + return vector[--top]; + } + + public void push(short pushed) { + if (vector.length == top) { + grow(); + } + vector[top++] = pushed; + } + + private void grow() { + vector = Arrays.copyOf(vector, vector.length << 1); + } + + public void clear() { + top = 0; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(">").append(value).append("<<"); + } else { + sb.append(value); + } + } + sb.append("]>"); + return sb.toString(); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TBase64Utils.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TBase64Utils.java new file mode 100644 index 0000000..6559da1 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TBase64Utils.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +/** + * Class for encoding and decoding Base64 data. + * + *

This class is kept at package level because the interface does no input validation and is + * therefore too low-level for generalized reuse. + * + *

Note also that the encoding does not pad with equal signs , as discussed in section 2.2 of the + * RFC (http://www.faqs.org/rfcs/rfc3548.html). Furthermore, bad data encountered when decoding is + * neither rejected or ignored but simply results in bad decoded data -- this is not in compliance + * with the RFC but is done in the interest of performance. + */ +class TBase64Utils { + + private static final String ENCODE_TABLE = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + + /** + * Encode len bytes of data in src at offset srcOff, storing the result into dst at offset dstOff. + * len must be 1, 2, or 3. dst must have at least len+1 bytes of space at dstOff. src and dst + * should not be the same object. This method does no validation of the input values in the + * interest of performance. + * + * @param src the source of bytes to encode + * @param srcOff the offset into the source to read the unencoded bytes + * @param len the number of bytes to encode (must be 1, 2, or 3). + * @param dst the destination for the encoding + * @param dstOff the offset into the destination to place the encoded bytes + */ + static void encode(byte[] src, int srcOff, int len, byte[] dst, int dstOff) { + dst[dstOff] = (byte) ENCODE_TABLE.charAt((src[srcOff] >> 2) & 0x3F); + if (len == 3) { + dst[dstOff + 1] = + (byte) ENCODE_TABLE.charAt(((src[srcOff] << 4) & 0x30) | ((src[srcOff + 1] >> 4) & 0x0F)); + dst[dstOff + 2] = + (byte) + ENCODE_TABLE.charAt( + ((src[srcOff + 1] << 2) & 0x3C) | ((src[srcOff + 2] >> 6) & 0x03)); + dst[dstOff + 3] = (byte) ENCODE_TABLE.charAt(src[srcOff + 2] & 0x3F); + } else if (len == 2) { + dst[dstOff + 1] = + (byte) ENCODE_TABLE.charAt(((src[srcOff] << 4) & 0x30) | ((src[srcOff + 1] >> 4) & 0x0F)); + dst[dstOff + 2] = (byte) ENCODE_TABLE.charAt((src[srcOff + 1] << 2) & 0x3C); + } else { // len == 1) { + dst[dstOff + 1] = (byte) ENCODE_TABLE.charAt((src[srcOff] << 4) & 0x30); + } + } + + private static final byte[] DECODE_TABLE = { + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, -1, 63, + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -1, -1, -1, + -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1, + -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + }; + + /** + * Decode len bytes of data in src at offset srcOff, storing the result into dst at offset dstOff. + * len must be 2, 3, or 4. dst must have at least len-1 bytes of space at dstOff. src and dst may + * be the same object as long as dstoff <= srcOff. This method does no validation of the input + * values in the interest of performance. + * + * @param src the source of bytes to decode + * @param srcOff the offset into the source to read the encoded bytes + * @param len the number of bytes to decode (must be 2, 3, or 4) + * @param dst the destination for the decoding + * @param dstOff the offset into the destination to place the decoded bytes + */ + static void decode(byte[] src, int srcOff, int len, byte[] dst, int dstOff) { + dst[dstOff] = + (byte) + ((DECODE_TABLE[src[srcOff] & 0x0FF] << 2) + | (DECODE_TABLE[src[srcOff + 1] & 0x0FF] >> 4)); + if (len > 2) { + dst[dstOff + 1] = + (byte) + (((DECODE_TABLE[src[srcOff + 1] & 0x0FF] << 4) & 0xF0) + | (DECODE_TABLE[src[srcOff + 2] & 0x0FF] >> 2)); + if (len > 3) { + dst[dstOff + 2] = + (byte) + (((DECODE_TABLE[src[srcOff + 2] & 0x0FF] << 6) & 0xC0) + | DECODE_TABLE[src[srcOff + 3] & 0x0FF]); + } + } + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TBinaryProtocol.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TBinaryProtocol.java new file mode 100644 index 0000000..6b016b8 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TBinaryProtocol.java @@ -0,0 +1,572 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.UUID; +import org.apache.thrift.TException; +import org.apache.thrift.partial.TFieldData; +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; + +/** Binary protocol implementation for thrift. */ +public class TBinaryProtocol extends TProtocol { + private static final TStruct ANONYMOUS_STRUCT = new TStruct(); + private static final long NO_LENGTH_LIMIT = -1; + + protected static final int VERSION_MASK = 0xffff0000; + protected static final int VERSION_1 = 0x80010000; + + /** + * The maximum number of bytes to read from the transport for variable-length fields (such as + * strings or binary) or {@link #NO_LENGTH_LIMIT} for unlimited. + */ + private final long stringLengthLimit_; + + /** + * The maximum number of elements to read from the network for containers (maps, sets, lists), or + * {@link #NO_LENGTH_LIMIT} for unlimited. + */ + private final long containerLengthLimit_; + + protected boolean strictRead_; + protected boolean strictWrite_; + + private final byte[] inoutTemp = new byte[16]; + + /** Factory */ + public static class Factory implements TProtocolFactory { + protected long stringLengthLimit_; + protected long containerLengthLimit_; + protected boolean strictRead_; + protected boolean strictWrite_; + + public Factory() { + this(false, true); + } + + public Factory(boolean strictRead, boolean strictWrite) { + this(strictRead, strictWrite, NO_LENGTH_LIMIT, NO_LENGTH_LIMIT); + } + + public Factory(long stringLengthLimit, long containerLengthLimit) { + this(false, true, stringLengthLimit, containerLengthLimit); + } + + public Factory( + boolean strictRead, + boolean strictWrite, + long stringLengthLimit, + long containerLengthLimit) { + stringLengthLimit_ = stringLengthLimit; + containerLengthLimit_ = containerLengthLimit; + strictRead_ = strictRead; + strictWrite_ = strictWrite; + } + + public TProtocol getProtocol(TTransport trans) { + return new TBinaryProtocol( + trans, stringLengthLimit_, containerLengthLimit_, strictRead_, strictWrite_); + } + } + + /** Constructor */ + public TBinaryProtocol(TTransport trans) { + this(trans, false, true); + } + + public TBinaryProtocol(TTransport trans, boolean strictRead, boolean strictWrite) { + this(trans, NO_LENGTH_LIMIT, NO_LENGTH_LIMIT, strictRead, strictWrite); + } + + public TBinaryProtocol(TTransport trans, long stringLengthLimit, long containerLengthLimit) { + this(trans, stringLengthLimit, containerLengthLimit, false, true); + } + + public TBinaryProtocol( + TTransport trans, + long stringLengthLimit, + long containerLengthLimit, + boolean strictRead, + boolean strictWrite) { + super(trans); + stringLengthLimit_ = stringLengthLimit; + containerLengthLimit_ = containerLengthLimit; + strictRead_ = strictRead; + strictWrite_ = strictWrite; + } + + @Override + public void writeMessageBegin(TMessage message) throws TException { + if (strictWrite_) { + int version = VERSION_1 | message.type; + writeI32(version); + writeString(message.name); + writeI32(message.seqid); + } else { + writeString(message.name); + writeByte(message.type); + writeI32(message.seqid); + } + } + + @Override + public void writeMessageEnd() throws TException {} + + @Override + public void writeStructBegin(TStruct struct) throws TException {} + + @Override + public void writeStructEnd() throws TException {} + + @Override + public void writeFieldBegin(TField field) throws TException { + writeByte(field.type); + writeI16(field.id); + } + + @Override + public void writeFieldEnd() throws TException {} + + @Override + public void writeFieldStop() throws TException { + writeByte(TType.STOP); + } + + @Override + public void writeMapBegin(TMap map) throws TException { + writeByte(map.keyType); + writeByte(map.valueType); + writeI32(map.size); + } + + @Override + public void writeMapEnd() throws TException {} + + @Override + public void writeListBegin(TList list) throws TException { + writeByte(list.elemType); + writeI32(list.size); + } + + @Override + public void writeListEnd() throws TException {} + + @Override + public void writeSetBegin(TSet set) throws TException { + writeByte(set.elemType); + writeI32(set.size); + } + + @Override + public void writeSetEnd() throws TException {} + + @Override + public void writeBool(boolean b) throws TException { + writeByte(b ? (byte) 1 : (byte) 0); + } + + @Override + public void writeByte(byte b) throws TException { + inoutTemp[0] = b; + trans_.write(inoutTemp, 0, 1); + } + + @Override + public void writeI16(short i16) throws TException { + inoutTemp[0] = (byte) (0xff & (i16 >> 8)); + inoutTemp[1] = (byte) (0xff & (i16)); + trans_.write(inoutTemp, 0, 2); + } + + @Override + public void writeI32(int i32) throws TException { + inoutTemp[0] = (byte) (0xff & (i32 >> 24)); + inoutTemp[1] = (byte) (0xff & (i32 >> 16)); + inoutTemp[2] = (byte) (0xff & (i32 >> 8)); + inoutTemp[3] = (byte) (0xff & (i32)); + trans_.write(inoutTemp, 0, 4); + } + + @Override + public void writeI64(long i64) throws TException { + inoutTemp[0] = (byte) (0xff & (i64 >> 56)); + inoutTemp[1] = (byte) (0xff & (i64 >> 48)); + inoutTemp[2] = (byte) (0xff & (i64 >> 40)); + inoutTemp[3] = (byte) (0xff & (i64 >> 32)); + inoutTemp[4] = (byte) (0xff & (i64 >> 24)); + inoutTemp[5] = (byte) (0xff & (i64 >> 16)); + inoutTemp[6] = (byte) (0xff & (i64 >> 8)); + inoutTemp[7] = (byte) (0xff & (i64)); + trans_.write(inoutTemp, 0, 8); + } + + @Override + public void writeUuid(UUID uuid) throws TException { + ByteBuffer bb = ByteBuffer.wrap(inoutTemp); + bb.putLong(uuid.getMostSignificantBits()); + bb.putLong(uuid.getLeastSignificantBits()); + trans_.write(inoutTemp, 0, 16); + } + + @Override + public void writeDouble(double dub) throws TException { + writeI64(Double.doubleToLongBits(dub)); + } + + @Override + public void writeString(String str) throws TException { + byte[] dat = str.getBytes(StandardCharsets.UTF_8); + writeI32(dat.length); + trans_.write(dat, 0, dat.length); + } + + @Override + public void writeBinary(ByteBuffer bin) throws TException { + int length = bin.limit() - bin.position(); + writeI32(length); + trans_.write(bin.array(), bin.position() + bin.arrayOffset(), length); + } + + /** Reading methods. */ + @Override + public TMessage readMessageBegin() throws TException { + int size = readI32(); + if (size < 0) { + int version = size & VERSION_MASK; + if (version != VERSION_1) { + throw new TProtocolException( + TProtocolException.BAD_VERSION, "Bad version in readMessageBegin"); + } + return new TMessage(readString(), (byte) (size & 0x000000ff), readI32()); + } else { + if (strictRead_) { + throw new TProtocolException( + TProtocolException.BAD_VERSION, "Missing version in readMessageBegin, old client?"); + } + return new TMessage(readStringBody(size), readByte(), readI32()); + } + } + + @Override + public void readMessageEnd() throws TException {} + + @Override + public TStruct readStructBegin() throws TException { + return ANONYMOUS_STRUCT; + } + + @Override + public void readStructEnd() throws TException {} + + @Override + public TField readFieldBegin() throws TException { + byte type = readByte(); + short id = type == TType.STOP ? 0 : readI16(); + return new TField("", type, id); + } + + @Override + public void readFieldEnd() throws TException {} + + @Override + public TMap readMapBegin() throws TException { + TMap map = new TMap(readByte(), readByte(), readI32()); + + checkReadBytesAvailable(map); + checkContainerReadLength(map.size); + return map; + } + + @Override + public void readMapEnd() throws TException {} + + @Override + public TList readListBegin() throws TException { + TList list = new TList(readByte(), readI32()); + + checkReadBytesAvailable(list); + checkContainerReadLength(list.size); + return list; + } + + @Override + public void readListEnd() throws TException {} + + @Override + public TSet readSetBegin() throws TException { + TSet set = new TSet(readByte(), readI32()); + + checkReadBytesAvailable(set); + checkContainerReadLength(set.size); + return set; + } + + @Override + public void readSetEnd() throws TException {} + + @Override + public boolean readBool() throws TException { + return (readByte() == 1); + } + + @Override + public byte readByte() throws TException { + if (trans_.getBytesRemainingInBuffer() >= 1) { + byte b = trans_.getBuffer()[trans_.getBufferPosition()]; + trans_.consumeBuffer(1); + return b; + } + readAll(inoutTemp, 0, 1); + return inoutTemp[0]; + } + + @Override + public short readI16() throws TException { + byte[] buf = inoutTemp; + int off = 0; + + if (trans_.getBytesRemainingInBuffer() >= 2) { + buf = trans_.getBuffer(); + off = trans_.getBufferPosition(); + trans_.consumeBuffer(2); + } else { + readAll(inoutTemp, 0, 2); + } + + return (short) (((buf[off] & 0xff) << 8) | ((buf[off + 1] & 0xff))); + } + + @Override + public int readI32() throws TException { + byte[] buf = inoutTemp; + int off = 0; + + if (trans_.getBytesRemainingInBuffer() >= 4) { + buf = trans_.getBuffer(); + off = trans_.getBufferPosition(); + trans_.consumeBuffer(4); + } else { + readAll(inoutTemp, 0, 4); + } + return ((buf[off] & 0xff) << 24) + | ((buf[off + 1] & 0xff) << 16) + | ((buf[off + 2] & 0xff) << 8) + | ((buf[off + 3] & 0xff)); + } + + @Override + public long readI64() throws TException { + byte[] buf = inoutTemp; + int off = 0; + + if (trans_.getBytesRemainingInBuffer() >= 8) { + buf = trans_.getBuffer(); + off = trans_.getBufferPosition(); + trans_.consumeBuffer(8); + } else { + readAll(inoutTemp, 0, 8); + } + + return ((long) (buf[off] & 0xff) << 56) + | ((long) (buf[off + 1] & 0xff) << 48) + | ((long) (buf[off + 2] & 0xff) << 40) + | ((long) (buf[off + 3] & 0xff) << 32) + | ((long) (buf[off + 4] & 0xff) << 24) + | ((long) (buf[off + 5] & 0xff) << 16) + | ((long) (buf[off + 6] & 0xff) << 8) + | ((long) (buf[off + 7] & 0xff)); + } + + @Override + public UUID readUuid() throws TException { + byte[] buf = inoutTemp; + int off = 0; + + if (trans_.getBytesRemainingInBuffer() >= 16) { + buf = trans_.getBuffer(); + off = trans_.getBufferPosition(); + trans_.consumeBuffer(16); + } else { + readAll(inoutTemp, 0, 16); + } + + ByteBuffer bb = ByteBuffer.wrap(buf, off, 16); + long msb = bb.getLong(); + long lsb = bb.getLong(); + return new UUID(msb, lsb); + } + + @Override + public double readDouble() throws TException { + return Double.longBitsToDouble(readI64()); + } + + @Override + public String readString() throws TException { + int size = readI32(); + + if (trans_.getBytesRemainingInBuffer() >= size) { + String s = + new String(trans_.getBuffer(), trans_.getBufferPosition(), size, StandardCharsets.UTF_8); + trans_.consumeBuffer(size); + return s; + } + + return readStringBody(size); + } + + public String readStringBody(int size) throws TException { + checkStringReadLength(size); + byte[] buf = new byte[size]; + trans_.readAll(buf, 0, size); + return new String(buf, StandardCharsets.UTF_8); + } + + @Override + public ByteBuffer readBinary() throws TException { + int size = readI32(); + + checkStringReadLength(size); + + if (trans_.getBytesRemainingInBuffer() >= size) { + ByteBuffer bb = ByteBuffer.wrap(trans_.getBuffer(), trans_.getBufferPosition(), size); + trans_.consumeBuffer(size); + return bb; + } + + byte[] buf = new byte[size]; + trans_.readAll(buf, 0, size); + return ByteBuffer.wrap(buf); + } + + private void checkStringReadLength(int length) throws TException { + if (length < 0) { + throw new TProtocolException(TProtocolException.NEGATIVE_SIZE, "Negative length: " + length); + } + + getTransport().checkReadBytesAvailable(length); + + if (stringLengthLimit_ != NO_LENGTH_LIMIT && length > stringLengthLimit_) { + throw new TProtocolException( + TProtocolException.SIZE_LIMIT, "Length exceeded max allowed: " + length); + } + } + + private void checkContainerReadLength(int length) throws TProtocolException { + if (length < 0) { + throw new TProtocolException(TProtocolException.NEGATIVE_SIZE, "Negative length: " + length); + } + if (containerLengthLimit_ != NO_LENGTH_LIMIT && length > containerLengthLimit_) { + throw new TProtocolException( + TProtocolException.SIZE_LIMIT, "Length exceeded max allowed: " + length); + } + } + + private int readAll(byte[] buf, int off, int len) throws TException { + return trans_.readAll(buf, off, len); + } + + /** Return the minimum number of bytes a type will consume on the wire */ + @Override + public int getMinSerializedSize(byte type) throws TTransportException { + switch (type) { + case 0: + return 1; // Stop - T_STOP needs to count itself + case 1: + return 1; // Void - T_VOID needs to count itself + case 2: + return 1; // Bool sizeof(byte) + case 3: + return 1; // Byte sizeof(byte) + case 4: + return 8; // Double sizeof(double) + case 6: + return 2; // I16 sizeof(short) + case 8: + return 4; // I32 sizeof(int) + case 10: + return 8; // I64 sizeof(long) + case 11: + return 4; // string length sizeof(int) + case 12: + return 1; // empty struct needs at least 1 byte for the T_STOP + case 13: + return 4; // element count Map sizeof(int) + case 14: + return 4; // element count Set sizeof(int) + case 15: + return 4; // element count List sizeof(int) + default: + throw new TTransportException(TTransportException.UNKNOWN, "unrecognized type code"); + } + } + + // ----------------------------------------------------------------- + // Additional methods to improve performance. + + @Override + public int readFieldBeginData() throws TException { + byte type = this.readByte(); + if (type == TType.STOP) { + return TFieldData.encode(type); + } + + short id = this.readI16(); + return TFieldData.encode(type, id); + } + + @Override + protected void skipBool() throws TException { + this.skipBytes(1); + } + + @Override + protected void skipByte() throws TException { + this.skipBytes(1); + } + + @Override + protected void skipI16() throws TException { + this.skipBytes(2); + } + + @Override + protected void skipI32() throws TException { + this.skipBytes(4); + } + + @Override + protected void skipI64() throws TException { + this.skipBytes(8); + } + + @Override + protected void skipDouble() throws TException { + this.skipBytes(8); + } + + @Override + protected void skipBinary() throws TException { + int size = readI32(); + this.skipBytes(size); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TCompactProtocol.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TCompactProtocol.java new file mode 100644 index 0000000..1976c00 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TCompactProtocol.java @@ -0,0 +1,968 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.UUID; +import org.apache.thrift.TException; +import org.apache.thrift.partial.TFieldData; +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; + +/** + * TCompactProtocol2 is the Java implementation of the compact protocol specified in THRIFT-110. The + * fundamental approach to reducing the overhead of structures is a) use variable-length integers + * all over the place and b) make use of unused bits wherever possible. Your savings will obviously + * vary based on the specific makeup of your structs, but in general, the more fields, nested + * structures, short strings and collections, and low-value i32 and i64 fields you have, the more + * benefit you'll see. + */ +public class TCompactProtocol extends TProtocol { + private static final byte[] EMPTY_BYTES = new byte[0]; + private static final ByteBuffer EMPTY_BUFFER = ByteBuffer.wrap(EMPTY_BYTES); + + private static final long NO_LENGTH_LIMIT = -1; + + private static final TStruct ANONYMOUS_STRUCT = new TStruct(""); + private static final TField TSTOP = new TField("", TType.STOP, (short) 0); + + private static final byte[] ttypeToCompactType = new byte[18]; + + static { + ttypeToCompactType[TType.STOP] = TType.STOP; + ttypeToCompactType[TType.BOOL] = Types.BOOLEAN_TRUE; + ttypeToCompactType[TType.BYTE] = Types.BYTE; + ttypeToCompactType[TType.I16] = Types.I16; + ttypeToCompactType[TType.I32] = Types.I32; + ttypeToCompactType[TType.I64] = Types.I64; + ttypeToCompactType[TType.DOUBLE] = Types.DOUBLE; + ttypeToCompactType[TType.STRING] = Types.BINARY; + ttypeToCompactType[TType.LIST] = Types.LIST; + ttypeToCompactType[TType.SET] = Types.SET; + ttypeToCompactType[TType.MAP] = Types.MAP; + ttypeToCompactType[TType.STRUCT] = Types.STRUCT; + ttypeToCompactType[TType.UUID] = Types.UUID; + } + + /** TProtocolFactory that produces TCompactProtocols. */ + public static class Factory implements TProtocolFactory { + private final long stringLengthLimit_; + private final long containerLengthLimit_; + + public Factory() { + this(NO_LENGTH_LIMIT, NO_LENGTH_LIMIT); + } + + public Factory(long stringLengthLimit) { + this(stringLengthLimit, NO_LENGTH_LIMIT); + } + + public Factory(long stringLengthLimit, long containerLengthLimit) { + this.containerLengthLimit_ = containerLengthLimit; + this.stringLengthLimit_ = stringLengthLimit; + } + + @Override + public TProtocol getProtocol(TTransport trans) { + return new TCompactProtocol(trans, stringLengthLimit_, containerLengthLimit_); + } + } + + private static final byte PROTOCOL_ID = (byte) 0x82; + private static final byte VERSION = 1; + private static final byte VERSION_MASK = 0x1f; // 0001 1111 + private static final byte TYPE_MASK = (byte) 0xE0; // 1110 0000 + private static final byte TYPE_BITS = 0x07; // 0000 0111 + private static final int TYPE_SHIFT_AMOUNT = 5; + + /** All of the on-wire type codes. */ + private static class Types { + public static final byte BOOLEAN_TRUE = 0x01; + public static final byte BOOLEAN_FALSE = 0x02; + public static final byte BYTE = 0x03; + public static final byte I16 = 0x04; + public static final byte I32 = 0x05; + public static final byte I64 = 0x06; + public static final byte DOUBLE = 0x07; + public static final byte BINARY = 0x08; + public static final byte LIST = 0x09; + public static final byte SET = 0x0A; + public static final byte MAP = 0x0B; + public static final byte STRUCT = 0x0C; + public static final byte UUID = 0x0D; + } + + /** + * Used to keep track of the last field for the current and previous structs, so we can do the + * delta stuff. + */ + private final ShortStack lastField_ = new ShortStack(15); + + private short lastFieldId_ = 0; + + /** + * If we encounter a boolean field begin, save the TField here so it can have the value + * incorporated. + */ + private TField booleanField_ = null; + + /** + * If we read a field header, and it's a boolean field, save the boolean value here so that + * readBool can use it. + */ + private Boolean boolValue_ = null; + + /** + * The maximum number of bytes to read from the transport for variable-length fields (such as + * strings or binary) or {@link #NO_LENGTH_LIMIT} for unlimited. + */ + private final long stringLengthLimit_; + + /** + * The maximum number of elements to read from the network for containers (maps, sets, lists), or + * {@link #NO_LENGTH_LIMIT} for unlimited. + */ + private final long containerLengthLimit_; + + /** + * Temporary buffer used for various operations that would otherwise require a small allocation. + */ + private final byte[] temp = new byte[16]; + + /** + * Create a TCompactProtocol. + * + * @param transport the TTransport object to read from or write to. + * @param stringLengthLimit the maximum number of bytes to read for variable-length fields. + * @param containerLengthLimit the maximum number of elements to read for containers. + */ + public TCompactProtocol(TTransport transport, long stringLengthLimit, long containerLengthLimit) { + super(transport); + this.stringLengthLimit_ = stringLengthLimit; + this.containerLengthLimit_ = containerLengthLimit; + } + + /** + * Create a TCompactProtocol. + * + * @param transport the TTransport object to read from or write to. + * @param stringLengthLimit the maximum number of bytes to read for variable-length fields. + * @deprecated Use constructor specifying both string limit and container limit instead + */ + @Deprecated + public TCompactProtocol(TTransport transport, long stringLengthLimit) { + this(transport, stringLengthLimit, NO_LENGTH_LIMIT); + } + + /** + * Create a TCompactProtocol. + * + * @param transport the TTransport object to read from or write to. + */ + public TCompactProtocol(TTransport transport) { + this(transport, NO_LENGTH_LIMIT, NO_LENGTH_LIMIT); + } + + @Override + public void reset() { + lastField_.clear(); + lastFieldId_ = 0; + } + + // + // Public Writing methods. + // + + /** + * Write a message header to the wire. Compact Protocol messages contain the protocol version so + * we can migrate forwards in the future if need be. + */ + @Override + public void writeMessageBegin(TMessage message) throws TException { + writeByteDirect(PROTOCOL_ID); + writeByteDirect((VERSION & VERSION_MASK) | ((message.type << TYPE_SHIFT_AMOUNT) & TYPE_MASK)); + writeVarint32(message.seqid); + writeString(message.name); + } + + /** + * Write a struct begin. This doesn't actually put anything on the wire. We use it as an + * opportunity to put special placeholder markers on the field stack so we can get the field id + * deltas correct. + */ + @Override + public void writeStructBegin(TStruct struct) throws TException { + lastField_.push(lastFieldId_); + lastFieldId_ = 0; + } + + /** + * Write a struct end. This doesn't actually put anything on the wire. We use this as an + * opportunity to pop the last field from the current struct off of the field stack. + */ + @Override + public void writeStructEnd() throws TException { + lastFieldId_ = lastField_.pop(); + } + + /** + * Write a field header containing the field id and field type. If the difference between the + * current field id and the last one is small (< 15), then the field id will be encoded in the + * 4 MSB as a delta. Otherwise, the field id will follow the type header as a zigzag varint. + */ + @Override + public void writeFieldBegin(TField field) throws TException { + if (field.type == TType.BOOL) { + // we want to possibly include the value, so we'll wait. + booleanField_ = field; + } else { + writeFieldBeginInternal(field, (byte) -1); + } + } + + /** + * The workhorse of writeFieldBegin. It has the option of doing a 'type override' of the type + * header. This is used specifically in the boolean field case. + */ + private void writeFieldBeginInternal(TField field, byte typeOverride) throws TException { + // short lastField = lastField_.pop(); + + // if there's a type override, use that. + byte typeToWrite = typeOverride == -1 ? getCompactType(field.type) : typeOverride; + + // check if we can use delta encoding for the field id + if (field.id > lastFieldId_ && field.id - lastFieldId_ <= 15) { + // write them together + writeByteDirect((field.id - lastFieldId_) << 4 | typeToWrite); + } else { + // write them separate + writeByteDirect(typeToWrite); + writeI16(field.id); + } + + lastFieldId_ = field.id; + // lastField_.push(field.id); + } + + /** Write the STOP symbol so we know there are no more fields in this struct. */ + @Override + public void writeFieldStop() throws TException { + writeByteDirect(TType.STOP); + } + + /** + * Write a map header. If the map is empty, omit the key and value type headers, as we don't need + * any additional information to skip it. + */ + @Override + public void writeMapBegin(TMap map) throws TException { + if (map.size == 0) { + writeByteDirect(0); + } else { + writeVarint32(map.size); + writeByteDirect(getCompactType(map.keyType) << 4 | getCompactType(map.valueType)); + } + } + + /** Write a list header. */ + @Override + public void writeListBegin(TList list) throws TException { + writeCollectionBegin(list.elemType, list.size); + } + + /** Write a set header. */ + @Override + public void writeSetBegin(TSet set) throws TException { + writeCollectionBegin(set.elemType, set.size); + } + + /** + * Write a boolean value. Potentially, this could be a boolean field, in which case the field + * header info isn't written yet. If so, decide what the right type header is for the value and + * then write the field header. Otherwise, write a single byte. + */ + @Override + public void writeBool(boolean b) throws TException { + if (booleanField_ != null) { + // we haven't written the field header yet + writeFieldBeginInternal(booleanField_, b ? Types.BOOLEAN_TRUE : Types.BOOLEAN_FALSE); + booleanField_ = null; + } else { + // we're not part of a field, so just write the value. + writeByteDirect(b ? Types.BOOLEAN_TRUE : Types.BOOLEAN_FALSE); + } + } + + /** Write a byte. Nothing to see here! */ + @Override + public void writeByte(byte b) throws TException { + writeByteDirect(b); + } + + /** Write an I16 as a zigzag varint. */ + @Override + public void writeI16(short i16) throws TException { + writeVarint32(intToZigZag(i16)); + } + + /** Write an i32 as a zigzag varint. */ + @Override + public void writeI32(int i32) throws TException { + writeVarint32(intToZigZag(i32)); + } + + /** Write an i64 as a zigzag varint. */ + @Override + public void writeI64(long i64) throws TException { + writeVarint64(longToZigzag(i64)); + } + + /** Write a double to the wire as 8 bytes. */ + @Override + public void writeDouble(double dub) throws TException { + fixedLongToBytes(Double.doubleToLongBits(dub), temp, 0); + trans_.write(temp, 0, 8); + } + + @Override + public void writeUuid(UUID uuid) throws TException { + ByteBuffer bb = ByteBuffer.wrap(temp); + bb.putLong(uuid.getMostSignificantBits()); + bb.putLong(uuid.getLeastSignificantBits()); + trans_.write(temp, 0, 16); + } + + /** Write a string to the wire with a varint size preceding. */ + @Override + public void writeString(String str) throws TException { + byte[] bytes = str.getBytes(StandardCharsets.UTF_8); + writeVarint32(bytes.length); + trans_.write(bytes, 0, bytes.length); + } + + /** Write a byte array, using a varint for the size. */ + @Override + public void writeBinary(ByteBuffer bin) throws TException { + ByteBuffer bb = bin.asReadOnlyBuffer(); + writeVarint32(bb.remaining()); + trans_.write(bb); + } + + // + // These methods are called by structs, but don't actually have any wire + // output or purpose. + // + + @Override + public void writeMessageEnd() throws TException {} + + @Override + public void writeMapEnd() throws TException {} + + @Override + public void writeListEnd() throws TException {} + + @Override + public void writeSetEnd() throws TException {} + + @Override + public void writeFieldEnd() throws TException {} + + // + // Internal writing methods + // + + /** + * Abstract method for writing the start of lists and sets. List and sets on the wire differ only + * by the type indicator. + */ + protected void writeCollectionBegin(byte elemType, int size) throws TException { + if (size <= 14) { + writeByteDirect(size << 4 | getCompactType(elemType)); + } else { + writeByteDirect(0xf0 | getCompactType(elemType)); + writeVarint32(size); + } + } + + /** + * Write an i32 as a varint. Results in 1-5 bytes on the wire. TODO: make a permanent buffer like + * writeVarint64? + */ + private void writeVarint32(int n) throws TException { + int idx = 0; + while (true) { + if ((n & ~0x7F) == 0) { + temp[idx++] = (byte) n; + // writeByteDirect((byte)n); + break; + // return; + } else { + temp[idx++] = (byte) ((n & 0x7F) | 0x80); + // writeByteDirect((byte)((n & 0x7F) | 0x80)); + n >>>= 7; + } + } + trans_.write(temp, 0, idx); + } + + /** Write an i64 as a varint. Results in 1-10 bytes on the wire. */ + private void writeVarint64(long n) throws TException { + int idx = 0; + while (true) { + if ((n & ~0x7FL) == 0) { + temp[idx++] = (byte) n; + break; + } else { + temp[idx++] = ((byte) ((n & 0x7F) | 0x80)); + n >>>= 7; + } + } + trans_.write(temp, 0, idx); + } + + /** + * Convert l into a zigzag long. This allows negative numbers to be represented compactly as a + * varint. + */ + private long longToZigzag(long l) { + return (l << 1) ^ (l >> 63); + } + + /** + * Convert n into a zigzag int. This allows negative numbers to be represented compactly as a + * varint. + */ + private int intToZigZag(int n) { + return (n << 1) ^ (n >> 31); + } + + /** Convert a long into little-endian bytes in buf starting at off and going until off+7. */ + private void fixedLongToBytes(long n, byte[] buf, int off) { + buf[off + 0] = (byte) (n & 0xff); + buf[off + 1] = (byte) ((n >> 8) & 0xff); + buf[off + 2] = (byte) ((n >> 16) & 0xff); + buf[off + 3] = (byte) ((n >> 24) & 0xff); + buf[off + 4] = (byte) ((n >> 32) & 0xff); + buf[off + 5] = (byte) ((n >> 40) & 0xff); + buf[off + 6] = (byte) ((n >> 48) & 0xff); + buf[off + 7] = (byte) ((n >> 56) & 0xff); + } + + /** + * Writes a byte without any possibility of all that field header nonsense. Used internally by + * other writing methods that know they need to write a byte. + */ + private void writeByteDirect(byte b) throws TException { + temp[0] = b; + trans_.write(temp, 0, 1); + } + + /** Writes a byte without any possibility of all that field header nonsense. */ + private void writeByteDirect(int n) throws TException { + writeByteDirect((byte) n); + } + + // + // Reading methods. + // + + /** Read a message header. */ + @Override + public TMessage readMessageBegin() throws TException { + byte protocolId = readByte(); + if (protocolId != PROTOCOL_ID) { + throw new TProtocolException( + "Expected protocol id " + + Integer.toHexString(PROTOCOL_ID) + + " but got " + + Integer.toHexString(protocolId)); + } + byte versionAndType = readByte(); + byte version = (byte) (versionAndType & VERSION_MASK); + if (version != VERSION) { + throw new TProtocolException("Expected version " + VERSION + " but got " + version); + } + byte type = (byte) ((versionAndType >> TYPE_SHIFT_AMOUNT) & TYPE_BITS); + int seqid = readVarint32(); + String messageName = readString(); + return new TMessage(messageName, type, seqid); + } + + /** + * Read a struct begin. There's nothing on the wire for this, but it is our opportunity to push a + * new struct begin marker onto the field stack. + */ + @Override + public TStruct readStructBegin() throws TException { + lastField_.push(lastFieldId_); + lastFieldId_ = 0; + return ANONYMOUS_STRUCT; + } + + /** + * Doesn't actually consume any wire data, just removes the last field for this struct from the + * field stack. + */ + @Override + public void readStructEnd() throws TException { + // consume the last field we read off the wire. + lastFieldId_ = lastField_.pop(); + } + + /** Read a field header off the wire. */ + @Override + public TField readFieldBegin() throws TException { + byte type = readByte(); + + // if it's a stop, then we can return immediately, as the struct is over. + if (type == TType.STOP) { + return TSTOP; + } + + return new TField("", getTType((byte) (type & 0x0f)), readFieldId(type)); + } + + /** + * Read a map header off the wire. If the size is zero, skip reading the key and value type. This + * means that 0-length maps will yield TMaps without the "correct" types. + */ + @Override + public TMap readMapBegin() throws TException { + int size = readVarint32(); + checkContainerReadLength(size); + byte keyAndValueType = size == 0 ? 0 : readByte(); + TMap map = + new TMap( + getTType((byte) (keyAndValueType >> 4)), + getTType((byte) (keyAndValueType & 0xf)), + size); + checkReadBytesAvailable(map); + return map; + } + + /** + * Read a list header off the wire. If the list size is 0-14, the size will be packed into the + * element type header. If it's a longer list, the 4 MSB of the element type header will be 0xF, + * and a varint will follow with the true size. + */ + @Override + public TList readListBegin() throws TException { + byte size_and_type = readByte(); + int size = (size_and_type >> 4) & 0x0f; + if (size == 15) { + size = readVarint32(); + } + checkContainerReadLength(size); + TList list = new TList(getTType(size_and_type), size); + checkReadBytesAvailable(list); + return list; + } + + /** + * Read a set header off the wire. If the set size is 0-14, the size will be packed into the + * element type header. If it's a longer set, the 4 MSB of the element type header will be 0xF, + * and a varint will follow with the true size. + */ + @Override + public TSet readSetBegin() throws TException { + return new TSet(readListBegin()); + } + + /** + * Read a boolean off the wire. If this is a boolean field, the value should already have been + * read during readFieldBegin, so we'll just consume the pre-stored value. Otherwise, read a byte. + */ + @Override + public boolean readBool() throws TException { + if (boolValue_ != null) { + boolean result = boolValue_; + boolValue_ = null; + return result; + } + return readByte() == Types.BOOLEAN_TRUE; + } + + /** Read a single byte off the wire. Nothing interesting here. */ + @Override + public byte readByte() throws TException { + byte b; + if (trans_.getBytesRemainingInBuffer() > 0) { + b = trans_.getBuffer()[trans_.getBufferPosition()]; + trans_.consumeBuffer(1); + } else { + trans_.readAll(temp, 0, 1); + b = temp[0]; + } + return b; + } + + /** Read an i16 from the wire as a zigzag varint. */ + @Override + public short readI16() throws TException { + return (short) zigzagToInt(readVarint32()); + } + + /** Read an i32 from the wire as a zigzag varint. */ + @Override + public int readI32() throws TException { + return zigzagToInt(readVarint32()); + } + + /** Read an i64 from the wire as a zigzag varint. */ + @Override + public long readI64() throws TException { + return zigzagToLong(readVarint64()); + } + + /** No magic here - just read a double off the wire. */ + @Override + public double readDouble() throws TException { + trans_.readAll(temp, 0, 8); + return Double.longBitsToDouble(bytesToLong(temp)); + } + + @Override + public UUID readUuid() throws TException { + trans_.readAll(temp, 0, 16); + ByteBuffer bb = ByteBuffer.wrap(temp, 0, 16); + return new UUID(bb.getLong(), bb.getLong()); + } + + /** Reads a byte[] (via readBinary), and then UTF-8 decodes it. */ + @Override + public String readString() throws TException { + int length = readVarint32(); + checkStringReadLength(length); + + if (length == 0) { + return ""; + } + + final String str; + if (trans_.getBytesRemainingInBuffer() >= length) { + str = + new String( + trans_.getBuffer(), trans_.getBufferPosition(), length, StandardCharsets.UTF_8); + trans_.consumeBuffer(length); + } else { + str = new String(readBinary(length), StandardCharsets.UTF_8); + } + return str; + } + + /** Read a ByteBuffer from the wire. */ + @Override + public ByteBuffer readBinary() throws TException { + int length = readVarint32(); + if (length == 0) { + return EMPTY_BUFFER; + } + getTransport().checkReadBytesAvailable(length); + if (trans_.getBytesRemainingInBuffer() >= length) { + ByteBuffer bb = ByteBuffer.wrap(trans_.getBuffer(), trans_.getBufferPosition(), length); + trans_.consumeBuffer(length); + return bb; + } + + byte[] buf = new byte[length]; + trans_.readAll(buf, 0, length); + return ByteBuffer.wrap(buf); + } + + /** Read a byte[] of a known length from the wire. */ + private byte[] readBinary(int length) throws TException { + if (length == 0) return EMPTY_BYTES; + + byte[] buf = new byte[length]; + trans_.readAll(buf, 0, length); + return buf; + } + + private void checkStringReadLength(int length) throws TException { + if (length < 0) { + throw new TProtocolException(TProtocolException.NEGATIVE_SIZE, "Negative length: " + length); + } + + getTransport().checkReadBytesAvailable(length); + + if (stringLengthLimit_ != NO_LENGTH_LIMIT && length > stringLengthLimit_) { + throw new TProtocolException( + TProtocolException.SIZE_LIMIT, "Length exceeded max allowed: " + length); + } + } + + private void checkContainerReadLength(int length) throws TProtocolException { + if (length < 0) { + throw new TProtocolException(TProtocolException.NEGATIVE_SIZE, "Negative length: " + length); + } + if (containerLengthLimit_ != NO_LENGTH_LIMIT && length > containerLengthLimit_) { + throw new TProtocolException( + TProtocolException.SIZE_LIMIT, "Length exceeded max allowed: " + length); + } + } + + // + // These methods are here for the struct to call, but don't have any wire + // encoding. + // + @Override + public void readMessageEnd() throws TException {} + + @Override + public void readFieldEnd() throws TException {} + + @Override + public void readMapEnd() throws TException {} + + @Override + public void readListEnd() throws TException {} + + @Override + public void readSetEnd() throws TException {} + + // + // Internal reading methods + // + + /** + * Read an i32 from the wire as a varint. The MSB of each byte is set if there is another byte to + * follow. This can read up to 5 bytes. + */ + private int readVarint32() throws TException { + int result = 0; + int shift = 0; + if (trans_.getBytesRemainingInBuffer() >= 5) { + byte[] buf = trans_.getBuffer(); + int pos = trans_.getBufferPosition(); + int off = 0; + while (true) { + byte b = buf[pos + off]; + result |= (b & 0x7f) << shift; + if ((b & 0x80) != 0x80) break; + shift += 7; + off++; + } + trans_.consumeBuffer(off + 1); + } else { + while (true) { + byte b = readByte(); + result |= (b & 0x7f) << shift; + if ((b & 0x80) != 0x80) break; + shift += 7; + } + } + return result; + } + + /** + * Read an i64 from the wire as a proper varint. The MSB of each byte is set if there is another + * byte to follow. This can read up to 10 bytes. + */ + private long readVarint64() throws TException { + int shift = 0; + long result = 0; + if (trans_.getBytesRemainingInBuffer() >= 10) { + byte[] buf = trans_.getBuffer(); + int pos = trans_.getBufferPosition(); + int off = 0; + while (true) { + byte b = buf[pos + off]; + result |= (long) (b & 0x7f) << shift; + if ((b & 0x80) != 0x80) break; + shift += 7; + off++; + } + trans_.consumeBuffer(off + 1); + } else { + while (true) { + byte b = readByte(); + result |= (long) (b & 0x7f) << shift; + if ((b & 0x80) != 0x80) break; + shift += 7; + } + } + return result; + } + + // + // encoding helpers + // + + /** Convert from zigzag int to int. */ + private int zigzagToInt(int n) { + return (n >>> 1) ^ -(n & 1); + } + + /** Convert from zigzag long to long. */ + private long zigzagToLong(long n) { + return (n >>> 1) ^ -(n & 1); + } + + /** + * Note that it's important that the mask bytes are long literals, otherwise they'll default to + * ints, and when you shift an int left 56 bits, you just get a messed up int. + */ + private long bytesToLong(byte[] bytes) { + return bytesToLong(bytes, 0); + } + + private long bytesToLong(byte[] bytes, int offset) { + return ((bytes[offset + 7] & 0xffL) << 56) + | ((bytes[offset + 6] & 0xffL) << 48) + | ((bytes[offset + 5] & 0xffL) << 40) + | ((bytes[offset + 4] & 0xffL) << 32) + | ((bytes[offset + 3] & 0xffL) << 24) + | ((bytes[offset + 2] & 0xffL) << 16) + | ((bytes[offset + 1] & 0xffL) << 8) + | ((bytes[offset + 0] & 0xffL)); + } + + // + // type testing and converting + // + + private boolean isBoolType(byte b) { + int lowerNibble = b & 0x0f; + return lowerNibble == Types.BOOLEAN_TRUE || lowerNibble == Types.BOOLEAN_FALSE; + } + + /** Given a TCompactProtocol.Types constant, convert it to its corresponding TType value. */ + private byte getTType(byte type) throws TProtocolException { + switch ((byte) (type & 0x0f)) { + case TType.STOP: + return TType.STOP; + case Types.BOOLEAN_FALSE: + case Types.BOOLEAN_TRUE: + return TType.BOOL; + case Types.BYTE: + return TType.BYTE; + case Types.I16: + return TType.I16; + case Types.I32: + return TType.I32; + case Types.I64: + return TType.I64; + case Types.UUID: + return TType.UUID; + case Types.DOUBLE: + return TType.DOUBLE; + case Types.BINARY: + return TType.STRING; + case Types.LIST: + return TType.LIST; + case Types.SET: + return TType.SET; + case Types.MAP: + return TType.MAP; + case Types.STRUCT: + return TType.STRUCT; + default: + throw new TProtocolException("don't know what type: " + (byte) (type & 0x0f)); + } + } + + /** Given a TType value, find the appropriate TCompactProtocol.Types constant. */ + private byte getCompactType(byte ttype) { + return ttypeToCompactType[ttype]; + } + + /** Return the minimum number of bytes a type will consume on the wire */ + @Override + public int getMinSerializedSize(byte type) throws TTransportException { + switch (type) { + case 0: + return 1; // Stop - T_STOP needs to count itself + case 1: + return 1; // Void - T_VOID needs to count itself + case 2: + return 1; // Bool sizeof(byte) + case 3: + return 1; // Byte sizeof(byte) + case 4: + return 8; // Double sizeof(double) + case 6: + return 1; // I16 sizeof(byte) + case 8: + return 1; // I32 sizeof(byte) + case 10: + return 1; // I64 sizeof(byte) + case 11: + return 1; // string length sizeof(byte) + case 12: + return 1; // empty struct needs at least 1 byte for the T_STOP + case 13: + return 1; // element count Map sizeof(byte) + case 14: + return 1; // element count Set sizeof(byte) + case 15: + return 1; // element count List sizeof(byte) + default: + throw new TTransportException(TTransportException.UNKNOWN, "unrecognized type code"); + } + } + + // ----------------------------------------------------------------- + // Additional methods to improve performance. + + @Override + public int readFieldBeginData() throws TException { + byte type = readByte(); + + // if it's a stop, then we can return immediately, as the struct is over. + if (type == TType.STOP) { + return TFieldData.encode(type); + } + + return TFieldData.encode(getTType((byte) (type & 0x0f)), readFieldId(type)); + } + + // Only makes sense to be called by readFieldBegin and readFieldBeginData + private short readFieldId(byte type) throws TException { + short fieldId; + + // mask off the 4 MSB of the type header. it could contain a field id delta. + short modifier = (short) ((type & 0xf0) >> 4); + if (modifier == 0) { + // not a delta. look ahead for the zigzag varint field id. + fieldId = readI16(); + } else { + // has a delta. add the delta to the last read field id. + fieldId = (short) (lastFieldId_ + modifier); + } + + // if this happens to be a boolean field, the value is encoded in the type + if (isBoolType(type)) { + // save the boolean value in a special instance variable. + boolValue_ = (byte) (type & 0x0f) == Types.BOOLEAN_TRUE ? Boolean.TRUE : Boolean.FALSE; + } + + // push the new field onto the field stack so we can keep the deltas going. + lastFieldId_ = fieldId; + return fieldId; + } + + @Override + protected void skipBinary() throws TException { + int size = intToZigZag(readI32()); + this.skipBytes(size); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TField.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TField.java new file mode 100644 index 0000000..9fde33c --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TField.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +/** + * Helper class that encapsulates field metadata. + * + *

Two fields are considered equal if they have the same type and id. + */ +public class TField { + public TField() { + this("", TType.STOP, (short) 0); + } + + public TField(String n, byte t, short i) { + name = n; + type = t; + id = i; + } + + public final String name; + public final byte type; + public final short id; + + public String toString() { + return ""; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + id; + result = prime * result + type; + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (getClass() != obj.getClass()) return false; + TField otherField = (TField) obj; + return type == otherField.type && id == otherField.id; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TJSONProtocol.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TJSONProtocol.java new file mode 100644 index 0000000..e32d70b --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TJSONProtocol.java @@ -0,0 +1,1004 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Stack; +import java.util.UUID; +import org.apache.thrift.TByteArrayOutputStream; +import org.apache.thrift.TException; +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; + +/** + * JSON protocol implementation for thrift. + * + *

This is a full-featured protocol supporting write and read. + * + *

Please see the C++ class header for a detailed description of the protocol's wire format. + */ +public class TJSONProtocol extends TProtocol { + + /** Factory for JSON protocol objects */ + public static class Factory implements TProtocolFactory { + protected boolean fieldNamesAsString_ = false; + + public Factory() {} + + public Factory(boolean fieldNamesAsString) { + fieldNamesAsString_ = fieldNamesAsString; + } + + public TProtocol getProtocol(TTransport trans) { + return new TJSONProtocol(trans, fieldNamesAsString_); + } + } + + private static final byte[] COMMA = new byte[] {','}; + private static final byte[] COLON = new byte[] {':'}; + private static final byte[] LBRACE = new byte[] {'{'}; + private static final byte[] RBRACE = new byte[] {'}'}; + private static final byte[] LBRACKET = new byte[] {'['}; + private static final byte[] RBRACKET = new byte[] {']'}; + private static final byte[] QUOTE = new byte[] {'"'}; + private static final byte[] BACKSLASH = new byte[] {'\\'}; + + private static final byte[] ESCSEQ = new byte[] {'\\', 'u', '0', '0'}; + + private static final long VERSION = 1; + + private static final byte[] JSON_CHAR_TABLE = { + /* 0 1 2 3 4 5 6 7 8 9 A B C D E F */ + 0, 0, 0, 0, 0, 0, 0, 0, 'b', 't', 'n', 0, 'f', 'r', 0, 0, // 0 + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1 + 1, 1, '"', 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // 2 + }; + + private static final String ESCAPE_CHARS = "\"\\/bfnrt"; + + private static final byte[] ESCAPE_CHAR_VALS = { + '"', '\\', '/', '\b', '\f', '\n', '\r', '\t', + }; + + private static final int DEF_STRING_SIZE = 16; + + private static final byte[] NAME_BOOL = new byte[] {'t', 'f'}; + private static final byte[] NAME_BYTE = new byte[] {'i', '8'}; + private static final byte[] NAME_I16 = new byte[] {'i', '1', '6'}; + private static final byte[] NAME_I32 = new byte[] {'i', '3', '2'}; + private static final byte[] NAME_I64 = new byte[] {'i', '6', '4'}; + private static final byte[] NAME_UUID = new byte[] {'u', 'i', 'd'}; + private static final byte[] NAME_DOUBLE = new byte[] {'d', 'b', 'l'}; + private static final byte[] NAME_STRUCT = new byte[] {'r', 'e', 'c'}; + private static final byte[] NAME_STRING = new byte[] {'s', 't', 'r'}; + private static final byte[] NAME_MAP = new byte[] {'m', 'a', 'p'}; + private static final byte[] NAME_LIST = new byte[] {'l', 's', 't'}; + private static final byte[] NAME_SET = new byte[] {'s', 'e', 't'}; + + private static final TStruct ANONYMOUS_STRUCT = new TStruct(); + + private static byte[] getTypeNameForTypeID(byte typeID) throws TException { + switch (typeID) { + case TType.BOOL: + return NAME_BOOL; + case TType.BYTE: + return NAME_BYTE; + case TType.I16: + return NAME_I16; + case TType.I32: + return NAME_I32; + case TType.I64: + return NAME_I64; + case TType.UUID: + return NAME_UUID; + case TType.DOUBLE: + return NAME_DOUBLE; + case TType.STRING: + return NAME_STRING; + case TType.STRUCT: + return NAME_STRUCT; + case TType.MAP: + return NAME_MAP; + case TType.SET: + return NAME_SET; + case TType.LIST: + return NAME_LIST; + default: + throw new TProtocolException(TProtocolException.NOT_IMPLEMENTED, "Unrecognized type"); + } + } + + private static byte getTypeIDForTypeName(byte[] name) throws TException { + byte result = TType.STOP; + if (name.length > 1) { + switch (name[0]) { + case 'd': + result = TType.DOUBLE; + break; + case 'i': + switch (name[1]) { + case '8': + result = TType.BYTE; + break; + case '1': + result = TType.I16; + break; + case '3': + result = TType.I32; + break; + case '6': + result = TType.I64; + break; + } + break; + case 'l': + result = TType.LIST; + break; + case 'm': + result = TType.MAP; + break; + case 'r': + result = TType.STRUCT; + break; + case 's': + if (name[1] == 't') { + result = TType.STRING; + } else if (name[1] == 'e') { + result = TType.SET; + } + break; + case 'u': + result = TType.UUID; + break; + case 't': + result = TType.BOOL; + break; + } + } + if (result == TType.STOP) { + throw new TProtocolException(TProtocolException.NOT_IMPLEMENTED, "Unrecognized type"); + } + return result; + } + + // Base class for tracking JSON contexts that may require inserting/reading + // additional JSON syntax characters + // This base context does nothing. + protected static class JSONBaseContext { + protected void write() throws TException {} + + protected void read() throws TException {} + + protected boolean escapeNum() { + return false; + } + } + + // Context for JSON lists. Will insert/read commas before each item except + // for the first one + protected class JSONListContext extends JSONBaseContext { + private boolean first_ = true; + + @Override + protected void write() throws TException { + if (first_) { + first_ = false; + } else { + trans_.write(COMMA); + } + } + + @Override + protected void read() throws TException { + if (first_) { + first_ = false; + } else { + readJSONSyntaxChar(COMMA); + } + } + } + + // Context for JSON records. Will insert/read colons before the value portion + // of each record pair, and commas before each key except the first. In + // addition, will indicate that numbers in the key position need to be + // escaped in quotes (since JSON keys must be strings). + protected class JSONPairContext extends JSONBaseContext { + private boolean first_ = true; + private boolean colon_ = true; + + @Override + protected void write() throws TException { + if (first_) { + first_ = false; + colon_ = true; + } else { + trans_.write(colon_ ? COLON : COMMA); + colon_ = !colon_; + } + } + + @Override + protected void read() throws TException { + if (first_) { + first_ = false; + colon_ = true; + } else { + readJSONSyntaxChar(colon_ ? COLON : COMMA); + colon_ = !colon_; + } + } + + @Override + protected boolean escapeNum() { + return colon_; + } + } + + // Holds up to one byte from the transport + protected class LookaheadReader { + + private boolean hasData_; + private final byte[] data_ = new byte[1]; + + // Return and consume the next byte to be read, either taking it from the + // data buffer if present or getting it from the transport otherwise. + protected byte read() throws TException { + if (hasData_) { + hasData_ = false; + } else { + trans_.readAll(data_, 0, 1); + } + return data_[0]; + } + + // Return the next byte to be read without consuming, filling the data + // buffer if it has not been filled already. + protected byte peek() throws TException { + if (!hasData_) { + trans_.readAll(data_, 0, 1); + } + hasData_ = true; + return data_[0]; + } + } + + // Stack of nested contexts that we may be in + private final Stack contextStack_ = new Stack<>(); + + // Current context that we are in + private JSONBaseContext context_ = new JSONBaseContext(); + + // Reader that manages a 1-byte buffer + private LookaheadReader reader_ = new LookaheadReader(); + + // Write out the TField names as a string instead of the default integer value + private boolean fieldNamesAsString_ = false; + + // Push a new JSON context onto the stack. + private void pushContext(JSONBaseContext c) { + contextStack_.push(context_); + context_ = c; + } + + // Pop the last JSON context off the stack + private void popContext() { + context_ = contextStack_.pop(); + } + + // Reset the context stack to its initial state + private void resetContext() { + while (!contextStack_.isEmpty()) { + popContext(); + } + } + + /** Constructor */ + public TJSONProtocol(TTransport trans) { + super(trans); + } + + public TJSONProtocol(TTransport trans, boolean fieldNamesAsString) { + super(trans); + fieldNamesAsString_ = fieldNamesAsString; + } + + @Override + public void reset() { + contextStack_.clear(); + context_ = new JSONBaseContext(); + reader_ = new LookaheadReader(); + } + + // Temporary buffer used by several methods + private final byte[] tmpbuf_ = new byte[4]; + + // Read a byte that must match b[0]; otherwise an exception is thrown. + // Marked protected to avoid synthetic accessor in JSONListContext.read + // and JSONPairContext.read + protected void readJSONSyntaxChar(byte[] b) throws TException { + byte ch = reader_.read(); + if (ch != b[0]) { + throw new TProtocolException( + TProtocolException.INVALID_DATA, "Unexpected character:" + (char) ch); + } + } + + // Convert a byte containing a hex char ('0'-'9' or 'a'-'f') into its + // corresponding hex value + private static byte hexVal(byte ch) throws TException { + if ((ch >= '0') && (ch <= '9')) { + return (byte) ((char) ch - '0'); + } else if ((ch >= 'a') && (ch <= 'f')) { + return (byte) ((char) ch - 'a' + 10); + } else { + throw new TProtocolException(TProtocolException.INVALID_DATA, "Expected hex character"); + } + } + + // Convert a byte containing a hex value to its corresponding hex character + private static byte hexChar(byte val) { + val &= 0x0F; + if (val < 10) { + return (byte) ((char) val + '0'); + } else { + return (byte) ((char) (val - 10) + 'a'); + } + } + + // Write the bytes in array buf as a JSON characters, escaping as needed + private void writeJSONString(byte[] b) throws TException { + context_.write(); + trans_.write(QUOTE); + int len = b.length; + for (int i = 0; i < len; i++) { + if ((b[i] & 0x00FF) >= 0x30) { + if (b[i] == BACKSLASH[0]) { + trans_.write(BACKSLASH); + trans_.write(BACKSLASH); + } else { + trans_.write(b, i, 1); + } + } else { + tmpbuf_[0] = JSON_CHAR_TABLE[b[i]]; + if (tmpbuf_[0] == 1) { + trans_.write(b, i, 1); + } else if (tmpbuf_[0] > 1) { + trans_.write(BACKSLASH); + trans_.write(tmpbuf_, 0, 1); + } else { + trans_.write(ESCSEQ); + tmpbuf_[0] = hexChar((byte) (b[i] >> 4)); + tmpbuf_[1] = hexChar(b[i]); + trans_.write(tmpbuf_, 0, 2); + } + } + } + trans_.write(QUOTE); + } + + // Write out number as a JSON value. If the context dictates so, it will be + // wrapped in quotes to output as a JSON string. + private void writeJSONInteger(long num) throws TException { + context_.write(); + String str = Long.toString(num); + boolean escapeNum = context_.escapeNum(); + if (escapeNum) { + trans_.write(QUOTE); + } + byte[] buf = str.getBytes(StandardCharsets.UTF_8); + trans_.write(buf); + if (escapeNum) { + trans_.write(QUOTE); + } + } + + // Write out a double as a JSON value. If it is NaN or infinity or if the + // context dictates escaping, write out as JSON string. + private void writeJSONDouble(double num) throws TException { + context_.write(); + String str = Double.toString(num); + boolean special = false; + switch (str.charAt(0)) { + case 'N': // NaN + case 'I': // Infinity + special = true; + break; + case '-': + if (str.charAt(1) == 'I') { // -Infinity + special = true; + } + break; + default: + break; + } + + boolean escapeNum = special || context_.escapeNum(); + if (escapeNum) { + trans_.write(QUOTE); + } + byte[] b = str.getBytes(StandardCharsets.UTF_8); + trans_.write(b, 0, b.length); + if (escapeNum) { + trans_.write(QUOTE); + } + } + + // Write out contents of byte array b as a JSON string with base-64 encoded + // data + private void writeJSONBase64(byte[] b, int offset, int length) throws TException { + context_.write(); + trans_.write(QUOTE); + int len = length; + int off = offset; + while (len >= 3) { + // Encode 3 bytes at a time + TBase64Utils.encode(b, off, 3, tmpbuf_, 0); + trans_.write(tmpbuf_, 0, 4); + off += 3; + len -= 3; + } + if (len > 0) { + // Encode remainder + TBase64Utils.encode(b, off, len, tmpbuf_, 0); + trans_.write(tmpbuf_, 0, len + 1); + } + trans_.write(QUOTE); + } + + private void writeJSONObjectStart() throws TException { + context_.write(); + trans_.write(LBRACE); + pushContext(new JSONPairContext()); + } + + private void writeJSONObjectEnd() throws TException { + popContext(); + trans_.write(RBRACE); + } + + private void writeJSONArrayStart() throws TException { + context_.write(); + trans_.write(LBRACKET); + pushContext(new JSONListContext()); + } + + private void writeJSONArrayEnd() throws TException { + popContext(); + trans_.write(RBRACKET); + } + + @Override + public void writeMessageBegin(TMessage message) throws TException { + resetContext(); // THRIFT-3743 + writeJSONArrayStart(); + writeJSONInteger(VERSION); + byte[] b = message.name.getBytes(StandardCharsets.UTF_8); + writeJSONString(b); + writeJSONInteger(message.type); + writeJSONInteger(message.seqid); + } + + @Override + public void writeMessageEnd() throws TException { + writeJSONArrayEnd(); + } + + @Override + public void writeStructBegin(TStruct struct) throws TException { + writeJSONObjectStart(); + } + + @Override + public void writeStructEnd() throws TException { + writeJSONObjectEnd(); + } + + @Override + public void writeFieldBegin(TField field) throws TException { + if (fieldNamesAsString_) { + writeString(field.name); + } else { + writeJSONInteger(field.id); + } + writeJSONObjectStart(); + writeJSONString(getTypeNameForTypeID(field.type)); + } + + @Override + public void writeFieldEnd() throws TException { + writeJSONObjectEnd(); + } + + @Override + public void writeFieldStop() {} + + @Override + public void writeMapBegin(TMap map) throws TException { + writeJSONArrayStart(); + writeJSONString(getTypeNameForTypeID(map.keyType)); + writeJSONString(getTypeNameForTypeID(map.valueType)); + writeJSONInteger(map.size); + writeJSONObjectStart(); + } + + @Override + public void writeMapEnd() throws TException { + writeJSONObjectEnd(); + writeJSONArrayEnd(); + } + + @Override + public void writeListBegin(TList list) throws TException { + writeJSONArrayStart(); + writeJSONString(getTypeNameForTypeID(list.elemType)); + writeJSONInteger(list.size); + } + + @Override + public void writeListEnd() throws TException { + writeJSONArrayEnd(); + } + + @Override + public void writeSetBegin(TSet set) throws TException { + writeJSONArrayStart(); + writeJSONString(getTypeNameForTypeID(set.elemType)); + writeJSONInteger(set.size); + } + + @Override + public void writeSetEnd() throws TException { + writeJSONArrayEnd(); + } + + @Override + public void writeBool(boolean b) throws TException { + writeJSONInteger(b ? (long) 1 : (long) 0); + } + + @Override + public void writeByte(byte b) throws TException { + writeJSONInteger(b); + } + + @Override + public void writeI16(short i16) throws TException { + writeJSONInteger(i16); + } + + @Override + public void writeI32(int i32) throws TException { + writeJSONInteger(i32); + } + + @Override + public void writeI64(long i64) throws TException { + writeJSONInteger(i64); + } + + @Override + public void writeUuid(UUID uuid) throws TException { + writeJSONString(uuid.toString().getBytes(StandardCharsets.UTF_8)); + } + + @Override + public void writeDouble(double dub) throws TException { + writeJSONDouble(dub); + } + + @Override + public void writeString(String str) throws TException { + byte[] b = str.getBytes(StandardCharsets.UTF_8); + writeJSONString(b); + } + + @Override + public void writeBinary(ByteBuffer bin) throws TException { + writeJSONBase64( + bin.array(), + bin.position() + bin.arrayOffset(), + bin.limit() - bin.position() - bin.arrayOffset()); + } + + /** Reading methods. */ + + // Read in a JSON string, unescaping as appropriate.. Skip reading from the + // context if skipContext is true. + private TByteArrayOutputStream readJSONString(boolean skipContext) throws TException { + TByteArrayOutputStream arr = new TByteArrayOutputStream(DEF_STRING_SIZE); + ArrayList codeunits = new ArrayList(); + if (!skipContext) { + context_.read(); + } + readJSONSyntaxChar(QUOTE); + while (true) { + byte ch = reader_.read(); + if (ch == QUOTE[0]) { + break; + } + if (ch == ESCSEQ[0]) { + ch = reader_.read(); + if (ch == ESCSEQ[1]) { + trans_.readAll(tmpbuf_, 0, 4); + short cu = + (short) + (((short) hexVal(tmpbuf_[0]) << 12) + + ((short) hexVal(tmpbuf_[1]) << 8) + + ((short) hexVal(tmpbuf_[2]) << 4) + + (short) hexVal(tmpbuf_[3])); + try { + if (Character.isHighSurrogate((char) cu)) { + if (codeunits.size() > 0) { + throw new TProtocolException( + TProtocolException.INVALID_DATA, "Expected low surrogate char"); + } + codeunits.add((char) cu); + } else if (Character.isLowSurrogate((char) cu)) { + if (codeunits.size() == 0) { + throw new TProtocolException( + TProtocolException.INVALID_DATA, "Expected high surrogate char"); + } + + codeunits.add((char) cu); + arr.write( + (new String(new int[] {codeunits.get(0), codeunits.get(1)}, 0, 2)) + .getBytes(StandardCharsets.UTF_8)); + codeunits.clear(); + } else { + arr.write((new String(new int[] {cu}, 0, 1)).getBytes(StandardCharsets.UTF_8)); + } + continue; + } catch (IOException ex) { + throw new TProtocolException( + TProtocolException.INVALID_DATA, "Invalid unicode sequence"); + } + } else { + int off = ESCAPE_CHARS.indexOf(ch); + if (off == -1) { + throw new TProtocolException(TProtocolException.INVALID_DATA, "Expected control char"); + } + ch = ESCAPE_CHAR_VALS[off]; + } + } + arr.write(ch); + } + return arr; + } + + // Return true if the given byte could be a valid part of a JSON number. + private boolean isJSONNumeric(byte b) { + switch (b) { + case '+': + case '-': + case '.': + case '0': + case '1': + case '2': + case '3': + case '4': + case '5': + case '6': + case '7': + case '8': + case '9': + case 'E': + case 'e': + return true; + } + return false; + } + + // Read in a sequence of characters that are all valid in JSON numbers. Does + // not do a complete regex check to validate that this is actually a number. + private String readJSONNumericChars() throws TException { + StringBuilder strbld = new StringBuilder(); + while (true) { + byte ch = reader_.peek(); + if (!isJSONNumeric(ch)) { + break; + } + strbld.append((char) reader_.read()); + } + return strbld.toString(); + } + + // Read in a JSON number. If the context dictates, read in enclosing quotes. + private long readJSONInteger() throws TException { + context_.read(); + if (context_.escapeNum()) { + readJSONSyntaxChar(QUOTE); + } + String str = readJSONNumericChars(); + if (context_.escapeNum()) { + readJSONSyntaxChar(QUOTE); + } + try { + return Long.parseLong(str); + } catch (NumberFormatException ex) { + throw new TProtocolException( + TProtocolException.INVALID_DATA, "Bad data encounted in numeric data"); + } + } + + // Read in a JSON double value. Throw if the value is not wrapped in quotes + // when expected or if wrapped in quotes when not expected. + private double readJSONDouble() throws TException { + context_.read(); + if (reader_.peek() == QUOTE[0]) { + TByteArrayOutputStream arr = readJSONString(true); + double dub = Double.parseDouble(arr.toString(StandardCharsets.UTF_8)); + if (!context_.escapeNum() && !Double.isNaN(dub) && !Double.isInfinite(dub)) { + // Throw exception -- we should not be in a string in this case + throw new TProtocolException( + TProtocolException.INVALID_DATA, "Numeric data unexpectedly quoted"); + } + return dub; + } else { + if (context_.escapeNum()) { + // This will throw - we should have had a quote if escapeNum == true + readJSONSyntaxChar(QUOTE); + } + try { + return Double.parseDouble(readJSONNumericChars()); + } catch (NumberFormatException ex) { + throw new TProtocolException( + TProtocolException.INVALID_DATA, "Bad data encounted in numeric data"); + } + } + } + + // Read in a JSON string containing base-64 encoded data and decode it. + private byte[] readJSONBase64() throws TException { + TByteArrayOutputStream arr = readJSONString(false); + byte[] b = arr.get(); + int len = arr.len(); + int off = 0; + int size = 0; + // Ignore padding + int bound = len >= 2 ? len - 2 : 0; + for (int i = len - 1; i >= bound && b[i] == '='; --i) { + --len; + } + while (len >= 4) { + // Decode 4 bytes at a time + TBase64Utils.decode(b, off, 4, b, size); // NB: decoded in place + off += 4; + len -= 4; + size += 3; + } + // Don't decode if we hit the end or got a single leftover byte (invalid + // base64 but legal for skip of regular string type) + if (len > 1) { + // Decode remainder + TBase64Utils.decode(b, off, len, b, size); // NB: decoded in place + size += len - 1; + } + // Sadly we must copy the byte[] (any way around this?) + byte[] result = new byte[size]; + System.arraycopy(b, 0, result, 0, size); + return result; + } + + private void readJSONObjectStart() throws TException { + context_.read(); + readJSONSyntaxChar(LBRACE); + pushContext(new JSONPairContext()); + } + + private void readJSONObjectEnd() throws TException { + readJSONSyntaxChar(RBRACE); + popContext(); + } + + private void readJSONArrayStart() throws TException { + context_.read(); + readJSONSyntaxChar(LBRACKET); + pushContext(new JSONListContext()); + } + + private void readJSONArrayEnd() throws TException { + readJSONSyntaxChar(RBRACKET); + popContext(); + } + + @Override + public TMessage readMessageBegin() throws TException { + resetContext(); // THRIFT-3743 + readJSONArrayStart(); + if (readJSONInteger() != VERSION) { + throw new TProtocolException( + TProtocolException.BAD_VERSION, "Message contained bad version."); + } + String name = readJSONString(false).toString(StandardCharsets.UTF_8); + byte type = (byte) readJSONInteger(); + int seqid = (int) readJSONInteger(); + return new TMessage(name, type, seqid); + } + + @Override + public void readMessageEnd() throws TException { + readJSONArrayEnd(); + } + + @Override + public TStruct readStructBegin() throws TException { + readJSONObjectStart(); + return ANONYMOUS_STRUCT; + } + + @Override + public void readStructEnd() throws TException { + readJSONObjectEnd(); + } + + @Override + public TField readFieldBegin() throws TException { + byte ch = reader_.peek(); + byte type; + short id = 0; + if (ch == RBRACE[0]) { + type = TType.STOP; + } else { + id = (short) readJSONInteger(); + readJSONObjectStart(); + type = getTypeIDForTypeName(readJSONString(false).get()); + } + return new TField("", type, id); + } + + @Override + public void readFieldEnd() throws TException { + readJSONObjectEnd(); + } + + @Override + public TMap readMapBegin() throws TException { + readJSONArrayStart(); + byte keyType = getTypeIDForTypeName(readJSONString(false).get()); + byte valueType = getTypeIDForTypeName(readJSONString(false).get()); + int size = (int) readJSONInteger(); + readJSONObjectStart(); + TMap map = new TMap(keyType, valueType, size); + + checkReadBytesAvailable(map); + return map; + } + + @Override + public void readMapEnd() throws TException { + readJSONObjectEnd(); + readJSONArrayEnd(); + } + + @Override + public TList readListBegin() throws TException { + readJSONArrayStart(); + byte elemType = getTypeIDForTypeName(readJSONString(false).get()); + int size = (int) readJSONInteger(); + TList list = new TList(elemType, size); + + checkReadBytesAvailable(list); + return list; + } + + @Override + public void readListEnd() throws TException { + readJSONArrayEnd(); + } + + @Override + public TSet readSetBegin() throws TException { + readJSONArrayStart(); + byte elemType = getTypeIDForTypeName(readJSONString(false).get()); + int size = (int) readJSONInteger(); + TSet set = new TSet(elemType, size); + + checkReadBytesAvailable(set); + return set; + } + + @Override + public void readSetEnd() throws TException { + readJSONArrayEnd(); + } + + @Override + public boolean readBool() throws TException { + return (readJSONInteger() != 0); + } + + @Override + public byte readByte() throws TException { + return (byte) readJSONInteger(); + } + + @Override + public short readI16() throws TException { + return (short) readJSONInteger(); + } + + @Override + public int readI32() throws TException { + return (int) readJSONInteger(); + } + + @Override + public long readI64() throws TException { + return readJSONInteger(); + } + + @Override + public UUID readUuid() throws TException { + return UUID.fromString(readString()); + } + + @Override + public double readDouble() throws TException { + return readJSONDouble(); + } + + @Override + public String readString() throws TException { + return readJSONString(false).toString(StandardCharsets.UTF_8); + } + + @Override + public ByteBuffer readBinary() throws TException { + return ByteBuffer.wrap(readJSONBase64()); + } + + /** Return the minimum number of bytes a type will consume on the wire */ + @Override + public int getMinSerializedSize(byte type) throws TTransportException { + switch (type) { + case 0: + return 1; // Stop - T_STOP needs to count itself + case 1: + return 1; // Void - T_VOID needs to count itself + case 2: + return 1; // Bool + case 3: + return 1; // Byte + case 4: + return 1; // Double + case 6: + return 1; // I16 + case 8: + return 1; // I32 + case 10: + return 1; // I64 + case 11: + return 2; // string length + case 12: + return 2; // empty struct + case 13: + return 2; // element count Map + case 14: + return 2; // element count Set + case 15: + return 2; // element count List + default: + throw new TTransportException(TTransportException.UNKNOWN, "unrecognized type code"); + } + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TLegacyUuidProtocolDecorator.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TLegacyUuidProtocolDecorator.java new file mode 100644 index 0000000..26b0924 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TLegacyUuidProtocolDecorator.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +import java.util.UUID; +import org.apache.thrift.TException; + +/** + * The TLegacyUuidProtocolDecorator that decorates an existing TProtocol to provide backwards + * compatibility with the old UUID format that the Java library used on the wire. + * + *

The initial UUID implementation in Java was not according to the protocol specification. This + * was fixed in THRIFT-5925 and as a result would break backwards compatibility with existing + * implementations that depends on that format. + * + *

This decorator is especially useful where migration is not needed and interop between other + * languages, that follows the specification, are not needed. + * + *

For usage see the TestTLegacyUuidProtocolDecorator tests. + */ +public class TLegacyUuidProtocolDecorator extends TProtocolDecorator { + + public TLegacyUuidProtocolDecorator(TProtocol protocol) { + super(protocol); + } + + @Override + public void writeUuid(UUID uuid) throws TException { + byte[] buf = new byte[16]; + + long lsb = uuid.getLeastSignificantBits(); + buf[0] = (byte) (0xff & (lsb >> 56)); + buf[1] = (byte) (0xff & (lsb >> 48)); + buf[2] = (byte) (0xff & (lsb >> 40)); + buf[3] = (byte) (0xff & (lsb >> 32)); + buf[4] = (byte) (0xff & (lsb >> 24)); + buf[5] = (byte) (0xff & (lsb >> 16)); + buf[6] = (byte) (0xff & (lsb >> 8)); + buf[7] = (byte) (0xff & (lsb)); + + long msb = uuid.getMostSignificantBits(); + buf[8] = (byte) (0xff & (msb >> 56)); + buf[9] = (byte) (0xff & (msb >> 48)); + buf[10] = (byte) (0xff & (msb >> 40)); + buf[11] = (byte) (0xff & (msb >> 32)); + buf[12] = (byte) (0xff & (msb >> 24)); + buf[13] = (byte) (0xff & (msb >> 16)); + buf[14] = (byte) (0xff & (msb >> 8)); + buf[15] = (byte) (0xff & (msb)); + + getTransport().write(buf, 0, 16); + } + + @Override + public UUID readUuid() throws TException { + byte[] buf = new byte[16]; + getTransport().readAll(buf, 0, 16); + + long lsb = + ((long) (buf[0] & 0xff) << 56) + | ((long) (buf[1] & 0xff) << 48) + | ((long) (buf[2] & 0xff) << 40) + | ((long) (buf[3] & 0xff) << 32) + | ((long) (buf[4] & 0xff) << 24) + | ((long) (buf[5] & 0xff) << 16) + | ((long) (buf[6] & 0xff) << 8) + | ((long) (buf[7] & 0xff)); + + long msb = + ((long) (buf[8] & 0xff) << 56) + | ((long) (buf[9] & 0xff) << 48) + | ((long) (buf[10] & 0xff) << 40) + | ((long) (buf[11] & 0xff) << 32) + | ((long) (buf[12] & 0xff) << 24) + | ((long) (buf[13] & 0xff) << 16) + | ((long) (buf[14] & 0xff) << 8) + | ((long) (buf[15] & 0xff)); + + return new UUID(msb, lsb); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TList.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TList.java new file mode 100644 index 0000000..9858b0c --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TList.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +/** Helper class that encapsulates list metadata. */ +public final class TList { + public TList() { + this(TType.STOP, 0); + } + + public TList(byte t, int s) { + elemType = t; + size = s; + } + + public final byte elemType; + public final int size; + + public byte getElemType() { + return elemType; + } + + public int getSize() { + return size; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TMap.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TMap.java new file mode 100644 index 0000000..a5d4fbd --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TMap.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +/** Helper class that encapsulates map metadata. */ +public final class TMap { + public TMap() { + this(TType.STOP, TType.STOP, 0); + } + + public TMap(byte k, byte v, int s) { + keyType = k; + valueType = v; + size = s; + } + + public final byte keyType; + public final byte valueType; + public final int size; + + public byte getKeyType() { + return keyType; + } + + public byte getValueType() { + return valueType; + } + + public int getSize() { + return size; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TMessage.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TMessage.java new file mode 100644 index 0000000..69c9dd1 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TMessage.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +/** Helper class that encapsulates struct metadata. */ +public final class TMessage { + public TMessage() { + this("", TType.STOP, 0); + } + + public TMessage(String n, byte t, int s) { + name = n; + type = t; + seqid = s; + } + + public final String name; + public final byte type; + public final int seqid; + + public String getName() { + return name; + } + + public byte getType() { + return type; + } + + public int getSeqid() { + return seqid; + } + + @Override + public String toString() { + return ""; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((name == null) ? 0 : name.hashCode()); + result = prime * result + seqid; + result = prime * result + type; + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null) return false; + if (getClass() != obj.getClass()) return false; + TMessage other = (TMessage) obj; + if (name == null) { + if (other.name != null) return false; + } else if (!name.equals(other.name)) return false; + if (seqid != other.seqid) return false; + if (type != other.type) return false; + return true; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TMessageType.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TMessageType.java new file mode 100644 index 0000000..a768cb6 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TMessageType.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +/** Message type constants in the Thrift protocol. */ +public final class TMessageType { + public static final byte CALL = 1; + public static final byte REPLY = 2; + public static final byte EXCEPTION = 3; + public static final byte ONEWAY = 4; +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TMultiplexedProtocol.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TMultiplexedProtocol.java new file mode 100644 index 0000000..0934330 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TMultiplexedProtocol.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +import org.apache.thrift.TException; + +/** + * TMultiplexedProtocol is a protocol-independent concrete decorator that allows a + * Thrift client to communicate with a multiplexing Thrift server, by prepending the service name to + * the function name during function calls. + * + *

NOTE: THIS IS NOT USED BY SERVERS. On the server, use {@link + * org.apache.thrift.TMultiplexedProcessor TMultiplexedProcessor} to handle requests from a + * multiplexing client. + * + *

This example uses a single socket transport to invoke two services: + * + * 

{@code
+ * TSocket transport = new TSocket("localhost", 9090);
+ * transport.open();
+ *
+ * TBinaryProtocol protocol = new TBinaryProtocol(transport);
+ *
+ * TMultiplexedProtocol mp = new TMultiplexedProtocol(protocol, "Calculator");
+ * Calculator.Client service = new Calculator.Client(mp);
+ *
+ * TMultiplexedProtocol mp2 = new TMultiplexedProtocol(protocol, "WeatherReport");
+ * WeatherReport.Client service2 = new WeatherReport.Client(mp2);
+ *
+ * System.out.println(service.add(2,2));
+ * System.out.println(service2.getTemperature());
+ * }
+ * + * @see org.apache.thrift.protocol.TProtocolDecorator + */ +public class TMultiplexedProtocol extends TProtocolDecorator { + + /** Used to delimit the service name from the function name */ + public static final String SEPARATOR = ":"; + + private final String SERVICE_NAME; + + /** + * Wrap the specified protocol, allowing it to be used to communicate with a multiplexing server. + * The serviceName is required as it is prepended to the message header so that the + * multiplexing server can broker the function call to the proper service. + * + * @param protocol Your communication protocol of choice, e.g. TBinaryProtocol. + * @param serviceName The service name of the service communicating via this protocol. + */ + public TMultiplexedProtocol(TProtocol protocol, String serviceName) { + super(protocol); + SERVICE_NAME = serviceName; + } + + /** + * Prepends the service name to the function name, separated by TMultiplexedProtocol.SEPARATOR. + * + * @param tMessage The original message. + * @throws TException Passed through from wrapped TProtocol instance. + */ + @Override + public void writeMessageBegin(TMessage tMessage) throws TException { + if (tMessage.type == TMessageType.CALL || tMessage.type == TMessageType.ONEWAY) { + super.writeMessageBegin( + new TMessage(SERVICE_NAME + SEPARATOR + tMessage.name, tMessage.type, tMessage.seqid)); + } else { + super.writeMessageBegin(tMessage); + } + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TProtocol.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TProtocol.java new file mode 100644 index 0000000..1238d57 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TProtocol.java @@ -0,0 +1,577 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.IntFunction; +import org.apache.thrift.TException; +import org.apache.thrift.partial.TFieldData; +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.StandardScheme; +import org.apache.thrift.transport.TTransport; + +/** Protocol interface definition. */ +public abstract class TProtocol implements TWriteProtocol, TReadProtocol { + + /** Prevent direct instantiation */ + @SuppressWarnings("unused") + private TProtocol() {} + + /** Transport */ + protected TTransport trans_; + + /** Current recursion depth during deserialization */ + private int recursionDepth_ = 0; + + /** Constructor */ + protected TProtocol(TTransport trans) { + trans_ = trans; + } + + /** + * Increment recursion depth, checking against the configured limit. + * + * @throws TProtocolException with DEPTH_LIMIT if limit exceeded + */ + public void incrementRecursionDepth() throws TProtocolException { + int limit = trans_.getConfiguration().getRecursionLimit(); + if (recursionDepth_ >= limit) { + throw new TProtocolException( + TProtocolException.DEPTH_LIMIT, + "Recursion depth " + (recursionDepth_ + 1) + " exceeds limit " + limit); + } + ++recursionDepth_; + } + + /** Decrement recursion depth. Call in finally block. */ + public void decrementRecursionDepth() { + --recursionDepth_; + } + + /** Transport accessor */ + public TTransport getTransport() { + return trans_; + } + + protected void checkReadBytesAvailable(TMap map) throws TException { + long elemSize = getMinSerializedSize(map.keyType) + getMinSerializedSize(map.valueType); + trans_.checkReadBytesAvailable(map.size * elemSize); + } + + protected void checkReadBytesAvailable(TList list) throws TException { + long size = list.getSize(); + trans_.checkReadBytesAvailable(size * getMinSerializedSize(list.elemType)); + } + + protected void checkReadBytesAvailable(TSet set) throws TException { + long size = set.getSize(); + trans_.checkReadBytesAvailable(size * getMinSerializedSize(set.elemType)); + } + + /** + * Return min serialized size in bytes + * + * @param type Returns the minimum amount of bytes needed to store the smallest possible instance + * of TType. + * @return min serialized size + * @throws TException when error happens + */ + public abstract int getMinSerializedSize(byte type) throws TException; + + public interface WriteCallback { + void call(T e) throws TException; + } + + public interface ReadCallback { + R accept(T t) throws TException; + } + + public interface ReadCollectionCallback { + R call() throws TException; + } + + public interface ReadMapEntryCallback { + K getKey() throws TException; + + V getValue() throws TException; + } + + public final void writeSet(byte elementType, Set set, WriteCallback callback) + throws TException { + writeSetBegin(new TSet(elementType, set.size())); + for (T t : set) { + callback.call(t); + } + writeSetEnd(); + } + + public final void writeList(byte elementType, List list, WriteCallback callback) + throws TException { + writeListBegin(new TList(elementType, list.size())); + for (T t : list) { + callback.call(t); + } + writeListEnd(); + } + + public final void writeMap( + byte keyType, byte valueType, Map map, WriteCallback> callback) + throws TException { + writeMapBegin(new TMap(keyType, valueType, map.size())); + for (Map.Entry entry : map.entrySet()) { + callback.call(entry); + } + writeMapEnd(); + } + + public final void writeField(TField field, WriteCallback callback) throws TException { + writeFieldBegin(field); + callback.call(null); + writeFieldEnd(); + } + + public final void writeStruct(TStruct struct, WriteCallback callback) throws TException { + writeStructBegin(struct); + callback.call(null); + writeStructEnd(); + } + + public final void writeMessage(TMessage message, WriteCallback callback) throws TException { + writeMessageBegin(message); + callback.call(null); + writeMessageEnd(); + } + + /** + * read a message by delegating to a callback, handles {@link #readMessageBegin() begin} and + * {@link #readMessageEnd() end} automatically. + * + * @param callback callback for actual reading + * @param result message type + * @return the message read + * @throws TException when any sub-operation failed + */ + public final T readMessage(ReadCallback callback) throws TException { + TMessage tMessage = readMessageBegin(); + T t = callback.accept(tMessage); + readMessageEnd(); + return t; + } + + /** + * read a struct by delegating to a callback, handles {@link #readStructBegin() begin} and {@link + * #readStructEnd() end} automatically. + * + * @param callback callback for actual reading + * @param result struct type + * @return the struct read + * @throws TException when any sub-operation failed + */ + public final T readStruct(ReadCallback callback) throws TException { + TStruct tStruct = readStructBegin(); + T t = callback.accept(tStruct); + readStructEnd(); + return t; + } + + /** + * read a field by delegating to a callback, handles {@link #readFieldBegin() begin} and {@link + * #readFieldEnd() end} automatically, and returns whether the {@link TType#STOP stop signal} was + * encountered. Because the value is not returned, you (the compiler generated code in most cases) + * are expected to set the field yourself within the callback. + * + * @param callback callback for reading a field + * @param result field type + * @return true if a stop signal was encountered, false otherwise + * @throws Exception when any sub-operation failed + */ + public final boolean readField(ReadCallback callback) throws Exception { + TField tField = readFieldBegin(); + if (tField.type == org.apache.thrift.protocol.TType.STOP) { + return true; + } + callback.accept(tField); + readFieldEnd(); + return false; + } + + /** + * read a {@link Map} of elements by delegating to the callback, handles {@link #readMapBegin() + * begin} and {@link #readMapEnd() end} automatically. + * + * @param callback callback for reading the map + * @param result map type + * @return the map read + * @throws TException when any sub-operation fails + */ + public final > T readMap(ReadCallback callback) throws TException { + TMap tMap = readMapBegin(); + T t = callback.accept(tMap); + readMapEnd(); + return t; + } + + /** + * read a {@link Map} of elements by delegating key and value reading to the callback, handles + * {@link #readMapBegin() begin} and {@link #readMapEnd() end} automatically. + * + * @param callback callback for reading keys and values, calls to {@link + * ReadMapEntryCallback#getKey()} and {@link ReadMapEntryCallback#getValue()} will be in + * alternating orders, i.e. k1, v1, k2, v2, .., k_n, v_n + * @param key type + * @param value type + * @return the map read + * @throws TException when any sub-operation fails + */ + public final Map readMap(ReadMapEntryCallback callback) throws TException { + return readMap(callback::getKey, callback::getValue); + } + + /** + * read a {@link Map} of elements by delegating key reading to the callback, handles {@link + * #readMapBegin() begin} and {@link #readMapEnd() end} automatically. Calls to keyCallback and + * valueCallback will be in alternating orders, i.e. k1, v1, k2, v2, .., k_n, v_n + * + * @param keyCallback callback for reading keys + * @param valueCallback callback for reading values + * @param key type + * @param value type + * @return the map read + * @throws TException when any sub-operation fails + */ + public final Map readMap( + ReadCollectionCallback keyCallback, ReadCollectionCallback valueCallback) + throws TException { + return readMap(keyCallback, valueCallback, size -> new HashMap<>(2 * size)); + } + + /** + * read a {@link Map} of elements by delegating key and value reading to the callback, handles + * {@link #readMapBegin() begin} and {@link #readMapEnd() end} automatically, with a specialized + * map creator given the size hint. + * + * @param callback callback for reading keys and values, calls to {@link + * ReadMapEntryCallback#getKey()} and {@link ReadMapEntryCallback#getValue()} will be in + * alternating orders, i.e. k1, v1, k2, v2, .., k_n, v_n + * @param mapCreator map creator given the size hint + * @param key type + * @param value type + * @return the map read + * @throws TException when any sub-operation fails + */ + public final Map readMap( + ReadMapEntryCallback callback, IntFunction> mapCreator) throws TException { + return readMap(callback::getKey, callback::getValue, mapCreator); + } + + /** + * read a {@link Map} of elements by delegating key and value reading to the callback, handles + * {@link #readMapBegin() begin} and {@link #readMapEnd() end} automatically, with a specialized + * map creator given the size hint. Calls to keyCallback and valueCallback will be in alternating + * orders, i.e. k1, v1, k2, v2, .., k_n, v_n + * + * @param keyCallback callback for reading keys + * @param valueCallback callback for reading values + * @param mapCreator map creator given the size hint + * @param key type + * @param value type + * @return the map read + * @throws TException when any sub-operation fails + */ + public final Map readMap( + ReadCollectionCallback keyCallback, + ReadCollectionCallback valueCallback, + IntFunction> mapCreator) + throws TException { + return readMap( + tMap -> { + Map map = mapCreator.apply(tMap.size); + for (int i = 0; i < tMap.size; i += 1) { + map.put(keyCallback.call(), valueCallback.call()); + } + return map; + }); + } + + /** + * read a {@link List} by delegating to the callback, handles {@link #readListBegin() begin} and + * {@link #readListEnd() end} automatically. + * + * @param callback callback for reading the list + * @param result list type + * @return the list read + * @throws TException when any sub-operation fails + */ + public final > T readList(ReadCallback callback) throws TException { + TList tList = readListBegin(); + T t = callback.accept(tList); + readListEnd(); + return t; + } + + /** + * read a {@link List} by delegating element reading to the callback, handles {@link + * #readListBegin() begin} and {@link #readListEnd() end} automatically. + * + * @param callback callback for reading one element + * @param element type + * @return list of elements read + * @throws TException when any sub-operation fails + */ + public final List readList(ReadCollectionCallback callback) throws TException { + return readList(callback, ArrayList::new); + } + + /** + * read a {@link List} by delegating element reading to the callback, handles {@link + * #readListBegin() begin} and {@link #readListEnd() end} automatically, with a specialized list + * creator given the size hint. + * + * @param callback callback for reading one element + * @param listCreator list creator given size hint + * @param element type + * @return list of elements read + * @throws TException when any sub-operation fails + */ + public final List readList( + ReadCollectionCallback callback, IntFunction> listCreator) throws TException { + return readList( + tList -> { + List list = listCreator.apply(tList.size); + for (int i = 0; i < tList.size; i += 1) { + list.add(callback.call()); + } + return list; + }); + } + + /** + * read a {@link Set} of elements by delegating to the callback, handles {@link #readSetBegin() + * begin} and {@link #readSetEnd() end} automatically + * + * @param callback callback for reading the set + * @param result set type + * @return the set read + * @throws TException when any sub-operation fails + */ + public final > T readSet(ReadCallback callback) throws TException { + TSet tSet = readSetBegin(); + T t = callback.accept(tSet); + readSetEnd(); + return t; + } + + /** + * read a {@link Set} of elements by delegating element reading to the callback, handles {@link + * #readSetBegin() begin} and {@link #readSetEnd() end} automatically + * + * @param callback callback for reading one element + * @param element type + * @return set of elements read + * @throws TException when any sub-operation fails + */ + public final Set readSet(ReadCollectionCallback callback) throws TException { + return readSet(callback, size -> new HashSet<>(2 * size)); + } + + /** + * read a {@link Set} of elements by delegating element reading to the callback, handles {@link + * #readSetBegin() begin} and {@link #readSetEnd() end} automatically, with a specialized set + * creator given the size hint. + * + * @param callback callback for reading one elment + * @param setCreator set creator given size hint + * @param element type + * @return set of elements read + * @throws TException when any sub-operation fails + */ + public final Set readSet( + ReadCollectionCallback callback, IntFunction> setCreator) throws TException { + return readSet( + tSet -> { + Set set = setCreator.apply(tSet.size); + for (int i = 0; i < tSet.size; i += 1) { + set.add(callback.call()); + } + return set; + }); + } + + /** + * Reset any internal state back to a blank slate. This method only needs to be implemented for + * stateful protocols. + */ + public void reset() {} + + /** Scheme accessor */ + public Class getScheme() { + return StandardScheme.class; + } + + // ----------------------------------------------------------------- + // Additional methods to improve performance. + + public int readFieldBeginData() throws TException { + // Derived classes should provide a more efficient version of this + // method if allowed by the encoding used by that protocol. + TField tfield = this.readFieldBegin(); + return TFieldData.encode(tfield.type, tfield.id); + } + + public void skip(byte fieldType) throws TException { + this.skip(fieldType, Integer.MAX_VALUE); + } + + public void skip(byte fieldType, int maxDepth) throws TException { + if (maxDepth <= 0) { + throw new TException("Maximum skip depth exceeded"); + } + + switch (fieldType) { + case TType.BOOL: + this.skipBool(); + break; + + case TType.BYTE: + this.skipByte(); + break; + + case TType.I16: + this.skipI16(); + break; + + case TType.I32: + this.skipI32(); + break; + + case TType.I64: + this.skipI64(); + break; + + case TType.DOUBLE: + this.skipDouble(); + break; + + case TType.STRING: + this.skipBinary(); + break; + + case TType.STRUCT: + this.readStructBegin(); + while (true) { + int tfieldData = this.readFieldBeginData(); + byte tfieldType = TFieldData.getType(tfieldData); + if (tfieldType == TType.STOP) { + break; + } + this.skip(tfieldType, maxDepth - 1); + this.readFieldEnd(); + } + this.readStructEnd(); + break; + + case TType.MAP: + TMap map = this.readMapBegin(); + for (int i = 0; i < map.size; i++) { + this.skip(map.keyType, maxDepth - 1); + this.skip(map.valueType, maxDepth - 1); + } + this.readMapEnd(); + break; + + case TType.SET: + TSet set = this.readSetBegin(); + for (int i = 0; i < set.size; i++) { + this.skip(set.elemType, maxDepth - 1); + } + this.readSetEnd(); + break; + + case TType.LIST: + TList list = this.readListBegin(); + for (int i = 0; i < list.size; i++) { + this.skip(list.elemType, maxDepth - 1); + } + this.readListEnd(); + break; + + default: + throw new TProtocolException( + TProtocolException.INVALID_DATA, "Unrecognized type " + fieldType); + } + } + + /** + * The default implementation of all skip() methods calls the corresponding read() method. + * Protocols that derive from this class are strongly encouraged to provide a more efficient + * alternative. + */ + protected void skipBool() throws TException { + this.readBool(); + } + + protected void skipByte() throws TException { + this.readByte(); + } + + protected void skipI16() throws TException { + this.readI16(); + } + + protected void skipI32() throws TException { + this.readI32(); + } + + protected void skipI64() throws TException { + this.readI64(); + } + + protected void skipDouble() throws TException { + this.readDouble(); + } + + protected void skipBinary() throws TException { + this.readBinary(); + } + + static final int MAX_SKIPPED_BYTES = 256; + protected byte[] skippedBytes = new byte[MAX_SKIPPED_BYTES]; + + protected void skipBytes(int numBytes) throws TException { + if (numBytes <= MAX_SKIPPED_BYTES) { + if (this.getTransport().getBytesRemainingInBuffer() >= numBytes) { + this.getTransport().consumeBuffer(numBytes); + } else { + this.getTransport().readAll(skippedBytes, 0, numBytes); + } + } else { + int remaining = numBytes; + while (remaining > 0) { + skipBytes(Math.min(remaining, MAX_SKIPPED_BYTES)); + remaining -= MAX_SKIPPED_BYTES; + } + } + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TProtocolDecorator.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TProtocolDecorator.java new file mode 100644 index 0000000..f84ce81 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TProtocolDecorator.java @@ -0,0 +1,320 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +import java.nio.ByteBuffer; +import java.util.UUID; +import org.apache.thrift.TException; + +/** + * TProtocolDecorator forwards all requests to an enclosed TProtocol + * instance, providing a way to author concise concrete decorator subclasses. While it has no + * abstract methods, it is marked abstract as a reminder that by itself, it does not modify the + * behaviour of the enclosed TProtocol. + * + *

See p.175 of Design Patterns (by Gamma et al.) + * + * @see org.apache.thrift.protocol.TMultiplexedProtocol + */ +public abstract class TProtocolDecorator extends TProtocol { + + private final TProtocol concreteProtocol; + + /** + * Encloses the specified protocol. + * + * @param protocol All operations will be forward to this protocol. Must be non-null. + */ + public TProtocolDecorator(TProtocol protocol) { + super(protocol.getTransport()); + concreteProtocol = protocol; + } + + @Override + public void writeMessageBegin(TMessage tMessage) throws TException { + concreteProtocol.writeMessageBegin(tMessage); + } + + @Override + public void writeMessageEnd() throws TException { + concreteProtocol.writeMessageEnd(); + } + + @Override + public void writeStructBegin(TStruct tStruct) throws TException { + concreteProtocol.writeStructBegin(tStruct); + } + + @Override + public UUID readUuid() throws TException { + return concreteProtocol.readUuid(); + } + + @Override + public void writeUuid(UUID uuid) throws TException { + concreteProtocol.writeUuid(uuid); + } + + @Override + public void writeStructEnd() throws TException { + concreteProtocol.writeStructEnd(); + } + + @Override + public void writeFieldBegin(TField tField) throws TException { + concreteProtocol.writeFieldBegin(tField); + } + + @Override + public void writeFieldEnd() throws TException { + concreteProtocol.writeFieldEnd(); + } + + @Override + public void writeFieldStop() throws TException { + concreteProtocol.writeFieldStop(); + } + + @Override + public void writeMapBegin(TMap tMap) throws TException { + concreteProtocol.writeMapBegin(tMap); + } + + @Override + public void writeMapEnd() throws TException { + concreteProtocol.writeMapEnd(); + } + + @Override + public void writeListBegin(TList tList) throws TException { + concreteProtocol.writeListBegin(tList); + } + + @Override + public void writeListEnd() throws TException { + concreteProtocol.writeListEnd(); + } + + @Override + public void writeSetBegin(TSet tSet) throws TException { + concreteProtocol.writeSetBegin(tSet); + } + + @Override + public void writeSetEnd() throws TException { + concreteProtocol.writeSetEnd(); + } + + @Override + public void writeBool(boolean b) throws TException { + concreteProtocol.writeBool(b); + } + + @Override + public void writeByte(byte b) throws TException { + concreteProtocol.writeByte(b); + } + + @Override + public void writeI16(short i) throws TException { + concreteProtocol.writeI16(i); + } + + @Override + public void writeI32(int i) throws TException { + concreteProtocol.writeI32(i); + } + + @Override + public void writeI64(long l) throws TException { + concreteProtocol.writeI64(l); + } + + @Override + public void writeDouble(double v) throws TException { + concreteProtocol.writeDouble(v); + } + + @Override + public void writeString(String s) throws TException { + concreteProtocol.writeString(s); + } + + @Override + public void writeBinary(ByteBuffer buf) throws TException { + concreteProtocol.writeBinary(buf); + } + + @Override + public TMessage readMessageBegin() throws TException { + return concreteProtocol.readMessageBegin(); + } + + @Override + public void readMessageEnd() throws TException { + concreteProtocol.readMessageEnd(); + } + + @Override + public TStruct readStructBegin() throws TException { + return concreteProtocol.readStructBegin(); + } + + @Override + public void readStructEnd() throws TException { + concreteProtocol.readStructEnd(); + } + + @Override + public TField readFieldBegin() throws TException { + return concreteProtocol.readFieldBegin(); + } + + @Override + public void readFieldEnd() throws TException { + concreteProtocol.readFieldEnd(); + } + + @Override + public TMap readMapBegin() throws TException { + return concreteProtocol.readMapBegin(); + } + + @Override + public void readMapEnd() throws TException { + concreteProtocol.readMapEnd(); + } + + @Override + public TList readListBegin() throws TException { + return concreteProtocol.readListBegin(); + } + + @Override + public void readListEnd() throws TException { + concreteProtocol.readListEnd(); + } + + @Override + public TSet readSetBegin() throws TException { + return concreteProtocol.readSetBegin(); + } + + @Override + public void readSetEnd() throws TException { + concreteProtocol.readSetEnd(); + } + + @Override + public boolean readBool() throws TException { + return concreteProtocol.readBool(); + } + + @Override + public byte readByte() throws TException { + return concreteProtocol.readByte(); + } + + @Override + public short readI16() throws TException { + return concreteProtocol.readI16(); + } + + @Override + public int readI32() throws TException { + return concreteProtocol.readI32(); + } + + @Override + public long readI64() throws TException { + return concreteProtocol.readI64(); + } + + @Override + public double readDouble() throws TException { + return concreteProtocol.readDouble(); + } + + @Override + public String readString() throws TException { + return concreteProtocol.readString(); + } + + @Override + public ByteBuffer readBinary() throws TException { + return concreteProtocol.readBinary(); + } + + @Override + public int readFieldBeginData() throws TException { + return concreteProtocol.readFieldBeginData(); + } + + @Override + protected void skipBool() throws TException { + concreteProtocol.skipBool(); + } + + @Override + protected void skipByte() throws TException { + concreteProtocol.skipByte(); + } + + @Override + protected void skipI16() throws TException { + concreteProtocol.skipI16(); + } + + @Override + protected void skipI32() throws TException { + concreteProtocol.skipI32(); + } + + @Override + protected void skipI64() throws TException { + concreteProtocol.skipI64(); + } + + @Override + protected void skipDouble() throws TException { + concreteProtocol.skipDouble(); + } + + @Override + protected void skipBinary() throws TException { + concreteProtocol.skipBinary(); + } + + /** + * @param type Returns the minimum amount of bytes needed to store the smallest possible instance + * of TType. + * @return size + * @throws TException if underlying protocol throws + */ + @Override + public int getMinSerializedSize(byte type) throws TException { + return concreteProtocol.getMinSerializedSize(type); + } + + @Override + public void reset() { + concreteProtocol.reset(); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TProtocolException.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TProtocolException.java new file mode 100644 index 0000000..3d645c4 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TProtocolException.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +import org.apache.thrift.TException; + +/** Protocol exceptions. */ +public class TProtocolException extends TException { + + private static final long serialVersionUID = 1L; + public static final int UNKNOWN = 0; + public static final int INVALID_DATA = 1; + public static final int NEGATIVE_SIZE = 2; + public static final int SIZE_LIMIT = 3; + public static final int BAD_VERSION = 4; + public static final int NOT_IMPLEMENTED = 5; + public static final int DEPTH_LIMIT = 6; + + protected int type_ = UNKNOWN; + + public TProtocolException() { + super(); + } + + public TProtocolException(int type) { + super(); + type_ = type; + } + + public TProtocolException(int type, String message) { + super(message); + type_ = type; + } + + public TProtocolException(String message) { + super(message); + } + + public TProtocolException(int type, Throwable cause) { + super(cause); + type_ = type; + } + + public TProtocolException(Throwable cause) { + super(cause); + } + + public TProtocolException(String message, Throwable cause) { + super(message, cause); + } + + public TProtocolException(int type, String message, Throwable cause) { + super(message, cause); + type_ = type; + } + + public int getType() { + return type_; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TProtocolFactory.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TProtocolFactory.java new file mode 100644 index 0000000..00b9fa6 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TProtocolFactory.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +import java.io.Serializable; +import org.apache.thrift.transport.TTransport; + +/** Factory interface for constructing protocol instances. */ +public interface TProtocolFactory extends Serializable { + TProtocol getProtocol(TTransport trans); +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TProtocolUtil.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TProtocolUtil.java new file mode 100644 index 0000000..7b44b10 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TProtocolUtil.java @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +import org.apache.thrift.TException; + +/** Utility class with static methods for interacting with protocol data streams. */ +public class TProtocolUtil { + + // no instantiation + private TProtocolUtil() {} + + /** The maximum recursive depth the skip() function will traverse before throwing a TException. */ + private static int maxSkipDepth = Integer.MAX_VALUE; + + /** + * Specifies the maximum recursive depth that the skip function will traverse before throwing a + * TException. This is a global setting, so any call to skip in this JVM will enforce this value. + * + * @param depth the maximum recursive depth. A value of 2 would allow the skip function to skip a + * structure or collection with basic children, but it would not permit skipping a struct that + * had a field containing a child struct. A value of 1 would only allow skipping of simple + * types and empty structs/collections. + */ + public static void setMaxSkipDepth(int depth) { + maxSkipDepth = depth; + } + + /** + * Skips over the next data element from the provided input TProtocol object. + * + * @param prot the protocol object to read from + * @param type the next value will be interpreted as this TType value. + */ + public static void skip(TProtocol prot, byte type) throws TException { + skip(prot, type, maxSkipDepth); + } + + /** + * Skips over the next data element from the provided input TProtocol object. + * + * @param prot the protocol object to read from + * @param type the next value will be interpreted as this TType value. + * @param maxDepth this function will only skip complex objects to this recursive depth, to + * prevent Java stack overflow. + */ + public static void skip(TProtocol prot, byte type, int maxDepth) throws TException { + if (maxDepth <= 0) { + throw new TException("Maximum skip depth exceeded"); + } + switch (type) { + case TType.BOOL: + prot.readBool(); + break; + + case TType.BYTE: + prot.readByte(); + break; + + case TType.I16: + prot.readI16(); + break; + + case TType.I32: + prot.readI32(); + break; + + case TType.I64: + prot.readI64(); + break; + + case TType.UUID: + prot.readUuid(); + break; + + case TType.DOUBLE: + prot.readDouble(); + break; + + case TType.STRING: + prot.readBinary(); + break; + + case TType.STRUCT: + prot.readStructBegin(); + while (true) { + TField field = prot.readFieldBegin(); + if (field.type == TType.STOP) { + break; + } + skip(prot, field.type, maxDepth - 1); + prot.readFieldEnd(); + } + prot.readStructEnd(); + break; + + case TType.MAP: + TMap map = prot.readMapBegin(); + for (int i = 0; i < map.size; i++) { + skip(prot, map.keyType, maxDepth - 1); + skip(prot, map.valueType, maxDepth - 1); + } + prot.readMapEnd(); + break; + + case TType.SET: + TSet set = prot.readSetBegin(); + for (int i = 0; i < set.size; i++) { + skip(prot, set.elemType, maxDepth - 1); + } + prot.readSetEnd(); + break; + + case TType.LIST: + TList list = prot.readListBegin(); + for (int i = 0; i < list.size; i++) { + skip(prot, list.elemType, maxDepth - 1); + } + prot.readListEnd(); + break; + + default: + throw new TProtocolException(TProtocolException.INVALID_DATA, "Unrecognized type " + type); + } + } + + /** + * Attempt to determine the protocol used to serialize some data. + * + *

The guess is based on known specificities of supported protocols. In some cases, no guess + * can be done, in that case we return the fallback TProtocolFactory. To be certain to correctly + * detect the protocol, the first encoded field should have a field id < 256 + * + * @param data The serialized data to guess the protocol for. + * @param fallback The TProtocol to return if no guess can be made. + * @return a Class implementing TProtocolFactory which can be used to create a deserializer. + */ + public static TProtocolFactory guessProtocolFactory(byte[] data, TProtocolFactory fallback) { + // + // If the first and last bytes are opening/closing curly braces we guess the protocol as + // being TJSONProtocol. + // It could not be a TCompactBinary encoding for a field of type 0xb (Map) + // with delta id 7 as the last byte for TCompactBinary is always 0. + // + + if ('{' == data[0] && '}' == data[data.length - 1]) { + return new TJSONProtocol.Factory(); + } + + // + // If the last byte is not 0, then it cannot be TCompactProtocol, it must be + // TBinaryProtocol. + // + + if (data[data.length - 1] != 0) { + return new TBinaryProtocol.Factory(); + } + + // + // A first byte of value > 16 indicates TCompactProtocol was used, and the first byte + // encodes a delta field id (id <= 15) and a field type. + // + + if (data[0] > 0x10) { + return new TCompactProtocol.Factory(); + } + + // + // If the second byte is 0 then it is a field id < 256 encoded by TBinaryProtocol. + // It cannot possibly be TCompactProtocol since a value of 0 would imply a field id + // of 0 as the zig zag varint encoding would end. + // + + if (data.length > 1 && 0 == data[1]) { + return new TBinaryProtocol.Factory(); + } + + // + // If bit 7 of the first byte of the field id is set then we have two choices: + // 1. A field id > 63 was encoded with TCompactProtocol. + // 2. A field id > 0x7fff (32767) was encoded with TBinaryProtocol and the last byte of the + // serialized data is 0. + // Option 2 is impossible since field ids are short and thus limited to 32767. + // + + if (data.length > 1 && (data[1] & 0x80) != 0) { + return new TCompactProtocol.Factory(); + } + + // + // The remaining case is either a field id <= 63 encoded as TCompactProtocol, + // one >= 256 encoded with TBinaryProtocol with a last byte at 0, or an empty structure. + // As we cannot really decide, we return the fallback protocol. + // + return fallback; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TReadProtocol.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TReadProtocol.java new file mode 100644 index 0000000..b15737a --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TReadProtocol.java @@ -0,0 +1,50 @@ +package org.apache.thrift.protocol; + +import java.nio.ByteBuffer; +import java.util.UUID; +import org.apache.thrift.TException; + +public interface TReadProtocol { + + TMessage readMessageBegin() throws TException; + + void readMessageEnd() throws TException; + + TStruct readStructBegin() throws TException; + + void readStructEnd() throws TException; + + TField readFieldBegin() throws TException; + + void readFieldEnd() throws TException; + + TMap readMapBegin() throws TException; + + void readMapEnd() throws TException; + + TList readListBegin() throws TException; + + void readListEnd() throws TException; + + TSet readSetBegin() throws TException; + + void readSetEnd() throws TException; + + boolean readBool() throws TException; + + byte readByte() throws TException; + + short readI16() throws TException; + + int readI32() throws TException; + + long readI64() throws TException; + + UUID readUuid() throws TException; + + double readDouble() throws TException; + + String readString() throws TException; + + ByteBuffer readBinary() throws TException; +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TSet.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TSet.java new file mode 100644 index 0000000..3443cac --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TSet.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +/** Helper class that encapsulates set metadata. */ +public final class TSet { + public TSet() { + this(TType.STOP, 0); + } + + public TSet(byte t, int s) { + elemType = t; + size = s; + } + + public TSet(TList list) { + this(list.elemType, list.size); + } + + public final byte elemType; + public final int size; + + public byte getElemType() { + return elemType; + } + + public int getSize() { + return size; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TSimpleJSONProtocol.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TSimpleJSONProtocol.java new file mode 100644 index 0000000..7b78a65 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TSimpleJSONProtocol.java @@ -0,0 +1,509 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Stack; +import java.util.UUID; +import org.apache.thrift.TException; +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; + +/** + * JSON protocol implementation for thrift. + * + *

This protocol is write-only and produces a simple output format suitable for parsing by + * scripting languages. It should not be confused with the full-featured TJSONProtocol. + */ +public class TSimpleJSONProtocol extends TProtocol { + + /** Factory */ + public static class Factory implements TProtocolFactory { + @Override + public TProtocol getProtocol(TTransport trans) { + return new TSimpleJSONProtocol(trans); + } + } + + private static final byte[] COMMA = new byte[] {','}; + private static final byte[] COLON = new byte[] {':'}; + private static final byte[] LBRACE = new byte[] {'{'}; + private static final byte[] RBRACE = new byte[] {'}'}; + private static final byte[] LBRACKET = new byte[] {'['}; + private static final byte[] RBRACKET = new byte[] {']'}; + private static final char QUOTE = '"'; + private static final String LIST = "list"; + private static final String SET = "set"; + private static final String MAP = "map"; + + protected static class Context { + protected void write() throws TException {} + + /** Returns whether the current value is a key in a map */ + protected boolean isMapKey() { + return false; + } + } + + protected class ListContext extends Context { + protected boolean first_ = true; + + protected void write() throws TException { + if (first_) { + first_ = false; + } else { + trans_.write(COMMA); + } + } + } + + protected class StructContext extends Context { + protected boolean first_ = true; + protected boolean colon_ = true; + + protected void write() throws TException { + if (first_) { + first_ = false; + colon_ = true; + } else { + trans_.write(colon_ ? COLON : COMMA); + colon_ = !colon_; + } + } + } + + protected class MapContext extends StructContext { + protected boolean isKey = true; + + @Override + protected void write() throws TException { + super.write(); + isKey = !isKey; + } + + protected boolean isMapKey() { + // we want to coerce map keys to json strings regardless + // of their type + return isKey; + } + } + + protected final Context BASE_CONTEXT = new Context(); + + /** Stack of nested contexts that we may be in. */ + protected Stack writeContextStack_ = new Stack(); + + /** Current context that we are in */ + protected Context writeContext_ = BASE_CONTEXT; + + /** Push a new write context onto the stack. */ + protected void pushWriteContext(Context c) { + writeContextStack_.push(writeContext_); + writeContext_ = c; + } + + /** Pop the last write context off the stack */ + protected void popWriteContext() { + writeContext_ = writeContextStack_.pop(); + } + + /** Reset the write context stack to its initial state. */ + protected void resetWriteContext() { + while (!writeContextStack_.isEmpty()) { + popWriteContext(); + } + } + + /** Used to make sure that we are not encountering a map whose keys are containers */ + protected void assertContextIsNotMapKey(String invalidKeyType) throws CollectionMapKeyException { + if (writeContext_.isMapKey()) { + throw new CollectionMapKeyException( + "Cannot serialize a map with keys that are of type " + invalidKeyType); + } + } + + /** Constructor */ + public TSimpleJSONProtocol(TTransport trans) { + super(trans); + } + + @Override + public void writeMessageBegin(TMessage message) throws TException { + resetWriteContext(); // THRIFT-3743 + trans_.write(LBRACKET); + pushWriteContext(new ListContext()); + writeString(message.name); + writeByte(message.type); + writeI32(message.seqid); + } + + @Override + public void writeMessageEnd() throws TException { + popWriteContext(); + trans_.write(RBRACKET); + } + + @Override + public void writeStructBegin(TStruct struct) throws TException { + writeContext_.write(); + trans_.write(LBRACE); + pushWriteContext(new StructContext()); + } + + @Override + public void writeStructEnd() throws TException { + popWriteContext(); + trans_.write(RBRACE); + } + + @Override + public void writeFieldBegin(TField field) throws TException { + // Note that extra type information is omitted in JSON! + writeString(field.name); + } + + @Override + public void writeFieldEnd() throws TException {} + + @Override + public void writeFieldStop() throws TException {} + + @Override + public void writeMapBegin(TMap map) throws TException { + assertContextIsNotMapKey(MAP); + writeContext_.write(); + trans_.write(LBRACE); + pushWriteContext(new MapContext()); + // No metadata! + } + + @Override + public void writeMapEnd() throws TException { + popWriteContext(); + trans_.write(RBRACE); + } + + @Override + public void writeListBegin(TList list) throws TException { + assertContextIsNotMapKey(LIST); + writeContext_.write(); + trans_.write(LBRACKET); + pushWriteContext(new ListContext()); + // No metadata! + } + + @Override + public void writeListEnd() throws TException { + popWriteContext(); + trans_.write(RBRACKET); + } + + @Override + public void writeSetBegin(TSet set) throws TException { + assertContextIsNotMapKey(SET); + writeContext_.write(); + trans_.write(LBRACKET); + pushWriteContext(new ListContext()); + // No metadata! + } + + @Override + public void writeSetEnd() throws TException { + popWriteContext(); + trans_.write(RBRACKET); + } + + @Override + public void writeBool(boolean b) throws TException { + writeByte(b ? (byte) 1 : (byte) 0); + } + + @Override + public void writeByte(byte b) throws TException { + writeI32(b); + } + + @Override + public void writeI16(short i16) throws TException { + writeI32(i16); + } + + @Override + public void writeI32(int i32) throws TException { + if (writeContext_.isMapKey()) { + writeString(Integer.toString(i32)); + } else { + writeContext_.write(); + _writeStringData(Integer.toString(i32)); + } + } + + public void _writeStringData(String s) throws TException { + byte[] b = s.getBytes(StandardCharsets.UTF_8); + trans_.write(b); + } + + @Override + public void writeI64(long i64) throws TException { + if (writeContext_.isMapKey()) { + writeString(Long.toString(i64)); + } else { + writeContext_.write(); + _writeStringData(Long.toString(i64)); + } + } + + @Override + public void writeUuid(UUID uuid) throws TException { + writeString(uuid.toString()); + } + + @Override + public void writeDouble(double dub) throws TException { + if (writeContext_.isMapKey()) { + writeString(Double.toString(dub)); + } else { + writeContext_.write(); + _writeStringData(Double.toString(dub)); + } + } + + @Override + public void writeString(String str) throws TException { + writeContext_.write(); + int length = str.length(); + StringBuilder escape = new StringBuilder(length + 16); + escape.append(QUOTE); + for (int i = 0; i < length; ++i) { + char c = str.charAt(i); + switch (c) { + case '"': + case '\\': + escape.append('\\'); + escape.append(c); + break; + case '\b': + escape.append('\\'); + escape.append('b'); + break; + case '\f': + escape.append('\\'); + escape.append('f'); + break; + case '\n': + escape.append('\\'); + escape.append('n'); + break; + case '\r': + escape.append('\\'); + escape.append('r'); + break; + case '\t': + escape.append('\\'); + escape.append('t'); + break; + default: + // Control characters! According to JSON RFC u0020 (space) + if (c < ' ') { + String hex = Integer.toHexString(c); + escape.append('\\'); + escape.append('u'); + for (int j = 4; j > hex.length(); --j) { + escape.append('0'); + } + escape.append(hex); + } else { + escape.append(c); + } + break; + } + } + escape.append(QUOTE); + _writeStringData(escape.toString()); + } + + @Override + public void writeBinary(ByteBuffer bin) throws TException { + // TODO(mcslee): Fix this + writeString( + new String( + bin.array(), + bin.position() + bin.arrayOffset(), + bin.limit() - bin.position() - bin.arrayOffset(), + StandardCharsets.UTF_8)); + } + + /** + * Reading methods. + * + *

simplejson is not meant to be read back into thrift - see ThriftUsageJava - use JSON instead + */ + @Override + public TMessage readMessageBegin() throws TException { + throw new TException("Not implemented"); + } + + @Override + public void readMessageEnd() throws TException { + throw new TException("Not implemented"); + } + + @Override + public TStruct readStructBegin() throws TException { + throw new TException("Not implemented"); + } + + @Override + public void readStructEnd() throws TException { + throw new TException("Not implemented"); + } + + @Override + public TField readFieldBegin() throws TException { + throw new TException("Not implemented"); + } + + @Override + public void readFieldEnd() throws TException { + throw new TException("Not implemented"); + } + + @Override + public TMap readMapBegin() throws TException { + throw new TException("Not implemented"); + } + + @Override + public void readMapEnd() throws TException { + throw new TException("Not implemented"); + } + + @Override + public TList readListBegin() throws TException { + throw new TException("Not implemented"); + } + + @Override + public void readListEnd() throws TException { + throw new TException("Not implemented"); + } + + @Override + public TSet readSetBegin() throws TException { + throw new TException("Not implemented"); + } + + @Override + public void readSetEnd() throws TException { + throw new TException("Not implemented"); + } + + @Override + public boolean readBool() throws TException { + throw new TException("Not implemented"); + } + + @Override + public byte readByte() throws TException { + throw new TException("Not implemented"); + } + + @Override + public short readI16() throws TException { + throw new TException("Not implemented"); + } + + @Override + public int readI32() throws TException { + throw new TException("Not implemented"); + } + + @Override + public long readI64() throws TException { + throw new TException("Not implemented"); + } + + @Override + public UUID readUuid() throws TException { + throw new TException("Not implemented"); + } + + @Override + public double readDouble() throws TException { + throw new TException("Not implemented"); + } + + @Override + public String readString() throws TException { + throw new TException("Not implemented"); + } + + public String readStringBody(int size) throws TException { + throw new TException("Not implemented"); + } + + @Override + public ByteBuffer readBinary() throws TException { + throw new TException("Not implemented"); + } + + public static class CollectionMapKeyException extends TException { + public CollectionMapKeyException(String message) { + super(message); + } + } + + /** Return the minimum number of bytes a type will consume on the wire */ + @Override + public int getMinSerializedSize(byte type) throws TException { + switch (type) { + case 0: + return 1; // Stop - T_STOP needs to count itself + case 1: + return 1; // Void - T_VOID needs to count itself + case 2: + return 1; // Bool + case 3: + return 1; // Byte + case 4: + return 1; // Double + case 6: + return 1; // I16 + case 8: + return 1; // I32 + case 10: + return 1; // I64 + case 11: + return 2; // string length + case 12: + return 2; // empty struct + case 13: + return 2; // element count Map + case 14: + return 2; // element count Set + case 15: + return 2; // element count List + default: + throw new TTransportException(TTransportException.UNKNOWN, "unrecognized type code"); + } + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TStruct.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TStruct.java new file mode 100644 index 0000000..a30fb5d --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TStruct.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +/** Helper class that encapsulates struct metadata. */ +public final class TStruct { + public TStruct() { + this(""); + } + + public TStruct(String n) { + name = n; + } + + public final String name; + + public String getName() { + return name; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TTupleProtocol.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TTupleProtocol.java new file mode 100644 index 0000000..19d05e4 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TTupleProtocol.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.thrift.protocol; + +import java.util.BitSet; +import org.apache.thrift.TException; +import org.apache.thrift.scheme.IScheme; +import org.apache.thrift.scheme.TupleScheme; +import org.apache.thrift.transport.TTransport; + +public final class TTupleProtocol extends TCompactProtocol { + public static class Factory implements TProtocolFactory { + public Factory() {} + + @Override + public TProtocol getProtocol(TTransport trans) { + return new TTupleProtocol(trans); + } + } + + public TTupleProtocol(TTransport transport) { + super(transport); + } + + @Override + public Class getScheme() { + return TupleScheme.class; + } + + public void writeBitSet(BitSet bs, int vectorWidth) throws TException { + byte[] bytes = toByteArray(bs, vectorWidth); + for (byte b : bytes) { + writeByte(b); + } + } + + public BitSet readBitSet(int i) throws TException { + int length = (int) Math.ceil(i / 8.0); + byte[] bytes = new byte[length]; + for (int j = 0; j < length; j++) { + bytes[j] = readByte(); + } + BitSet bs = fromByteArray(bytes); + return bs; + } + + /** Returns a bitset containing the values in bytes. The byte-ordering must be big-endian. */ + public static BitSet fromByteArray(byte[] bytes) { + BitSet bits = new BitSet(); + for (int i = 0; i < bytes.length * 8; i++) { + if ((bytes[bytes.length - i / 8 - 1] & (1 << (i % 8))) > 0) { + bits.set(i); + } + } + return bits; + } + + /** + * Returns a byte array of at least length 1. The most significant bit in the result is guaranteed + * not to be a 1 (since BitSet does not support sign extension). The byte-ordering of the result + * is big-endian which means the most significant bit is in element 0. The bit at index 0 of the + * bit set is assumed to be the least significant bit. + * + * @param bits bit set + * @param vectorWidth width of the vector + * @return a byte array of at least length 1 + */ + public static byte[] toByteArray(BitSet bits, int vectorWidth) { + byte[] bytes = new byte[(int) Math.ceil(vectorWidth / 8.0)]; + for (int i = 0; i < bits.length(); i++) { + if (bits.get(i)) { + bytes[bytes.length - i / 8 - 1] |= 1 << (i % 8); + } + } + return bytes; + } + + public TMap readMapBegin(byte keyType, byte valTyep) throws TException { + int size = super.readI32(); + TMap map = new TMap(keyType, valTyep, size); + + checkReadBytesAvailable(map); + return map; + } + + public TList readListBegin(byte type) throws TException { + int size = super.readI32(); + TList list = new TList(type, size); + + checkReadBytesAvailable(list); + return list; + } + + public TSet readSetBegin(byte type) throws TException { + return new TSet(readListBegin(type)); + } + + @Override + public void readMapEnd() throws TException {} + + @Override + public void readListEnd() throws TException {} + + @Override + public void readSetEnd() throws TException {} +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TType.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TType.java new file mode 100644 index 0000000..fbcdac3 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TType.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.protocol; + +/** Type constants in the Thrift protocol. */ +public final class TType { + public static final byte STOP = 0; + public static final byte VOID = 1; + public static final byte BOOL = 2; + public static final byte BYTE = 3; + public static final byte DOUBLE = 4; + public static final byte I16 = 6; + public static final byte I32 = 8; + public static final byte I64 = 10; + public static final byte STRING = 11; + public static final byte STRUCT = 12; + public static final byte MAP = 13; + public static final byte SET = 14; + public static final byte LIST = 15; + public static final byte UUID = 16; + + /** This is not part of the TBinaryProtocol spec but Java specific implementation detail */ + public static final byte ENUM = -1; +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TWriteProtocol.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TWriteProtocol.java new file mode 100644 index 0000000..e44ac8e --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TWriteProtocol.java @@ -0,0 +1,52 @@ +package org.apache.thrift.protocol; + +import java.nio.ByteBuffer; +import java.util.UUID; +import org.apache.thrift.TException; + +public interface TWriteProtocol { + + void writeMessageBegin(TMessage message) throws TException; + + void writeMessageEnd() throws TException; + + void writeStructBegin(TStruct struct) throws TException; + + void writeStructEnd() throws TException; + + void writeFieldBegin(TField field) throws TException; + + void writeFieldEnd() throws TException; + + void writeFieldStop() throws TException; + + void writeMapBegin(TMap map) throws TException; + + void writeMapEnd() throws TException; + + void writeListBegin(TList list) throws TException; + + void writeListEnd() throws TException; + + void writeSetBegin(TSet set) throws TException; + + void writeSetEnd() throws TException; + + void writeBool(boolean b) throws TException; + + void writeByte(byte b) throws TException; + + void writeI16(short i16) throws TException; + + void writeI32(int i32) throws TException; + + void writeI64(long i64) throws TException; + + void writeUuid(UUID uuid) throws TException; + + void writeDouble(double dub) throws TException; + + void writeString(String str) throws TException; + + void writeBinary(ByteBuffer buf) throws TException; +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/scheme/IScheme.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/scheme/IScheme.java new file mode 100644 index 0000000..ecf41fc --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/scheme/IScheme.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.thrift.scheme; + +import org.apache.thrift.TBase; + +public interface IScheme { + + void read(org.apache.thrift.protocol.TProtocol iproto, T struct) + throws org.apache.thrift.TException; + + void write(org.apache.thrift.protocol.TProtocol oproto, T struct) + throws org.apache.thrift.TException; +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/scheme/SchemeFactory.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/scheme/SchemeFactory.java new file mode 100644 index 0000000..94b7a11 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/scheme/SchemeFactory.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.thrift.scheme; + +public interface SchemeFactory { + + S getScheme(); +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/scheme/StandardScheme.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/scheme/StandardScheme.java new file mode 100644 index 0000000..2d2683b --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/scheme/StandardScheme.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.thrift.scheme; + +import org.apache.thrift.TBase; + +public abstract class StandardScheme implements IScheme {} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/scheme/TupleScheme.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/scheme/TupleScheme.java new file mode 100644 index 0000000..0efa4cf --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/scheme/TupleScheme.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.thrift.scheme; + +import org.apache.thrift.TBase; + +public abstract class TupleScheme implements IScheme {} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/AbstractNonblockingServer.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/AbstractNonblockingServer.java new file mode 100644 index 0000000..0d940a6 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/AbstractNonblockingServer.java @@ -0,0 +1,626 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.server; + +import java.io.IOException; +import java.net.SocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.SelectionKey; +import java.nio.channels.Selector; +import java.nio.channels.spi.SelectorProvider; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.thrift.TAsyncProcessor; +import org.apache.thrift.TByteArrayOutputStream; +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.transport.SocketAddressProvider; +import org.apache.thrift.transport.TIOStreamTransport; +import org.apache.thrift.transport.TMemoryInputTransport; +import org.apache.thrift.transport.TNonblockingServerTransport; +import org.apache.thrift.transport.TNonblockingTransport; +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; +import org.apache.thrift.transport.layered.TFramedTransport; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Provides common methods and classes used by nonblocking TServer implementations. */ +public abstract class AbstractNonblockingServer extends TServer { + protected final Logger LOGGER = LoggerFactory.getLogger(getClass().getName()); + + public abstract static class AbstractNonblockingServerArgs< + T extends AbstractNonblockingServerArgs> + extends AbstractServerArgs { + public long maxReadBufferBytes = 256 * 1024 * 1024; + + public AbstractNonblockingServerArgs(TNonblockingServerTransport transport) { + super(transport); + transportFactory(new TFramedTransport.Factory()); + } + } + + /** + * The maximum amount of memory we will allocate to client IO buffers at a time. Without this + * limit, the server will gladly allocate client buffers right into an out of memory exception, + * rather than waiting. + */ + final long MAX_READ_BUFFER_BYTES; + + /** How many bytes are currently allocated to read buffers. */ + final AtomicLong readBufferBytesAllocated = new AtomicLong(0); + + public AbstractNonblockingServer(AbstractNonblockingServerArgs args) { + super(args); + MAX_READ_BUFFER_BYTES = args.maxReadBufferBytes; + } + + /** Begin accepting connections and processing invocations. */ + public void serve() { + // start any IO threads + if (!startThreads()) { + return; + } + + // start listening, or exit + if (!startListening()) { + return; + } + + setServing(true); + + // this will block while we serve + waitForShutdown(); + + setServing(false); + + // do a little cleanup + stopListening(); + } + + /** + * Starts any threads required for serving. + * + * @return true if everything went ok, false if threads could not be started. + */ + protected abstract boolean startThreads(); + + /** A method that will block until when threads handling the serving have been shut down. */ + protected abstract void waitForShutdown(); + + /** + * Have the server transport start accepting connections. + * + * @return true if we started listening successfully, false if something went wrong. + */ + protected boolean startListening() { + try { + serverTransport_.listen(); + return true; + } catch (TTransportException ttx) { + LOGGER.error("Failed to start listening on server socket!", ttx); + return false; + } + } + + /** Stop listening for connections. */ + protected void stopListening() { + serverTransport_.close(); + } + + /** + * Perform an invocation. This method could behave several different ways - invoke immediately + * inline, queue for separate execution, etc. + * + * @return true if invocation was successfully requested, which is not a guarantee that invocation + * has completed. False if the request failed. + */ + protected abstract boolean requestInvoke(FrameBuffer frameBuffer); + + /** + * An abstract thread that handles selecting on a set of transports and {@link FrameBuffer + * FrameBuffers} associated with selected keys corresponding to requests. + */ + protected abstract class AbstractSelectThread extends Thread { + protected Selector selector; + + // List of FrameBuffers that want to change their selection interests. + protected final Set selectInterestChanges = new HashSet(); + + public AbstractSelectThread() throws IOException { + this.selector = SelectorProvider.provider().openSelector(); + } + + /** If the selector is blocked, wake it up. */ + public void wakeupSelector() { + selector.wakeup(); + } + + /** + * Add FrameBuffer to the list of select interest changes and wake up the selector if it's + * blocked. When the select() call exits, it'll give the FrameBuffer a chance to change its + * interests. + */ + public void requestSelectInterestChange(FrameBuffer frameBuffer) { + synchronized (selectInterestChanges) { + selectInterestChanges.add(frameBuffer); + } + // wakeup the selector, if it's currently blocked. + selector.wakeup(); + } + + /** + * Check to see if there are any FrameBuffers that have switched their interest type from read + * to write or vice versa. + */ + protected void processInterestChanges() { + synchronized (selectInterestChanges) { + for (FrameBuffer fb : selectInterestChanges) { + fb.changeSelectInterests(); + } + selectInterestChanges.clear(); + } + } + + /** + * Do the work required to read from a readable client. If the frame is fully read, then invoke + * the method call. + */ + protected void handleRead(SelectionKey key) { + FrameBuffer buffer = (FrameBuffer) key.attachment(); + if (!buffer.read()) { + cleanupSelectionKey(key); + return; + } + + // if the buffer's frame read is complete, invoke the method. + if (buffer.isFrameFullyRead() && !requestInvoke(buffer)) { + cleanupSelectionKey(key); + } + } + + /** Let a writable client get written, if there's data to be written. */ + protected void handleWrite(SelectionKey key) { + FrameBuffer buffer = (FrameBuffer) key.attachment(); + if (!buffer.write()) { + cleanupSelectionKey(key); + } + } + + /** Do connection-close cleanup on a given SelectionKey. */ + protected void cleanupSelectionKey(SelectionKey key) { + // remove the records from the two maps + FrameBuffer buffer = (FrameBuffer) key.attachment(); + if (buffer != null) { + // close the buffer + buffer.close(); + } + // cancel the selection key + key.cancel(); + } + } // SelectThread + + /** Possible states for the FrameBuffer state machine. */ + private enum FrameBufferState { + // in the midst of reading the frame size off the wire + READING_FRAME_SIZE, + // reading the actual frame data now, but not all the way done yet + READING_FRAME, + // completely read the frame, so an invocation can now happen + READ_FRAME_COMPLETE, + // waiting to get switched to listening for write events + AWAITING_REGISTER_WRITE, + // started writing response data, not fully complete yet + WRITING, + // another thread wants this framebuffer to go back to reading + AWAITING_REGISTER_READ, + // we want our transport and selection key invalidated in the selector + // thread + AWAITING_CLOSE + } + + /** + * Class that implements a sort of state machine around the interaction with a client and an + * invoker. It manages reading the frame size and frame data, getting it handed off as wrapped + * transports, and then the writing of response data back to the client. In the process it manages + * flipping the read and write bits on the selection key for its client. + */ + public class FrameBuffer { + private final Logger LOGGER = LoggerFactory.getLogger(getClass().getName()); + + // the actual transport hooked up to the client. + protected final TNonblockingTransport trans_; + + // the SelectionKey that corresponds to our transport + protected SelectionKey selectionKey_; + + // the SelectThread that owns the registration of our transport + protected final AbstractSelectThread selectThread_; + + // where in the process of reading/writing are we? + protected FrameBufferState state_ = FrameBufferState.READING_FRAME_SIZE; + + // the ByteBuffer we'll be using to write and read, depending on the state + protected ByteBuffer buffer_; + + protected final TByteArrayOutputStream response_; + + // the frame that the TTransport should wrap. + protected final TMemoryInputTransport frameTrans_; + + // the transport that should be used to connect to clients + protected final TTransport inTrans_; + + protected final TTransport outTrans_; + + // the input protocol to use on frames + protected final TProtocol inProt_; + + // the output protocol to use on frames + protected final TProtocol outProt_; + + // context associated with this connection + protected final ServerContext context_; + + public FrameBuffer( + final TNonblockingTransport trans, + final SelectionKey selectionKey, + final AbstractSelectThread selectThread) + throws TTransportException { + trans_ = trans; + selectionKey_ = selectionKey; + selectThread_ = selectThread; + buffer_ = ByteBuffer.allocate(4); + + frameTrans_ = new TMemoryInputTransport(trans_.getConfiguration()); + response_ = new TByteArrayOutputStream(); + inTrans_ = inputTransportFactory_.getTransport(frameTrans_); + outTrans_ = + outputTransportFactory_.getTransport( + new TIOStreamTransport(trans_.getConfiguration(), response_)); + inProt_ = inputProtocolFactory_.getProtocol(inTrans_); + outProt_ = outputProtocolFactory_.getProtocol(outTrans_); + + if (eventHandler_ != null) { + context_ = eventHandler_.createContext(inProt_, outProt_); + SocketAddress remoteAddress = + trans_ instanceof SocketAddressProvider + ? ((SocketAddressProvider) trans_).getRemoteSocketAddress() + : null; + context_.setRemoteAddress(remoteAddress); + } else { + context_ = null; + } + } + + /** + * Sets the selection key (this is not thread safe). + * + * @param selectionKey the new key to set. + */ + public void setSelectionKey(SelectionKey selectionKey) { + selectionKey_ = selectionKey; + } + + /** + * @return the amount of memory currently used to read data from clients. This information can + * be useful for debugging, metrics, and configuring the maximum memory limit. + */ + public final long getReadBufferBytesAllocated() { + return readBufferBytesAllocated.get(); + } + + /** + * Give this FrameBuffer a chance to read. The selector loop should have received a read event + * for this FrameBuffer. + * + * @return true if the connection should live on, false if it should be closed + */ + public boolean read() { + if (state_ == FrameBufferState.READING_FRAME_SIZE) { + // try to read the frame size completely + if (!internalRead()) { + return false; + } + + // if the frame size has been read completely, then prepare to read the + // actual frame. + if (buffer_.remaining() == 0) { + // pull out the frame size as an integer. + int frameSize = buffer_.getInt(0); + if (frameSize <= 0) { + LOGGER.error( + "Read an invalid frame size of " + + frameSize + + ". Are you using TFramedTransport on the client side?"); + return false; + } + + // if this frame will always be too large for this server, log the + // error and close the connection. + if (frameSize > trans_.getMaxFrameSize()) { + LOGGER.error( + "Read a frame size of " + + frameSize + + ", which is bigger than the maximum allowable frame size " + + trans_.getMaxFrameSize() + + " for ALL connections."); + return false; + } + + // if this frame will push us over the memory limit, then return. + // with luck, more memory will free up the next time around. + long currentAllocated = getReadBufferBytesAllocated(); + if (currentAllocated + frameSize > MAX_READ_BUFFER_BYTES) { + LOGGER.trace( + "Deferring reading frame of size {} because {} is already buffered and {} is the limit.", + frameSize, + currentAllocated, + MAX_READ_BUFFER_BYTES); + return true; + } + + // increment the amount of memory allocated to read buffers + readBufferBytesAllocated.addAndGet(frameSize + 4); + + // reallocate the readbuffer as a frame-sized buffer + buffer_ = ByteBuffer.allocate(frameSize + 4); + buffer_.putInt(frameSize); + + state_ = FrameBufferState.READING_FRAME; + } else { + // this skips the check of READING_FRAME state below, since we can't + // possibly go on to that state if there's data left to be read at + // this one. + return true; + } + } + + // it is possible to fall through from the READING_FRAME_SIZE section + // to READING_FRAME if there's already some frame data available once + // READING_FRAME_SIZE is complete. + + if (state_ == FrameBufferState.READING_FRAME) { + if (!internalRead()) { + return false; + } + + // since we're already in the select loop here for sure, we can just + // modify our selection key directly. + if (buffer_.remaining() == 0) { + // get rid of the read select interests + if (selectionKey_.isValid()) { + selectionKey_.interestOps(0); + } else { + LOGGER.warn("SelectionKey was invalidated during read"); + } + state_ = FrameBufferState.READ_FRAME_COMPLETE; + } + + return true; + } + + // if we fall through to this point, then the state must be invalid. + LOGGER.error("Read was called but state is invalid (" + state_ + ")"); + return false; + } + + /** Give this FrameBuffer a chance to write its output to the final client. */ + public boolean write() { + if (state_ == FrameBufferState.WRITING) { + try { + if (trans_.write(buffer_) < 0) { + return false; + } + } catch (TTransportException e) { + LOGGER.warn("Got an Exception during write", e); + return false; + } + + // we're done writing. now we need to switch back to reading. + if (buffer_.remaining() == 0) { + prepareRead(); + } + return true; + } + + LOGGER.error("Write was called, but state is invalid (" + state_ + ")"); + return false; + } + + /** Give this FrameBuffer a chance to set its interest to write, once data has come in. */ + public void changeSelectInterests() { + switch (state_) { + case AWAITING_REGISTER_WRITE: + // set the OP_WRITE interest + if (selectionKey_.isValid()) { + selectionKey_.interestOps(SelectionKey.OP_WRITE); + state_ = FrameBufferState.WRITING; + } else { + LOGGER.warn("SelectionKey was invalidated before write"); + } + break; + case AWAITING_REGISTER_READ: + prepareRead(); + break; + case AWAITING_CLOSE: + close(); + selectionKey_.cancel(); + break; + default: + LOGGER.error("changeSelectInterest was called, but state is invalid ({})", state_); + } + } + + /** Shut the connection down. */ + public void close() { + // if we're being closed due to an error, we might have allocated a + // buffer that we need to subtract for our memory accounting. + if (state_ == FrameBufferState.READING_FRAME + || state_ == FrameBufferState.READ_FRAME_COMPLETE + || state_ == FrameBufferState.AWAITING_CLOSE) { + readBufferBytesAllocated.addAndGet(-buffer_.array().length); + } + try { + if (eventHandler_ != null) { + eventHandler_.deleteContext(context_, inProt_, outProt_); + } + } finally { + trans_.close(); + } + } + + /** Check if this FrameBuffer has a full frame read. */ + public boolean isFrameFullyRead() { + return state_ == FrameBufferState.READ_FRAME_COMPLETE; + } + + /** + * After the processor has processed the invocation, whatever thread is managing invocations + * should call this method on this FrameBuffer so we know it's time to start trying to write + * again. Also, if it turns out that there actually isn't any data in the response buffer, we'll + * skip trying to write and instead go back to reading. + */ + public void responseReady() { + // the read buffer is definitely no longer in use, so we will decrement + // our read buffer count. we do this here as well as in close because + // we'd like to free this read memory up as quickly as possible for other + // clients. + readBufferBytesAllocated.addAndGet(-buffer_.array().length); + + if (response_.len() == 0) { + // go straight to reading again. this was probably an oneway method + state_ = FrameBufferState.AWAITING_REGISTER_READ; + buffer_ = null; + } else { + buffer_ = ByteBuffer.wrap(response_.get(), 0, response_.len()); + + // set state that we're waiting to be switched to write. we do this + // asynchronously through requestSelectInterestChange() because there is + // a possibility that we're not in the main thread, and thus currently + // blocked in select(). (this functionality is in place for the sake of + // the HsHa server.) + state_ = FrameBufferState.AWAITING_REGISTER_WRITE; + } + requestSelectInterestChange(); + } + + /** Actually invoke the method signified by this FrameBuffer. */ + public void invoke() { + frameTrans_.reset(buffer_.array()); + response_.reset(); + + try { + if (eventHandler_ != null) { + eventHandler_.processContext(context_, inTrans_, outTrans_); + } + processorFactory_.getProcessor(inTrans_).process(inProt_, outProt_); + responseReady(); + return; + } catch (TException te) { + LOGGER.warn("Exception while invoking!", te); + } catch (Throwable t) { + LOGGER.error("Unexpected throwable while invoking!", t); + } + // This will only be reached when there is a throwable. + state_ = FrameBufferState.AWAITING_CLOSE; + requestSelectInterestChange(); + } + + /** + * Perform a read into buffer. + * + * @return true if the read succeeded, false if there was an error or the connection closed. + */ + private boolean internalRead() { + try { + return trans_.read(buffer_) >= 0; + } catch (TTransportException e) { + LOGGER.warn("Got an Exception in internalRead", e); + return false; + } + } + + /** We're done writing, so reset our interest ops and change state accordingly. */ + private void prepareRead() { + // we can set our interest directly without using the queue because + // we're in the select thread. + if (selectionKey_.isValid()) { + selectionKey_.interestOps(SelectionKey.OP_READ); + } else { + LOGGER.warn("SelectionKey was invalidated before read"); + } + // get ready for another go-around + buffer_ = ByteBuffer.allocate(4); + state_ = FrameBufferState.READING_FRAME_SIZE; + } + + /** + * When this FrameBuffer needs to change its select interests and execution might not be in its + * select thread, then this method will make sure the interest change gets done when the select + * thread wakes back up. When the current thread is this FrameBuffer's select thread, then it + * just does the interest change immediately. + */ + protected void requestSelectInterestChange() { + if (Thread.currentThread() == this.selectThread_) { + changeSelectInterests(); + } else { + this.selectThread_.requestSelectInterestChange(this); + } + } + } // FrameBuffer + + public class AsyncFrameBuffer extends FrameBuffer { + public AsyncFrameBuffer( + TNonblockingTransport trans, SelectionKey selectionKey, AbstractSelectThread selectThread) + throws TTransportException { + super(trans, selectionKey, selectThread); + } + + public TProtocol getInputProtocol() { + return inProt_; + } + + public TProtocol getOutputProtocol() { + return outProt_; + } + + public void invoke() { + frameTrans_.reset(buffer_.array()); + response_.reset(); + + try { + if (eventHandler_ != null) { + eventHandler_.processContext(context_, inTrans_, outTrans_); + } + ((TAsyncProcessor) processorFactory_.getProcessor(inTrans_)).process(this); + return; + } catch (TException te) { + LOGGER.warn("Exception while invoking!", te); + } catch (Throwable t) { + LOGGER.error("Unexpected throwable while invoking!", t); + } + // This will only be reached when there is a throwable. + state_ = FrameBufferState.AWAITING_CLOSE; + requestSelectInterestChange(); + } + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/Invocation.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/Invocation.java new file mode 100644 index 0000000..423b40b --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/Invocation.java @@ -0,0 +1,21 @@ +package org.apache.thrift.server; + +import org.apache.thrift.server.AbstractNonblockingServer.FrameBuffer; + +/** + * An Invocation represents a method call that is prepared to execute, given an idle worker thread. + * It contains the input and output protocols the thread's processor should use to perform the usual + * Thrift invocation. + */ +class Invocation implements Runnable { + private final FrameBuffer frameBuffer; + + public Invocation(final FrameBuffer frameBuffer) { + this.frameBuffer = frameBuffer; + } + + @Override + public void run() { + frameBuffer.invoke(); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/ServerContext.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/ServerContext.java new file mode 100644 index 0000000..8cc95aa --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/ServerContext.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/** Interface for storing server's connection context. */ +package org.apache.thrift.server; + +import java.net.SocketAddress; + +public interface ServerContext { + + /** + * Returns an object that implements the given interface to allow access to application specific + * contexts. + * + * @param iface A Class defining an interface that the result must implement + * @return an object that implements the interface + * @throws RuntimeException If the context cannot be unwrapped to the provided class + */ + T unwrap(Class iface); + + /** + * Returns true if this server context is a wrapper for the provided application specific context + * interface argument or returns false otherwise. + * + * @param iface a Class defining the underlying context + * @return true if this implements the interface can be unwrapped to the provided class + * @throws RuntimeException if an error occurs while determining whether the provided class can be + * unwrapped from this context. + */ + boolean isWrapperFor(Class iface); + + /** + * Set the remote socket address for this ServerContext. The remoteAddress is null when transport + * is not socket based + * + * @param remoteAddress The remote socket address, may be null. + */ + default void setRemoteAddress(SocketAddress remoteAddress) {} +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TExtensibleServlet.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TExtensibleServlet.java new file mode 100644 index 0000000..8e7e3f3 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TExtensibleServlet.java @@ -0,0 +1,168 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.server; + +import javax.servlet.ServletConfig; +import javax.servlet.ServletContext; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Map; +import org.apache.thrift.TException; +import org.apache.thrift.TProcessor; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.protocol.TProtocolFactory; +import org.apache.thrift.transport.TIOStreamTransport; +import org.apache.thrift.transport.TTransport; + +/** + * Servlet implementation class ThriftServer, that allows {@link TProcessor} and {@link + * TProtocolFactory} to be supplied after the {@link #init()} method has finished.
+ * Subclasses must implement the abstract methods that return the TProcessor and two + * TProtocolFactory. Those methods are guaranteed to be called exactly once, and that {@link + * ServletContext} is available. + */ +public abstract class TExtensibleServlet extends HttpServlet { + private static final long serialVersionUID = 1L; + + private TProcessor processor; + + private TProtocolFactory inFactory; + + private Collection> customHeaders; + + /** + * Returns the appropriate {@link TProcessor}. This will be called once just after the + * {@link #init()} method + * + * @return the appropriate {@link TProcessor} + */ + protected abstract TProcessor getProcessor(); + + /** + * Returns the appropriate in {@link TProtocolFactory}. This will be called once just after + * the {@link #init()} method + * + * @return the appropriate in {@link TProtocolFactory} + */ + protected abstract TProtocolFactory getInProtocolFactory(); + + /** + * Returns the appropriate out {@link TProtocolFactory}. This will be called once just + * after the {@link #init()} method + * + * @return the appropriate out {@link TProtocolFactory} + */ + protected abstract TProtocolFactory getOutProtocolFactory(); + + @Override + public final void init(ServletConfig config) throws ServletException { + super.init(config); // no-args init() happens here + this.processor = getProcessor(); + this.inFactory = getInProtocolFactory(); + TProtocolFactory outFactory = getOutProtocolFactory(); + this.customHeaders = new ArrayList>(); + + if (processor == null) { + throw new ServletException("processor must be set"); + } + if (inFactory == null) { + throw new ServletException("inFactory must be set"); + } + if (outFactory == null) { + throw new ServletException("outFactory must be set"); + } + } + + /** + * @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response) + */ + @Override + protected void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + TTransport inTransport; + TTransport outTransport; + + try { + response.setContentType("application/x-thrift"); + + if (null != this.customHeaders) { + for (Map.Entry header : this.customHeaders) { + response.addHeader(header.getKey(), header.getValue()); + } + } + + InputStream in = request.getInputStream(); + OutputStream out = response.getOutputStream(); + + TTransport transport = new TIOStreamTransport(in, out); + inTransport = transport; + outTransport = transport; + + TProtocol inProtocol = inFactory.getProtocol(inTransport); + TProtocol outProtocol = inFactory.getProtocol(outTransport); + + processor.process(inProtocol, outProtocol); + out.flush(); + } catch (TException te) { + throw new ServletException(te); + } + } + + /** + * @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response) + */ + @Override + protected void doGet(HttpServletRequest req, HttpServletResponse resp) + throws ServletException, IOException { + doPost(req, resp); + } + + public void addCustomHeader(final String key, final String value) { + this.customHeaders.add( + new Map.Entry() { + @Override + public String getKey() { + return key; + } + + @Override + public String getValue() { + return value; + } + + @Override + public String setValue(String value) { + return null; + } + }); + } + + public void setCustomHeaders(Collection> headers) { + this.customHeaders.clear(); + this.customHeaders.addAll(headers); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/THsHaServer.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/THsHaServer.java new file mode 100644 index 0000000..6e59638 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/THsHaServer.java @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.server; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import org.apache.thrift.transport.TNonblockingServerTransport; + +/** + * An extension of the TNonblockingServer to a Half-Sync/Half-Async server. Like TNonblockingServer, + * it relies on the use of TFramedTransport. + */ +public class THsHaServer extends TNonblockingServer { + + public static class Args extends AbstractNonblockingServerArgs { + public int minWorkerThreads = 5; + public int maxWorkerThreads = Integer.MAX_VALUE; + private int stopTimeoutVal = 60; + private TimeUnit stopTimeoutUnit = TimeUnit.SECONDS; + private ExecutorService executorService = null; + + public Args(TNonblockingServerTransport transport) { + super(transport); + } + + /** + * Sets the min and max threads. + * + * @deprecated use {@link #minWorkerThreads(int)} and {@link #maxWorkerThreads(int)} instead. + */ + @Deprecated + public Args workerThreads(int n) { + minWorkerThreads = n; + maxWorkerThreads = n; + return this; + } + + /** + * @return what the min threads was set to. + * @deprecated use {@link #getMinWorkerThreads()} and {@link #getMaxWorkerThreads()} instead. + */ + @Deprecated + public int getWorkerThreads() { + return minWorkerThreads; + } + + public Args minWorkerThreads(int n) { + minWorkerThreads = n; + return this; + } + + public Args maxWorkerThreads(int n) { + maxWorkerThreads = n; + return this; + } + + public int getMinWorkerThreads() { + return minWorkerThreads; + } + + public int getMaxWorkerThreads() { + return maxWorkerThreads; + } + + public int getStopTimeoutVal() { + return stopTimeoutVal; + } + + public Args stopTimeoutVal(int stopTimeoutVal) { + this.stopTimeoutVal = stopTimeoutVal; + return this; + } + + public TimeUnit getStopTimeoutUnit() { + return stopTimeoutUnit; + } + + public Args stopTimeoutUnit(TimeUnit stopTimeoutUnit) { + this.stopTimeoutUnit = stopTimeoutUnit; + return this; + } + + public ExecutorService getExecutorService() { + return executorService; + } + + public Args executorService(ExecutorService executorService) { + this.executorService = executorService; + return this; + } + } + + // This wraps all the functionality of queueing and thread pool management + // for the passing of Invocations from the Selector to workers. + private final ExecutorService invoker; + + private final Args args; + + /** Create the server with the specified Args configuration */ + public THsHaServer(Args args) { + super(args); + + invoker = args.executorService == null ? createInvokerPool(args) : args.executorService; + this.args = args; + } + + /** {@inheritDoc} */ + @Override + protected void waitForShutdown() { + joinSelector(); + gracefullyShutdownInvokerPool(); + } + + /** Helper to create an invoker pool */ + protected static ExecutorService createInvokerPool(Args options) { + int minWorkerThreads = options.minWorkerThreads; + int maxWorkerThreads = options.maxWorkerThreads; + int stopTimeoutVal = options.stopTimeoutVal; + TimeUnit stopTimeoutUnit = options.stopTimeoutUnit; + + LinkedBlockingQueue queue = new LinkedBlockingQueue(); + ExecutorService invoker = + new ThreadPoolExecutor( + minWorkerThreads, maxWorkerThreads, stopTimeoutVal, stopTimeoutUnit, queue); + + return invoker; + } + + protected ExecutorService getInvoker() { + return invoker; + } + + protected void gracefullyShutdownInvokerPool() { + // try to gracefully shut down the executor service + invoker.shutdown(); + + // Loop until awaitTermination finally does return without a interrupted + // exception. If we don't do this, then we'll shut down prematurely. We want + // to let the executorService clear it's task queue, closing client sockets + // appropriately. + long timeoutMS = args.stopTimeoutUnit.toMillis(args.stopTimeoutVal); + long now = System.currentTimeMillis(); + while (timeoutMS >= 0) { + try { + invoker.awaitTermination(timeoutMS, TimeUnit.MILLISECONDS); + break; + } catch (InterruptedException ix) { + long newnow = System.currentTimeMillis(); + timeoutMS -= (newnow - now); + now = newnow; + } + } + } + + /** + * We override the standard invoke method here to queue the invocation for invoker service instead + * of immediately invoking. The thread pool takes care of the rest. + */ + @Override + protected boolean requestInvoke(FrameBuffer frameBuffer) { + try { + Runnable invocation = getRunnable(frameBuffer); + invoker.execute(invocation); + return true; + } catch (RejectedExecutionException rx) { + LOGGER.warn("ExecutorService rejected execution!", rx); + return false; + } + } + + protected Runnable getRunnable(FrameBuffer frameBuffer) { + return new Invocation(frameBuffer); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TNonblockingServer.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TNonblockingServer.java new file mode 100644 index 0000000..5c015eb --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TNonblockingServer.java @@ -0,0 +1,234 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.server; + +import java.io.IOException; +import java.nio.channels.SelectionKey; +import java.util.Iterator; +import org.apache.thrift.transport.TNonblockingServerTransport; +import org.apache.thrift.transport.TNonblockingTransport; +import org.apache.thrift.transport.TTransportException; + +/** + * A nonblocking TServer implementation. This allows for fairness amongst all connected clients in + * terms of invocations. + * + *

This server is inherently single-threaded. If you want a limited thread pool coupled with + * invocation-fairness, see THsHaServer. + * + *

To use this server, you MUST use a TFramedTransport at the outermost transport, otherwise this + * server will be unable to determine when a whole method call has been read off the wire. Clients + * must also use TFramedTransport. + */ +public class TNonblockingServer extends AbstractNonblockingServer { + + public static class Args extends AbstractNonblockingServerArgs { + public Args(TNonblockingServerTransport transport) { + super(transport); + } + } + + private SelectAcceptThread selectAcceptThread_; + + public TNonblockingServer(AbstractNonblockingServerArgs args) { + super(args); + } + + /** + * Start the selector thread to deal with accepts and client messages. + * + * @return true if everything went ok, false if we couldn't start for some reason. + */ + @Override + protected boolean startThreads() { + // start the selector + try { + selectAcceptThread_ = new SelectAcceptThread((TNonblockingServerTransport) serverTransport_); + selectAcceptThread_.start(); + return true; + } catch (IOException e) { + LOGGER.error("Failed to start selector thread!", e); + return false; + } + } + + @Override + protected void waitForShutdown() { + joinSelector(); + } + + /** Block until the selector thread exits. */ + protected void joinSelector() { + // wait until the selector thread exits + try { + selectAcceptThread_.join(); + } catch (InterruptedException e) { + LOGGER.debug("Interrupted while waiting for accept thread", e); + Thread.currentThread().interrupt(); + } + } + + /** Stop serving and shut everything down. */ + @Override + public void stop() { + stopped_ = true; + if (selectAcceptThread_ != null) { + selectAcceptThread_.wakeupSelector(); + } + } + + /** + * Perform an invocation. This method could behave several different ways - invoke immediately + * inline, queue for separate execution, etc. + */ + @Override + protected boolean requestInvoke(FrameBuffer frameBuffer) { + frameBuffer.invoke(); + return true; + } + + public boolean isStopped() { + return selectAcceptThread_.isStopped(); + } + + /** + * The thread that will be doing all the selecting, managing new connections and those that still + * need to be read. + */ + protected class SelectAcceptThread extends AbstractSelectThread { + + // The server transport on which new client transports will be accepted + private final TNonblockingServerTransport serverTransport; + + /** Set up the thread that will handle the non-blocking accepts, reads, and writes. */ + public SelectAcceptThread(final TNonblockingServerTransport serverTransport) + throws IOException { + this.serverTransport = serverTransport; + serverTransport.registerSelector(selector); + } + + public boolean isStopped() { + return stopped_; + } + + /** + * The work loop. Handles both selecting (all IO operations) and managing the selection + * preferences of all existing connections. + */ + @Override + public void run() { + try { + if (eventHandler_ != null) { + eventHandler_.preServe(); + } + + while (!stopped_) { + select(); + processInterestChanges(); + } + for (SelectionKey selectionKey : selector.keys()) { + cleanupSelectionKey(selectionKey); + } + } catch (Throwable t) { + LOGGER.error("run() exiting due to uncaught error", t); + } finally { + try { + selector.close(); + } catch (IOException e) { + LOGGER.error("Got an IOException while closing selector!", e); + } + stopped_ = true; + } + } + + /** + * Select and process IO events appropriately: If there are connections to be accepted, accept + * them. If there are existing connections with data waiting to be read, read it, buffering + * until a whole frame has been read. If there are any pending responses, buffer them until + * their target client is available, and then send the data. + */ + private void select() { + try { + // wait for io events. + selector.select(); + + // process the io events we received + Iterator selectedKeys = selector.selectedKeys().iterator(); + while (!stopped_ && selectedKeys.hasNext()) { + SelectionKey key = selectedKeys.next(); + selectedKeys.remove(); + + // skip if not valid + if (!key.isValid()) { + cleanupSelectionKey(key); + continue; + } + + // if the key is marked Accept, then it has to be the server + // transport. + if (key.isAcceptable()) { + handleAccept(); + } else if (key.isReadable()) { + // deal with reads + handleRead(key); + } else if (key.isWritable()) { + // deal with writes + handleWrite(key); + } else { + LOGGER.warn("Unexpected state in select! " + key.interestOps()); + } + } + } catch (IOException e) { + LOGGER.warn("Got an IOException while selecting!", e); + } + } + + protected FrameBuffer createFrameBuffer( + final TNonblockingTransport trans, + final SelectionKey selectionKey, + final AbstractSelectThread selectThread) + throws TTransportException { + return processorFactory_.isAsyncProcessor() + ? new AsyncFrameBuffer(trans, selectionKey, selectThread) + : new FrameBuffer(trans, selectionKey, selectThread); + } + + /** Accept a new connection. */ + private void handleAccept() throws IOException { + SelectionKey clientKey = null; + TNonblockingTransport client = null; + try { + // accept the connection + client = serverTransport.accept(); + clientKey = client.registerSelector(selector, SelectionKey.OP_READ); + + // add this key to the map + FrameBuffer frameBuffer = createFrameBuffer(client, clientKey, SelectAcceptThread.this); + + clientKey.attach(frameBuffer); + } catch (TTransportException tte) { + // something went wrong accepting. + LOGGER.warn("Exception trying to accept!", tte); + if (clientKey != null) cleanupSelectionKey(clientKey); + if (client != null) client.close(); + } + } + } // SelectAcceptThread +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TSaslNonblockingServer.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TSaslNonblockingServer.java new file mode 100644 index 0000000..8c899d5 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TSaslNonblockingServer.java @@ -0,0 +1,488 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.server; + +import java.io.IOException; +import java.nio.channels.SelectionKey; +import java.nio.channels.Selector; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import javax.security.auth.callback.CallbackHandler; +import org.apache.thrift.TProcessor; +import org.apache.thrift.transport.TNonblockingServerSocket; +import org.apache.thrift.transport.TNonblockingServerTransport; +import org.apache.thrift.transport.TNonblockingTransport; +import org.apache.thrift.transport.TTransportException; +import org.apache.thrift.transport.sasl.NonblockingSaslHandler; +import org.apache.thrift.transport.sasl.NonblockingSaslHandler.Phase; +import org.apache.thrift.transport.sasl.TBaseSaslProcessorFactory; +import org.apache.thrift.transport.sasl.TSaslProcessorFactory; +import org.apache.thrift.transport.sasl.TSaslServerFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** TServer with sasl support, using asynchronous execution and nonblocking io. */ +public class TSaslNonblockingServer extends TServer { + private static final Logger LOGGER = LoggerFactory.getLogger(TSaslNonblockingServer.class); + + private static final int DEFAULT_NETWORK_THREADS = 1; + private static final int DEFAULT_AUTHENTICATION_THREADS = 1; + private static final int DEFAULT_PROCESSING_THREADS = Runtime.getRuntime().availableProcessors(); + + private final AcceptorThread acceptor; + private final NetworkThreadPool networkThreadPool; + private final ExecutorService authenticationExecutor; + private final ExecutorService processingExecutor; + private final TSaslServerFactory saslServerFactory; + private final TSaslProcessorFactory saslProcessorFactory; + + public TSaslNonblockingServer(Args args) throws IOException { + super(args); + acceptor = new AcceptorThread((TNonblockingServerSocket) serverTransport_); + networkThreadPool = new NetworkThreadPool(args.networkThreads); + authenticationExecutor = Executors.newFixedThreadPool(args.saslThreads); + processingExecutor = Executors.newFixedThreadPool(args.processingThreads); + saslServerFactory = args.saslServerFactory; + saslProcessorFactory = args.saslProcessorFactory; + } + + @Override + public void serve() { + if (eventHandler_ != null) { + eventHandler_.preServe(); + } + networkThreadPool.start(); + acceptor.start(); + setServing(true); + } + + /** Trigger a graceful shutdown, but it does not block to wait for the shutdown to finish. */ + @Override + public void stop() { + if (!stopped_) { + setServing(false); + stopped_ = true; + acceptor.wakeup(); + networkThreadPool.wakeupAll(); + authenticationExecutor.shutdownNow(); + processingExecutor.shutdownNow(); + } + } + + /** + * Gracefully shut down the server and block until all threads are stopped. + * + * @throws InterruptedException if is interrupted while waiting for shutdown. + */ + public void shutdown() throws InterruptedException { + stop(); + acceptor.join(); + for (NetworkThread networkThread : networkThreadPool.networkThreads) { + networkThread.join(); + } + while (!authenticationExecutor.isTerminated()) { + authenticationExecutor.awaitTermination(10, TimeUnit.SECONDS); + } + while (!processingExecutor.isTerminated()) { + processingExecutor.awaitTermination(10, TimeUnit.SECONDS); + } + } + + private class AcceptorThread extends Thread { + + private final TNonblockingServerTransport serverTransport; + private final Selector acceptSelector; + + private AcceptorThread(TNonblockingServerSocket serverTransport) throws IOException { + super("acceptor-thread"); + this.serverTransport = serverTransport; + acceptSelector = Selector.open(); + serverTransport.registerSelector(acceptSelector); + } + + @Override + public void run() { + try { + serverTransport.listen(); + while (!stopped_) { + select(); + acceptNewConnection(); + } + } catch (TTransportException e) { + // Failed to listen. + LOGGER.error("Failed to listen on server socket, error " + e.getType(), e); + } catch (Throwable e) { + // Unexpected errors. + LOGGER.error("Unexpected error in acceptor thread.", e); + } finally { + TSaslNonblockingServer.this.stop(); + close(); + } + } + + void wakeup() { + acceptSelector.wakeup(); + } + + private void acceptNewConnection() { + Iterator selectedKeyItr = acceptSelector.selectedKeys().iterator(); + while (!stopped_ && selectedKeyItr.hasNext()) { + SelectionKey selected = selectedKeyItr.next(); + selectedKeyItr.remove(); + if (selected.isAcceptable()) { + try { + while (true) { + // Accept all available connections from the backlog. + TNonblockingTransport connection = serverTransport.accept(); + if (connection == null) { + break; + } + if (!networkThreadPool.acceptNewConnection(connection)) { + LOGGER.error("Network thread does not accept: " + connection); + connection.close(); + } + } + } catch (TTransportException e) { + LOGGER.warn("Failed to accept incoming connection.", e); + } + } else { + LOGGER.error("Not acceptable selection: " + selected.channel()); + } + } + } + + private void select() { + try { + acceptSelector.select(); + } catch (IOException e) { + LOGGER.error("Failed to select on the server socket.", e); + } + } + + private void close() { + LOGGER.info("Closing acceptor thread."); + serverTransport.close(); + try { + acceptSelector.close(); + } catch (IOException e) { + LOGGER.error("Failed to close accept selector.", e); + } + } + } + + private class NetworkThread extends Thread { + private final BlockingQueue incomingConnections = + new LinkedBlockingQueue<>(); + private final BlockingQueue stateTransitions = + new LinkedBlockingQueue<>(); + private final Selector ioSelector; + + NetworkThread(String name) throws IOException { + super(name); + ioSelector = Selector.open(); + } + + @Override + public void run() { + try { + while (!stopped_) { + handleIncomingConnections(); + handleStateChanges(); + select(); + handleIO(); + } + } catch (Throwable e) { + LOGGER.error("Unreoverable error in " + getName(), e); + } finally { + close(); + } + } + + private void handleStateChanges() { + while (true) { + NonblockingSaslHandler statemachine = stateTransitions.poll(); + if (statemachine == null) { + return; + } + tryRunNextPhase(statemachine); + } + } + + private void select() { + try { + ioSelector.select(); + } catch (IOException e) { + LOGGER.error("Failed to select in " + getName(), e); + } + } + + private void handleIO() { + Iterator selectedKeyItr = ioSelector.selectedKeys().iterator(); + while (!stopped_ && selectedKeyItr.hasNext()) { + SelectionKey selected = selectedKeyItr.next(); + selectedKeyItr.remove(); + if (!selected.isValid()) { + closeChannel(selected); + } + NonblockingSaslHandler saslHandler = (NonblockingSaslHandler) selected.attachment(); + if (selected.isReadable()) { + saslHandler.handleRead(); + } else if (selected.isWritable()) { + saslHandler.handleWrite(); + } else { + LOGGER.error("Invalid interest op " + selected.interestOps()); + closeChannel(selected); + continue; + } + if (saslHandler.isCurrentPhaseDone()) { + tryRunNextPhase(saslHandler); + } + } + } + + // The following methods are modifying the registered channel set on the selector, which itself + // is not thread safe. Thus we need a lock to protect it from race condition. + + private synchronized void handleIncomingConnections() { + while (true) { + TNonblockingTransport connection = incomingConnections.poll(); + if (connection == null) { + return; + } + if (!connection.isOpen()) { + LOGGER.warn("Incoming connection is already closed"); + continue; + } + try { + SelectionKey selectionKey = connection.registerSelector(ioSelector, SelectionKey.OP_READ); + if (selectionKey.isValid()) { + NonblockingSaslHandler saslHandler = + new NonblockingSaslHandler( + selectionKey, + connection, + saslServerFactory, + saslProcessorFactory, + inputProtocolFactory_, + outputProtocolFactory_, + eventHandler_); + selectionKey.attach(saslHandler); + } + } catch (IOException e) { + LOGGER.error("Failed to register connection for the selector, close it.", e); + connection.close(); + } + } + } + + private synchronized void close() { + LOGGER.warn("Closing " + getName()); + while (true) { + TNonblockingTransport incomingConnection = incomingConnections.poll(); + if (incomingConnection == null) { + break; + } + incomingConnection.close(); + } + Set registered = ioSelector.keys(); + for (SelectionKey selection : registered) { + closeChannel(selection); + } + try { + ioSelector.close(); + } catch (IOException e) { + LOGGER.error("Failed to close io selector " + getName(), e); + } + } + + private synchronized void closeChannel(SelectionKey selectionKey) { + if (selectionKey.attachment() == null) { + try { + selectionKey.channel().close(); + } catch (IOException e) { + LOGGER.error("Failed to close channel.", e); + } finally { + selectionKey.cancel(); + } + } else { + NonblockingSaslHandler saslHandler = (NonblockingSaslHandler) selectionKey.attachment(); + saslHandler.close(); + } + } + + private void tryRunNextPhase(NonblockingSaslHandler saslHandler) { + Phase nextPhase = saslHandler.getNextPhase(); + saslHandler.stepToNextPhase(); + switch (nextPhase) { + case EVALUATING_SASL_RESPONSE: + authenticationExecutor.submit(new Computation(saslHandler)); + break; + case PROCESSING: + processingExecutor.submit(new Computation(saslHandler)); + break; + case CLOSING: + saslHandler.runCurrentPhase(); + break; + default: // waiting for next io event for the current state machine + } + } + + public boolean accept(TNonblockingTransport connection) { + if (stopped_) { + return false; + } + if (incomingConnections.offer(connection)) { + wakeup(); + return true; + } + return false; + } + + private void wakeup() { + ioSelector.wakeup(); + } + + private class Computation implements Runnable { + + private final NonblockingSaslHandler statemachine; + + private Computation(NonblockingSaslHandler statemachine) { + this.statemachine = statemachine; + } + + @Override + public void run() { + try { + while (!statemachine.isCurrentPhaseDone()) { + statemachine.runCurrentPhase(); + } + stateTransitions.add(statemachine); + wakeup(); + } catch (Throwable e) { + LOGGER.error("Damn it!", e); + } + } + } + } + + private class NetworkThreadPool { + private final List networkThreads; + private int accepted = 0; + + NetworkThreadPool(int size) throws IOException { + networkThreads = new ArrayList<>(size); + int digits = (int) Math.log10(size) + 1; + String threadNamePattern = "network-thread-%0" + digits + "d"; + for (int i = 0; i < size; i++) { + networkThreads.add(new NetworkThread(String.format(threadNamePattern, i))); + } + } + + /** + * Round robin new connection among all the network threads. + * + * @param connection incoming connection. + * @return true if the incoming connection is accepted by network thread pool. + */ + boolean acceptNewConnection(TNonblockingTransport connection) { + return networkThreads.get((accepted++) % networkThreads.size()).accept(connection); + } + + public void start() { + for (NetworkThread thread : networkThreads) { + thread.start(); + } + } + + void wakeupAll() { + for (NetworkThread networkThread : networkThreads) { + networkThread.wakeup(); + } + } + } + + public static class Args extends AbstractServerArgs { + + private int networkThreads = DEFAULT_NETWORK_THREADS; + private int saslThreads = DEFAULT_AUTHENTICATION_THREADS; + private int processingThreads = DEFAULT_PROCESSING_THREADS; + private TSaslServerFactory saslServerFactory = new TSaslServerFactory(); + private TSaslProcessorFactory saslProcessorFactory; + + public Args(TNonblockingServerTransport transport) { + super(transport); + } + + public Args networkThreads(int networkThreads) { + this.networkThreads = networkThreads <= 0 ? DEFAULT_NETWORK_THREADS : networkThreads; + return this; + } + + public Args saslThreads(int authenticationThreads) { + this.saslThreads = + authenticationThreads <= 0 ? DEFAULT_AUTHENTICATION_THREADS : authenticationThreads; + return this; + } + + public Args processingThreads(int processingThreads) { + this.processingThreads = + processingThreads <= 0 ? DEFAULT_PROCESSING_THREADS : processingThreads; + return this; + } + + public Args processor(TProcessor processor) { + saslProcessorFactory = new TBaseSaslProcessorFactory(processor); + return this; + } + + public Args saslProcessorFactory(TSaslProcessorFactory saslProcessorFactory) { + if (saslProcessorFactory == null) { + throw new NullPointerException("Processor factory cannot be null"); + } + this.saslProcessorFactory = saslProcessorFactory; + return this; + } + + public Args addSaslMechanism( + String mechanism, + String protocol, + String serverName, + Map props, + CallbackHandler cbh) { + saslServerFactory.addSaslMechanism(mechanism, protocol, serverName, props, cbh); + return this; + } + + public Args saslServerFactory(TSaslServerFactory saslServerFactory) { + if (saslServerFactory == null) { + throw new NullPointerException("saslServerFactory cannot be null"); + } + this.saslServerFactory = saslServerFactory; + return this; + } + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TServer.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TServer.java new file mode 100644 index 0000000..1a4f198 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TServer.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.server; + +import org.apache.thrift.TProcessor; +import org.apache.thrift.TProcessorFactory; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.protocol.TProtocolFactory; +import org.apache.thrift.transport.TServerTransport; +import org.apache.thrift.transport.TTransportFactory; + +/** Generic interface for a Thrift server. */ +public abstract class TServer { + + public static class Args extends AbstractServerArgs { + public Args(TServerTransport transport) { + super(transport); + } + } + + public abstract static class AbstractServerArgs> { + final TServerTransport serverTransport; + TProcessorFactory processorFactory; + TTransportFactory inputTransportFactory = new TTransportFactory(); + TTransportFactory outputTransportFactory = new TTransportFactory(); + TProtocolFactory inputProtocolFactory = new TBinaryProtocol.Factory(); + TProtocolFactory outputProtocolFactory = new TBinaryProtocol.Factory(); + + public AbstractServerArgs(TServerTransport transport) { + serverTransport = transport; + } + + public T processorFactory(TProcessorFactory factory) { + this.processorFactory = factory; + return (T) this; + } + + public T processor(TProcessor processor) { + this.processorFactory = new TProcessorFactory(processor); + return (T) this; + } + + public T transportFactory(TTransportFactory factory) { + this.inputTransportFactory = factory; + this.outputTransportFactory = factory; + return (T) this; + } + + public T inputTransportFactory(TTransportFactory factory) { + this.inputTransportFactory = factory; + return (T) this; + } + + public T outputTransportFactory(TTransportFactory factory) { + this.outputTransportFactory = factory; + return (T) this; + } + + public T protocolFactory(TProtocolFactory factory) { + this.inputProtocolFactory = factory; + this.outputProtocolFactory = factory; + return (T) this; + } + + public T inputProtocolFactory(TProtocolFactory factory) { + this.inputProtocolFactory = factory; + return (T) this; + } + + public T outputProtocolFactory(TProtocolFactory factory) { + this.outputProtocolFactory = factory; + return (T) this; + } + } + + /** Core processor */ + protected TProcessorFactory processorFactory_; + + /** Server transport */ + protected TServerTransport serverTransport_; + + /** Input Transport Factory */ + protected TTransportFactory inputTransportFactory_; + + /** Output Transport Factory */ + protected TTransportFactory outputTransportFactory_; + + /** Input Protocol Factory */ + protected TProtocolFactory inputProtocolFactory_; + + /** Output Protocol Factory */ + protected TProtocolFactory outputProtocolFactory_; + + private volatile boolean isServing; + + protected TServerEventHandler eventHandler_; + + // Flag for stopping the server + // Please see THRIFT-1795 for the usage of this flag + protected volatile boolean stopped_ = false; + + protected TServer(AbstractServerArgs args) { + processorFactory_ = args.processorFactory; + serverTransport_ = args.serverTransport; + inputTransportFactory_ = args.inputTransportFactory; + outputTransportFactory_ = args.outputTransportFactory; + inputProtocolFactory_ = args.inputProtocolFactory; + outputProtocolFactory_ = args.outputProtocolFactory; + } + + /** The run method fires up the server and gets things going. */ + public abstract void serve(); + + /** + * Stop the server. This is optional on a per-implementation basis. Not all servers are required + * to be cleanly stoppable. + */ + public void stop() {} + + public boolean isServing() { + return isServing; + } + + protected void setServing(boolean serving) { + isServing = serving; + } + + public void setServerEventHandler(TServerEventHandler eventHandler) { + eventHandler_ = eventHandler; + } + + public TServerEventHandler getEventHandler() { + return eventHandler_; + } + + public boolean getShouldStop() { + return this.stopped_; + } + + public void setShouldStop(boolean shouldStop) { + this.stopped_ = shouldStop; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TServerEventHandler.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TServerEventHandler.java new file mode 100644 index 0000000..4ff043a --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TServerEventHandler.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.server; + +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.transport.TTransport; + +/** + * Interface that can handle events from the server core. To use this you should subclass it and + * implement the methods that you care about. Your subclass can also store local data that you may + * care about, such as additional "arguments" to these methods (stored in the object instance's + * state). + * + *

TODO: It seems this is a custom code entry point created for some resource management purpose + * in hive. But when looking into hive code, we see that the argments of TProtocol and TTransport + * are never used. We probably should remove these arguments from all the methods. + */ +public interface TServerEventHandler { + + /** Called before the server begins. */ + void preServe(); + + /** Called when a new client has connected and is about to being processing. */ + ServerContext createContext(TProtocol input, TProtocol output); + + /** Called when a client has finished request-handling to delete server context. */ + void deleteContext(ServerContext serverContext, TProtocol input, TProtocol output); + + /** Called when a client is about to call the processor. */ + void processContext( + ServerContext serverContext, TTransport inputTransport, TTransport outputTransport); +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TServlet.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TServlet.java new file mode 100644 index 0000000..984f53b --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TServlet.java @@ -0,0 +1,120 @@ +package org.apache.thrift.server; + +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Map; +import org.apache.thrift.TException; +import org.apache.thrift.TProcessor; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.protocol.TProtocolFactory; +import org.apache.thrift.transport.TIOStreamTransport; +import org.apache.thrift.transport.TTransport; + +/** Servlet implementation class ThriftServer */ +public class TServlet extends HttpServlet { + + private final TProcessor processor; + + private final TProtocolFactory inProtocolFactory; + + private final TProtocolFactory outProtocolFactory; + + private final Collection> customHeaders; + + /** + * @see HttpServlet#HttpServlet() + */ + public TServlet( + TProcessor processor, + TProtocolFactory inProtocolFactory, + TProtocolFactory outProtocolFactory) { + super(); + this.processor = processor; + this.inProtocolFactory = inProtocolFactory; + this.outProtocolFactory = outProtocolFactory; + this.customHeaders = new ArrayList>(); + } + + /** + * @see HttpServlet#HttpServlet() + */ + public TServlet(TProcessor processor, TProtocolFactory protocolFactory) { + this(processor, protocolFactory, protocolFactory); + } + + /** + * @see HttpServlet#doPost(HttpServletRequest request, HttpServletResponse response) + */ + @Override + protected void doPost(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + + TTransport inTransport = null; + TTransport outTransport = null; + + try { + response.setContentType("application/x-thrift"); + + if (null != this.customHeaders) { + for (Map.Entry header : this.customHeaders) { + response.addHeader(header.getKey(), header.getValue()); + } + } + InputStream in = request.getInputStream(); + OutputStream out = response.getOutputStream(); + + TTransport transport = new TIOStreamTransport(in, out); + inTransport = transport; + outTransport = transport; + + TProtocol inProtocol = inProtocolFactory.getProtocol(inTransport); + TProtocol outProtocol = outProtocolFactory.getProtocol(outTransport); + + processor.process(inProtocol, outProtocol); + out.flush(); + } catch (TException te) { + throw new ServletException(te); + } + } + + /** + * @see HttpServlet#doGet(HttpServletRequest request, HttpServletResponse response) + */ + @Override + protected void doGet(HttpServletRequest request, HttpServletResponse response) + throws ServletException, IOException { + doPost(request, response); + } + + public void addCustomHeader(final String key, final String value) { + this.customHeaders.add( + new Map.Entry() { + @Override + public String getKey() { + return key; + } + + @Override + public String getValue() { + return value; + } + + @Override + public String setValue(String value) { + return null; + } + }); + } + + public void setCustomHeaders(Collection> headers) { + this.customHeaders.clear(); + this.customHeaders.addAll(headers); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TSimpleServer.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TSimpleServer.java new file mode 100644 index 0000000..db1e57f --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TSimpleServer.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.server; + +import java.net.SocketAddress; +import org.apache.thrift.TException; +import org.apache.thrift.TProcessor; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.transport.SocketAddressProvider; +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Simple singlethreaded server for testing. */ +public class TSimpleServer extends TServer { + + private static final Logger LOGGER = LoggerFactory.getLogger(TSimpleServer.class.getName()); + + public TSimpleServer(AbstractServerArgs args) { + super(args); + } + + @Override + public void serve() { + try { + serverTransport_.listen(); + } catch (TTransportException ttx) { + LOGGER.error("Error occurred during listening.", ttx); + return; + } + + // Run the preServe event + if (eventHandler_ != null) { + eventHandler_.preServe(); + } + + setServing(true); + + while (!stopped_) { + TTransport client = null; + TProcessor processor = null; + TTransport inputTransport = null; + TTransport outputTransport = null; + TProtocol inputProtocol = null; + TProtocol outputProtocol = null; + ServerContext connectionContext = null; + try { + client = serverTransport_.accept(); + if (client != null) { + processor = processorFactory_.getProcessor(client); + inputTransport = inputTransportFactory_.getTransport(client); + outputTransport = outputTransportFactory_.getTransport(client); + inputProtocol = inputProtocolFactory_.getProtocol(inputTransport); + outputProtocol = outputProtocolFactory_.getProtocol(outputTransport); + if (eventHandler_ != null) { + connectionContext = eventHandler_.createContext(inputProtocol, outputProtocol); + SocketAddress remoteAddress = + client instanceof SocketAddressProvider + ? ((SocketAddressProvider) client).getRemoteSocketAddress() + : null; + connectionContext.setRemoteAddress(remoteAddress); + } + while (true) { + if (eventHandler_ != null) { + eventHandler_.processContext(connectionContext, inputTransport, outputTransport); + } + processor.process(inputProtocol, outputProtocol); + } + } + } catch (TTransportException ttx) { + // Client died, just move on + LOGGER.debug("Client Transportation Exception", ttx); + } catch (TException tx) { + if (!stopped_) { + LOGGER.error("Thrift error occurred during processing of message.", tx); + } + } catch (Exception x) { + if (!stopped_) { + LOGGER.error("Error occurred during processing of message.", x); + } + } + + if (eventHandler_ != null) { + eventHandler_.deleteContext(connectionContext, inputProtocol, outputProtocol); + } + + if (inputTransport != null) { + inputTransport.close(); + } + + if (outputTransport != null) { + outputTransport.close(); + } + } + setServing(false); + } + + public void stop() { + stopped_ = true; + serverTransport_.interrupt(); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TThreadPoolServer.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TThreadPoolServer.java new file mode 100644 index 0000000..073f1bc --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TThreadPoolServer.java @@ -0,0 +1,316 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.server; + +import java.net.SocketAddress; +import java.net.SocketException; +import java.util.Optional; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.thrift.TException; +import org.apache.thrift.TProcessor; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.transport.SocketAddressProvider; +import org.apache.thrift.transport.TServerTransport; +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Server which uses Java's built in ThreadPool management to spawn off a worker pool that deals + * with client connections in blocking way. + */ +public class TThreadPoolServer extends TServer { + private static final Logger LOGGER = LoggerFactory.getLogger(TThreadPoolServer.class); + + public static class Args extends AbstractServerArgs { + public int minWorkerThreads = 5; + public int maxWorkerThreads = Integer.MAX_VALUE; + public ExecutorService executorService; + public int stopTimeoutVal = 60; + public TimeUnit stopTimeoutUnit = TimeUnit.SECONDS; + + public Args(TServerTransport transport) { + super(transport); + } + + public Args minWorkerThreads(int n) { + minWorkerThreads = n; + return this; + } + + public Args maxWorkerThreads(int n) { + maxWorkerThreads = n; + return this; + } + + public Args stopTimeoutVal(int n) { + stopTimeoutVal = n; + return this; + } + + public Args stopTimeoutUnit(TimeUnit tu) { + stopTimeoutUnit = tu; + return this; + } + + public Args executorService(ExecutorService executorService) { + this.executorService = executorService; + return this; + } + } + + // Executor service for handling client connections + private final ExecutorService executorService_; + + private final TimeUnit stopTimeoutUnit; + + private final long stopTimeoutVal; + + public TThreadPoolServer(Args args) { + super(args); + + stopTimeoutUnit = args.stopTimeoutUnit; + stopTimeoutVal = args.stopTimeoutVal; + + executorService_ = + args.executorService != null ? args.executorService : createDefaultExecutorService(args); + } + + private static ExecutorService createDefaultExecutorService(Args args) { + return new ThreadPoolExecutor( + args.minWorkerThreads, + args.maxWorkerThreads, + 60L, + TimeUnit.SECONDS, + new SynchronousQueue<>(), + new ThreadFactory() { + final AtomicLong count = new AtomicLong(); + + @Override + public Thread newThread(Runnable r) { + Thread thread = new Thread(r); + thread.setDaemon(true); + thread.setName( + String.format("TThreadPoolServer WorkerProcess-%d", count.getAndIncrement())); + return thread; + } + }); + } + + protected ExecutorService getExecutorService() { + return executorService_; + } + + protected boolean preServe() { + try { + serverTransport_.listen(); + } catch (TTransportException ttx) { + LOGGER.error("Error occurred during listening.", ttx); + return false; + } + + // Run the preServe event + if (eventHandler_ != null) { + eventHandler_.preServe(); + } + stopped_ = false; + setServing(true); + return true; + } + + @Override + public void serve() { + if (!preServe()) { + return; + } + + execute(); + + executorService_.shutdownNow(); + + if (!waitForShutdown()) { + LOGGER.error("Shutdown is not done after " + stopTimeoutVal + stopTimeoutUnit); + } + + setServing(false); + } + + protected void execute() { + while (!stopped_) { + try { + TTransport client = serverTransport_.accept(); + try { + executorService_.execute(new WorkerProcess(client)); + } catch (RejectedExecutionException ree) { + if (!stopped_) { + LOGGER.warn( + "ThreadPool is saturated with incoming requests. Closing latest connection."); + } + client.close(); + } + } catch (TTransportException ttx) { + if (!stopped_) { + LOGGER.warn("Transport error occurred during acceptance of message", ttx); + } + } + } + } + + protected boolean waitForShutdown() { + // Loop until awaitTermination finally does return without a interrupted + // exception. If we don't do this, then we'll shut down prematurely. We want + // to let the executorService clear it's task queue, closing client sockets + // appropriately. + long timeoutMS = stopTimeoutUnit.toMillis(stopTimeoutVal); + long now = System.currentTimeMillis(); + while (timeoutMS >= 0) { + try { + return executorService_.awaitTermination(timeoutMS, TimeUnit.MILLISECONDS); + } catch (InterruptedException ix) { + long newnow = System.currentTimeMillis(); + timeoutMS -= (newnow - now); + now = newnow; + } + } + return false; + } + + @Override + public void stop() { + stopped_ = true; + serverTransport_.interrupt(); + } + + private class WorkerProcess implements Runnable { + + /** Client that this services. */ + private final TTransport client_; + + /** + * Default constructor. + * + * @param client Transport to process + */ + private WorkerProcess(TTransport client) { + client_ = client; + } + + /** Loops on processing a client forever */ + @Override + public void run() { + TProcessor processor = null; + TTransport inputTransport = null; + TTransport outputTransport = null; + TProtocol inputProtocol = null; + TProtocol outputProtocol = null; + + Optional eventHandler = Optional.empty(); + ServerContext connectionContext = null; + + try { + processor = processorFactory_.getProcessor(client_); + inputTransport = inputTransportFactory_.getTransport(client_); + outputTransport = outputTransportFactory_.getTransport(client_); + inputProtocol = inputProtocolFactory_.getProtocol(inputTransport); + outputProtocol = outputProtocolFactory_.getProtocol(outputTransport); + + eventHandler = Optional.ofNullable(getEventHandler()); + + if (eventHandler.isPresent()) { + connectionContext = eventHandler_.createContext(inputProtocol, outputProtocol); + SocketAddress remoteAddress = + client_ instanceof SocketAddressProvider + ? ((SocketAddressProvider) client_).getRemoteSocketAddress() + : null; + connectionContext.setRemoteAddress(remoteAddress); + } + + while (true) { + if (Thread.currentThread().isInterrupted()) { + LOGGER.debug("WorkerProcess requested to shutdown"); + break; + } + if (eventHandler.isPresent()) { + eventHandler.get().processContext(connectionContext, inputTransport, outputTransport); + } + // This process cannot be interrupted by Interrupting the Thread. This + // will return once a message has been processed or the socket timeout + // has elapsed, at which point it will return and check the interrupt + // state of the thread. + processor.process(inputProtocol, outputProtocol); + } + } catch (Exception x) { + logException(x); + } finally { + if (eventHandler.isPresent()) { + eventHandler.get().deleteContext(connectionContext, inputProtocol, outputProtocol); + } + if (inputTransport != null) { + inputTransport.close(); + } + if (outputTransport != null) { + outputTransport.close(); + } + if (client_.isOpen()) { + client_.close(); + } + } + } + + private void logException(Exception x) { + // We'll usually receive RuntimeException types here + // Need to unwrap to ascertain real causing exception before we choose to ignore + LOGGER.debug("Error processing request", x); + TTransportException tTransportException = null; + + if (x instanceof TTransportException) { + tTransportException = (TTransportException) x; + } else if (x.getCause() instanceof TTransportException) { + tTransportException = (TTransportException) x.getCause(); + } + + if (tTransportException != null) { + switch (tTransportException.getType()) { + case TTransportException.END_OF_FILE: + case TTransportException.TIMED_OUT: + return; // don't log these + } + if (tTransportException.getCause() instanceof SocketException) { + LOGGER.warn( + "SocketException occurred during processing of message.", + tTransportException.getCause()); + return; + } + } + // Log the exception at error level and continue + LOGGER.error( + (x instanceof TException ? "Thrift " : "") + + "Error occurred during processing of message.", + x); + } + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TThreadedSelectorServer.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TThreadedSelectorServer.java new file mode 100644 index 0000000..33b4686 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TThreadedSelectorServer.java @@ -0,0 +1,730 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.server; + +import java.io.IOException; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.SelectableChannel; +import java.nio.channels.SelectionKey; +import java.nio.channels.Selector; +import java.nio.channels.spi.SelectorProvider; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import org.apache.thrift.transport.TNonblockingServerTransport; +import org.apache.thrift.transport.TNonblockingTransport; +import org.apache.thrift.transport.TTransportException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A Half-Sync/Half-Async server with a separate pool of threads to handle non-blocking I/O. Accepts + * are handled on a single thread, and a configurable number of nonblocking selector threads manage + * reading and writing of client connections. A synchronous worker thread pool handles processing of + * requests. + * + *

Performs better than TNonblockingServer/THsHaServer in multi-core environments when the + * bottleneck is CPU on the single selector thread handling I/O. In addition, because the accept + * handling is decoupled from reads/writes and invocation, the server has better ability to handle + * back-pressure from new connections (e.g. stop accepting when busy). + * + *

Like TNonblockingServer, it relies on the use of TFramedTransport. + */ +public class TThreadedSelectorServer extends AbstractNonblockingServer { + private static final Logger LOGGER = + LoggerFactory.getLogger(TThreadedSelectorServer.class.getName()); + + public static class Args extends AbstractNonblockingServerArgs { + + /** The number of threads for selecting on already-accepted connections */ + public int selectorThreads = 2; + + /** + * The size of the executor service (if none is specified) that will handle invocations. This + * may be set to 0, in which case invocations will be handled directly on the selector threads + * (as is in TNonblockingServer) + */ + private int workerThreads = 5; + + /** Time to wait for server to stop gracefully */ + private int stopTimeoutVal = 60; + + private TimeUnit stopTimeoutUnit = TimeUnit.SECONDS; + + /** The ExecutorService for handling dispatched requests */ + private ExecutorService executorService = null; + + /** + * The size of the blocking queue per selector thread for passing accepted connections to the + * selector thread + */ + private int acceptQueueSizePerThread = 4; + + /** Determines the strategy for handling new accepted connections. */ + public static enum AcceptPolicy { + /** + * Require accepted connection registration to be handled by the executor. If the worker pool + * is saturated, further accepts will be closed immediately. Slightly increases latency due to + * an extra scheduling. + */ + FAIR_ACCEPT, + /** + * Handle the accepts as fast as possible, disregarding the status of the executor service. + */ + FAST_ACCEPT + } + + private AcceptPolicy acceptPolicy = AcceptPolicy.FAST_ACCEPT; + + public Args(TNonblockingServerTransport transport) { + super(transport); + } + + public Args selectorThreads(int i) { + selectorThreads = i; + return this; + } + + public int getSelectorThreads() { + return selectorThreads; + } + + public Args workerThreads(int i) { + workerThreads = i; + return this; + } + + public int getWorkerThreads() { + return workerThreads; + } + + public int getStopTimeoutVal() { + return stopTimeoutVal; + } + + public Args stopTimeoutVal(int stopTimeoutVal) { + this.stopTimeoutVal = stopTimeoutVal; + return this; + } + + public TimeUnit getStopTimeoutUnit() { + return stopTimeoutUnit; + } + + public Args stopTimeoutUnit(TimeUnit stopTimeoutUnit) { + this.stopTimeoutUnit = stopTimeoutUnit; + return this; + } + + public ExecutorService getExecutorService() { + return executorService; + } + + public Args executorService(ExecutorService executorService) { + this.executorService = executorService; + return this; + } + + public int getAcceptQueueSizePerThread() { + return acceptQueueSizePerThread; + } + + public Args acceptQueueSizePerThread(int acceptQueueSizePerThread) { + this.acceptQueueSizePerThread = acceptQueueSizePerThread; + return this; + } + + public AcceptPolicy getAcceptPolicy() { + return acceptPolicy; + } + + public Args acceptPolicy(AcceptPolicy acceptPolicy) { + this.acceptPolicy = acceptPolicy; + return this; + } + + public void validate() { + if (selectorThreads <= 0) { + throw new IllegalArgumentException("selectorThreads must be positive."); + } + if (workerThreads < 0) { + throw new IllegalArgumentException("workerThreads must be non-negative."); + } + if (acceptQueueSizePerThread <= 0) { + throw new IllegalArgumentException("acceptQueueSizePerThread must be positive."); + } + } + } + + // The thread handling all accepts + private AcceptThread acceptThread; + + // Threads handling events on client transports + private final Set selectorThreads = new HashSet<>(); + + // This wraps all the functionality of queueing and thread pool management + // for the passing of Invocations from the selector thread(s) to the workers + // (if any). + private final ExecutorService invoker; + + private final Args args; + + /** Create the server with the specified Args configuration */ + public TThreadedSelectorServer(Args args) { + super(args); + args.validate(); + invoker = args.executorService == null ? createDefaultExecutor(args) : args.executorService; + this.args = args; + } + + /** + * Start the accept and selector threads running to deal with clients. + * + * @return true if everything went ok, false if we couldn't start for some reason. + */ + @Override + protected boolean startThreads() { + try { + for (int i = 0; i < args.selectorThreads; ++i) { + selectorThreads.add(new SelectorThread(args.acceptQueueSizePerThread)); + } + acceptThread = + new AcceptThread( + (TNonblockingServerTransport) serverTransport_, + createSelectorThreadLoadBalancer(selectorThreads)); + for (SelectorThread thread : selectorThreads) { + thread.start(); + } + acceptThread.start(); + return true; + } catch (IOException e) { + LOGGER.error("Failed to start threads!", e); + return false; + } + } + + /** Joins the accept and selector threads and shuts down the executor service. */ + @Override + protected void waitForShutdown() { + try { + joinThreads(); + } catch (InterruptedException e) { + // Non-graceful shutdown occurred + LOGGER.error("Interrupted while joining threads!", e); + } + gracefullyShutdownInvokerPool(); + } + + protected void joinThreads() throws InterruptedException { + // wait until the io threads exit + acceptThread.join(); + for (SelectorThread thread : selectorThreads) { + thread.join(); + } + } + + /** Stop serving and shut everything down. */ + @Override + public void stop() { + stopped_ = true; + + // Stop queuing connect attempts asap + stopListening(); + + if (acceptThread != null) { + acceptThread.wakeupSelector(); + } + for (SelectorThread thread : selectorThreads) { + if (thread != null) thread.wakeupSelector(); + } + } + + protected void gracefullyShutdownInvokerPool() { + // try to gracefully shut down the executor service + invoker.shutdown(); + + // Loop until awaitTermination finally does return without a interrupted + // exception. If we don't do this, then we'll shut down prematurely. We want + // to let the executorService clear it's task queue, closing client sockets + // appropriately. + long timeoutMS = args.stopTimeoutUnit.toMillis(args.stopTimeoutVal); + long now = System.currentTimeMillis(); + while (timeoutMS >= 0) { + try { + invoker.awaitTermination(timeoutMS, TimeUnit.MILLISECONDS); + break; + } catch (InterruptedException ix) { + long newnow = System.currentTimeMillis(); + timeoutMS -= (newnow - now); + now = newnow; + } + } + } + + /** + * We override the standard invoke method here to queue the invocation for invoker service instead + * of immediately invoking. If there is no thread pool, handle the invocation inline on this + * thread + */ + @Override + protected boolean requestInvoke(FrameBuffer frameBuffer) { + Runnable invocation = getRunnable(frameBuffer); + if (invoker != null) { + try { + invoker.execute(invocation); + return true; + } catch (RejectedExecutionException rx) { + LOGGER.warn("ExecutorService rejected execution!", rx); + return false; + } + } else { + // Invoke on the caller's thread + invocation.run(); + return true; + } + } + + protected Runnable getRunnable(FrameBuffer frameBuffer) { + return new Invocation(frameBuffer); + } + + /** Helper to create the invoker if one is not specified */ + protected static ExecutorService createDefaultExecutor(Args options) { + return (options.workerThreads > 0) ? Executors.newFixedThreadPool(options.workerThreads) : null; + } + + private static BlockingQueue createDefaultAcceptQueue(int queueSize) { + if (queueSize == 0) { + // Unbounded queue + return new LinkedBlockingQueue(); + } + return new ArrayBlockingQueue(queueSize); + } + + /** + * The thread that selects on the server transport (listen socket) and accepts new connections to + * hand off to the IO selector threads + */ + protected class AcceptThread extends Thread { + + // The listen socket to accept on + private final TNonblockingServerTransport serverTransport; + private final Selector acceptSelector; + + private final SelectorThreadLoadBalancer threadChooser; + + /** + * Set up the AcceptThead + * + * @throws IOException if failed to register selector + */ + public AcceptThread( + TNonblockingServerTransport serverTransport, SelectorThreadLoadBalancer threadChooser) + throws IOException { + this.serverTransport = serverTransport; + this.threadChooser = threadChooser; + this.acceptSelector = SelectorProvider.provider().openSelector(); + this.serverTransport.registerSelector(acceptSelector); + } + + /** + * The work loop. Selects on the server transport and accepts. If there was a server transport + * that had blocking accepts, and returned on blocking client transports, that should be used + * instead + */ + public void run() { + try { + if (eventHandler_ != null) { + eventHandler_.preServe(); + } + + while (!stopped_) { + select(); + } + } catch (Throwable t) { + LOGGER.error("run() on AcceptThread exiting due to uncaught error", t); + } finally { + try { + acceptSelector.close(); + } catch (IOException e) { + LOGGER.error("Got an IOException while closing accept selector!", e); + } + // This will wake up the selector threads + TThreadedSelectorServer.this.stop(); + } + } + + /** If the selector is blocked, wake it up. */ + public void wakeupSelector() { + acceptSelector.wakeup(); + } + + /** + * Select and process IO events appropriately: If there are connections to be accepted, accept + * them. + */ + private void select() { + try { + // wait for connect events. + acceptSelector.select(); + + // process the io events we received + Iterator selectedKeys = acceptSelector.selectedKeys().iterator(); + while (!stopped_ && selectedKeys.hasNext()) { + SelectionKey key = selectedKeys.next(); + selectedKeys.remove(); + + // skip if not valid + if (!key.isValid()) { + continue; + } + + if (key.isAcceptable()) { + handleAccept(); + } else { + LOGGER.warn("Unexpected state in select! " + key.interestOps()); + } + } + } catch (IOException e) { + LOGGER.warn("Got an IOException while selecting!", e); + } + } + + /** Accept a new connection. */ + private void handleAccept() { + final TNonblockingTransport client = doAccept(); + if (client != null) { + // Pass this connection to a selector thread + final SelectorThread targetThread = threadChooser.nextThread(); + + if (args.acceptPolicy == Args.AcceptPolicy.FAST_ACCEPT || invoker == null) { + doAddAccept(targetThread, client); + } else { + // FAIR_ACCEPT + try { + invoker.submit( + new Runnable() { + public void run() { + doAddAccept(targetThread, client); + } + }); + } catch (RejectedExecutionException rx) { + LOGGER.warn("ExecutorService rejected accept registration!", rx); + // close immediately + client.close(); + } + } + } + } + + private TNonblockingTransport doAccept() { + try { + return serverTransport.accept(); + } catch (TTransportException tte) { + // something went wrong accepting. + LOGGER.warn("Exception trying to accept!", tte); + return null; + } + } + + private void doAddAccept(SelectorThread thread, TNonblockingTransport client) { + if (!thread.addAcceptedConnection(client)) { + client.close(); + } + } + } // AcceptThread + + /** The SelectorThread(s) will be doing all the selecting on accepted active connections. */ + protected class SelectorThread extends AbstractSelectThread { + + // Accepted connections added by the accept thread. + private final BlockingQueue acceptedQueue; + private static final int SELECTOR_AUTO_REBUILD_THRESHOLD = 512; + private static final long MONITOR_PERIOD = 1000L; + private int jvmBug = 0; + + /** + * Set up the SelectorThread with an unbounded queue for incoming accepts. + * + * @throws IOException if a selector cannot be created + */ + public SelectorThread() throws IOException { + this(new LinkedBlockingQueue()); + } + + /** + * Set up the SelectorThread with an bounded queue for incoming accepts. + * + * @throws IOException if a selector cannot be created + */ + public SelectorThread(int maxPendingAccepts) throws IOException { + this(createDefaultAcceptQueue(maxPendingAccepts)); + } + + /** + * Set up the SelectorThread with a specified queue for connections. + * + * @param acceptedQueue The BlockingQueue implementation for holding incoming accepted + * connections. + * @throws IOException if a selector cannot be created. + */ + public SelectorThread(BlockingQueue acceptedQueue) throws IOException { + this.acceptedQueue = acceptedQueue; + } + + /** + * Hands off an accepted connection to be handled by this thread. This method will block if the + * queue for new connections is at capacity. + * + * @param accepted The connection that has been accepted. + * @return true if the connection has been successfully added. + */ + public boolean addAcceptedConnection(TNonblockingTransport accepted) { + try { + acceptedQueue.put(accepted); + } catch (InterruptedException e) { + LOGGER.warn("Interrupted while adding accepted connection!", e); + return false; + } + selector.wakeup(); + return true; + } + + /** + * The work loop. Handles selecting (read/write IO), dispatching, and managing the selection + * preferences of all existing connections. + */ + public void run() { + try { + while (!stopped_) { + select(); + processAcceptedConnections(); + processInterestChanges(); + } + for (SelectionKey selectionKey : selector.keys()) { + cleanupSelectionKey(selectionKey); + } + } catch (Throwable t) { + LOGGER.error("run() on SelectorThread exiting due to uncaught error", t); + } finally { + try { + selector.close(); + } catch (IOException e) { + LOGGER.error("Got an IOException while closing selector!", e); + } + // This will wake up the accept thread and the other selector threads + TThreadedSelectorServer.this.stop(); + } + } + + /** + * Select and process IO events appropriately: If there are existing connections with data + * waiting to be read, read it, buffering until a whole frame has been read. If there are any + * pending responses, buffer them until their target client is available, and then send the + * data. + */ + private void select() { + try { + + doSelect(); + + // process the io events we received + Iterator selectedKeys = selector.selectedKeys().iterator(); + while (!stopped_ && selectedKeys.hasNext()) { + SelectionKey key = selectedKeys.next(); + selectedKeys.remove(); + + // skip if not valid + if (!key.isValid()) { + cleanupSelectionKey(key); + continue; + } + + if (key.isReadable()) { + // deal with reads + handleRead(key); + } else if (key.isWritable()) { + // deal with writes + handleWrite(key); + } else { + LOGGER.warn("Unexpected state in select! " + key.interestOps()); + } + } + } catch (IOException e) { + LOGGER.warn("Got an IOException while selecting!", e); + } + } + + /** + * Do select and judge epoll bug happen. See THRIFT-4251 + */ + private void doSelect() throws IOException { + long beforeSelect = System.currentTimeMillis(); + int selectedNums = selector.select(); + long afterSelect = System.currentTimeMillis(); + + if (selectedNums == 0) { + jvmBug++; + } else { + jvmBug = 0; + } + + long selectedTime = afterSelect - beforeSelect; + if (selectedTime >= MONITOR_PERIOD) { + jvmBug = 0; + } else if (jvmBug > SELECTOR_AUTO_REBUILD_THRESHOLD) { + LOGGER.warn( + "In {} ms happen {} times jvm bug; rebuilding selector.", MONITOR_PERIOD, jvmBug); + rebuildSelector(); + selector.selectNow(); + jvmBug = 0; + } + } + + /** + * Replaces the current Selector of this SelectorThread with newly created Selector to work + * around the infamous epoll 100% CPU bug. + */ + private synchronized void rebuildSelector() { + final Selector oldSelector = selector; + if (oldSelector == null) { + return; + } + Selector newSelector = null; + try { + newSelector = Selector.open(); + LOGGER.warn("Created new Selector."); + } catch (IOException e) { + LOGGER.error("Create new Selector error.", e); + } + + for (SelectionKey key : oldSelector.keys()) { + if (!key.isValid() || key.interestOps() == 0 || key.channel().keyFor(newSelector) != null) { + continue; + } + SelectableChannel channel = key.channel(); + Object attachment = key.attachment(); + + int interestOps = key.interestOps(); + SelectionKey newKey; + try { + if (attachment == null) { + newKey = channel.register(newSelector, interestOps); + } else { + newKey = channel.register(newSelector, interestOps, attachment); + if (attachment instanceof FrameBuffer) { + ((FrameBuffer) attachment).setSelectionKey(newKey); + } + } + } catch (ClosedChannelException e) { + LOGGER.error("Register new selector key error.", e); + } + } + + selector = newSelector; + try { + oldSelector.close(); + } catch (IOException e) { + LOGGER.error("Close old selector error.", e); + } + LOGGER.warn("Replace new selector success."); + } + + private void processAcceptedConnections() { + // Register accepted connections + while (!stopped_) { + TNonblockingTransport accepted = acceptedQueue.poll(); + if (accepted == null) { + break; + } + registerAccepted(accepted); + } + } + + protected FrameBuffer createFrameBuffer( + final TNonblockingTransport trans, + final SelectionKey selectionKey, + final AbstractSelectThread selectThread) + throws TTransportException { + return processorFactory_.isAsyncProcessor() + ? new AsyncFrameBuffer(trans, selectionKey, selectThread) + : new FrameBuffer(trans, selectionKey, selectThread); + } + + private void registerAccepted(TNonblockingTransport accepted) { + SelectionKey clientKey = null; + try { + clientKey = accepted.registerSelector(selector, SelectionKey.OP_READ); + + FrameBuffer frameBuffer = createFrameBuffer(accepted, clientKey, SelectorThread.this); + + clientKey.attach(frameBuffer); + } catch (IOException | TTransportException e) { + LOGGER.warn("Failed to register accepted connection to selector!", e); + if (clientKey != null) { + cleanupSelectionKey(clientKey); + } + accepted.close(); + } + } + } // SelectorThread + + /** + * Creates a SelectorThreadLoadBalancer to be used by the accept thread for assigning newly + * accepted connections across the threads. + */ + protected SelectorThreadLoadBalancer createSelectorThreadLoadBalancer( + Collection threads) { + return new SelectorThreadLoadBalancer(threads); + } + + /** A round robin load balancer for choosing selector threads for new connections. */ + protected static class SelectorThreadLoadBalancer { + private final Collection threads; + private Iterator nextThreadIterator; + + public SelectorThreadLoadBalancer(Collection threads) { + if (threads.isEmpty()) { + throw new IllegalArgumentException("At least one selector thread is required"); + } + this.threads = Collections.unmodifiableList(new ArrayList(threads)); + nextThreadIterator = this.threads.iterator(); + } + + public SelectorThread nextThread() { + // Choose a selector thread (round robin) + if (!nextThreadIterator.hasNext()) { + nextThreadIterator = threads.iterator(); + } + return nextThreadIterator.next(); + } + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/AutoExpandingBuffer.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/AutoExpandingBuffer.java new file mode 100644 index 0000000..b1c815f --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/AutoExpandingBuffer.java @@ -0,0 +1,49 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.thrift.transport; + +import java.util.Arrays; + +/** + * Helper class that wraps a byte[] so that it can expand and be reused. Users should call + * resizeIfNecessary to make sure the buffer has suitable capacity, and then use the array as + * needed. Note that the internal array will grow at a rate slightly faster than the requested + * capacity with the (untested) objective of avoiding expensive buffer allocations and copies. + */ +public class AutoExpandingBuffer { + private byte[] array; + + public AutoExpandingBuffer(int initialCapacity) { + this.array = new byte[initialCapacity]; + } + + public void resizeIfNecessary(int size) { + final int currentCapacity = this.array.length; + if (currentCapacity < size) { + // Increase by a factor of 1.5x + int growCapacity = currentCapacity + (currentCapacity >> 1); + int newCapacity = Math.max(growCapacity, size); + this.array = Arrays.copyOf(array, newCapacity); + } + } + + public byte[] array() { + return this.array; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/AutoExpandingBufferReadTransport.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/AutoExpandingBufferReadTransport.java new file mode 100644 index 0000000..d59c657 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/AutoExpandingBufferReadTransport.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.thrift.transport; + +import org.apache.thrift.TConfiguration; + +/** TTransport for reading from an AutoExpandingBuffer. */ +public class AutoExpandingBufferReadTransport extends TEndpointTransport { + + private final AutoExpandingBuffer buf; + + private int pos = 0; + private int limit = 0; + + public AutoExpandingBufferReadTransport(TConfiguration config, int initialCapacity) + throws TTransportException { + super(config); + this.buf = new AutoExpandingBuffer(initialCapacity); + } + + public void fill(TTransport inTrans, int length) throws TTransportException { + buf.resizeIfNecessary(length); + inTrans.readAll(buf.array(), 0, length); + pos = 0; + limit = length; + } + + @Override + public void close() {} + + @Override + public boolean isOpen() { + return true; + } + + @Override + public void open() throws TTransportException {} + + @Override + public final int read(byte[] target, int off, int len) throws TTransportException { + int amtToRead = Math.min(len, getBytesRemainingInBuffer()); + if (amtToRead > 0) { + System.arraycopy(buf.array(), pos, target, off, amtToRead); + consumeBuffer(amtToRead); + } + return amtToRead; + } + + @Override + public void write(byte[] buf, int off, int len) throws TTransportException { + throw new UnsupportedOperationException(); + } + + @Override + public final void consumeBuffer(int len) { + pos += len; + } + + @Override + public final byte[] getBuffer() { + return buf.array(); + } + + @Override + public final int getBufferPosition() { + return pos; + } + + @Override + public final int getBytesRemainingInBuffer() { + return limit - pos; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/AutoExpandingBufferWriteTransport.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/AutoExpandingBufferWriteTransport.java new file mode 100644 index 0000000..4888481 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/AutoExpandingBufferWriteTransport.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.thrift.transport; + +import org.apache.thrift.TConfiguration; + +/** TTransport for writing to an AutoExpandingBuffer. */ +public final class AutoExpandingBufferWriteTransport extends TEndpointTransport { + + private final AutoExpandingBuffer buf; + private int pos; + private int res; + + /** + * Constructor. + * + * @param config the configuration to use. Currently used for defining the maximum message size. + * @param initialCapacity the initial capacity of the buffer + * @param frontReserve space, if any, to reserve at the beginning such that the first write is + * after this reserve. This allows framed transport to reserve space for the frame buffer + * length. + * @throws IllegalArgumentException if initialCapacity is less than one + * @throws IllegalArgumentException if frontReserve is less than zero + * @throws IllegalArgumentException if frontReserve is greater than initialCapacity + */ + public AutoExpandingBufferWriteTransport( + TConfiguration config, int initialCapacity, int frontReserve) throws TTransportException { + super(config); + if (initialCapacity < 1) { + throw new IllegalArgumentException("initialCapacity"); + } + if (frontReserve < 0 || initialCapacity < frontReserve) { + throw new IllegalArgumentException("frontReserve"); + } + this.buf = new AutoExpandingBuffer(initialCapacity); + this.pos = frontReserve; + this.res = frontReserve; + } + + @Override + public void close() {} + + @Override + public boolean isOpen() { + return true; + } + + @Override + public void open() throws TTransportException {} + + @Override + public int read(byte[] buf, int off, int len) throws TTransportException { + throw new UnsupportedOperationException(); + } + + @Override + public void write(byte[] toWrite, int off, int len) throws TTransportException { + buf.resizeIfNecessary(pos + len); + System.arraycopy(toWrite, off, buf.array(), pos, len); + pos += len; + } + + public AutoExpandingBuffer getBuf() { + return buf; + } + + /** + * @return length of the buffer, including any front reserve + */ + public int getLength() { + return pos; + } + + public void reset() { + pos = res; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/SocketAddressProvider.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/SocketAddressProvider.java new file mode 100644 index 0000000..1f79941 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/SocketAddressProvider.java @@ -0,0 +1,30 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +import java.net.SocketAddress; + +/** Interface that can retrieve the socket address. */ +public interface SocketAddressProvider { + + SocketAddress getRemoteSocketAddress(); + + SocketAddress getLocalSocketAddress(); +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TByteBuffer.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TByteBuffer.java new file mode 100644 index 0000000..226f02a --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TByteBuffer.java @@ -0,0 +1,111 @@ +package org.apache.thrift.transport; + +import java.nio.BufferOverflowException; +import java.nio.BufferUnderflowException; +import java.nio.ByteBuffer; +import org.apache.thrift.TConfiguration; + +/** ByteBuffer-backed implementation of TTransport. */ +public final class TByteBuffer extends TEndpointTransport { + private final ByteBuffer byteBuffer; + + /** + * Creates a new TByteBuffer wrapping a given NIO ByteBuffer and custom TConfiguration. + * + * @param configuration the custom TConfiguration. + * @param byteBuffer the NIO ByteBuffer to wrap. + * @throws TTransportException on error. + */ + public TByteBuffer(TConfiguration configuration, ByteBuffer byteBuffer) + throws TTransportException { + super(configuration); + this.byteBuffer = byteBuffer; + updateKnownMessageSize(byteBuffer.capacity()); + } + + /** + * Creates a new TByteBuffer wrapping a given NIO ByteBuffer. + * + * @param byteBuffer the NIO ByteBuffer to wrap. + * @throws TTransportException on error. + */ + public TByteBuffer(ByteBuffer byteBuffer) throws TTransportException { + this(new TConfiguration(), byteBuffer); + } + + @Override + public boolean isOpen() { + return true; + } + + @Override + public void open() {} + + @Override + public void close() {} + + @Override + public int read(byte[] buf, int off, int len) throws TTransportException { + // + checkReadBytesAvailable(len); + + final int n = Math.min(byteBuffer.remaining(), len); + if (n > 0) { + try { + byteBuffer.get(buf, off, n); + } catch (BufferUnderflowException e) { + throw new TTransportException("Unexpected end of input buffer", e); + } + } + return n; + } + + @Override + public void write(byte[] buf, int off, int len) throws TTransportException { + try { + byteBuffer.put(buf, off, len); + } catch (BufferOverflowException e) { + throw new TTransportException("Not enough room in output buffer", e); + } + } + + /** + * Gets the underlying NIO ByteBuffer. + * + * @return the underlying NIO ByteBuffer. + */ + public ByteBuffer getByteBuffer() { + return byteBuffer; + } + + /** + * Convenience method to call clear() on the underlying NIO ByteBuffer. + * + * @return this instance. + */ + public TByteBuffer clear() { + byteBuffer.clear(); + return this; + } + + /** + * Convenience method to call flip() on the underlying NIO ByteBuffer. + * + * @return this instance. + */ + public TByteBuffer flip() { + byteBuffer.flip(); + return this; + } + + /** + * Convenience method to convert the underlying NIO ByteBuffer to a plain old byte array. + * + * @return the byte array backing the underlying NIO ByteBuffer. + */ + public byte[] toByteArray() { + final byte[] data = new byte[byteBuffer.remaining()]; + byteBuffer.slice().get(data); + return data; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TEOFException.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TEOFException.java new file mode 100644 index 0000000..52a5cf7 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TEOFException.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +/** End of file, especially, the underlying socket is closed. */ +public class TEOFException extends TTransportException { + + public TEOFException(String message) { + super(TTransportException.END_OF_FILE, message); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TEndpointTransport.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TEndpointTransport.java new file mode 100644 index 0000000..a839488 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TEndpointTransport.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.thrift.transport; + +import java.util.Objects; +import org.apache.thrift.TConfiguration; + +public abstract class TEndpointTransport extends TTransport { + + protected long getMaxMessageSize() { + return getConfiguration().getMaxMessageSize(); + } + + public int getMaxFrameSize() { + return getConfiguration().getMaxFrameSize(); + } + + public void setMaxFrameSize(int maxFrameSize) { + getConfiguration().setMaxFrameSize(maxFrameSize); + } + + public void setMaxMessageSize(int maxMessageSize) { + getConfiguration().setMaxMessageSize(maxMessageSize); + } + + protected long knownMessageSize; + protected long remainingMessageSize; + + private TConfiguration _configuration; + + public TConfiguration getConfiguration() { + return _configuration; + } + + public TEndpointTransport(TConfiguration config) throws TTransportException { + _configuration = Objects.isNull(config) ? new TConfiguration() : config; + + resetConsumedMessageSize(-1); + } + + /** + * Resets RemainingMessageSize to the configured maximum + * + * @param newSize + */ + protected void resetConsumedMessageSize(long newSize) throws TTransportException { + // full reset + if (newSize < 0) { + knownMessageSize = getMaxMessageSize(); + remainingMessageSize = getMaxMessageSize(); + return; + } + + // update only: message size can shrink, but not grow + if (newSize > knownMessageSize) + throw new TTransportException( + TTransportException.MESSAGE_SIZE_LIMIT, + "Message size exceeds limit: " + getMaxMessageSize()); + + knownMessageSize = newSize; + remainingMessageSize = newSize; + } + + /** + * Updates RemainingMessageSize to reflect then known real message size (e.g. framed transport). + * Will throw if we already consumed too many bytes or if the new size is larger than allowed. + * + * @param size + */ + public void updateKnownMessageSize(long size) throws TTransportException { + long consumed = knownMessageSize - remainingMessageSize; + resetConsumedMessageSize(size == 0 ? -1 : size); + countConsumedMessageBytes(consumed); + } + + /** + * Throws if there are not enough bytes in the input stream to satisfy a read of numBytes bytes of + * data + * + * @param numBytes + */ + public void checkReadBytesAvailable(long numBytes) throws TTransportException { + if (remainingMessageSize < numBytes || numBytes < 0) + throw new TTransportException( + TTransportException.MESSAGE_SIZE_LIMIT, + "Message size exceeds limit: " + getMaxMessageSize()); + } + + /** + * Consumes numBytes from the RemainingMessageSize. + * + * @param numBytes + */ + protected void countConsumedMessageBytes(long numBytes) throws TTransportException { + if (remainingMessageSize >= numBytes) { + remainingMessageSize -= numBytes; + } else { + remainingMessageSize = 0; + throw new TTransportException( + TTransportException.MESSAGE_SIZE_LIMIT, + "Message size exceeds limit: " + getMaxMessageSize()); + } + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TFileProcessor.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TFileProcessor.java new file mode 100644 index 0000000..e447aca --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TFileProcessor.java @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +import org.apache.thrift.TException; +import org.apache.thrift.TProcessor; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.protocol.TProtocolFactory; + +/** + * FileProcessor: helps in processing files generated by TFileTransport. Port of original cpp + * implementation + */ +public class TFileProcessor { + + private TProcessor processor_; + private TProtocolFactory inputProtocolFactory_; + private TProtocolFactory outputProtocolFactory_; + private TFileTransport inputTransport_; + private TTransport outputTransport_; + + public TFileProcessor( + TProcessor processor, + TProtocolFactory protocolFactory, + TFileTransport inputTransport, + TTransport outputTransport) { + processor_ = processor; + inputProtocolFactory_ = outputProtocolFactory_ = protocolFactory; + inputTransport_ = inputTransport; + outputTransport_ = outputTransport; + } + + public TFileProcessor( + TProcessor processor, + TProtocolFactory inputProtocolFactory, + TProtocolFactory outputProtocolFactory, + TFileTransport inputTransport, + TTransport outputTransport) { + processor_ = processor; + inputProtocolFactory_ = inputProtocolFactory; + outputProtocolFactory_ = outputProtocolFactory; + inputTransport_ = inputTransport; + outputTransport_ = outputTransport; + } + + private void processUntil(int lastChunk) throws TException { + TProtocol ip = inputProtocolFactory_.getProtocol(inputTransport_); + TProtocol op = outputProtocolFactory_.getProtocol(outputTransport_); + int curChunk = inputTransport_.getCurChunk(); + + try { + while (lastChunk >= curChunk) { + processor_.process(ip, op); + int newChunk = inputTransport_.getCurChunk(); + curChunk = newChunk; + } + } catch (TTransportException e) { + // if we are processing the last chunk - we could have just hit EOF + // on EOF - trap the error and stop processing. + if (e.getType() != TTransportException.END_OF_FILE) throw e; + else { + return; + } + } + } + + /** + * Process from start to last chunk both inclusive where chunks begin from 0 + * + * @param startChunkNum first chunk to be processed + * @param endChunkNum last chunk to be processed + * @throws TException if endChunkNum is less than startChunkNum. + */ + public void processChunk(int startChunkNum, int endChunkNum) throws TException { + int numChunks = inputTransport_.getNumChunks(); + if (endChunkNum < 0) endChunkNum += numChunks; + + if (startChunkNum < 0) startChunkNum += numChunks; + + if (endChunkNum < startChunkNum) + throw new TException("endChunkNum " + endChunkNum + " is less than " + startChunkNum); + + inputTransport_.seekToChunk(startChunkNum); + processUntil(endChunkNum); + } + + /** + * Process a single chunk + * + * @param chunkNum chunk to be processed + * @throws TException on error while processing the given chunk. + */ + public void processChunk(int chunkNum) throws TException { + processChunk(chunkNum, chunkNum); + } + + /** + * Process a current chunk + * + * @throws TException on error while processing the given chunk. + */ + public void processChunk() throws TException { + processChunk(inputTransport_.getCurChunk()); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TFileTransport.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TFileTransport.java new file mode 100644 index 0000000..b067272 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TFileTransport.java @@ -0,0 +1,612 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Random; +import org.apache.thrift.TConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * FileTransport implementation of the TTransport interface. Currently this is a straightforward + * port of the cpp implementation + * + *

It may make better sense to provide a basic stream access on top of the framed file format The + * FileTransport can then be a user of this framed file format with some additional logic for + * chunking. + */ +public class TFileTransport extends TTransport { + + private static final Logger LOGGER = LoggerFactory.getLogger(TFileTransport.class.getName()); + + public static class TruncableBufferedInputStream extends BufferedInputStream { + public void trunc() { + pos = count = 0; + } + + public TruncableBufferedInputStream(InputStream in) { + super(in); + } + + public TruncableBufferedInputStream(InputStream in, int size) { + super(in, size); + } + } + + public static class Event { + private byte[] buf_; + private int nread_; + private int navailable_; + + /** + * Initialize an event. Initially, it has no valid contents + * + * @param buf byte array buffer to store event + */ + public Event(byte[] buf) { + buf_ = buf; + nread_ = navailable_ = 0; + } + + public byte[] getBuf() { + return buf_; + } + + public int getSize() { + return buf_.length; + } + + public void setAvailable(int sz) { + nread_ = 0; + navailable_ = sz; + } + + public int getRemaining() { + return (navailable_ - nread_); + } + + public int emit(byte[] buf, int offset, int ndesired) { + if ((ndesired == 0) || (ndesired > getRemaining())) ndesired = getRemaining(); + + if (ndesired <= 0) return (ndesired); + + System.arraycopy(buf_, nread_, buf, offset, ndesired); + nread_ += ndesired; + + return (ndesired); + } + } + + public static class ChunkState { + /** Chunk Size. Must be same across all implementations */ + public static final int DEFAULT_CHUNK_SIZE = 16 * 1024 * 1024; + + private int chunk_size_ = DEFAULT_CHUNK_SIZE; + private long offset_ = 0; + + public ChunkState() {} + + public ChunkState(int chunk_size) { + chunk_size_ = chunk_size; + } + + public void skip(int size) { + offset_ += size; + } + + public void seek(long offset) { + offset_ = offset; + } + + public int getChunkSize() { + return chunk_size_; + } + + public int getChunkNum() { + return ((int) (offset_ / chunk_size_)); + } + + public int getRemaining() { + return (chunk_size_ - ((int) (offset_ % chunk_size_))); + } + + public long getOffset() { + return (offset_); + } + } + + public enum TailPolicy { + NOWAIT(0, 0), + WAIT_FOREVER(500, -1); + + /** Time in milliseconds to sleep before next read If 0, no sleep */ + public final int timeout_; + + /** Number of retries before giving up if 0, no retries if -1, retry forever */ + public final int retries_; + + /** + * ctor for policy + * + * @param timeout sleep time for this particular policy + * @param retries number of retries + */ + TailPolicy(int timeout, int retries) { + timeout_ = timeout; + retries_ = retries; + } + } + + /** Current tailing policy */ + TailPolicy currentPolicy_ = TailPolicy.NOWAIT; + + /** Underlying file being read */ + protected TSeekableFile inputFile_ = null; + + /** Underlying outputStream */ + protected OutputStream outputStream_ = null; + + /** Event currently read in */ + Event currentEvent_ = null; + + /** InputStream currently being used for reading */ + InputStream inputStream_ = null; + + /** current Chunk state */ + ChunkState cs = null; + + /** is read only? */ + private boolean readOnly_ = false; + + /** + * Get File Tailing Policy + * + * @return current read policy + */ + public TailPolicy getTailPolicy() { + return (currentPolicy_); + } + + /** + * Set file Tailing Policy + * + * @param policy New policy to set + * @return Old policy + */ + public TailPolicy setTailPolicy(TailPolicy policy) { + TailPolicy old = currentPolicy_; + currentPolicy_ = policy; + return (old); + } + + /** + * Initialize read input stream + * + * @return input stream to read from file + */ + private InputStream createInputStream() throws TTransportException { + InputStream is; + try { + if (inputStream_ != null) { + ((TruncableBufferedInputStream) inputStream_).trunc(); + is = inputStream_; + } else { + is = new TruncableBufferedInputStream(inputFile_.getInputStream()); + } + } catch (IOException iox) { + throw new TTransportException(iox.getMessage(), iox); + } + return (is); + } + + /** + * Read (potentially tailing) an input stream + * + * @param is InputStream to read from + * @param buf Buffer to read into + * @param off Offset in buffer to read into + * @param len Number of bytes to read + * @param tp policy to use if we hit EOF + * @return number of bytes read + */ + private int tailRead(InputStream is, byte[] buf, int off, int len, TailPolicy tp) + throws TTransportException { + int orig_len = len; + try { + int retries = 0; + while (len > 0) { + int cnt = is.read(buf, off, len); + if (cnt > 0) { + off += cnt; + len -= cnt; + retries = 0; + cs.skip(cnt); // remember that we read so many bytes + } else if (cnt == -1) { + // EOF + retries++; + + if ((tp.retries_ != -1) && tp.retries_ < retries) return (orig_len - len); + + if (tp.timeout_ > 0) { + try { + Thread.sleep(tp.timeout_); + } catch (InterruptedException e) { + } + } + } else { + // either non-zero or -1 is what the contract says! + throw new TTransportException("Unexpected return from InputStream.read = " + cnt); + } + } + } catch (IOException iox) { + throw new TTransportException(iox.getMessage(), iox); + } + + return (orig_len - len); + } + + /** + * Event is corrupted. Do recovery + * + * @return true if recovery could be performed and we can read more data false is returned only + * when nothing more can be read + */ + private boolean performRecovery() throws TTransportException { + int numChunks = getNumChunks(); + int curChunk = cs.getChunkNum(); + + if (curChunk >= (numChunks - 1)) { + return false; + } + seekToChunk(curChunk + 1); + return true; + } + + /** + * Read event from underlying file + * + * @return true if event could be read, false otherwise (on EOF) + */ + private boolean readEvent() throws TTransportException { + byte[] ebytes = new byte[4]; + int esize; + int nread; + int nrequested; + + retry: + do { + // corner case. read to end of chunk + nrequested = cs.getRemaining(); + if (nrequested < 4) { + nread = tailRead(inputStream_, ebytes, 0, nrequested, currentPolicy_); + if (nread != nrequested) { + return (false); + } + } + + // assuming serialized on little endian machine + nread = tailRead(inputStream_, ebytes, 0, 4, currentPolicy_); + if (nread != 4) { + return (false); + } + + esize = 0; + for (int i = 3; i >= 0; i--) { + int val = (0x000000ff & (int) ebytes[i]); + esize |= (val << (i * 8)); + } + + // check if event is corrupted and do recovery as required + if (esize > cs.getRemaining()) { + throw new TTransportException("FileTransport error: bad event size"); + /* + if(performRecovery()) { + esize=0; + } else { + return false; + } + */ + } + } while (esize == 0); + + // reset existing event or get a larger one + if (currentEvent_.getSize() < esize) currentEvent_ = new Event(new byte[esize]); + + // populate the event + byte[] buf = currentEvent_.getBuf(); + nread = tailRead(inputStream_, buf, 0, esize, currentPolicy_); + if (nread != esize) { + return (false); + } + currentEvent_.setAvailable(esize); + return (true); + } + + /** + * open if both input/output open unless readonly + * + * @return true + */ + public boolean isOpen() { + return ((inputStream_ != null) && (readOnly_ || (outputStream_ != null))); + } + + /** + * Diverging from the cpp model and sticking to the TSocket model Files are not opened in ctor - + * but in explicit open call + */ + public void open() throws TTransportException { + if (isOpen()) throw new TTransportException(TTransportException.ALREADY_OPEN); + + try { + inputStream_ = createInputStream(); + cs = new ChunkState(); + currentEvent_ = new Event(new byte[256]); + + if (!readOnly_) outputStream_ = new BufferedOutputStream(inputFile_.getOutputStream()); + } catch (IOException iox) { + throw new TTransportException(TTransportException.NOT_OPEN, iox); + } + } + + /** Closes the transport. */ + public void close() { + if (inputFile_ != null) { + try { + inputFile_.close(); + } catch (IOException iox) { + LOGGER.warn("WARNING: Error closing input file: " + iox.getMessage()); + } + inputFile_ = null; + } + if (outputStream_ != null) { + try { + outputStream_.close(); + } catch (IOException iox) { + LOGGER.warn("WARNING: Error closing output stream: " + iox.getMessage()); + } + outputStream_ = null; + } + } + + /** + * File Transport ctor + * + * @param path File path to read and write from + * @param readOnly Whether this is a read-only transport + * @throws IOException if there is an error accessing the file. + */ + public TFileTransport(final String path, boolean readOnly) throws IOException { + inputFile_ = new TStandardFile(path); + readOnly_ = readOnly; + } + + /** + * File Transport ctor + * + * @param inputFile open TSeekableFile to read/write from + * @param readOnly Whether this is a read-only transport + */ + public TFileTransport(TSeekableFile inputFile, boolean readOnly) { + inputFile_ = inputFile; + readOnly_ = readOnly; + } + + /** + * Cloned from TTransport.java:readAll(). Only difference is throwing an EOF exception where one + * is detected. + */ + public int readAll(byte[] buf, int off, int len) throws TTransportException { + int got = 0; + int ret = 0; + while (got < len) { + ret = read(buf, off + got, len - got); + if (ret < 0) { + throw new TTransportException("Error in reading from file"); + } + if (ret == 0) { + throw new TTransportException(TTransportException.END_OF_FILE, "End of File reached"); + } + got += ret; + } + return got; + } + + /** + * Reads up to len bytes into buffer buf, starting at offset off. + * + * @param buf Array to read into + * @param off Index to start reading at + * @param len Maximum number of bytes to read + * @return The number of bytes actually read + * @throws TTransportException if there was an error reading data + */ + public int read(byte[] buf, int off, int len) throws TTransportException { + if (!isOpen()) + throw new TTransportException(TTransportException.NOT_OPEN, "Must open before reading"); + + if (currentEvent_.getRemaining() == 0 && !readEvent()) { + return 0; + } + + return currentEvent_.emit(buf, off, len); + } + + public int getNumChunks() throws TTransportException { + if (!isOpen()) + throw new TTransportException(TTransportException.NOT_OPEN, "Must open before getNumChunks"); + try { + long len = inputFile_.length(); + if (len == 0) return 0; + else return (((int) (len / cs.getChunkSize())) + 1); + + } catch (IOException iox) { + throw new TTransportException(iox.getMessage(), iox); + } + } + + public int getCurChunk() throws TTransportException { + if (!isOpen()) + throw new TTransportException(TTransportException.NOT_OPEN, "Must open before getCurChunk"); + return (cs.getChunkNum()); + } + + public void seekToChunk(int chunk) throws TTransportException { + if (!isOpen()) + throw new TTransportException(TTransportException.NOT_OPEN, "Must open before seeking"); + + int numChunks = getNumChunks(); + + // file is empty, seeking to chunk is pointless + if (numChunks == 0) { + return; + } + + // negative indicates reverse seek (from the end) + if (chunk < 0) { + chunk += numChunks; + } + + // too large a value for reverse seek, just seek to beginning + if (chunk < 0) { + chunk = 0; + } + + long eofOffset = 0; + boolean seekToEnd = (chunk >= numChunks); + if (seekToEnd) { + chunk = chunk - 1; + try { + eofOffset = inputFile_.length(); + } catch (IOException iox) { + throw new TTransportException(iox.getMessage(), iox); + } + } + + if (chunk * cs.getChunkSize() != cs.getOffset()) { + try { + inputFile_.seek((long) chunk * cs.getChunkSize()); + } catch (IOException iox) { + throw new TTransportException("Seek to chunk " + chunk + " " + iox.getMessage(), iox); + } + + cs.seek((long) chunk * cs.getChunkSize()); + currentEvent_.setAvailable(0); + inputStream_ = createInputStream(); + } + + if (seekToEnd) { + // waiting forever here - otherwise we can hit EOF and end up + // having consumed partial data from the data stream. + TailPolicy old = setTailPolicy(TailPolicy.WAIT_FOREVER); + while (cs.getOffset() < eofOffset) { + readEvent(); + } + currentEvent_.setAvailable(0); + setTailPolicy(old); + } + } + + public void seekToEnd() throws TTransportException { + if (!isOpen()) + throw new TTransportException(TTransportException.NOT_OPEN, "Must open before seeking"); + seekToChunk(getNumChunks()); + } + + /** + * Writes up to len bytes from the buffer. + * + * @param buf The output data buffer + * @param off The offset to start writing from + * @param len The number of bytes to write + * @throws TTransportException if there was an error writing data + */ + public void write(byte[] buf, int off, int len) throws TTransportException { + throw new TTransportException("Not Supported"); + } + + /** + * Flush any pending data out of a transport buffer. + * + * @throws TTransportException if there was an error writing out data. + */ + public void flush() throws TTransportException { + throw new TTransportException("Not Supported"); + } + + @Override + public TConfiguration getConfiguration() { + return null; + } + + @Override + public void updateKnownMessageSize(long size) throws TTransportException {} + + @Override + public void checkReadBytesAvailable(long numBytes) throws TTransportException {} + + /** test program */ + public static void main(String[] args) throws Exception { + + int num_chunks = 10; + + if ((args.length < 1) + || args[0].equals("--help") + || args[0].equals("-h") + || args[0].equals("-?")) { + printUsage(); + } + + if (args.length > 1) { + try { + num_chunks = Integer.parseInt(args[1]); + } catch (Exception e) { + LOGGER.error("Cannot parse " + args[1]); + printUsage(); + } + } + + TFileTransport t = new TFileTransport(args[0], true); + t.open(); + LOGGER.info("NumChunks=" + t.getNumChunks()); + + Random r = new Random(); + for (int j = 0; j < num_chunks; j++) { + byte[] buf = new byte[4096]; + int cnum = r.nextInt(t.getNumChunks() - 1); + LOGGER.info("Reading chunk " + cnum); + t.seekToChunk(cnum); + for (int i = 0; i < 4096; i++) { + t.read(buf, 0, 4096); + } + } + } + + private static void printUsage() { + LOGGER.error("Usage: TFileTransport [num_chunks]"); + LOGGER.error(" (Opens and reads num_chunks chunks from file randomly)"); + System.exit(1); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/THttpClient.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/THttpClient.java new file mode 100644 index 0000000..4b87d9c --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/THttpClient.java @@ -0,0 +1,329 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URL; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import org.apache.http.HttpHost; +import org.apache.http.client.HttpClient; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.ByteArrayEntity; +import org.apache.thrift.TConfiguration; +import org.apache.thrift.THttpClientResponseHandler; + +/** + * HTTP implementation of the TTransport interface. Used for working with a Thrift web services + * implementation (using for example TServlet). + * + *

This class offers two implementations of the HTTP transport. One uses HttpURLConnection + * instances, the other HttpClient from Apache Http Components. The chosen implementation depends on + * the constructor used to create the THttpClient instance. Using the THttpClient(String url) + * constructor or passing null as the HttpClient to THttpClient(String url, HttpClient client) will + * create an instance which will use HttpURLConnection. + * + *

When using HttpClient, the following configuration leads to 5-15% better performance than the + * HttpURLConnection implementation: + * + *

http.protocol.version=HttpVersion.HTTP_1_1 http.protocol.content-charset=UTF-8 + * http.protocol.expect-continue=false http.connection.stalecheck=false + * + *

Also note that under high load, the HttpURLConnection implementation may exhaust the open file + * descriptor limit. + * + * @see THRIFT-970 + */ +public class THttpClient extends TEndpointTransport { + + private final URL url_; + + private final ByteArrayOutputStream requestBuffer_ = new ByteArrayOutputStream(); + + private InputStream inputStream_ = null; + + private int connectTimeout_ = 0; + + private int readTimeout_ = 0; + + private Map customHeaders_ = null; + + private final HttpHost host; + + private final HttpClient client; + + private static final Map DEFAULT_HEADERS = + Collections.unmodifiableMap(getDefaultHeaders()); + + public static class Factory extends TTransportFactory { + + private final String url; + private final HttpClient client; + + public Factory(String url) { + this.url = url; + this.client = null; + } + + public Factory(String url, HttpClient client) { + this.url = url; + this.client = client; + } + + @Override + public TTransport getTransport(TTransport trans) { + try { + if (null != client) { + return new THttpClient(trans.getConfiguration(), url, client); + } else { + return new THttpClient(trans.getConfiguration(), url); + } + } catch (TTransportException tte) { + return null; + } + } + } + + public THttpClient(TConfiguration config, String url) throws TTransportException { + super(config); + try { + url_ = new URL(url); + this.client = null; + this.host = null; + } catch (IOException iox) { + throw new TTransportException(iox); + } + } + + public THttpClient(String url) throws TTransportException { + super(new TConfiguration()); + try { + url_ = new URL(url); + this.client = null; + this.host = null; + } catch (IOException iox) { + throw new TTransportException(iox); + } + } + + public THttpClient(TConfiguration config, String url, HttpClient client) + throws TTransportException { + super(config); + try { + url_ = new URL(url); + this.client = client; + this.host = + new HttpHost( + url_.getHost(), + -1 == url_.getPort() ? url_.getDefaultPort() : url_.getPort(), + url_.getProtocol()); + } catch (IOException iox) { + throw new TTransportException(iox); + } + } + + public THttpClient(String url, HttpClient client) throws TTransportException { + super(new TConfiguration()); + try { + url_ = new URL(url); + this.client = client; + this.host = + new HttpHost( + url_.getHost(), + -1 == url_.getPort() ? url_.getDefaultPort() : url_.getPort(), + url_.getProtocol()); + } catch (IOException iox) { + throw new TTransportException(iox); + } + } + + public void setConnectTimeout(int timeout) { + connectTimeout_ = timeout; + } + + /** + * Use instead a {@link RequestConfig} pre-configured on the underlying Apache HttpClient (e.g. + * via {@code HttpClientBuilder#setDefaultRequestConfig}). + */ + @Deprecated + public void setReadTimeout(int timeout) { + readTimeout_ = timeout; + } + + public void setCustomHeaders(Map headers) { + customHeaders_ = new HashMap<>(headers); + } + + public void setCustomHeader(String key, String value) { + if (customHeaders_ == null) { + customHeaders_ = new HashMap<>(); + } + customHeaders_.put(key, value); + } + + @Override + public void open() {} + + @Override + public void close() { + if (null != inputStream_) { + try { + inputStream_.close(); + } catch (IOException ioe) { + } + inputStream_ = null; + } + } + + @Override + public boolean isOpen() { + return true; + } + + @Override + public int read(byte[] buf, int off, int len) throws TTransportException { + if (inputStream_ == null) { + throw new TTransportException("Response buffer is empty, no request."); + } + + checkReadBytesAvailable(len); + + try { + int ret = inputStream_.read(buf, off, len); + if (ret == -1) { + throw new TTransportException("No more data available."); + } + countConsumedMessageBytes(ret); + + return ret; + } catch (IOException iox) { + throw new TTransportException(iox); + } + } + + @Override + public void write(byte[] buf, int off, int len) { + requestBuffer_.write(buf, off, len); + } + + private RequestConfig getRequestConfig() { + RequestConfig.Builder builder = RequestConfig.custom(); + if (connectTimeout_ > 0) { + builder.setConnectTimeout(connectTimeout_).setConnectionRequestTimeout(connectTimeout_); + } + if (readTimeout_ > 0) { + builder.setSocketTimeout(readTimeout_); + } + return builder.build(); + } + + private static Map getDefaultHeaders() { + Map headers = new HashMap<>(); + headers.put("Content-Type", "application/x-thrift"); + headers.put("Accept", "application/x-thrift"); + headers.put("User-Agent", "Java/THttpClient/HC"); + return headers; + } + + private void flushUsingHttpClient() throws TTransportException { + if (null == this.client) { + throw new TTransportException("Null HttpClient, aborting."); + } + + // Extract request and reset buffer + byte[] data = requestBuffer_.toByteArray(); + requestBuffer_.reset(); + + HttpPost post = new HttpPost(this.url_.getFile()); + try { + // Set request to path + query string + post.setConfig(getRequestConfig()); + DEFAULT_HEADERS.forEach(post::addHeader); + if (null != customHeaders_) { + customHeaders_.forEach(post::addHeader); + } + post.setEntity(new ByteArrayEntity(data)); + inputStream_ = client.execute(this.host, post, new THttpClientResponseHandler()); + } catch (IOException ioe) { + // Abort method so the connection gets released back to the connection manager + post.abort(); + throw new TTransportException(ioe); + } finally { + resetConsumedMessageSize(-1); + } + } + + public void flush() throws TTransportException { + + if (null != this.client) { + flushUsingHttpClient(); + return; + } + + // Extract request and reset buffer + byte[] data = requestBuffer_.toByteArray(); + requestBuffer_.reset(); + + try { + // Create connection object + HttpURLConnection connection = (HttpURLConnection) url_.openConnection(); + + // Timeouts, only if explicitly set + if (connectTimeout_ > 0) { + connection.setConnectTimeout(connectTimeout_); + } + if (readTimeout_ > 0) { + connection.setReadTimeout(readTimeout_); + } + + // Make the request + connection.setRequestMethod("POST"); + connection.setRequestProperty("Content-Type", "application/x-thrift"); + connection.setRequestProperty("Accept", "application/x-thrift"); + connection.setRequestProperty("User-Agent", "Java/THttpClient"); + if (customHeaders_ != null) { + for (Map.Entry header : customHeaders_.entrySet()) { + connection.setRequestProperty(header.getKey(), header.getValue()); + } + } + connection.setDoOutput(true); + connection.connect(); + connection.getOutputStream().write(data); + + int responseCode = connection.getResponseCode(); + if (responseCode != HttpURLConnection.HTTP_OK) { + throw new TTransportException("HTTP Response code: " + responseCode); + } + + // Read the responses + inputStream_ = connection.getInputStream(); + + } catch (IOException iox) { + throw new TTransportException(iox); + } finally { + resetConsumedMessageSize(-1); + } + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TIOStreamTransport.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TIOStreamTransport.java new file mode 100644 index 0000000..44dd2b0 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TIOStreamTransport.java @@ -0,0 +1,209 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.SocketTimeoutException; +import org.apache.thrift.TConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This is the most commonly used base transport. It takes an InputStream or an OutputStream or both + * and uses it/them to perform transport operations. This allows for compatibility with all the nice + * constructs Java already has to provide a variety of types of streams. + */ +public class TIOStreamTransport extends TEndpointTransport { + + private static final Logger LOGGER = LoggerFactory.getLogger(TIOStreamTransport.class.getName()); + + /** Underlying inputStream */ + protected InputStream inputStream_ = null; + + /** Underlying outputStream */ + protected OutputStream outputStream_ = null; + + /** + * Subclasses can invoke the default constructor and then assign the input streams in the open + * method. + */ + protected TIOStreamTransport(TConfiguration config) throws TTransportException { + super(config); + } + + /** + * Subclasses can invoke the default constructor and then assign the input streams in the open + * method. + */ + protected TIOStreamTransport() throws TTransportException { + super(new TConfiguration()); + } + + /** + * Input stream constructor, constructs an input only transport. + * + * @param config + * @param is Input stream to read from + */ + public TIOStreamTransport(TConfiguration config, InputStream is) throws TTransportException { + super(config); + inputStream_ = is; + } + + /** + * Input stream constructor, constructs an input only transport. + * + * @param is Input stream to read from + */ + public TIOStreamTransport(InputStream is) throws TTransportException { + super(new TConfiguration()); + inputStream_ = is; + } + + /** + * Output stream constructor, constructs an output only transport. + * + * @param config + * @param os Output stream to write to + */ + public TIOStreamTransport(TConfiguration config, OutputStream os) throws TTransportException { + super(config); + outputStream_ = os; + } + + /** + * Output stream constructor, constructs an output only transport. + * + * @param os Output stream to write to + */ + public TIOStreamTransport(OutputStream os) throws TTransportException { + super(new TConfiguration()); + outputStream_ = os; + } + + /** + * Two-way stream constructor. + * + * @param config + * @param is Input stream to read from + * @param os Output stream to read from + */ + public TIOStreamTransport(TConfiguration config, InputStream is, OutputStream os) + throws TTransportException { + super(config); + inputStream_ = is; + outputStream_ = os; + } + + /** + * Two-way stream constructor. + * + * @param is Input stream to read from + * @param os Output stream to read from + */ + public TIOStreamTransport(InputStream is, OutputStream os) throws TTransportException { + super(new TConfiguration()); + inputStream_ = is; + outputStream_ = os; + } + + /** + * @return false after close is called. + */ + public boolean isOpen() { + return inputStream_ != null || outputStream_ != null; + } + + /** The streams must already be open. This method does nothing. */ + public void open() throws TTransportException {} + + /** Closes both the input and output streams. */ + public void close() { + try { + if (inputStream_ != null) { + try { + inputStream_.close(); + } catch (IOException iox) { + LOGGER.warn("Error closing input stream.", iox); + } + } + if (outputStream_ != null) { + try { + outputStream_.close(); + } catch (IOException iox) { + LOGGER.warn("Error closing output stream.", iox); + } + } + } finally { + inputStream_ = null; + outputStream_ = null; + } + } + + /** Reads from the underlying input stream if not null. */ + public int read(byte[] buf, int off, int len) throws TTransportException { + if (inputStream_ == null) { + throw new TTransportException( + TTransportException.NOT_OPEN, "Cannot read from null inputStream"); + } + int bytesRead; + try { + bytesRead = inputStream_.read(buf, off, len); + } catch (SocketTimeoutException ste) { + throw new TTransportException(TTransportException.TIMED_OUT, ste); + } catch (IOException iox) { + throw new TTransportException(TTransportException.UNKNOWN, iox); + } + if (bytesRead < 0) { + throw new TTransportException(TTransportException.END_OF_FILE, "Socket is closed by peer."); + } + return bytesRead; + } + + /** Writes to the underlying output stream if not null. */ + public void write(byte[] buf, int off, int len) throws TTransportException { + if (outputStream_ == null) { + throw new TTransportException( + TTransportException.NOT_OPEN, "Cannot write to null outputStream"); + } + try { + outputStream_.write(buf, off, len); + } catch (IOException iox) { + throw new TTransportException(TTransportException.UNKNOWN, iox); + } + } + + /** Flushes the underlying output stream if not null. */ + public void flush() throws TTransportException { + if (outputStream_ == null) { + throw new TTransportException(TTransportException.NOT_OPEN, "Cannot flush null outputStream"); + } + try { + outputStream_.flush(); + + resetConsumedMessageSize(-1); + + } catch (IOException iox) { + throw new TTransportException(TTransportException.UNKNOWN, iox); + } + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TMemoryBuffer.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TMemoryBuffer.java new file mode 100644 index 0000000..9ce1b21 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TMemoryBuffer.java @@ -0,0 +1,120 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +import java.nio.charset.Charset; +import org.apache.thrift.TByteArrayOutputStream; +import org.apache.thrift.TConfiguration; + +/** Memory buffer-based implementation of the TTransport interface. */ +public class TMemoryBuffer extends TEndpointTransport { + /** + * Create a TMemoryBuffer with an initial buffer size of size. The internal buffer will + * grow as necessary to accommodate the size of the data being written to it. + * + * @param size the initial size of the buffer + * @throws TTransportException on error initializing the underlying transport. + */ + public TMemoryBuffer(int size) throws TTransportException { + super(new TConfiguration()); + arr_ = new TByteArrayOutputStream(size); + updateKnownMessageSize(size); + } + + /** + * Create a TMemoryBuffer with an initial buffer size of size. The internal buffer will + * grow as necessary to accommodate the size of the data being written to it. + * + * @param config the configuration to use. + * @param size the initial size of the buffer + * @throws TTransportException on error initializing the underlying transport. + */ + public TMemoryBuffer(TConfiguration config, int size) throws TTransportException { + super(config); + arr_ = new TByteArrayOutputStream(size); + updateKnownMessageSize(size); + } + + @Override + public boolean isOpen() { + return true; + } + + @Override + public void open() { + /* Do nothing */ + } + + @Override + public void close() { + /* Do nothing */ + } + + @Override + public int read(byte[] buf, int off, int len) throws TTransportException { + checkReadBytesAvailable(len); + byte[] src = arr_.get(); + int amtToRead = (len > arr_.len() - pos_ ? arr_.len() - pos_ : len); + + if (amtToRead > 0) { + System.arraycopy(src, pos_, buf, off, amtToRead); + pos_ += amtToRead; + } + return amtToRead; + } + + @Override + public void write(byte[] buf, int off, int len) { + arr_.write(buf, off, len); + } + + /** + * Output the contents of the memory buffer as a String, using the supplied encoding + * + * @param charset the encoding to use + * @return the contents of the memory buffer as a String + */ + public String toString(Charset charset) { + return arr_.toString(charset); + } + + public String inspect() { + StringBuilder buf = new StringBuilder(); + byte[] bytes = arr_.toByteArray(); + for (int i = 0; i < bytes.length; i++) { + buf.append(pos_ == i ? "==>" : "").append(Integer.toHexString(bytes[i] & 0xff)).append(" "); + } + return buf.toString(); + } + + // The contents of the buffer + private TByteArrayOutputStream arr_; + + // Position to read next byte from + private int pos_; + + public int length() { + return arr_.size(); + } + + public byte[] getArray() { + return arr_.get(); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TMemoryInputTransport.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TMemoryInputTransport.java new file mode 100644 index 0000000..375e2b7 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TMemoryInputTransport.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.thrift.transport; + +import org.apache.thrift.TConfiguration; + +public final class TMemoryInputTransport extends TEndpointTransport { + + private byte[] buf_; + private int pos_; + private int endPos_; + + public TMemoryInputTransport() throws TTransportException { + this(new TConfiguration()); + } + + public TMemoryInputTransport(TConfiguration _configuration) throws TTransportException { + this(_configuration, new byte[0]); + } + + public TMemoryInputTransport(byte[] buf) throws TTransportException { + this(new TConfiguration(), buf); + } + + public TMemoryInputTransport(TConfiguration _configuration, byte[] buf) + throws TTransportException { + this(_configuration, buf, 0, buf.length); + } + + public TMemoryInputTransport(byte[] buf, int offset, int length) throws TTransportException { + this(new TConfiguration(), buf, offset, length); + } + + public TMemoryInputTransport(TConfiguration _configuration, byte[] buf, int offset, int length) + throws TTransportException { + super(_configuration); + reset(buf, offset, length); + updateKnownMessageSize(length); + } + + public void reset(byte[] buf) { + reset(buf, 0, buf.length); + } + + public void reset(byte[] buf, int offset, int length) { + buf_ = buf; + pos_ = offset; + endPos_ = offset + length; + try { + resetConsumedMessageSize(-1); + } catch (TTransportException e) { + // ignore + } + } + + public void clear() { + buf_ = null; + try { + resetConsumedMessageSize(-1); + } catch (TTransportException e) { + // ignore + } + } + + @Override + public void close() {} + + @Override + public boolean isOpen() { + return true; + } + + @Override + public void open() throws TTransportException {} + + @Override + public int read(byte[] buf, int off, int len) throws TTransportException { + int bytesRemaining = getBytesRemainingInBuffer(); + int amtToRead = (len > bytesRemaining ? bytesRemaining : len); + if (amtToRead > 0) { + System.arraycopy(buf_, pos_, buf, off, amtToRead); + consumeBuffer(amtToRead); + countConsumedMessageBytes(amtToRead); + } + return amtToRead; + } + + @Override + public void write(byte[] buf, int off, int len) throws TTransportException { + throw new UnsupportedOperationException("No writing allowed!"); + } + + @Override + public byte[] getBuffer() { + return buf_; + } + + public int getBufferPosition() { + return pos_; + } + + public int getBytesRemainingInBuffer() { + return endPos_ - pos_; + } + + public void consumeBuffer(int len) { + pos_ += len; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TMemoryTransport.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TMemoryTransport.java new file mode 100644 index 0000000..2e38588 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TMemoryTransport.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +import java.nio.ByteBuffer; +import org.apache.thrift.TByteArrayOutputStream; +import org.apache.thrift.TConfiguration; + +/** In memory transport with separate buffers for input and output. */ +public class TMemoryTransport extends TEndpointTransport { + + private final ByteBuffer inputBuffer; + private final TByteArrayOutputStream outputBuffer; + + public TMemoryTransport(byte[] input) throws TTransportException { + super(new TConfiguration()); + inputBuffer = ByteBuffer.wrap(input); + outputBuffer = new TByteArrayOutputStream(1024); + updateKnownMessageSize(input.length); + } + + public TMemoryTransport(TConfiguration config, byte[] input) throws TTransportException { + super(config); + inputBuffer = ByteBuffer.wrap(input); + outputBuffer = new TByteArrayOutputStream(1024); + updateKnownMessageSize(input.length); + } + + @Override + public boolean isOpen() { + return true; + } + + /** Opening on an in memory transport should have no effect. */ + @Override + public void open() { + // Do nothing. + } + + @Override + public void close() { + // Do nothing. + } + + @Override + public int read(byte[] buf, int off, int len) throws TTransportException { + checkReadBytesAvailable(len); + int remaining = inputBuffer.remaining(); + if (remaining < len) { + throw new TTransportException( + TTransportException.END_OF_FILE, + "There's only " + remaining + "bytes, but it asks for " + len); + } + inputBuffer.get(buf, off, len); + return len; + } + + @Override + public void write(byte[] buf, int off, int len) throws TTransportException { + outputBuffer.write(buf, off, len); + } + + /** + * Get all the bytes written by thrift output protocol. + * + * @return a byte array. + */ + public TByteArrayOutputStream getOutput() { + return outputBuffer; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TNonblockingSSLSocket.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TNonblockingSSLSocket.java new file mode 100644 index 0000000..71204f6 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TNonblockingSSLSocket.java @@ -0,0 +1,263 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.SelectionKey; +import java.nio.channels.Selector; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLEngine; +import javax.net.ssl.SSLEngineResult; +import javax.net.ssl.SSLEngineResult.HandshakeStatus; +import javax.net.ssl.SSLException; +import javax.net.ssl.SSLParameters; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Transport for use with async ssl client. */ +public class TNonblockingSSLSocket extends TNonblockingSocket implements SocketAddressProvider { + + private static final Logger LOGGER = + LoggerFactory.getLogger(TNonblockingSSLSocket.class.getName()); + + private final SSLEngine sslEngine_; + + private final ByteBuffer appUnwrap; + private final ByteBuffer netUnwrap; + + private final ByteBuffer netWrap; + + private boolean isHandshakeCompleted; + + private SelectionKey selectionKey; + + private final ExecutorService executorService = Executors.newSingleThreadExecutor(); + + protected TNonblockingSSLSocket(String host, int port, int timeout, SSLContext sslContext) + throws IOException, TTransportException { + super(host, port, timeout); + sslEngine_ = sslContext.createSSLEngine(host, port); + sslEngine_.setUseClientMode(true); + SSLParameters sslParams = sslEngine_.getSSLParameters(); + sslParams.setEndpointIdentificationAlgorithm("HTTPS"); + sslEngine_.setSSLParameters(sslParams); + + int appBufferSize = sslEngine_.getSession().getApplicationBufferSize(); + int netBufferSize = sslEngine_.getSession().getPacketBufferSize(); + appUnwrap = ByteBuffer.allocate(appBufferSize); + netUnwrap = ByteBuffer.allocate(netBufferSize); + netWrap = ByteBuffer.allocate(netBufferSize); + isHandshakeCompleted = false; + } + + /** {@inheritDoc} */ + @Override + public SelectionKey registerSelector(Selector selector, int interests) throws IOException { + selectionKey = super.registerSelector(selector, interests); + return selectionKey; + } + + /** {@inheritDoc} */ + @Override + public boolean isOpen() { + // isConnected() does not return false after close(), but isOpen() does + return super.isOpen() && isHandshakeCompleted; + } + + /** {@inheritDoc} */ + @Override + public void open() throws TTransportException { + throw new RuntimeException("open() is not implemented for TNonblockingSSLSocket"); + } + + /** {@inheritDoc} */ + @Override + public synchronized int read(ByteBuffer buffer) throws TTransportException { + int numBytes = buffer.remaining(); + while (appUnwrap.limit() == appUnwrap.capacity() + || appUnwrap.remaining() < buffer.remaining()) { + if (appUnwrap.limit() < appUnwrap.capacity() && appUnwrap.hasRemaining()) { + buffer.put(appUnwrap); + } + try { + if (doUnwrap() == -1) { + throw new IOException("Unable to read " + numBytes + " bytes"); + } + } catch (IOException iox) { + throw new TTransportException(TTransportException.UNKNOWN, iox); + } + } + if (buffer.hasRemaining()) { + int originLimit = appUnwrap.limit(); + appUnwrap.limit(appUnwrap.position() + buffer.remaining()); + buffer.put(appUnwrap); + appUnwrap.limit(originLimit); + } + // In SSL mode, the Thrift server may merge the frame size and body into a single TLS package. + // Setting OP_WRITE to trigger subsequent read operations in the Thrift async client. + selectionKey.interestOps(SelectionKey.OP_WRITE); + return numBytes; + } + + /** {@inheritDoc} */ + @Override + public synchronized int write(ByteBuffer buffer) throws TTransportException { + int numBytes = buffer.remaining(); + + while (buffer.hasRemaining()) { + try { + if (doWrap(buffer) == -1) { + throw new IOException("Unable to write " + numBytes + " bytes"); + } + } catch (IOException iox) { + throw new TTransportException(TTransportException.UNKNOWN, iox); + } + } + return numBytes; + } + + /** {@inheritDoc} */ + @Override + public void close() { + executorService.shutdown(); + sslEngine_.closeOutbound(); + super.close(); + } + + /** {@inheritDoc} */ + @Override + public boolean startConnect() throws IOException { + if (this.isOpen()) { + return true; + } + sslEngine_.beginHandshake(); + return super.startConnect() && doHandShake(); + } + + /** {@inheritDoc} */ + @Override + public boolean finishConnect() throws IOException { + return super.finishConnect() && doHandShake(); + } + + private synchronized boolean doHandShake() throws IOException { + while (true) { + HandshakeStatus hs = sslEngine_.getHandshakeStatus(); + switch (hs) { + case NEED_UNWRAP: + if (doUnwrap() == -1) { + LOGGER.error("Unexpected. Handshake failed abruptly during unwrap"); + return false; + } + break; + case NEED_WRAP: + if (doWrap(ByteBuffer.wrap(new byte[0])) == -1) { + LOGGER.error("Unexpected. Handshake failed abruptly during wrap"); + return false; + } + break; + case NEED_TASK: + doTask(); + break; + case FINISHED: + case NOT_HANDSHAKING: + isHandshakeCompleted = true; + return true; + default: + LOGGER.error("Unknown handshake status. Handshake failed"); + return false; + } + } + } + + private void doTask() { + Runnable runnable; + while ((runnable = sslEngine_.getDelegatedTask()) != null) { + executorService.submit(runnable); + } + } + + private int doUnwrap() throws IOException { + int num = getSocketChannel().read(netUnwrap); + netUnwrap.flip(); + if (num < 0) { + LOGGER.error("Failed during read operation. Probably server is down"); + return -1; + } + SSLEngineResult unwrapResult; + + try { + appUnwrap.clear(); + unwrapResult = sslEngine_.unwrap(netUnwrap, appUnwrap); + netUnwrap.compact(); + } catch (SSLException ex) { + LOGGER.error(ex.getMessage()); + throw ex; + } + + switch (unwrapResult.getStatus()) { + case OK: + if (appUnwrap.position() > 0) { + appUnwrap.flip(); + } + break; + case CLOSED: + return -1; + case BUFFER_OVERFLOW: + throw new IllegalStateException("Failed to unwrap"); + case BUFFER_UNDERFLOW: + break; + } + return num; + } + + private int doWrap(ByteBuffer appWrap) throws IOException { + int num = 0; + SSLEngineResult wrapResult; + try { + wrapResult = sslEngine_.wrap(appWrap, netWrap); + } catch (SSLException exc) { + LOGGER.error(exc.getMessage()); + throw exc; + } + + switch (wrapResult.getStatus()) { + case OK: + if (netWrap.position() > 0) { + netWrap.flip(); + num = getSocketChannel().write(netWrap); + netWrap.clear(); + } + break; + case BUFFER_UNDERFLOW: + // try again later + break; + case BUFFER_OVERFLOW: + throw new IllegalStateException("Failed to wrap"); + case CLOSED: + LOGGER.error("SSL session is closed"); + return -1; + } + return num; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TNonblockingServerSocket.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TNonblockingServerSocket.java new file mode 100644 index 0000000..0bcf601 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TNonblockingServerSocket.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.SocketException; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.SelectionKey; +import java.nio.channels.Selector; +import java.nio.channels.ServerSocketChannel; +import java.nio.channels.SocketChannel; +import org.apache.thrift.TConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Wrapper around ServerSocketChannel */ +public class TNonblockingServerSocket extends TNonblockingServerTransport { + private static final Logger LOGGER = + LoggerFactory.getLogger(TNonblockingServerSocket.class.getName()); + + /** This channel is where all the nonblocking magic happens. */ + private ServerSocketChannel serverSocketChannel = null; + + /** Underlying ServerSocket object */ + private ServerSocket serverSocket_ = null; + + /** Timeout for client sockets from accept */ + private int clientTimeout_ = 0; + + /** Limit for client sockets request size */ + private int maxFrameSize_ = 0; + + /** Max message size */ + private int maxMessageSize_ = 0; + + public static class NonblockingAbstractServerSocketArgs + extends AbstractServerTransportArgs {} + + /** Creates just a port listening server socket */ + public TNonblockingServerSocket(int port) throws TTransportException { + this(port, 0); + } + + /** Creates just a port listening server socket */ + public TNonblockingServerSocket(int port, int clientTimeout) throws TTransportException { + this(port, clientTimeout, TConfiguration.DEFAULT_MAX_FRAME_SIZE); + } + + public TNonblockingServerSocket(int port, int clientTimeout, int maxFrameSize) + throws TTransportException { + this( + new NonblockingAbstractServerSocketArgs() + .port(port) + .clientTimeout(clientTimeout) + .maxFrameSize(maxFrameSize)); + } + + public TNonblockingServerSocket(InetSocketAddress bindAddr) throws TTransportException { + this(bindAddr, 0); + } + + public TNonblockingServerSocket(InetSocketAddress bindAddr, int clientTimeout) + throws TTransportException { + this(bindAddr, clientTimeout, TConfiguration.DEFAULT_MAX_FRAME_SIZE); + } + + public TNonblockingServerSocket(InetSocketAddress bindAddr, int clientTimeout, int maxFrameSize) + throws TTransportException { + this( + new NonblockingAbstractServerSocketArgs() + .bindAddr(bindAddr) + .clientTimeout(clientTimeout) + .maxFrameSize(maxFrameSize)); + } + + public TNonblockingServerSocket(NonblockingAbstractServerSocketArgs args) + throws TTransportException { + clientTimeout_ = args.clientTimeout; + maxFrameSize_ = args.maxFrameSize; + maxMessageSize_ = args.maxMessageSize; + try { + serverSocketChannel = ServerSocketChannel.open(); + serverSocketChannel.configureBlocking(false); + + // Make server socket + serverSocket_ = serverSocketChannel.socket(); + // Prevent 2MSL delay problem on server restarts + serverSocket_.setReuseAddress(true); + // Bind to listening port + serverSocket_.bind(args.bindAddr, args.backlog); + } catch (IOException ioe) { + serverSocket_ = null; + throw new TTransportException( + "Could not create ServerSocket on address " + args.bindAddr.toString() + ".", ioe); + } + } + + public void listen() throws TTransportException { + // Make sure not to block on accept + if (serverSocket_ != null) { + try { + serverSocket_.setSoTimeout(0); + } catch (SocketException sx) { + LOGGER.error("Socket exception while setting socket timeout", sx); + } + } + } + + @Override + public TNonblockingSocket accept() throws TTransportException { + if (serverSocket_ == null) { + throw new TTransportException(TTransportException.NOT_OPEN, "No underlying server socket."); + } + try { + SocketChannel socketChannel = serverSocketChannel.accept(); + if (socketChannel == null) { + return null; + } + + TNonblockingSocket tsocket = new TNonblockingSocket(socketChannel); + tsocket.setTimeout(clientTimeout_); + tsocket.setMaxFrameSize(maxFrameSize_); + tsocket.setMaxMessageSize(maxMessageSize_); + return tsocket; + } catch (IOException iox) { + throw new TTransportException(iox); + } + } + + public void registerSelector(Selector selector) { + try { + // Register the server socket channel, indicating an interest in + // accepting new connections + serverSocketChannel.register(selector, SelectionKey.OP_ACCEPT); + } catch (ClosedChannelException e) { + // this shouldn't happen, ideally... + // TODO: decide what to do with this. + } + } + + public void close() { + if (serverSocket_ != null) { + try { + serverSocket_.close(); + } catch (IOException iox) { + LOGGER.warn("WARNING: Could not close server socket: " + iox.getMessage()); + } + serverSocket_ = null; + } + } + + public void interrupt() { + // The thread-safeness of this is dubious, but Java documentation suggests + // that it is safe to do this from a different thread context + close(); + } + + public int getPort() { + if (serverSocket_ == null) return -1; + return serverSocket_.getLocalPort(); + } + + // Expose it for test purpose. + ServerSocketChannel getServerSocketChannel() { + return serverSocketChannel; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TNonblockingServerTransport.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TNonblockingServerTransport.java new file mode 100644 index 0000000..8dfdefb --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TNonblockingServerTransport.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +import java.nio.channels.Selector; + +/** Server transport that can be operated in a nonblocking fashion. */ +public abstract class TNonblockingServerTransport extends TServerTransport { + + public abstract void registerSelector(Selector selector); + + /** + * @return an incoming connection or null if there is none. + * @throws TTransportException on error during this operation. + */ + @Override + public abstract TNonblockingTransport accept() throws TTransportException; +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TNonblockingSocket.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TNonblockingSocket.java new file mode 100644 index 0000000..40146c8 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TNonblockingSocket.java @@ -0,0 +1,214 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.net.SocketAddress; +import java.net.SocketException; +import java.nio.ByteBuffer; +import java.nio.channels.SelectionKey; +import java.nio.channels.Selector; +import java.nio.channels.SocketChannel; +import org.apache.thrift.TConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Transport for use with async client. */ +public class TNonblockingSocket extends TNonblockingTransport implements SocketAddressProvider { + + private static final Logger LOGGER = LoggerFactory.getLogger(TNonblockingSocket.class.getName()); + + /** Host and port if passed in, used for lazy non-blocking connect. */ + private final SocketAddress socketAddress_; + + private final SocketChannel socketChannel_; + + public TNonblockingSocket(String host, int port) throws IOException, TTransportException { + this(host, port, 0); + } + + /** + * Create a new nonblocking socket transport that will be connected to host:port. + * + * @param host + * @param port + * @throws IOException + */ + public TNonblockingSocket(String host, int port, int timeout) + throws IOException, TTransportException { + this(SocketChannel.open(), timeout, new InetSocketAddress(host, port)); + } + + /** + * Constructor that takes an already created socket. + * + * @param socketChannel Already created SocketChannel object + * @throws IOException if there is an error setting up the streams + */ + public TNonblockingSocket(SocketChannel socketChannel) throws IOException, TTransportException { + this(socketChannel, 0, null); + if (!socketChannel.isConnected()) throw new IOException("Socket must already be connected"); + } + + private TNonblockingSocket(SocketChannel socketChannel, int timeout, SocketAddress socketAddress) + throws IOException, TTransportException { + this(new TConfiguration(), socketChannel, timeout, socketAddress); + } + + private TNonblockingSocket( + TConfiguration config, SocketChannel socketChannel, int timeout, SocketAddress socketAddress) + throws IOException, TTransportException { + super(config); + socketChannel_ = socketChannel; + socketAddress_ = socketAddress; + + // make it a nonblocking channel + socketChannel.configureBlocking(false); + + // set options + Socket socket = socketChannel.socket(); + socket.setSoLinger(false, 0); + socket.setTcpNoDelay(true); + socket.setKeepAlive(true); + setTimeout(timeout); + } + + /** + * Register the new SocketChannel with our Selector, indicating we'd like to be notified when it's + * ready for I/O. + * + * @param selector + * @return the selection key for this socket. + */ + public SelectionKey registerSelector(Selector selector, int interests) throws IOException { + return socketChannel_.register(selector, interests); + } + + /** + * Sets the socket timeout, although this implementation never uses blocking operations so it is + * unused. + * + * @param timeout Milliseconds timeout + */ + public void setTimeout(int timeout) { + try { + socketChannel_.socket().setSoTimeout(timeout); + } catch (SocketException sx) { + LOGGER.warn("Could not set socket timeout.", sx); + } + } + + /** Returns a reference to the underlying SocketChannel. */ + public SocketChannel getSocketChannel() { + return socketChannel_; + } + + /** Checks whether the socket is connected. */ + public boolean isOpen() { + // isConnected() does not return false after close(), but isOpen() does + return socketChannel_.isOpen() && socketChannel_.isConnected(); + } + + /** Do not call, the implementation provides its own lazy non-blocking connect. */ + public void open() throws TTransportException { + throw new RuntimeException("open() is not implemented for TNonblockingSocket"); + } + + /** Perform a nonblocking read into buffer. */ + public int read(ByteBuffer buffer) throws TTransportException { + try { + return socketChannel_.read(buffer); + } catch (IOException iox) { + throw new TTransportException(TTransportException.UNKNOWN, iox); + } + } + + /** Reads from the underlying input stream if not null. */ + public int read(byte[] buf, int off, int len) throws TTransportException { + if ((socketChannel_.validOps() & SelectionKey.OP_READ) != SelectionKey.OP_READ) { + throw new TTransportException( + TTransportException.NOT_OPEN, "Cannot read from write-only socket channel"); + } + return read(ByteBuffer.wrap(buf, off, len)); + } + + /** Perform a nonblocking write of the data in buffer; */ + public int write(ByteBuffer buffer) throws TTransportException { + try { + return socketChannel_.write(buffer); + } catch (IOException iox) { + throw new TTransportException(TTransportException.UNKNOWN, iox); + } + } + + /** Writes to the underlying output stream if not null. */ + public void write(byte[] buf, int off, int len) throws TTransportException { + if ((socketChannel_.validOps() & SelectionKey.OP_WRITE) != SelectionKey.OP_WRITE) { + throw new TTransportException( + TTransportException.NOT_OPEN, "Cannot write to write-only socket channel"); + } + write(ByteBuffer.wrap(buf, off, len)); + } + + /** Noop. */ + public void flush() throws TTransportException { + // Not supported by SocketChannel. + } + + /** Closes the socket. */ + public void close() { + try { + socketChannel_.close(); + } catch (IOException iox) { + LOGGER.warn("Could not close socket.", iox); + } + } + + /** {@inheritDoc} */ + public boolean startConnect() throws IOException { + return socketChannel_.connect(socketAddress_); + } + + /** {@inheritDoc} */ + public boolean finishConnect() throws IOException { + return socketChannel_.finishConnect(); + } + + @Override + public String toString() { + return "[remote: " + + socketChannel_.socket().getRemoteSocketAddress() + + ", local: " + + socketChannel_.socket().getLocalAddress() + + "]"; + } + + @Override + public SocketAddress getRemoteSocketAddress() { + return socketChannel_.socket().getRemoteSocketAddress(); + } + + @Override + public SocketAddress getLocalSocketAddress() { + return socketChannel_.socket().getLocalSocketAddress(); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TNonblockingTransport.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TNonblockingTransport.java new file mode 100644 index 0000000..ab3028c --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TNonblockingTransport.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +import java.io.IOException; +import java.net.SocketAddress; +import java.nio.channels.SelectionKey; +import java.nio.channels.Selector; +import org.apache.thrift.TConfiguration; + +public abstract class TNonblockingTransport extends TEndpointTransport { + + public TNonblockingTransport(TConfiguration config) throws TTransportException { + super(config); + } + + /** + * Non-blocking connection initialization. + * + * @see java.nio.channels.SocketChannel#connect(SocketAddress remote) + */ + public abstract boolean startConnect() throws IOException; + + /** + * Non-blocking connection completion. + * + * @see java.nio.channels.SocketChannel#finishConnect() + */ + public abstract boolean finishConnect() throws IOException; + + public abstract SelectionKey registerSelector(Selector selector, int interests) + throws IOException; +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TSSLTransportFactory.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TSSLTransportFactory.java new file mode 100644 index 0000000..9a08b19 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TSSLTransportFactory.java @@ -0,0 +1,528 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.net.InetAddress; +import java.net.MalformedURLException; +import java.net.URL; +import java.security.KeyStore; +import java.util.Arrays; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLParameters; +import javax.net.ssl.SSLServerSocket; +import javax.net.ssl.SSLServerSocketFactory; +import javax.net.ssl.SSLSocket; +import javax.net.ssl.SSLSocketFactory; +import javax.net.ssl.TrustManagerFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A Factory for providing and setting up Client and Server SSL wrapped TSocket and TServerSocket + */ +public class TSSLTransportFactory { + + private static final Logger LOGGER = LoggerFactory.getLogger(TSSLTransportFactory.class); + + /** + * Get a SSL wrapped TServerSocket bound to the specified port. In this configuration the default + * settings are used. Default settings are retrieved from System properties that are set. + * + *

Example system properties: -Djavax.net.ssl.trustStore=<truststore location> + * -Djavax.net.ssl.trustStorePassword=password -Djavax.net.ssl.keyStore=<keystore location> + * -Djavax.net.ssl.keyStorePassword=password + * + * @param port server port + * @return A SSL wrapped TServerSocket + * @throws TTransportException when failed to create server socket + */ + public static TServerSocket getServerSocket(int port) throws TTransportException { + return getServerSocket(port, 0); + } + + /** + * Get a default SSL wrapped TServerSocket bound to the specified port + * + * @param port + * @param clientTimeout + * @return A SSL wrapped TServerSocket + * @throws TTransportException + */ + public static TServerSocket getServerSocket(int port, int clientTimeout) + throws TTransportException { + return getServerSocket(port, clientTimeout, false, null); + } + + /** + * Get a default SSL wrapped TServerSocket bound to the specified port and interface + * + * @param port + * @param clientTimeout + * @param ifAddress + * @return A SSL wrapped TServerSocket + * @throws TTransportException + */ + public static TServerSocket getServerSocket( + int port, int clientTimeout, boolean clientAuth, InetAddress ifAddress) + throws TTransportException { + SSLServerSocketFactory factory = (SSLServerSocketFactory) SSLServerSocketFactory.getDefault(); + return createServer(factory, port, clientTimeout, clientAuth, ifAddress, null); + } + + /** + * Get a configured SSL wrapped TServerSocket bound to the specified port and interface. Here the + * TSSLTransportParameters are used to set the values for the algorithms, keystore, truststore and + * other settings + * + * @param port + * @param clientTimeout + * @param ifAddress + * @param params + * @return A SSL wrapped TServerSocket + * @throws TTransportException + */ + public static TServerSocket getServerSocket( + int port, int clientTimeout, InetAddress ifAddress, TSSLTransportParameters params) + throws TTransportException { + if (params == null || !(params.isKeyStoreSet || params.isTrustStoreSet)) { + throw new TTransportException( + "Either one of the KeyStore or TrustStore must be set for SSLTransportParameters"); + } + + SSLContext ctx = createSSLContext(params); + return createServer( + ctx.getServerSocketFactory(), port, clientTimeout, params.clientAuth, ifAddress, params); + } + + private static TServerSocket createServer( + SSLServerSocketFactory factory, + int port, + int timeout, + boolean clientAuth, + InetAddress ifAddress, + TSSLTransportParameters params) + throws TTransportException { + try { + SSLServerSocket serverSocket = + (SSLServerSocket) factory.createServerSocket(port, 100, ifAddress); + serverSocket.setSoTimeout(timeout); + serverSocket.setNeedClientAuth(clientAuth); + if (params != null && params.cipherSuites != null) { + serverSocket.setEnabledCipherSuites(params.cipherSuites); + } + return new TServerSocket( + new TServerSocket.ServerSocketTransportArgs() + .serverSocket(serverSocket) + .clientTimeout(timeout)); + } catch (Exception e) { + throw new TTransportException("Could not bind to port " + port, e); + } + } + + /** + * Get a default SSL wrapped TSocket connected to the specified host and port. All the client + * methods return a bound connection. So there is no need to call open() on the TTransport. + * + * @param host + * @param port + * @param timeout + * @return A SSL wrapped TSocket + * @throws TTransportException + */ + public static TSocket getClientSocket(String host, int port, int timeout) + throws TTransportException { + SSLSocketFactory factory = (SSLSocketFactory) SSLSocketFactory.getDefault(); + return createClient(factory, host, port, timeout); + } + + /** + * Get a default SSL wrapped TSocket connected to the specified host and port. + * + * @param host + * @param port + * @return A SSL wrapped TSocket + * @throws TTransportException + */ + public static TSocket getClientSocket(String host, int port) throws TTransportException { + return getClientSocket(host, port, 0); + } + + /** + * Get a custom configured SSL wrapped TSocket. The SSL settings are obtained from the passed in + * TSSLTransportParameters. + * + * @param host + * @param port + * @param timeout + * @param params + * @return A SSL wrapped TSocket + * @throws TTransportException + */ + public static TSocket getClientSocket( + String host, int port, int timeout, TSSLTransportParameters params) + throws TTransportException { + if (params == null || !(params.isKeyStoreSet || params.isTrustStoreSet)) { + throw new TTransportException( + TTransportException.NOT_OPEN, + "Either one of the KeyStore or TrustStore must be set for SSLTransportParameters"); + } + + SSLContext ctx = createSSLContext(params); + return createClient(ctx.getSocketFactory(), host, port, timeout); + } + + /** + * Get a default SSL wrapped TNonblockingTransport connected to the specified host and port. + * + * @param host + * @param port + * @return A SSL wrapped TNonblockingSocket + * @throws TTransportException + */ + public static TNonblockingSocket getNonblockingClientSocket(String host, int port) + throws TTransportException, IOException { + return getNonblockingClientSocket(host, port, 0); + } + + /** + * Get a default SSL wrapped TNonblockingTransport connected to the specified host and port. + * + * @param host + * @param port + * @param timeout + * @return A SSL wrapped TNonblockingSocket + * @throws TTransportException + */ + public static TNonblockingSocket getNonblockingClientSocket(String host, int port, int timeout) + throws TTransportException, IOException { + SSLContext ctx; + try { + ctx = SSLContext.getDefault(); + } catch (Exception e) { + throw new TTransportException( + TTransportException.NOT_OPEN, "Error creating the transport", e); + } + return new TNonblockingSSLSocket(host, port, timeout, ctx); + } + + /** + * Get a custom configured TNonblockingTransport. The SSL settings are obtained from the passed in + * TSSLTransportParameters. + * + * @param host + * @param port + * @param timeout + * @param params + * @return A SSL wrapped TNonblockingSocket + * @throws TTransportException + */ + public static TNonblockingSocket getNonblockingClientSocket( + String host, int port, int timeout, TSSLTransportParameters params) + throws TTransportException, IOException { + if (params == null || !(params.isKeyStoreSet || params.isTrustStoreSet)) { + throw new TTransportException( + "Either one of the KeyStore or TrustStore must be set for SSLTransportParameters"); + } + SSLContext ctx = createSSLContext(params); + return new TNonblockingSSLSocket(host, port, timeout, ctx); + } + + private static SSLContext createSSLContext(TSSLTransportParameters params) + throws TTransportException { + SSLContext ctx; + InputStream in = null; + InputStream is = null; + + try { + ctx = SSLContext.getInstance(params.protocol); + TrustManagerFactory tmf = null; + KeyManagerFactory kmf = null; + + if (params.isTrustStoreSet) { + tmf = TrustManagerFactory.getInstance(params.trustManagerType); + KeyStore ts = KeyStore.getInstance(params.trustStoreType); + if (params.trustStoreStream != null) { + in = params.trustStoreStream; + } else { + in = getStoreAsStream(params.trustStore); + } + ts.load(in, (params.trustPass != null ? params.trustPass.toCharArray() : null)); + tmf.init(ts); + } + + if (params.isKeyStoreSet) { + kmf = KeyManagerFactory.getInstance(params.keyManagerType); + KeyStore ks = KeyStore.getInstance(params.keyStoreType); + if (params.keyStoreStream != null) { + is = params.keyStoreStream; + } else { + is = getStoreAsStream(params.keyStore); + } + ks.load(is, params.keyPass.toCharArray()); + kmf.init(ks, params.keyPass.toCharArray()); + } + + if (params.isKeyStoreSet && params.isTrustStoreSet) { + ctx.init(kmf.getKeyManagers(), tmf.getTrustManagers(), null); + } else if (params.isKeyStoreSet) { + ctx.init(kmf.getKeyManagers(), null, null); + } else { + ctx.init(null, tmf.getTrustManagers(), null); + } + + } catch (Exception e) { + throw new TTransportException( + TTransportException.NOT_OPEN, "Error creating the transport", e); + } finally { + if (in != null) { + try { + in.close(); + } catch (IOException e) { + LOGGER.warn("Unable to close stream", e); + } + } + if (is != null) { + try { + is.close(); + } catch (IOException e) { + LOGGER.warn("Unable to close stream", e); + } + } + } + + return ctx; + } + + private static InputStream getStoreAsStream(String store) throws IOException { + try { + return new FileInputStream(store); + } catch (FileNotFoundException e) { + } + + InputStream storeStream; + try { + storeStream = new URL(store).openStream(); + if (storeStream != null) { + return storeStream; + } + } catch (MalformedURLException e) { + } + + storeStream = Thread.currentThread().getContextClassLoader().getResourceAsStream(store); + + if (storeStream != null) { + return storeStream; + } else { + throw new IOException("Could not load file: " + store); + } + } + + private static TSocket createClient(SSLSocketFactory factory, String host, int port, int timeout) + throws TTransportException { + try { + SSLSocket socket = (SSLSocket) factory.createSocket(host, port); + socket.setSoTimeout(timeout); + SSLParameters sslParams = socket.getSSLParameters(); + sslParams.setEndpointIdentificationAlgorithm("HTTPS"); + socket.setSSLParameters(sslParams); + return new TSocket(socket); + } catch (TTransportException tte) { + throw tte; + } catch (Exception e) { + throw new TTransportException( + TTransportException.NOT_OPEN, "Could not connect to " + host + " on port " + port, e); + } + } + + /** A Class to hold all the SSL parameters */ + public static class TSSLTransportParameters { + protected String protocol = "TLS"; + protected String keyStore; + protected InputStream keyStoreStream; + protected String keyPass; + protected String keyManagerType = KeyManagerFactory.getDefaultAlgorithm(); + protected String keyStoreType = "JKS"; + protected String trustStore; + protected InputStream trustStoreStream; + protected String trustPass; + protected String trustManagerType = TrustManagerFactory.getDefaultAlgorithm(); + protected String trustStoreType = "JKS"; + protected String[] cipherSuites; + protected boolean clientAuth = false; + protected boolean isKeyStoreSet = false; + protected boolean isTrustStoreSet = false; + + public TSSLTransportParameters() {} + + /** + * Create parameters specifying the protocol and cipher suites + * + * @param protocol The specific protocol (TLS/SSL) can be specified with versions + * @param cipherSuites + */ + public TSSLTransportParameters(String protocol, String[] cipherSuites) { + this(protocol, cipherSuites, false); + } + + /** + * Create parameters specifying the protocol, cipher suites and if client authentication is + * required + * + * @param protocol The specific protocol (TLS/SSL) can be specified with versions + * @param cipherSuites + * @param clientAuth + */ + public TSSLTransportParameters(String protocol, String[] cipherSuites, boolean clientAuth) { + if (protocol != null) { + this.protocol = protocol; + } + this.cipherSuites = + cipherSuites != null ? Arrays.copyOf(cipherSuites, cipherSuites.length) : null; + this.clientAuth = clientAuth; + } + + /** + * Set the keystore, password, certificate type and the store type + * + * @param keyStore Location of the Keystore on disk + * @param keyPass Keystore password + * @param keyManagerType The default is X509 + * @param keyStoreType The default is JKS + */ + public void setKeyStore( + String keyStore, String keyPass, String keyManagerType, String keyStoreType) { + this.keyStore = keyStore; + this.keyPass = keyPass; + if (keyManagerType != null) { + this.keyManagerType = keyManagerType; + } + if (keyStoreType != null) { + this.keyStoreType = keyStoreType; + } + isKeyStoreSet = true; + } + + /** + * Set the keystore, password, certificate type and the store type + * + * @param keyStoreStream Keystore content input stream + * @param keyPass Keystore password + * @param keyManagerType The default is X509 + * @param keyStoreType The default is JKS + */ + public void setKeyStore( + InputStream keyStoreStream, String keyPass, String keyManagerType, String keyStoreType) { + this.keyStoreStream = keyStoreStream; + setKeyStore("", keyPass, keyManagerType, keyStoreType); + } + + /** + * Set the keystore and password + * + * @param keyStore Location of the Keystore on disk + * @param keyPass Keystore password + */ + public void setKeyStore(String keyStore, String keyPass) { + setKeyStore(keyStore, keyPass, null, null); + } + + /** + * Set the keystore and password + * + * @param keyStoreStream Keystore content input stream + * @param keyPass Keystore password + */ + public void setKeyStore(InputStream keyStoreStream, String keyPass) { + setKeyStore(keyStoreStream, keyPass, null, null); + } + + /** + * Set the truststore, password, certificate type and the store type + * + * @param trustStore Location of the Truststore on disk + * @param trustPass Truststore password + * @param trustManagerType The default is X509 + * @param trustStoreType The default is JKS + */ + public void setTrustStore( + String trustStore, String trustPass, String trustManagerType, String trustStoreType) { + this.trustStore = trustStore; + this.trustPass = trustPass; + if (trustManagerType != null) { + this.trustManagerType = trustManagerType; + } + if (trustStoreType != null) { + this.trustStoreType = trustStoreType; + } + isTrustStoreSet = true; + } + + /** + * Set the truststore, password, certificate type and the store type + * + * @param trustStoreStream Truststore content input stream + * @param trustPass Truststore password + * @param trustManagerType The default is X509 + * @param trustStoreType The default is JKS + */ + public void setTrustStore( + InputStream trustStoreStream, + String trustPass, + String trustManagerType, + String trustStoreType) { + this.trustStoreStream = trustStoreStream; + setTrustStore("", trustPass, trustManagerType, trustStoreType); + } + + /** + * Set the truststore and password + * + * @param trustStore Location of the Truststore on disk + * @param trustPass Truststore password + */ + public void setTrustStore(String trustStore, String trustPass) { + setTrustStore(trustStore, trustPass, null, null); + } + + /** + * Set the truststore and password + * + * @param trustStoreStream Truststore content input stream + * @param trustPass Truststore password + */ + public void setTrustStore(InputStream trustStoreStream, String trustPass) { + setTrustStore(trustStoreStream, trustPass, null, null); + } + + /** + * Set if client authentication is required + * + * @param clientAuth + */ + public void requireClientAuth(boolean clientAuth) { + this.clientAuth = clientAuth; + } + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TSaslClientTransport.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TSaslClientTransport.java new file mode 100644 index 0000000..19cc41e --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TSaslClientTransport.java @@ -0,0 +1,109 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +import java.nio.charset.StandardCharsets; +import java.util.Map; +import javax.security.auth.callback.CallbackHandler; +import javax.security.sasl.Sasl; +import javax.security.sasl.SaslClient; +import javax.security.sasl.SaslException; +import org.apache.thrift.transport.sasl.NegotiationStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Wraps another Thrift TTransport, but performs SASL client negotiation on the call to + * open(). This class will wrap ensuing communication over it, if a SASL QOP is + * negotiated with the other party. + */ +public class TSaslClientTransport extends TSaslTransport { + + private static final Logger LOGGER = LoggerFactory.getLogger(TSaslClientTransport.class); + + /** The name of the mechanism this client supports. */ + private final String mechanism; + + /** + * Uses the given SaslClient. + * + * @param saslClient The SaslClient to use for the subsequent SASL negotiation. + * @param transport Transport underlying this one. + */ + public TSaslClientTransport(SaslClient saslClient, TTransport transport) + throws TTransportException { + super(saslClient, transport); + mechanism = saslClient.getMechanismName(); + } + + /** + * Creates a SaslClient using the given SASL-specific parameters. See the Java + * documentation for Sasl.createSaslClient for the details of the parameters. + * + * @param transport The underlying Thrift transport. + * @throws SaslException + */ + public TSaslClientTransport( + String mechanism, + String authorizationId, + String protocol, + String serverName, + Map props, + CallbackHandler cbh, + TTransport transport) + throws SaslException, TTransportException { + super( + Sasl.createSaslClient( + new String[] {mechanism}, authorizationId, protocol, serverName, props, cbh), + transport); + this.mechanism = mechanism; + } + + @Override + protected SaslRole getRole() { + return SaslRole.CLIENT; + } + + /** + * Performs the client side of the initial portion of the Thrift SASL protocol. Generates and + * sends the initial response to the server, including which mechanism this client wants to use. + */ + @Override + protected void handleSaslStartMessage() throws TTransportException, SaslException { + SaslClient saslClient = getSaslClient(); + + byte[] initialResponse = new byte[0]; + if (saslClient.hasInitialResponse()) + initialResponse = saslClient.evaluateChallenge(initialResponse); + + LOGGER.debug( + "Sending mechanism name {} and initial response of length {}", + mechanism, + initialResponse.length); + + byte[] mechanismBytes = mechanism.getBytes(StandardCharsets.UTF_8); + sendSaslMessage(NegotiationStatus.START, mechanismBytes); + // Send initial response + sendSaslMessage( + saslClient.isComplete() ? NegotiationStatus.COMPLETE : NegotiationStatus.OK, + initialResponse); + underlyingTransport.flush(); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TSaslServerTransport.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TSaslServerTransport.java new file mode 100644 index 0000000..42a838c --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TSaslServerTransport.java @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +import java.lang.ref.WeakReference; +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.WeakHashMap; +import javax.security.auth.callback.CallbackHandler; +import javax.security.sasl.Sasl; +import javax.security.sasl.SaslException; +import javax.security.sasl.SaslServer; +import org.apache.thrift.transport.sasl.NegotiationStatus; +import org.apache.thrift.transport.sasl.TSaslServerDefinition; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Wraps another Thrift TTransport, but performs SASL server negotiation on the call to + * open(). This class will wrap ensuing communication over it, if a SASL QOP is + * negotiated with the other party. + */ +public class TSaslServerTransport extends TSaslTransport { + + private static final Logger LOGGER = LoggerFactory.getLogger(TSaslServerTransport.class); + + /** + * Mapping from SASL mechanism name -> all the parameters required to instantiate a SASL server. + */ + private Map serverDefinitionMap = + new HashMap(); + + /** + * Uses the given underlying transport. Assumes that addServerDefinition is called later. + * + * @param transport Transport underlying this one. + */ + public TSaslServerTransport(TTransport transport) throws TTransportException { + super(transport); + } + + /** + * Creates a SaslServer using the given SASL-specific parameters. See the Java + * documentation for Sasl.createSaslServer for the details of the parameters. + * + * @param transport The underlying Thrift transport. + */ + public TSaslServerTransport( + String mechanism, + String protocol, + String serverName, + Map props, + CallbackHandler cbh, + TTransport transport) + throws TTransportException { + super(transport); + addServerDefinition(mechanism, protocol, serverName, props, cbh); + } + + private TSaslServerTransport( + Map serverDefinitionMap, TTransport transport) + throws TTransportException { + super(transport); + this.serverDefinitionMap.putAll(serverDefinitionMap); + } + + /** + * Add a supported server definition to this transport. See the Java documentation for + * Sasl.createSaslServer for the details of the parameters. + */ + public void addServerDefinition( + String mechanism, + String protocol, + String serverName, + Map props, + CallbackHandler cbh) { + serverDefinitionMap.put( + mechanism, new TSaslServerDefinition(mechanism, protocol, serverName, props, cbh)); + } + + @Override + protected SaslRole getRole() { + return SaslRole.SERVER; + } + + /** + * Performs the server side of the initial portion of the Thrift SASL protocol. Receives the + * initial response from the client, creates a SASL server using the mechanism requested by the + * client (if this server supports it), and sends the first challenge back to the client. + */ + @Override + protected void handleSaslStartMessage() throws TTransportException, SaslException { + SaslResponse message = receiveSaslMessage(); + + LOGGER.debug("Received start message with status {}", message.status); + if (message.status != NegotiationStatus.START) { + throw sendAndThrowMessage( + NegotiationStatus.ERROR, "Expecting START status, received " + message.status); + } + + // Get the mechanism name. + String mechanismName = new String(message.payload, StandardCharsets.UTF_8); + TSaslServerDefinition serverDefinition = serverDefinitionMap.get(mechanismName); + LOGGER.debug("Received mechanism name '{}'", mechanismName); + + if (serverDefinition == null) { + throw sendAndThrowMessage( + NegotiationStatus.BAD, "Unsupported mechanism type " + mechanismName); + } + SaslServer saslServer = + Sasl.createSaslServer( + serverDefinition.mechanism, + serverDefinition.protocol, + serverDefinition.serverName, + serverDefinition.props, + serverDefinition.cbh); + setSaslServer(saslServer); + } + + /** + * TTransportFactory to create TSaslServerTransports. Ensures that a + * given underlying TTransport instance receives the same TSaslServerTransport + * . This is kind of an awful hack to work around the fact that Thrift is designed assuming + * that TTransport instances are stateless, and thus the existing TServers + * use different TTransport instances for input and output. + */ + public static class Factory extends TTransportFactory { + + /** + * This is the implementation of the awful hack described above. WeakHashMap is + * used to ensure that we don't leak memory. + */ + private static Map> transportMap = + Collections.synchronizedMap( + new WeakHashMap>()); + + /** + * Mapping from SASL mechanism name -> all the parameters required to instantiate a SASL server. + */ + private Map serverDefinitionMap = + new HashMap(); + + /** Create a new Factory. Assumes that addServerDefinition will be called later. */ + public Factory() { + super(); + } + + /** + * Create a new Factory, initially with the single server definition given. You may + * still call addServerDefinition later. See the Java documentation for + * Sasl.createSaslServer for the details of the parameters. + */ + public Factory( + String mechanism, + String protocol, + String serverName, + Map props, + CallbackHandler cbh) { + super(); + addServerDefinition(mechanism, protocol, serverName, props, cbh); + } + + /** + * Add a supported server definition to the transports created by this factory. See the Java + * documentation for Sasl.createSaslServer for the details of the parameters. + */ + public void addServerDefinition( + String mechanism, + String protocol, + String serverName, + Map props, + CallbackHandler cbh) { + serverDefinitionMap.put( + mechanism, new TSaslServerDefinition(mechanism, protocol, serverName, props, cbh)); + } + + /** + * Get a new TSaslServerTransport instance, or reuse the existing one if a + * TSaslServerTransport has already been created before using the given TTransport + * as an underlying transport. This ensures that a given underlying transport instance + * receives the same TSaslServerTransport. + */ + @Override + public TTransport getTransport(TTransport base) throws TTransportException { + WeakReference ret = transportMap.get(base); + if (ret == null || ret.get() == null) { + LOGGER.debug("transport map does not contain key", base); + ret = + new WeakReference( + new TSaslServerTransport(serverDefinitionMap, base)); + try { + ret.get().open(); + } catch (TTransportException e) { + LOGGER.debug("failed to open server transport", e); + throw new RuntimeException(e); + } + transportMap.put(base, ret); // No need for putIfAbsent(). + // Concurrent calls to getTransport() will pass in different TTransports. + } else { + LOGGER.debug("transport map does contain key {}", base); + } + return ret.get(); + } + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TSaslTransport.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TSaslTransport.java new file mode 100644 index 0000000..0a280d6 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TSaslTransport.java @@ -0,0 +1,510 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +import java.nio.charset.StandardCharsets; +import java.util.Objects; +import javax.security.sasl.Sasl; +import javax.security.sasl.SaslClient; +import javax.security.sasl.SaslException; +import javax.security.sasl.SaslServer; +import org.apache.thrift.EncodingUtils; +import org.apache.thrift.TByteArrayOutputStream; +import org.apache.thrift.TConfiguration; +import org.apache.thrift.transport.layered.TFramedTransport; +import org.apache.thrift.transport.sasl.NegotiationStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A superclass for SASL client/server thrift transports. A subclass need only implement the + * open method. + */ +abstract class TSaslTransport extends TEndpointTransport { + + private static final Logger LOGGER = LoggerFactory.getLogger(TSaslTransport.class); + + protected static final int DEFAULT_MAX_LENGTH = 0x7FFFFFFF; + + protected static final int MECHANISM_NAME_BYTES = 1; + protected static final int STATUS_BYTES = 1; + protected static final int PAYLOAD_LENGTH_BYTES = 4; + + protected static enum SaslRole { + SERVER, + CLIENT; + } + + /** Transport underlying this one. */ + protected TTransport underlyingTransport; + + /** Either a SASL client or a SASL server. */ + private SaslParticipant sasl; + + /** + * Whether or not we should wrap/unwrap reads/writes. Determined by whether or not a QOP is + * negotiated during the SASL handshake. + */ + private boolean shouldWrap = false; + + /** Buffer for input. */ + private TMemoryInputTransport readBuffer; + + /** Buffer for output. */ + private final TByteArrayOutputStream writeBuffer = new TByteArrayOutputStream(1024); + + /** + * Create a TSaslTransport. It's assumed that setSaslServer will be called later to initialize the + * SASL endpoint underlying this transport. + * + * @param underlyingTransport The thrift transport which this transport is wrapping. + */ + protected TSaslTransport(TTransport underlyingTransport) throws TTransportException { + super( + Objects.isNull(underlyingTransport.getConfiguration()) + ? new TConfiguration() + : underlyingTransport.getConfiguration()); + this.underlyingTransport = underlyingTransport; + this.readBuffer = new TMemoryInputTransport(underlyingTransport.getConfiguration()); + } + + /** + * Create a TSaslTransport which acts as a client. + * + * @param saslClient The SaslClient which this transport will use for SASL + * negotiation. + * @param underlyingTransport The thrift transport which this transport is wrapping. + */ + protected TSaslTransport(SaslClient saslClient, TTransport underlyingTransport) + throws TTransportException { + super( + Objects.isNull(underlyingTransport.getConfiguration()) + ? new TConfiguration() + : underlyingTransport.getConfiguration()); + sasl = new SaslParticipant(saslClient); + this.underlyingTransport = underlyingTransport; + this.readBuffer = new TMemoryInputTransport(underlyingTransport.getConfiguration()); + } + + protected void setSaslServer(SaslServer saslServer) { + sasl = new SaslParticipant(saslServer); + } + + // Used to read the status byte and payload length. + private final byte[] messageHeader = new byte[STATUS_BYTES + PAYLOAD_LENGTH_BYTES]; + + /** + * Send a complete Thrift SASL message. + * + * @param status The status to send. + * @param payload The data to send as the payload of this message. + * @throws TTransportException + */ + protected void sendSaslMessage(NegotiationStatus status, byte[] payload) + throws TTransportException { + if (payload == null) payload = new byte[0]; + + messageHeader[0] = status.getValue(); + EncodingUtils.encodeBigEndian(payload.length, messageHeader, STATUS_BYTES); + + LOGGER.debug( + "{}: Writing message with status {} and payload length {}", + getRole(), + status, + payload.length); + + underlyingTransport.write(messageHeader); + underlyingTransport.write(payload); + underlyingTransport.flush(); + } + + /** + * Read a complete Thrift SASL message. + * + * @return The SASL status and payload from this message. + * @throws TTransportException Thrown if there is a failure reading from the underlying transport, + * or if a status code of BAD or ERROR is encountered. + */ + protected SaslResponse receiveSaslMessage() throws TTransportException { + underlyingTransport.readAll(messageHeader, 0, messageHeader.length); + + byte statusByte = messageHeader[0]; + + NegotiationStatus status = NegotiationStatus.byValue(statusByte); + if (status == null) { + throw sendAndThrowMessage(NegotiationStatus.ERROR, "Invalid status " + statusByte); + } + + int payloadBytes = EncodingUtils.decodeBigEndian(messageHeader, STATUS_BYTES); + if (payloadBytes < 0 || payloadBytes > getConfiguration().getMaxMessageSize() /* 100 MB */) { + throw sendAndThrowMessage( + NegotiationStatus.ERROR, "Invalid payload header length: " + payloadBytes); + } + + byte[] payload = new byte[payloadBytes]; + underlyingTransport.readAll(payload, 0, payload.length); + + if (status == NegotiationStatus.BAD || status == NegotiationStatus.ERROR) { + String remoteMessage = new String(payload, StandardCharsets.UTF_8); + throw new TTransportException("Peer indicated failure: " + remoteMessage); + } + LOGGER.debug( + "{}: Received message with status {} and payload length {}", + getRole(), + status, + payload.length); + return new SaslResponse(status, payload); + } + + /** + * Send a Thrift SASL message with the given status (usually BAD or ERROR) and string message, and + * then throw a TTransportException with the given message. + * + * @param status The Thrift SASL status code to send. Usually BAD or ERROR. + * @param message The optional message to send to the other side. + * @throws TTransportException Always thrown with the message provided. + * @return always throws TTransportException but declares return type to allow throw + * sendAndThrowMessage(...) to inform compiler control flow + */ + protected TTransportException sendAndThrowMessage(NegotiationStatus status, String message) + throws TTransportException { + try { + sendSaslMessage(status, message.getBytes(StandardCharsets.UTF_8)); + } catch (Exception e) { + LOGGER.warn("Could not send failure response", e); + message += "\nAlso, could not send response: " + e.toString(); + } + throw new TTransportException(message); + } + + /** + * Implemented by subclasses to start the Thrift SASL handshake process. When this method + * completes, the SaslParticipant in this class is assumed to be initialized. + * + * @throws TTransportException + * @throws SaslException + */ + protected abstract void handleSaslStartMessage() throws TTransportException, SaslException; + + protected abstract SaslRole getRole(); + + /** + * Opens the underlying transport if it's not already open and then performs SASL negotiation. If + * a QOP is negotiated during this SASL handshake, it used for all communication on this transport + * after this call is complete. + */ + @Override + public void open() throws TTransportException { + /* + * readSaslHeader is used to tag whether the SASL header has been read properly. + * If there is a problem in reading the header, there might not be any + * data in the stream, possibly a TCP health check from load balancer. + */ + boolean readSaslHeader = false; + + LOGGER.debug("opening transport {}", this); + if (sasl != null && sasl.isComplete()) + throw new TTransportException("SASL transport already open"); + + if (!underlyingTransport.isOpen()) underlyingTransport.open(); + + try { + // Negotiate a SASL mechanism. The client also sends its + // initial response, or an empty one. + handleSaslStartMessage(); + readSaslHeader = true; + LOGGER.debug("{}: Start message handled", getRole()); + + SaslResponse message = null; + while (!sasl.isComplete()) { + message = receiveSaslMessage(); + if (message.status != NegotiationStatus.COMPLETE + && message.status != NegotiationStatus.OK) { + throw new TTransportException("Expected COMPLETE or OK, got " + message.status); + } + + byte[] challenge = sasl.evaluateChallengeOrResponse(message.payload); + + // If we are the client, and the server indicates COMPLETE, we don't need to + // send back any further response. + if (message.status == NegotiationStatus.COMPLETE && getRole() == SaslRole.CLIENT) { + LOGGER.debug("{}: All done!", getRole()); + continue; + } + + sendSaslMessage( + sasl.isComplete() ? NegotiationStatus.COMPLETE : NegotiationStatus.OK, challenge); + } + LOGGER.debug("{}: Main negotiation loop complete", getRole()); + + // If we're the client, and we're complete, but the server isn't + // complete yet, we need to wait for its response. This will occur + // with ANONYMOUS auth, for example, where we send an initial response + // and are immediately complete. + if (getRole() == SaslRole.CLIENT + && (message == null || message.status == NegotiationStatus.OK)) { + LOGGER.debug("{}: SASL Client receiving last message", getRole()); + message = receiveSaslMessage(); + if (message.status != NegotiationStatus.COMPLETE) { + throw new TTransportException("Expected SASL COMPLETE, but got " + message.status); + } + } + } catch (SaslException e) { + try { + LOGGER.error("SASL negotiation failure", e); + throw sendAndThrowMessage(NegotiationStatus.BAD, e.getMessage()); + } finally { + underlyingTransport.close(); + } + } catch (TTransportException e) { + // If there is no-data or no-sasl header in the stream, + // log the failure, and clean up the underlying transport. + if (!readSaslHeader && e.getType() == TTransportException.END_OF_FILE) { + underlyingTransport.close(); + LOGGER.debug("No data or no sasl data in the stream during negotiation"); + } + throw e; + } + + String qop = (String) sasl.getNegotiatedProperty(Sasl.QOP); + if (qop != null && !qop.equalsIgnoreCase("auth")) shouldWrap = true; + } + + /** + * Get the underlying SaslClient. + * + * @return The SaslClient, or null if this transport is backed by a + * SaslServer. + */ + public SaslClient getSaslClient() { + return sasl.saslClient; + } + + /** + * Get the underlying transport that Sasl is using. + * + * @return The TTransport transport + */ + public TTransport getUnderlyingTransport() { + return underlyingTransport; + } + + /** + * Get the underlying SaslServer. + * + * @return The SaslServer, or null if this transport is backed by a + * SaslClient. + */ + public SaslServer getSaslServer() { + return sasl.saslServer; + } + + /** + * Read a 4-byte word from the underlying transport and interpret it as an integer. + * + * @return The length prefix of the next SASL message to read. + * @throws TTransportException Thrown if reading from the underlying transport fails. + */ + protected int readLength() throws TTransportException { + byte[] lenBuf = new byte[4]; + underlyingTransport.readAll(lenBuf, 0, lenBuf.length); + return EncodingUtils.decodeBigEndian(lenBuf); + } + + /** + * Write the given integer as 4 bytes to the underlying transport. + * + * @param length The length prefix of the next SASL message to write. + * @throws TTransportException Thrown if writing to the underlying transport fails. + */ + protected void writeLength(int length) throws TTransportException { + byte[] lenBuf = new byte[4]; + TFramedTransport.encodeFrameSize(length, lenBuf); + underlyingTransport.write(lenBuf); + } + + // Below is the SASL implementation of the TTransport interface. + + /** + * Closes the underlying transport and disposes of the SASL implementation underlying this + * transport. + */ + @Override + public void close() { + underlyingTransport.close(); + try { + sasl.dispose(); + } catch (SaslException e) { + LOGGER.warn("Failed to dispose sasl participant.", e); + } + } + + /** True if the underlying transport is open and the SASL handshake is complete. */ + @Override + public boolean isOpen() { + return underlyingTransport.isOpen() && sasl != null && sasl.isComplete(); + } + + /** + * Read from the underlying transport. Unwraps the contents if a QOP was negotiated during the + * SASL handshake. + */ + @Override + public int read(byte[] buf, int off, int len) throws TTransportException { + if (!isOpen()) throw new TTransportException("SASL authentication not complete"); + + int got = readBuffer.read(buf, off, len); + if (got > 0) { + return got; + } + + // Read another frame of data + try { + readFrame(); + } catch (SaslException e) { + throw new TTransportException(e); + } catch (TTransportException transportException) { + // If there is no-data or no-sasl header in the stream, log the failure, and rethrow. + if (transportException.getType() == TTransportException.END_OF_FILE) { + LOGGER.debug("No data or no sasl data in the stream during negotiation"); + } + throw transportException; + } + + return readBuffer.read(buf, off, len); + } + + /** + * Read a single frame of data from the underlying transport, unwrapping if necessary. + * + * @throws TTransportException Thrown if there's an error reading from the underlying transport. + * @throws SaslException Thrown if there's an error unwrapping the data. + */ + private void readFrame() throws TTransportException, SaslException { + int dataLength = readLength(); + + if (dataLength < 0) + throw new TTransportException("Read a negative frame size (" + dataLength + ")!"); + + byte[] buff = new byte[dataLength]; + LOGGER.debug("{}: reading data length: {}", getRole(), dataLength); + underlyingTransport.readAll(buff, 0, dataLength); + if (shouldWrap) { + buff = sasl.unwrap(buff, 0, buff.length); + LOGGER.debug("data length after unwrap: {}", buff.length); + } + readBuffer.reset(buff); + } + + /** Write to the underlying transport. */ + @Override + public void write(byte[] buf, int off, int len) throws TTransportException { + if (!isOpen()) throw new TTransportException("SASL authentication not complete"); + + writeBuffer.write(buf, off, len); + } + + /** + * Flushes to the underlying transport. Wraps the contents if a QOP was negotiated during the SASL + * handshake. + */ + @Override + public void flush() throws TTransportException { + byte[] buf = writeBuffer.get(); + int dataLength = writeBuffer.len(); + writeBuffer.reset(); + + if (shouldWrap) { + LOGGER.debug("data length before wrap: {}", dataLength); + try { + buf = sasl.wrap(buf, 0, dataLength); + } catch (SaslException e) { + throw new TTransportException(e); + } + dataLength = buf.length; + } + LOGGER.debug("writing data length: {}", dataLength); + writeLength(dataLength); + underlyingTransport.write(buf, 0, dataLength); + underlyingTransport.flush(); + } + + /** Used exclusively by readSaslMessage to return both a status and data. */ + protected static class SaslResponse { + public NegotiationStatus status; + public byte[] payload; + + public SaslResponse(NegotiationStatus status, byte[] payload) { + this.status = status; + this.payload = payload; + } + } + + /** + * Used to abstract over the SaslServer and SaslClient classes, which + * share a lot of their interface, but unfortunately don't share a common superclass. + */ + private static class SaslParticipant { + // One of these will always be null. + public SaslServer saslServer; + public SaslClient saslClient; + + public SaslParticipant(SaslServer saslServer) { + this.saslServer = saslServer; + } + + public SaslParticipant(SaslClient saslClient) { + this.saslClient = saslClient; + } + + public byte[] evaluateChallengeOrResponse(byte[] challengeOrResponse) throws SaslException { + if (saslClient != null) { + return saslClient.evaluateChallenge(challengeOrResponse); + } else { + return saslServer.evaluateResponse(challengeOrResponse); + } + } + + public boolean isComplete() { + if (saslClient != null) return saslClient.isComplete(); + else return saslServer.isComplete(); + } + + public void dispose() throws SaslException { + if (saslClient != null) saslClient.dispose(); + else saslServer.dispose(); + } + + public byte[] unwrap(byte[] buf, int off, int len) throws SaslException { + if (saslClient != null) return saslClient.unwrap(buf, off, len); + else return saslServer.unwrap(buf, off, len); + } + + public byte[] wrap(byte[] buf, int off, int len) throws SaslException { + if (saslClient != null) return saslClient.wrap(buf, off, len); + else return saslServer.wrap(buf, off, len); + } + + public Object getNegotiatedProperty(String propName) { + if (saslClient != null) return saslClient.getNegotiatedProperty(propName); + else return saslServer.getNegotiatedProperty(propName); + } + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TSeekableFile.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TSeekableFile.java new file mode 100644 index 0000000..6200dad --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TSeekableFile.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +public interface TSeekableFile { + + public InputStream getInputStream() throws IOException; + + public OutputStream getOutputStream() throws IOException; + + public void close() throws IOException; + + public long length() throws IOException; + + public void seek(long pos) throws IOException; +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TServerSocket.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TServerSocket.java new file mode 100644 index 0000000..59cef20 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TServerSocket.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.ServerSocket; +import java.net.Socket; +import java.net.SocketException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Wrapper around ServerSocket for Thrift. */ +public class TServerSocket extends TServerTransport { + + private static final Logger LOGGER = LoggerFactory.getLogger(TServerSocket.class.getName()); + + /** Underlying ServerSocket object */ + private ServerSocket serverSocket_ = null; + + /** Timeout for client sockets from accept */ + private int clientTimeout_ = 0; + + /** Max message size */ + private int maxMessageSize_ = 0; + + public static class ServerSocketTransportArgs + extends AbstractServerTransportArgs { + ServerSocket serverSocket; + + public ServerSocketTransportArgs serverSocket(ServerSocket serverSocket) { + this.serverSocket = serverSocket; + return this; + } + } + + /** Creates a server socket from underlying socket object */ + public TServerSocket(ServerSocket serverSocket) throws TTransportException { + this(serverSocket, 0); + } + + /** Creates a server socket from underlying socket object */ + public TServerSocket(ServerSocket serverSocket, int clientTimeout) throws TTransportException { + this(new ServerSocketTransportArgs().serverSocket(serverSocket).clientTimeout(clientTimeout)); + } + + /** Creates just a port listening server socket */ + public TServerSocket(int port) throws TTransportException { + this(port, 0); + } + + /** Creates just a port listening server socket */ + public TServerSocket(int port, int clientTimeout) throws TTransportException { + this(new InetSocketAddress(port), clientTimeout); + } + + public TServerSocket(InetSocketAddress bindAddr) throws TTransportException { + this(bindAddr, 0); + } + + public TServerSocket(InetSocketAddress bindAddr, int clientTimeout) throws TTransportException { + this(new ServerSocketTransportArgs().bindAddr(bindAddr).clientTimeout(clientTimeout)); + } + + public TServerSocket(ServerSocketTransportArgs args) throws TTransportException { + clientTimeout_ = args.clientTimeout; + maxMessageSize_ = args.maxMessageSize; + if (args.serverSocket != null) { + this.serverSocket_ = args.serverSocket; + return; + } + try { + // Make server socket + serverSocket_ = new ServerSocket(); + // Prevent 2MSL delay problem on server restarts + serverSocket_.setReuseAddress(true); + // Bind to listening port + serverSocket_.bind(args.bindAddr, args.backlog); + } catch (IOException ioe) { + close(); + throw new TTransportException( + "Could not create ServerSocket on address " + args.bindAddr.toString() + ".", ioe); + } + } + + public void listen() throws TTransportException { + // Make sure to block on accept + if (serverSocket_ != null) { + try { + serverSocket_.setSoTimeout(0); + } catch (SocketException sx) { + LOGGER.error("Could not set socket timeout.", sx); + } + } + } + + @Override + public TSocket accept() throws TTransportException { + if (serverSocket_ == null) { + throw new TTransportException(TTransportException.NOT_OPEN, "No underlying server socket."); + } + Socket result; + try { + result = serverSocket_.accept(); + } catch (Exception e) { + throw new TTransportException(e); + } + if (result == null) { + throw new TTransportException("Blocking server's accept() may not return NULL"); + } + TSocket socket = new TSocket(result); + socket.setTimeout(clientTimeout_); + socket.setMaxMessageSize(maxMessageSize_); + return socket; + } + + public void close() { + if (serverSocket_ != null) { + try { + serverSocket_.close(); + } catch (IOException iox) { + LOGGER.warn("Could not close server socket.", iox); + } + serverSocket_ = null; + } + } + + public void interrupt() { + // The thread-safeness of this is dubious, but Java documentation suggests + // that it is safe to do this from a different thread context + close(); + } + + public ServerSocket getServerSocket() { + return serverSocket_; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TServerTransport.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TServerTransport.java new file mode 100644 index 0000000..05a3f09 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TServerTransport.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +import java.io.Closeable; +import java.net.InetSocketAddress; +import org.apache.thrift.TConfiguration; + +/** Server transport. Object which provides client transports. */ +public abstract class TServerTransport implements Closeable { + + public abstract static class AbstractServerTransportArgs< + T extends AbstractServerTransportArgs> { + int backlog = 0; // A value of 0 means the default value will be used (currently set at 50) + int clientTimeout = 0; + InetSocketAddress bindAddr; + int maxFrameSize = TConfiguration.DEFAULT_MAX_FRAME_SIZE; + int maxMessageSize = TConfiguration.DEFAULT_MAX_MESSAGE_SIZE; + + public T backlog(int backlog) { + this.backlog = backlog; + return (T) this; + } + + public T clientTimeout(int clientTimeout) { + this.clientTimeout = clientTimeout; + return (T) this; + } + + public T port(int port) { + this.bindAddr = new InetSocketAddress(port); + return (T) this; + } + + public T bindAddr(InetSocketAddress bindAddr) { + this.bindAddr = bindAddr; + return (T) this; + } + + public T maxFrameSize(int maxFrameSize) { + this.maxFrameSize = maxFrameSize; + return (T) this; + } + + public T maxMessageSize(int maxMessageSize) { + this.maxMessageSize = maxMessageSize; + return (T) this; + } + } + + public abstract void listen() throws TTransportException; + + /** + * Accept incoming connection on the server socket. When there is no incoming connection + * available: either it should block infinitely in a blocking implementation, either it should + * return null in a nonblocking implementation. + * + * @return new connection + * @throws TTransportException if IO error. + */ + public abstract TTransport accept() throws TTransportException; + + public abstract void close(); + + /** + * Optional method implementation. This signals to the server transport that it should break out + * of any accept() or listen() that it is currently blocked on. This method, if implemented, MUST + * be thread safe, as it may be called from a different thread context than the other + * TServerTransport methods. + */ + public void interrupt() {} +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TSimpleFileTransport.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TSimpleFileTransport.java new file mode 100644 index 0000000..cd6be66 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TSimpleFileTransport.java @@ -0,0 +1,225 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.thrift.transport; + +import java.io.IOException; +import java.io.RandomAccessFile; +import org.apache.thrift.TConfiguration; + +/** Basic file support for the TTransport interface */ +public final class TSimpleFileTransport extends TEndpointTransport { + + private RandomAccessFile file = null; + private final boolean readable; + private final boolean writable; + private final String path_; + + /** + * Create a transport backed by a simple file + * + * @param path the path to the file to open/create + * @param read true to support read operations + * @param write true to support write operations + * @param openFile true to open the file on construction + * @throws TTransportException if file open fails + */ + public TSimpleFileTransport(String path, boolean read, boolean write, boolean openFile) + throws TTransportException { + this(new TConfiguration(), path, read, write, openFile); + } + + /** + * Create a transport backed by a simple file + * + * @param config + * @param path the path to the file to open/create + * @param read true to support read operations + * @param write true to support write operations + * @param openFile true to open the file on construction + * @throws TTransportException if file open fails + */ + public TSimpleFileTransport( + TConfiguration config, String path, boolean read, boolean write, boolean openFile) + throws TTransportException { + super(config); + if (path.length() <= 0) { + throw new TTransportException("No path specified"); + } + if (!read && !write) { + throw new TTransportException("Neither READ nor WRITE specified"); + } + readable = read; + writable = write; + path_ = path; + if (openFile) { + open(); + } + } + + /** + * Create a transport backed by a simple file Implicitly opens file to conform to C++ behavior. + * + * @param path the path to the file to open/create + * @param read true to support read operations + * @param write true to support write operations + * @throws TTransportException if file open fails + */ + public TSimpleFileTransport(String path, boolean read, boolean write) throws TTransportException { + this(path, read, write, true); + } + + /** + * Create a transport backed by a simple read only disk file (implicitly opens file) + * + * @param path the path to the file to open/create + * @throws TTransportException if file open fails + */ + public TSimpleFileTransport(String path) throws TTransportException { + this(path, true, false, true); + } + + /** + * Test file status + * + * @return true if open, otherwise false + */ + @Override + public boolean isOpen() { + return (file != null); + } + + /** + * Open file if not previously opened. + * + * @throws TTransportException if open fails + */ + @Override + public void open() throws TTransportException { + if (file == null) { + try { + String access = "r"; // RandomAccessFile objects must be readable + if (writable) { + access += "w"; + } + file = new RandomAccessFile(path_, access); + } catch (IOException ioe) { + file = null; + throw new TTransportException(ioe.getMessage()); + } + } + } + + /** Close file, subsequent read/write activity will throw exceptions */ + @Override + public void close() { + if (file != null) { + try { + file.close(); + } catch (Exception e) { + // Nothing to do + } + file = null; + } + } + + /** + * Read up to len many bytes into buf at offset + * + * @param buf houses bytes read + * @param off offset into buff to begin writing to + * @param len maximum number of bytes to read + * @return number of bytes actually read + * @throws TTransportException on read failure + */ + @Override + public int read(byte[] buf, int off, int len) throws TTransportException { + if (!readable) { + throw new TTransportException("Read operation on write only file"); + } + checkReadBytesAvailable(len); + int iBytesRead = 0; + try { + iBytesRead = file.read(buf, off, len); + } catch (IOException ioe) { + file = null; + throw new TTransportException(ioe.getMessage()); + } + return iBytesRead; + } + + /** + * Write len many bytes from buff starting at offset + * + * @param buf buffer containing bytes to write + * @param off offset into buffer to begin writing from + * @param len number of bytes to write + * @throws TTransportException on write failure + */ + @Override + public void write(byte[] buf, int off, int len) throws TTransportException { + try { + file.write(buf, off, len); + } catch (IOException ioe) { + file = null; + throw new TTransportException(ioe.getMessage()); + } + } + + /** + * Move file pointer to specified offset, new read/write calls will act here + * + * @param offset bytes from beginning of file to move pointer to + * @throws TTransportException is seek fails + */ + public void seek(long offset) throws TTransportException { + try { + file.seek(offset); + } catch (IOException ex) { + throw new TTransportException(ex.getMessage()); + } + } + + /** + * Return the length of the file in bytes + * + * @return length of the file in bytes + * @throws TTransportException if file access fails + */ + public long length() throws TTransportException { + try { + return file.length(); + } catch (IOException ex) { + throw new TTransportException(ex.getMessage()); + } + } + + /** + * Return current file pointer position in bytes from beginning of file + * + * @return file pointer position + * @throws TTransportException if file access fails + */ + public long getFilePointer() throws TTransportException { + try { + return file.getFilePointer(); + } catch (IOException ex) { + throw new TTransportException(ex.getMessage()); + } + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TSocket.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TSocket.java new file mode 100644 index 0000000..2458d0f --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TSocket.java @@ -0,0 +1,261 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.net.SocketAddress; +import java.net.SocketException; +import org.apache.thrift.TConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Socket implementation of the TTransport interface. To be commented soon! */ +public class TSocket extends TIOStreamTransport implements SocketAddressProvider { + + private static final Logger LOGGER = LoggerFactory.getLogger(TSocket.class.getName()); + + /** Wrapped Socket object */ + private Socket socket_; + + /** Remote host */ + private String host_; + + /** Remote port */ + private int port_; + + /** Socket timeout - read timeout on the socket */ + private int socketTimeout_; + + /** Connection timeout */ + private int connectTimeout_; + + /** + * Constructor that takes an already created socket. + * + * @param socket Already created socket object + * @throws TTransportException if there is an error setting up the streams + */ + public TSocket(Socket socket) throws TTransportException { + super(new TConfiguration()); + socket_ = socket; + try { + socket_.setSoLinger(false, 0); + socket_.setTcpNoDelay(true); + socket_.setKeepAlive(true); + } catch (SocketException sx) { + LOGGER.warn("Could not configure socket.", sx); + } + + if (isOpen()) { + try { + inputStream_ = new BufferedInputStream(socket_.getInputStream()); + outputStream_ = new BufferedOutputStream(socket_.getOutputStream()); + } catch (IOException iox) { + close(); + throw new TTransportException(TTransportException.NOT_OPEN, iox); + } + } + } + + /** + * Creates a new unconnected socket that will connect to the given host on the given port. + * + * @param config check config + * @param host Remote host + * @param port Remote port + */ + public TSocket(TConfiguration config, String host, int port) throws TTransportException { + this(config, host, port, 0); + } + + /** + * Creates a new unconnected socket that will connect to the given host on the given port. + * + * @param host Remote host + * @param port Remote port + */ + public TSocket(String host, int port) throws TTransportException { + this(new TConfiguration(), host, port, 0); + } + + /** + * Creates a new unconnected socket that will connect to the given host on the given port. + * + * @param host Remote host + * @param port Remote port + * @param timeout Socket timeout and connection timeout + */ + public TSocket(String host, int port, int timeout) throws TTransportException { + this(new TConfiguration(), host, port, timeout, timeout); + } + + /** + * Creates a new unconnected socket that will connect to the given host on the given port. + * + * @param config check config + * @param host Remote host + * @param port Remote port + * @param timeout Socket timeout and connection timeout + */ + public TSocket(TConfiguration config, String host, int port, int timeout) + throws TTransportException { + this(config, host, port, timeout, timeout); + } + + /** + * Creates a new unconnected socket that will connect to the given host on the given port, with a + * specific connection timeout and a specific socket timeout. + * + * @param config check config + * @param host Remote host + * @param port Remote port + * @param socketTimeout Socket timeout + * @param connectTimeout Connection timeout + */ + public TSocket( + TConfiguration config, String host, int port, int socketTimeout, int connectTimeout) + throws TTransportException { + super(config); + host_ = host; + port_ = port; + socketTimeout_ = socketTimeout; + connectTimeout_ = connectTimeout; + initSocket(); + } + + /** Initializes the socket object */ + private void initSocket() { + socket_ = new Socket(); + try { + socket_.setSoLinger(false, 0); + socket_.setTcpNoDelay(true); + socket_.setKeepAlive(true); + socket_.setSoTimeout(socketTimeout_); + } catch (SocketException sx) { + LOGGER.error("Could not configure socket.", sx); + } + } + + /** + * Sets the socket timeout and connection timeout. + * + * @param timeout Milliseconds timeout + */ + public void setTimeout(int timeout) { + this.setConnectTimeout(timeout); + this.setSocketTimeout(timeout); + } + + /** + * Sets the time after which the connection attempt will time out + * + * @param timeout Milliseconds timeout + */ + public void setConnectTimeout(int timeout) { + connectTimeout_ = timeout; + } + + /** + * Sets the socket timeout + * + * @param timeout Milliseconds timeout + */ + public void setSocketTimeout(int timeout) { + socketTimeout_ = timeout; + try { + socket_.setSoTimeout(timeout); + } catch (SocketException sx) { + LOGGER.warn("Could not set socket timeout.", sx); + } + } + + /** Returns a reference to the underlying socket. */ + public Socket getSocket() { + if (socket_ == null) { + initSocket(); + } + return socket_; + } + + /** Checks whether the socket is connected. */ + public boolean isOpen() { + if (socket_ == null) { + return false; + } + return socket_.isConnected(); + } + + /** Connects the socket, creating a new socket object if necessary. */ + public void open() throws TTransportException { + if (isOpen()) { + throw new TTransportException(TTransportException.ALREADY_OPEN, "Socket already connected."); + } + + if (host_ == null || host_.length() == 0) { + throw new TTransportException(TTransportException.NOT_OPEN, "Cannot open null host."); + } + if (port_ <= 0 || port_ > 65535) { + throw new TTransportException(TTransportException.NOT_OPEN, "Invalid port " + port_); + } + + if (socket_ == null) { + initSocket(); + } + + try { + socket_.connect(new InetSocketAddress(host_, port_), connectTimeout_); + inputStream_ = new BufferedInputStream(socket_.getInputStream()); + outputStream_ = new BufferedOutputStream(socket_.getOutputStream()); + } catch (IOException iox) { + close(); + throw new TTransportException(TTransportException.NOT_OPEN, iox); + } + } + + /** Closes the socket. */ + public void close() { + // Close the underlying streams + super.close(); + + // Close the socket + if (socket_ != null) { + try { + socket_.close(); + } catch (IOException iox) { + LOGGER.warn("Could not close socket.", iox); + } + socket_ = null; + } + } + + @Override + public SocketAddress getRemoteSocketAddress() { + return socket_.getRemoteSocketAddress(); + } + + @Override + public SocketAddress getLocalSocketAddress() { + return socket_.getLocalSocketAddress(); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TStandardFile.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TStandardFile.java new file mode 100644 index 0000000..65c29a5 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TStandardFile.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.RandomAccessFile; + +public class TStandardFile implements TSeekableFile { + + protected String path_ = null; + protected RandomAccessFile inputFile_ = null; + + public TStandardFile(String path) throws IOException { + path_ = path; + inputFile_ = new RandomAccessFile(path_, "r"); + } + + public InputStream getInputStream() throws IOException { + return new FileInputStream(inputFile_.getFD()); + } + + public OutputStream getOutputStream() throws IOException { + return new FileOutputStream(path_); + } + + public void close() throws IOException { + if (inputFile_ != null) { + inputFile_.close(); + } + } + + public long length() throws IOException { + return inputFile_.length(); + } + + public void seek(long pos) throws IOException { + inputFile_.seek(pos); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TTransport.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TTransport.java new file mode 100644 index 0000000..dc771fc --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TTransport.java @@ -0,0 +1,202 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +import java.io.Closeable; +import java.nio.ByteBuffer; +import org.apache.thrift.TConfiguration; + +/** + * Generic class that encapsulates the I/O layer. This is basically a thin wrapper around the + * combined functionality of Java input/output streams. + */ +public abstract class TTransport implements Closeable { + + /** + * Queries whether the transport is open. + * + * @return True if the transport is open. + */ + public abstract boolean isOpen(); + + /** + * Is there more data to be read? + * + * @return True if the remote side is still alive and feeding us + */ + public boolean peek() { + return isOpen(); + } + + /** + * Opens the transport for reading/writing. + * + * @throws TTransportException if the transport could not be opened + */ + public abstract void open() throws TTransportException; + + /** Closes the transport. */ + public abstract void close(); + + /** + * Reads a sequence of bytes from this channel into the given buffer. An attempt is made to read + * up to the number of bytes remaining in the buffer, that is, dst.remaining(), at the moment this + * method is invoked. Upon return the buffer's position will move forward the number of bytes + * read; its limit will not have changed. Subclasses are encouraged to provide a more efficient + * implementation of this method. + * + * @param dst The buffer into which bytes are to be transferred + * @return The number of bytes read, possibly zero, or -1 if the channel has reached end-of-stream + * @throws TTransportException if there was an error reading data + */ + public int read(ByteBuffer dst) throws TTransportException { + byte[] arr = new byte[dst.remaining()]; + int n = read(arr, 0, arr.length); + dst.put(arr, 0, n); + return n; + } + + /** + * Reads up to len bytes into buffer buf, starting at offset off. + * + * @param buf Array to read into + * @param off Index to start reading at + * @param len Maximum number of bytes to read + * @return The number of bytes actually read + * @throws TTransportException if there was an error reading data + */ + public abstract int read(byte[] buf, int off, int len) throws TTransportException; + + /** + * Guarantees that all of len bytes are actually read off the transport. + * + * @param buf Array to read into + * @param off Index to start reading at + * @param len Maximum number of bytes to read + * @return The number of bytes actually read, which must be equal to len + * @throws TTransportException if there was an error reading data + */ + public int readAll(byte[] buf, int off, int len) throws TTransportException { + int got = 0; + int ret = 0; + while (got < len) { + ret = read(buf, off + got, len - got); + if (ret <= 0) { + throw new TTransportException( + "Cannot read. Remote side has closed. Tried to read " + + len + + " bytes, but only got " + + got + + " bytes. (This is often indicative of an internal error on the server side. Please check your server logs.)"); + } + got += ret; + } + return got; + } + + /** + * Writes the buffer to the output + * + * @param buf The output data buffer + * @throws TTransportException if an error occurs writing data + */ + public void write(byte[] buf) throws TTransportException { + write(buf, 0, buf.length); + } + + /** + * Writes up to len bytes from the buffer. + * + * @param buf The output data buffer + * @param off The offset to start writing from + * @param len The number of bytes to write + * @throws TTransportException if there was an error writing data + */ + public abstract void write(byte[] buf, int off, int len) throws TTransportException; + + /** + * Writes a sequence of bytes to the buffer. An attempt is made to write all remaining bytes in + * the buffer, that is, src.remaining(), at the moment this method is invoked. Upon return the + * buffer's position will updated; its limit will not have changed. Subclasses are encouraged to + * provide a more efficient implementation of this method. + * + * @param src The buffer from which bytes are to be retrieved + * @return The number of bytes written, possibly zero + * @throws TTransportException if there was an error writing data + */ + public int write(ByteBuffer src) throws TTransportException { + byte[] arr = new byte[src.remaining()]; + src.get(arr); + write(arr, 0, arr.length); + return arr.length; + } + + /** + * Flush any pending data out of a transport buffer. + * + * @throws TTransportException if there was an error writing out data. + */ + public void flush() throws TTransportException {} + + /** + * Access the protocol's underlying buffer directly. If this is not a buffered transport, return + * null. + * + * @return protocol's Underlying buffer + */ + public byte[] getBuffer() { + return null; + } + + /** + * Return the index within the underlying buffer that specifies the next spot that should be read + * from. + * + * @return index within the underlying buffer that specifies the next spot that should be read + * from + */ + public int getBufferPosition() { + return 0; + } + + /** + * Get the number of bytes remaining in the underlying buffer. Returns -1 if this is a + * non-buffered transport. + * + * @return the number of bytes remaining in the underlying buffer.
+ * Returns -1 if this is a non-buffered transport. + */ + public int getBytesRemainingInBuffer() { + return -1; + } + + /** + * Consume len bytes from the underlying buffer. + * + * @param len the number of bytes to consume from the underlying buffer. + */ + public void consumeBuffer(int len) {} + + public abstract TConfiguration getConfiguration(); + + public abstract void updateKnownMessageSize(long size) throws TTransportException; + + public abstract void checkReadBytesAvailable(long numBytes) throws TTransportException; +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TTransportException.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TTransportException.java new file mode 100644 index 0000000..a7ebed6 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TTransportException.java @@ -0,0 +1,78 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +import org.apache.thrift.TException; + +/** Transport exceptions. */ +public class TTransportException extends TException { + + private static final long serialVersionUID = 1L; + + public static final int UNKNOWN = 0; + public static final int NOT_OPEN = 1; + public static final int ALREADY_OPEN = 2; + public static final int TIMED_OUT = 3; + public static final int END_OF_FILE = 4; + public static final int CORRUPTED_DATA = 5; + public static final int MESSAGE_SIZE_LIMIT = 6; + + protected int type_ = UNKNOWN; + + public TTransportException() { + super(); + } + + public TTransportException(int type) { + super(); + type_ = type; + } + + public TTransportException(int type, String message) { + super(message); + type_ = type; + } + + public TTransportException(String message) { + super(message); + } + + public TTransportException(int type, Throwable cause) { + super(cause); + type_ = type; + } + + public TTransportException(Throwable cause) { + super(cause); + } + + public TTransportException(String message, Throwable cause) { + super(message, cause); + } + + public TTransportException(int type, String message, Throwable cause) { + super(message, cause); + type_ = type; + } + + public int getType() { + return type_; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TTransportFactory.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TTransportFactory.java new file mode 100644 index 0000000..ded5be4 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TTransportFactory.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport; + +/** + * Factory class used to create wrapped instance of Transports. This is used primarily in servers, + * which get Transports from a ServerTransport and then may want to mutate them (i.e. create a + * BufferedTransport from the underlying base transport) + */ +public class TTransportFactory { + + /** + * Return a wrapped instance of the base Transport. + * + * @param trans The base transport + * @return Wrapped Transport + */ + public TTransport getTransport(TTransport trans) throws TTransportException { + return trans; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TZlibTransport.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TZlibTransport.java new file mode 100644 index 0000000..e0b5b90 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TZlibTransport.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.thrift.transport; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.Objects; +import java.util.zip.Deflater; +import java.util.zip.DeflaterOutputStream; +import java.util.zip.Inflater; +import java.util.zip.InflaterInputStream; +import org.apache.thrift.TConfiguration; + +/** TZlibTransport deflates on write and inflates on read. */ +public class TZlibTransport extends TIOStreamTransport { + + private TTransport transport_ = null; + + public static class Factory extends TTransportFactory { + public Factory() {} + + @Override + public TTransport getTransport(TTransport base) throws TTransportException { + return new TZlibTransport(base); + } + } + + /** + * Constructs a new TZlibTransport instance. + * + * @param transport the underlying transport to read from and write to + */ + public TZlibTransport(TTransport transport) throws TTransportException { + this(transport, Deflater.BEST_COMPRESSION); + } + + /** + * Constructs a new TZlibTransport instance. + * + * @param transport the underlying transport to read from and write to + * @param compressionLevel 0 for no compression, 9 for maximum compression + */ + public TZlibTransport(TTransport transport, int compressionLevel) throws TTransportException { + super( + Objects.isNull(transport.getConfiguration()) + ? new TConfiguration() + : transport.getConfiguration()); + transport_ = transport; + inputStream_ = new InflaterInputStream(new TTransportInputStream(transport_), new Inflater()); + outputStream_ = + new DeflaterOutputStream( + new TTransportOutputStream(transport_), new Deflater(compressionLevel, false), true); + } + + @Override + public boolean isOpen() { + return transport_.isOpen(); + } + + @Override + public void open() throws TTransportException { + transport_.open(); + } + + @Override + public void close() { + super.close(); + if (transport_.isOpen()) { + transport_.close(); + } + } +} + +class TTransportInputStream extends InputStream { + + private TTransport transport = null; + + public TTransportInputStream(TTransport transport) { + this.transport = transport; + } + + @Override + public int read() throws IOException { + try { + byte[] buf = new byte[1]; + transport.read(buf, 0, 1); + return buf[0]; + } catch (TTransportException e) { + throw new IOException(e); + } + } + + @Override + public int read(byte b[], int off, int len) throws IOException { + try { + return transport.read(b, off, len); + } catch (TTransportException e) { + throw new IOException(e); + } + } +} + +class TTransportOutputStream extends OutputStream { + + private TTransport transport = null; + + public TTransportOutputStream(TTransport transport) { + this.transport = transport; + } + + @Override + public void write(final int b) throws IOException { + try { + transport.write(new byte[] {(byte) b}); + } catch (TTransportException e) { + throw new IOException(e); + } + } + + @Override + public void write(byte b[], int off, int len) throws IOException { + try { + transport.write(b, off, len); + } catch (TTransportException e) { + throw new IOException(e); + } + } + + @Override + public void flush() throws IOException { + try { + transport.flush(); + } catch (TTransportException e) { + throw new IOException(e); + } + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/layered/TFastFramedTransport.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/layered/TFastFramedTransport.java new file mode 100644 index 0000000..45f4e97 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/layered/TFastFramedTransport.java @@ -0,0 +1,201 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.thrift.transport.layered; + +import java.util.Objects; +import org.apache.thrift.TConfiguration; +import org.apache.thrift.transport.*; + +/** + * This transport is wire compatible with {@link TFramedTransport}, but makes use of reusable, + * expanding read and write buffers in order to avoid allocating new byte[]s all the time. Since the + * buffers only expand, you should probably only use this transport if your messages are not too + * variably large, unless the persistent memory cost is not an issue. + * + *

This implementation is NOT threadsafe. + */ +public class TFastFramedTransport extends TLayeredTransport { + + public static class Factory extends TTransportFactory { + private final int initialCapacity; + private final int maxLength; + + public Factory() { + this(DEFAULT_BUF_CAPACITY, TConfiguration.DEFAULT_MAX_FRAME_SIZE); + } + + public Factory(int initialCapacity) { + this(initialCapacity, TConfiguration.DEFAULT_MAX_FRAME_SIZE); + } + + public Factory(int initialCapacity, int maxLength) { + this.initialCapacity = initialCapacity; + this.maxLength = maxLength; + } + + @Override + public TTransport getTransport(TTransport trans) throws TTransportException { + return new TFastFramedTransport(trans, initialCapacity, maxLength); + } + } + + /** How big should the default read and write buffers be? */ + public static final int DEFAULT_BUF_CAPACITY = 1024; + + private final AutoExpandingBufferWriteTransport writeBuffer; + private AutoExpandingBufferReadTransport readBuffer; + private final int initialBufferCapacity; + private final byte[] i32buf = new byte[4]; + private final int maxLength; + + /** + * Create a new {@link TFastFramedTransport}. Use the defaults for initial buffer size and max + * frame length. + * + * @param underlying Transport that real reads and writes will go through to. + */ + public TFastFramedTransport(TTransport underlying) throws TTransportException { + this(underlying, DEFAULT_BUF_CAPACITY, TConfiguration.DEFAULT_MAX_FRAME_SIZE); + } + + /** + * Create a new {@link TFastFramedTransport}. Use the specified initial buffer capacity and the + * default max frame length. + * + * @param underlying Transport that real reads and writes will go through to. + * @param initialBufferCapacity The initial size of the read and write buffers. In practice, it's + * not critical to set this unless you know in advance that your messages are going to be very + * large. + */ + public TFastFramedTransport(TTransport underlying, int initialBufferCapacity) + throws TTransportException { + this(underlying, initialBufferCapacity, TConfiguration.DEFAULT_MAX_FRAME_SIZE); + } + + /** + * @param underlying Transport that real reads and writes will go through to. + * @param initialBufferCapacity The initial size of the read and write buffers. In practice, it's + * not critical to set this unless you know in advance that your messages are going to be very + * large. (You can pass TFramedTransportWithReusableBuffer.DEFAULT_BUF_CAPACITY if you're only + * using this constructor because you want to set the maxLength.) + * @param maxLength The max frame size you are willing to read. You can use this parameter to + * limit how much memory can be allocated. + */ + public TFastFramedTransport(TTransport underlying, int initialBufferCapacity, int maxLength) + throws TTransportException { + super(underlying); + TConfiguration config = + Objects.isNull(underlying.getConfiguration()) + ? new TConfiguration() + : underlying.getConfiguration(); + this.maxLength = maxLength; + config.setMaxFrameSize(maxLength); + this.initialBufferCapacity = initialBufferCapacity; + readBuffer = new AutoExpandingBufferReadTransport(config, initialBufferCapacity); + writeBuffer = new AutoExpandingBufferWriteTransport(config, initialBufferCapacity, 4); + } + + @Override + public void close() { + getInnerTransport().close(); + } + + @Override + public boolean isOpen() { + return getInnerTransport().isOpen(); + } + + @Override + public void open() throws TTransportException { + getInnerTransport().open(); + } + + @Override + public int read(byte[] buf, int off, int len) throws TTransportException { + int got = readBuffer.read(buf, off, len); + if (got > 0) { + return got; + } + + // Read another frame of data + readFrame(); + + return readBuffer.read(buf, off, len); + } + + private void readFrame() throws TTransportException { + getInnerTransport().readAll(i32buf, 0, 4); + int size = TFramedTransport.decodeFrameSize(i32buf); + + if (size < 0) { + close(); + throw new TTransportException( + TTransportException.CORRUPTED_DATA, "Read a negative frame size (" + size + ")!"); + } + + if (size > getInnerTransport().getConfiguration().getMaxFrameSize()) { + close(); + throw new TTransportException( + TTransportException.CORRUPTED_DATA, + "Frame size (" + size + ") larger than max length (" + maxLength + ")!"); + } + + readBuffer.fill(getInnerTransport(), size); + } + + @Override + public void write(byte[] buf, int off, int len) throws TTransportException { + writeBuffer.write(buf, off, len); + } + + @Override + public void consumeBuffer(int len) { + readBuffer.consumeBuffer(len); + } + + /** Only clears the read buffer! */ + public void clear() throws TTransportException { + readBuffer = new AutoExpandingBufferReadTransport(getConfiguration(), initialBufferCapacity); + } + + @Override + public void flush() throws TTransportException { + int payloadLength = writeBuffer.getLength() - 4; + byte[] data = writeBuffer.getBuf().array(); + TFramedTransport.encodeFrameSize(payloadLength, data); + getInnerTransport().write(data, 0, payloadLength + 4); + writeBuffer.reset(); + getInnerTransport().flush(); + } + + @Override + public byte[] getBuffer() { + return readBuffer.getBuffer(); + } + + @Override + public int getBufferPosition() { + return readBuffer.getBufferPosition(); + } + + @Override + public int getBytesRemainingInBuffer() { + return readBuffer.getBytesRemainingInBuffer(); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/layered/TFramedTransport.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/layered/TFramedTransport.java new file mode 100644 index 0000000..a3c1bcc --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/layered/TFramedTransport.java @@ -0,0 +1,186 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport.layered; + +import java.util.Objects; +import org.apache.thrift.TByteArrayOutputStream; +import org.apache.thrift.TConfiguration; +import org.apache.thrift.transport.TMemoryInputTransport; +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; +import org.apache.thrift.transport.TTransportFactory; + +/** + * TFramedTransport is a buffered TTransport that ensures a fully read message every time by + * preceding messages with a 4-byte frame size. + */ +public class TFramedTransport extends TLayeredTransport { + + /** Buffer for output */ + private final TByteArrayOutputStream writeBuffer_ = new TByteArrayOutputStream(1024); + + /** Buffer for input */ + private final TMemoryInputTransport readBuffer_; + + public static class Factory extends TTransportFactory { + private final int maxLength_; + + public Factory() { + maxLength_ = TConfiguration.DEFAULT_MAX_FRAME_SIZE; + } + + public Factory(int maxLength) { + maxLength_ = maxLength; + } + + @Override + public TTransport getTransport(TTransport base) throws TTransportException { + return new TFramedTransport(base, maxLength_); + } + } + + /** + * Something to fill in the first four bytes of the buffer to make room for the frame size. This + * allows the implementation to write once instead of twice. + */ + private static final byte[] sizeFiller_ = new byte[] {0x00, 0x00, 0x00, 0x00}; + + /** Constructor wraps around another transport */ + public TFramedTransport(TTransport transport, int maxLength) throws TTransportException { + super(transport); + TConfiguration _configuration = + Objects.isNull(transport.getConfiguration()) + ? new TConfiguration() + : transport.getConfiguration(); + _configuration.setMaxFrameSize(maxLength); + writeBuffer_.write(sizeFiller_, 0, 4); + readBuffer_ = new TMemoryInputTransport(_configuration, new byte[0]); + } + + public TFramedTransport(TTransport transport) throws TTransportException { + this(transport, TConfiguration.DEFAULT_MAX_FRAME_SIZE); + } + + public void open() throws TTransportException { + getInnerTransport().open(); + } + + public boolean isOpen() { + return getInnerTransport().isOpen(); + } + + public void close() { + getInnerTransport().close(); + } + + public int read(byte[] buf, int off, int len) throws TTransportException { + int got = readBuffer_.read(buf, off, len); + if (got > 0) { + return got; + } + + // Read another frame of data + readFrame(); + + return readBuffer_.read(buf, off, len); + } + + @Override + public byte[] getBuffer() { + return readBuffer_.getBuffer(); + } + + @Override + public int getBufferPosition() { + return readBuffer_.getBufferPosition(); + } + + @Override + public int getBytesRemainingInBuffer() { + return readBuffer_.getBytesRemainingInBuffer(); + } + + @Override + public void consumeBuffer(int len) { + readBuffer_.consumeBuffer(len); + } + + public void clear() { + readBuffer_.clear(); + } + + private final byte[] i32buf = new byte[4]; + + private void readFrame() throws TTransportException { + getInnerTransport().readAll(i32buf, 0, 4); + int size = decodeFrameSize(i32buf); + + if (size < 0) { + close(); + throw new TTransportException( + TTransportException.CORRUPTED_DATA, "Read a negative frame size (" + size + ")!"); + } + + if (size > getInnerTransport().getConfiguration().getMaxFrameSize()) { + close(); + throw new TTransportException( + TTransportException.CORRUPTED_DATA, + "Frame size (" + + size + + ") larger than max length (" + + getInnerTransport().getConfiguration().getMaxFrameSize() + + ")!"); + } + + byte[] buff = new byte[size]; + getInnerTransport().readAll(buff, 0, size); + readBuffer_.reset(buff); + } + + public void write(byte[] buf, int off, int len) throws TTransportException { + writeBuffer_.write(buf, off, len); + } + + @Override + public void flush() throws TTransportException { + byte[] buf = writeBuffer_.get(); + int len = writeBuffer_.len() - 4; // account for the prepended frame size + writeBuffer_.reset(); + writeBuffer_.write(sizeFiller_, 0, 4); // make room for the next frame's size data + + encodeFrameSize(len, buf); // this is the frame length without the filler + getInnerTransport().write(buf, 0, len + 4); // we have to write the frame size and frame data + getInnerTransport().flush(); + } + + public static void encodeFrameSize(final int frameSize, final byte[] buf) { + buf[0] = (byte) (0xff & (frameSize >> 24)); + buf[1] = (byte) (0xff & (frameSize >> 16)); + buf[2] = (byte) (0xff & (frameSize >> 8)); + buf[3] = (byte) (0xff & (frameSize)); + } + + public static int decodeFrameSize(final byte[] buf) { + return ((buf[0] & 0xff) << 24) + | ((buf[1] & 0xff) << 16) + | ((buf[2] & 0xff) << 8) + | ((buf[3] & 0xff)); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/layered/TLayeredTransport.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/layered/TLayeredTransport.java new file mode 100644 index 0000000..5ab4c95 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/layered/TLayeredTransport.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.thrift.transport.layered; + +import java.util.Objects; +import org.apache.thrift.TConfiguration; +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; + +public abstract class TLayeredTransport extends TTransport { + + private final TTransport innerTransport; + + @Override + public TConfiguration getConfiguration() { + return innerTransport.getConfiguration(); + } + + public TLayeredTransport(TTransport transport) { + Objects.requireNonNull(transport, "TTransport cannot be null."); + innerTransport = transport; + } + + @Override + public void updateKnownMessageSize(long size) throws TTransportException { + innerTransport.updateKnownMessageSize(size); + } + + @Override + public void checkReadBytesAvailable(long numBytes) throws TTransportException { + innerTransport.checkReadBytesAvailable(numBytes); + } + + public TTransport getInnerTransport() { + return innerTransport; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/DataFrameHeaderReader.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/DataFrameHeaderReader.java new file mode 100644 index 0000000..399ccc6 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/DataFrameHeaderReader.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport.sasl; + +/** The header for data frame, it only contains a 4-byte payload size. */ +public class DataFrameHeaderReader extends FixedSizeHeaderReader { + public static final int PAYLOAD_LENGTH_BYTES = 4; + + private int payloadSize; + + @Override + protected int headerSize() { + return PAYLOAD_LENGTH_BYTES; + } + + @Override + protected void onComplete() throws TInvalidSaslFrameException { + payloadSize = byteBuffer.getInt(0); + if (payloadSize < 0) { + throw new TInvalidSaslFrameException("Payload size is negative: " + payloadSize); + } + } + + @Override + public int payloadSize() { + return payloadSize; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/DataFrameReader.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/DataFrameReader.java new file mode 100644 index 0000000..cd5c002 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/DataFrameReader.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport.sasl; + +/** Frames for thrift (serialized) messages. */ +public class DataFrameReader extends FrameReader { + + public DataFrameReader() { + super(new DataFrameHeaderReader()); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/DataFrameWriter.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/DataFrameWriter.java new file mode 100644 index 0000000..9f2168a --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/DataFrameWriter.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport.sasl; + +import static org.apache.thrift.transport.sasl.DataFrameHeaderReader.PAYLOAD_LENGTH_BYTES; + +import java.nio.ByteBuffer; +import org.apache.thrift.EncodingUtils; +import org.apache.thrift.utils.StringUtils; + +/** + * Write frames of thrift messages. It expects an empty/null header to be provided with a payload to + * be written out. Non empty headers are considered as error. + */ +public class DataFrameWriter extends FrameWriter { + + @Override + public void withOnlyPayload(byte[] payload, int offset, int length) { + if (!isComplete()) { + throw new IllegalStateException( + "Previous write is not yet complete, with " + frameBytes.remaining() + " bytes left."); + } + frameBytes = buildFrameWithPayload(payload, offset, length); + } + + @Override + protected ByteBuffer buildFrame( + byte[] header, + int headerOffset, + int headerLength, + byte[] payload, + int payloadOffset, + int payloadLength) { + if (header != null && headerLength > 0) { + throw new IllegalArgumentException( + "Extra header [" + + StringUtils.bytesToHexString(header) + + "] offset " + + payloadOffset + + " length " + + payloadLength); + } + return buildFrameWithPayload(payload, payloadOffset, payloadLength); + } + + private ByteBuffer buildFrameWithPayload(byte[] payload, int offset, int length) { + byte[] bytes = new byte[PAYLOAD_LENGTH_BYTES + length]; + EncodingUtils.encodeBigEndian(length, bytes, 0); + System.arraycopy(payload, offset, bytes, PAYLOAD_LENGTH_BYTES, length); + return ByteBuffer.wrap(bytes); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/FixedSizeHeaderReader.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/FixedSizeHeaderReader.java new file mode 100644 index 0000000..b89484a --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/FixedSizeHeaderReader.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport.sasl; + +import java.nio.ByteBuffer; +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; +import org.apache.thrift.utils.StringUtils; + +/** Headers' size should be predefined. */ +public abstract class FixedSizeHeaderReader implements FrameHeaderReader { + + protected final ByteBuffer byteBuffer = ByteBuffer.allocate(headerSize()); + + @Override + public boolean isComplete() { + return !byteBuffer.hasRemaining(); + } + + @Override + public void clear() { + byteBuffer.clear(); + } + + @Override + public byte[] toBytes() { + if (!isComplete()) { + throw new IllegalStateException( + "Header is not yet complete " + + StringUtils.bytesToHexString(byteBuffer.array(), 0, byteBuffer.position())); + } + return byteBuffer.array(); + } + + @Override + public boolean read(TTransport transport) throws TTransportException { + FrameReader.readAvailable(transport, byteBuffer); + if (byteBuffer.hasRemaining()) { + return false; + } + onComplete(); + return true; + } + + /** + * @return Size of the header. + */ + protected abstract int headerSize(); + + /** + * Actions (e.g. validation) to carry out when the header is complete. + * + * @throws TTransportException + */ + protected abstract void onComplete() throws TTransportException; +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/FrameHeaderReader.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/FrameHeaderReader.java new file mode 100644 index 0000000..dd79bc4 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/FrameHeaderReader.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport.sasl; + +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; + +/** + * Read headers for a frame. For each frame, the header contains payload size and other metadata. + */ +public interface FrameHeaderReader { + + /** + * As the thrift sasl specification states, all sasl messages (both for negotiatiing and for + * sending data) should have a header to indicate the size of the payload. + * + * @return size of the payload. + */ + int payloadSize(); + + /** + * @return The received bytes for the header. + * @throws IllegalStateException if isComplete returns false. + */ + byte[] toBytes(); + + /** + * @return true if this header has all its fields set. + */ + boolean isComplete(); + + /** Clear the header and make it available to read a new header. */ + void clear(); + + /** + * (Nonblocking) Read fields from underlying transport layer. + * + * @param transport underlying transport. + * @return true if header is complete after read. + * @throws TSaslNegotiationException if fail to read a valid header of a sasl negotiation message. + * @throws TTransportException if io error. + */ + boolean read(TTransport transport) throws TSaslNegotiationException, TTransportException; +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/FrameReader.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/FrameReader.java new file mode 100644 index 0000000..5491ed8 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/FrameReader.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport.sasl; + +import java.nio.ByteBuffer; +import org.apache.thrift.transport.TEOFException; +import org.apache.thrift.transport.TTransport; +import org.apache.thrift.transport.TTransportException; + +/** + * Read frames from a transport. Each frame has a header and a payload. A header will indicate the + * size of the payload and other informations about how to decode payload. Implementations should + * subclass it by providing a header reader implementation. + * + * @param Header type. + */ +public abstract class FrameReader { + private final T header; + private ByteBuffer payload; + + protected FrameReader(T header) { + this.header = header; + } + + /** + * (Nonblocking) Read available bytes out of the transport without blocking to wait for incoming + * data. + * + * @param transport TTransport + * @return true if current frame is complete after read. + * @throws TSaslNegotiationException if fail to read back a valid sasl negotiation message. + * @throws TTransportException if io error. + */ + public boolean read(TTransport transport) throws TSaslNegotiationException, TTransportException { + if (!header.isComplete()) { + if (readHeader(transport)) { + payload = ByteBuffer.allocate(header.payloadSize()); + } else { + return false; + } + } + if (header.payloadSize() == 0) { + return true; + } + return readPayload(transport); + } + + /** + * (Nonblocking) Try to read available header bytes from transport. + * + * @return true if header is complete after read. + * @throws TSaslNegotiationException if fail to read back a validd sasl negotiation header. + * @throws TTransportException if io error. + */ + private boolean readHeader(TTransport transport) + throws TSaslNegotiationException, TTransportException { + return header.read(transport); + } + + /** + * (Nonblocking) Try to read available + * + * @param transport underlying transport. + * @return true if payload is complete after read. + * @throws TTransportException if io error. + */ + private boolean readPayload(TTransport transport) throws TTransportException { + readAvailable(transport, payload); + return payload.hasRemaining(); + } + + /** + * @return header of the frame + */ + public T getHeader() { + return header; + } + + /** + * @return number of bytes of the header + */ + public int getHeaderSize() { + return header.toBytes().length; + } + + /** + * @return byte array of the payload + */ + public byte[] getPayload() { + return payload.array(); + } + + /** + * @return size of the payload + */ + public int getPayloadSize() { + return header.payloadSize(); + } + + /** + * @return true if the reader has fully read a frame + */ + public boolean isComplete() { + return !(payload == null || payload.hasRemaining()); + } + + /** Reset the state of the reader so that it can be reused to read a new frame. */ + public void clear() { + header.clear(); + payload = null; + } + + /** + * Read immediately available bytes from the transport into the byte buffer. + * + * @param transport TTransport + * @param recipient ByteBuffer + * @return number of bytes read out of the transport + * @throws TTransportException if io error + */ + static int readAvailable(TTransport transport, ByteBuffer recipient) throws TTransportException { + if (!recipient.hasRemaining()) { + throw new IllegalStateException( + "Trying to fill a full recipient with " + recipient.limit() + " bytes"); + } + int currentPosition = recipient.position(); + byte[] bytes = recipient.array(); + int offset = recipient.arrayOffset() + currentPosition; + int expectedLength = recipient.remaining(); + int got = transport.read(bytes, offset, expectedLength); + if (got < 0) { + throw new TEOFException( + "Transport is closed, while trying to read " + expectedLength + " bytes"); + } + recipient.position(currentPosition + got); + return got; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/FrameWriter.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/FrameWriter.java new file mode 100644 index 0000000..4af4aa0 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/FrameWriter.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport.sasl; + +import java.nio.ByteBuffer; +import org.apache.thrift.transport.TNonblockingTransport; +import org.apache.thrift.transport.TTransportException; + +/** Write frame (header and payload) to transport in a nonblocking way. */ +public abstract class FrameWriter { + + protected ByteBuffer frameBytes; + + /** + * Provide (maybe empty) header and payload to the frame. This can be called only when isComplete + * returns true (last frame has been written out). + * + * @param header Some extra header bytes (without the 4 bytes for payload length), which will be + * the start of the frame. It can be empty, depending on the message format + * @param payload Payload as a byte array + * @throws IllegalStateException if it is called when isComplete returns false + * @throws IllegalArgumentException if header or payload is invalid + */ + public void withHeaderAndPayload(byte[] header, byte[] payload) { + if (payload == null) { + payload = new byte[0]; + } + if (header == null) { + withOnlyPayload(payload); + } else { + withHeaderAndPayload(header, 0, header.length, payload, 0, payload.length); + } + } + + /** + * Provide extra header and payload to the frame. + * + * @param header byte array containing the extra header + * @param headerOffset starting offset of the header portition + * @param headerLength length of the extra header + * @param payload byte array containing the payload + * @param payloadOffset starting offset of the payload portion + * @param payloadLength length of the payload + * @throws IllegalStateException if preivous frame is not yet complete (isComplete returns fals) + * @throws IllegalArgumentException if header or payload is invalid + */ + public void withHeaderAndPayload( + byte[] header, + int headerOffset, + int headerLength, + byte[] payload, + int payloadOffset, + int payloadLength) { + if (!isComplete()) { + throw new IllegalStateException( + "Previsous write is not yet complete, with " + frameBytes.remaining() + " bytes left."); + } + frameBytes = + buildFrame(header, headerOffset, headerLength, payload, payloadOffset, payloadLength); + } + + /** + * Provide only payload to the frame. Throws UnsupportedOperationException if the frame expects a + * header. + * + * @param payload payload as a byte array + */ + public void withOnlyPayload(byte[] payload) { + withOnlyPayload(payload, 0, payload.length); + } + + /** + * Provide only payload to the frame. Throws UnsupportedOperationException if the frame expects a + * header. + * + * @param payload The underlying byte array as a recipient of the payload + * @param offset The offset in the byte array starting from where the payload is located + * @param length The length of the payload + */ + public abstract void withOnlyPayload(byte[] payload, int offset, int length); + + protected abstract ByteBuffer buildFrame( + byte[] header, + int headerOffset, + int headerLength, + byte[] payload, + int payloadOffset, + int payloadLength); + + /** + * Nonblocking write to the underlying transport. + * + * @throws TTransportException + */ + public void write(TNonblockingTransport transport) throws TTransportException { + transport.write(frameBytes); + } + + /** + * @return true when no more data needs to be written out + */ + public boolean isComplete() { + return frameBytes == null || !frameBytes.hasRemaining(); + } + + /** Release the byte buffer. */ + public void clear() { + frameBytes = null; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/NegotiationStatus.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/NegotiationStatus.java new file mode 100644 index 0000000..9b0ddcb --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/NegotiationStatus.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport.sasl; + +import static org.apache.thrift.transport.sasl.TSaslNegotiationException.ErrorType.PROTOCOL_ERROR; + +import java.util.HashMap; +import java.util.Map; + +/** Status bytes used during the initial Thrift SASL handshake. */ +public enum NegotiationStatus { + START((byte) 0x01), + OK((byte) 0x02), + BAD((byte) 0x03), + ERROR((byte) 0x04), + COMPLETE((byte) 0x05); + + private static final Map reverseMap = new HashMap<>(); + + static { + for (NegotiationStatus s : NegotiationStatus.values()) { + reverseMap.put(s.getValue(), s); + } + } + + private final byte value; + + NegotiationStatus(byte val) { + this.value = val; + } + + public byte getValue() { + return value; + } + + public static NegotiationStatus byValue(byte val) throws TSaslNegotiationException { + if (!reverseMap.containsKey(val)) { + throw new TSaslNegotiationException(PROTOCOL_ERROR, "Invalid status " + val); + } + return reverseMap.get(val); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/NonblockingSaslHandler.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/NonblockingSaslHandler.java new file mode 100644 index 0000000..33291aa --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/NonblockingSaslHandler.java @@ -0,0 +1,537 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport.sasl; + +import static org.apache.thrift.transport.sasl.NegotiationStatus.COMPLETE; +import static org.apache.thrift.transport.sasl.NegotiationStatus.OK; + +import java.net.SocketAddress; +import java.nio.channels.SelectionKey; +import java.nio.charset.StandardCharsets; +import javax.security.sasl.SaslServer; +import org.apache.thrift.TByteArrayOutputStream; +import org.apache.thrift.TProcessor; +import org.apache.thrift.protocol.TProtocol; +import org.apache.thrift.protocol.TProtocolFactory; +import org.apache.thrift.server.ServerContext; +import org.apache.thrift.server.TServerEventHandler; +import org.apache.thrift.transport.SocketAddressProvider; +import org.apache.thrift.transport.TMemoryTransport; +import org.apache.thrift.transport.TNonblockingTransport; +import org.apache.thrift.transport.TTransportException; +import org.apache.thrift.transport.sasl.TSaslNegotiationException.ErrorType; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** State machine managing one sasl connection in a nonblocking way. */ +public class NonblockingSaslHandler { + private static final Logger LOGGER = LoggerFactory.getLogger(NonblockingSaslHandler.class); + + private static final int INTEREST_NONE = 0; + private static final int INTEREST_READ = SelectionKey.OP_READ; + private static final int INTEREST_WRITE = SelectionKey.OP_WRITE; + + // Tracking the current running phase + private Phase currentPhase = Phase.INITIIALIIZING; + // Tracking the next phase on the next invocation of the state machine. + // It should be the same as current phase if current phase is not yet finished. + // Otherwise, if it is different from current phase, the statemachine is in a transition state: + // current phase is done, and next phase is not yet started. + private Phase nextPhase = currentPhase; + + // Underlying nonblocking transport + private SelectionKey selectionKey; + private TNonblockingTransport underlyingTransport; + + // APIs for intercepting event / customizing behaviors: + // Factories (decorating the base implementations) & EventHandler (intercepting) + private TSaslServerFactory saslServerFactory; + private TSaslProcessorFactory processorFactory; + private TProtocolFactory inputProtocolFactory; + private TProtocolFactory outputProtocolFactory; + private TServerEventHandler eventHandler; + private ServerContext serverContext; + // It turns out the event handler implementation in hive sometimes creates a null ServerContext. + // In order to know whether TServerEventHandler#createContext is called we use such a flag. + private boolean serverContextCreated = false; + + // Wrapper around sasl server + private ServerSaslPeer saslPeer; + + // Sasl negotiation io + private SaslNegotiationFrameReader saslResponse; + private SaslNegotiationFrameWriter saslChallenge; + // IO for request from and response to the socket + private DataFrameReader requestReader; + private DataFrameWriter responseWriter; + // If sasl is negotiated for integrity/confidentiality protection + private boolean dataProtected; + + public NonblockingSaslHandler( + SelectionKey selectionKey, + TNonblockingTransport underlyingTransport, + TSaslServerFactory saslServerFactory, + TSaslProcessorFactory processorFactory, + TProtocolFactory inputProtocolFactory, + TProtocolFactory outputProtocolFactory, + TServerEventHandler eventHandler) { + this.selectionKey = selectionKey; + this.underlyingTransport = underlyingTransport; + this.saslServerFactory = saslServerFactory; + this.processorFactory = processorFactory; + this.inputProtocolFactory = inputProtocolFactory; + this.outputProtocolFactory = outputProtocolFactory; + this.eventHandler = eventHandler; + + saslResponse = new SaslNegotiationFrameReader(); + saslChallenge = new SaslNegotiationFrameWriter(); + requestReader = new DataFrameReader(); + responseWriter = new DataFrameWriter(); + } + + /** + * Get current phase of the state machine. + * + * @return current phase. + */ + public Phase getCurrentPhase() { + return currentPhase; + } + + /** + * Get next phase of the state machine. It is different from current phase iff current phase is + * done (and next phase not yet started). + * + * @return next phase. + */ + public Phase getNextPhase() { + return nextPhase; + } + + /** + * @return underlying nonblocking socket + */ + public TNonblockingTransport getUnderlyingTransport() { + return underlyingTransport; + } + + /** + * @return SaslServer instance + */ + public SaslServer getSaslServer() { + return saslPeer.getSaslServer(); + } + + /** + * @return true if current phase is done. + */ + public boolean isCurrentPhaseDone() { + return currentPhase != nextPhase; + } + + /** + * Run state machine. + * + * @throws IllegalStateException if current state is already done. + */ + public void runCurrentPhase() { + currentPhase.runStateMachine(this); + } + + /** + * When current phase is intrested in read selection, calling this will run the current phase and + * its following phases if the following ones are interested to read, until there is nothing + * available in the underlying transport. + * + * @throws IllegalStateException if is called in an irrelevant phase. + */ + public void handleRead() { + handleOps(INTEREST_READ); + } + + /** + * Similiar to handleRead. But it is for write ops. + * + * @throws IllegalStateException if it is called in an irrelevant phase. + */ + public void handleWrite() { + handleOps(INTEREST_WRITE); + } + + private void handleOps(int interestOps) { + if (currentPhase.selectionInterest != interestOps) { + throw new IllegalStateException( + "Current phase " + currentPhase + " but got interest " + interestOps); + } + runCurrentPhase(); + if (isCurrentPhaseDone() && nextPhase.selectionInterest == interestOps) { + stepToNextPhase(); + handleOps(interestOps); + } + } + + /** + * When current phase is finished, it's expected to call this method first before running the + * state machine again. By calling this, "next phase" is marked as started (and not done), thus is + * ready to run. + * + * @throws IllegalArgumentException if current phase is not yet done. + */ + public void stepToNextPhase() { + if (!isCurrentPhaseDone()) { + throw new IllegalArgumentException("Not yet done with current phase: " + currentPhase); + } + LOGGER.debug("Switch phase {} to {}", currentPhase, nextPhase); + switch (nextPhase) { + case INITIIALIIZING: + throw new IllegalStateException("INITIALIZING cannot be the next phase of " + currentPhase); + default: + } + // If next phase's interest is not the same as current, nor the same as the selection key, + // we need to change interest on the selector. + if (!(nextPhase.selectionInterest == currentPhase.selectionInterest + || nextPhase.selectionInterest == selectionKey.interestOps())) { + changeSelectionInterest(nextPhase.selectionInterest); + } + currentPhase = nextPhase; + } + + private void changeSelectionInterest(int selectionInterest) { + selectionKey.interestOps(selectionInterest); + } + + // sasl negotiation failure handling + private void failSaslNegotiation(TSaslNegotiationException e) { + LOGGER.error("Sasl negotiation failed", e); + String errorMsg = e.getDetails(); + saslChallenge.withHeaderAndPayload( + new byte[] {e.getErrorType().code.getValue()}, errorMsg.getBytes(StandardCharsets.UTF_8)); + nextPhase = Phase.WRITING_FAILURE_MESSAGE; + } + + private void fail(Exception e) { + LOGGER.error("Failed io in " + currentPhase, e); + nextPhase = Phase.CLOSING; + } + + private void failIO(TTransportException e) { + StringBuilder errorMsg = + new StringBuilder("IO failure ").append(e.getType()).append(" in ").append(currentPhase); + if (e.getMessage() != null) { + errorMsg.append(": ").append(e.getMessage()); + } + LOGGER.error(errorMsg.toString(), e); + nextPhase = Phase.CLOSING; + } + + // Read handlings + + private void handleInitializing() { + try { + saslResponse.read(underlyingTransport); + if (saslResponse.isComplete()) { + SaslNegotiationHeaderReader startHeader = saslResponse.getHeader(); + if (startHeader.getStatus() != NegotiationStatus.START) { + throw new TInvalidSaslFrameException( + "Expecting START status but got " + startHeader.getStatus()); + } + String mechanism = new String(saslResponse.getPayload(), StandardCharsets.UTF_8); + saslPeer = saslServerFactory.getSaslPeer(mechanism); + saslResponse.clear(); + nextPhase = Phase.READING_SASL_RESPONSE; + } + } catch (TSaslNegotiationException e) { + failSaslNegotiation(e); + } catch (TTransportException e) { + failIO(e); + } + } + + private void handleReadingSaslResponse() { + try { + saslResponse.read(underlyingTransport); + if (saslResponse.isComplete()) { + nextPhase = Phase.EVALUATING_SASL_RESPONSE; + } + } catch (TSaslNegotiationException e) { + failSaslNegotiation(e); + } catch (TTransportException e) { + failIO(e); + } + } + + private void handleReadingRequest() { + try { + requestReader.read(underlyingTransport); + if (requestReader.isComplete()) { + nextPhase = Phase.PROCESSING; + } + } catch (TTransportException e) { + failIO(e); + } + } + + // Computation executions + + private void executeEvaluatingSaslResponse() { + if (!(saslResponse.getHeader().getStatus() == OK + || saslResponse.getHeader().getStatus() == COMPLETE)) { + String error = + "Expect status OK or COMPLETE, but got " + saslResponse.getHeader().getStatus(); + failSaslNegotiation(new TSaslNegotiationException(ErrorType.PROTOCOL_ERROR, error)); + return; + } + try { + byte[] response = saslResponse.getPayload(); + saslResponse.clear(); + byte[] newChallenge = saslPeer.evaluate(response); + if (saslPeer.isAuthenticated()) { + dataProtected = saslPeer.isDataProtected(); + saslChallenge.withHeaderAndPayload(new byte[] {COMPLETE.getValue()}, newChallenge); + nextPhase = Phase.WRITING_SUCCESS_MESSAGE; + } else { + saslChallenge.withHeaderAndPayload(new byte[] {OK.getValue()}, newChallenge); + nextPhase = Phase.WRITING_SASL_CHALLENGE; + } + } catch (TSaslNegotiationException e) { + failSaslNegotiation(e); + } + } + + private void executeProcessing() { + try { + byte[] inputPayload = requestReader.getPayload(); + requestReader.clear(); + byte[] rawInput = dataProtected ? saslPeer.unwrap(inputPayload) : inputPayload; + TMemoryTransport memoryTransport = + new TMemoryTransport(underlyingTransport.getConfiguration(), rawInput); + TProtocol requestProtocol = inputProtocolFactory.getProtocol(memoryTransport); + TProtocol responseProtocol = outputProtocolFactory.getProtocol(memoryTransport); + + if (eventHandler != null) { + if (!serverContextCreated) { + serverContext = eventHandler.createContext(requestProtocol, responseProtocol); + SocketAddress remoteAddress = + underlyingTransport instanceof SocketAddressProvider + ? ((SocketAddressProvider) underlyingTransport).getRemoteSocketAddress() + : null; + serverContext.setRemoteAddress(remoteAddress); + serverContextCreated = true; + } + eventHandler.processContext(serverContext, memoryTransport, memoryTransport); + } + + TProcessor processor = processorFactory.getProcessor(this); + processor.process(requestProtocol, responseProtocol); + TByteArrayOutputStream rawOutput = memoryTransport.getOutput(); + if (rawOutput.len() == 0) { + // This is a oneway request, no response to send back. Waiting for next incoming request. + nextPhase = Phase.READING_REQUEST; + return; + } + if (dataProtected) { + byte[] outputPayload = saslPeer.wrap(rawOutput.get(), 0, rawOutput.len()); + responseWriter.withOnlyPayload(outputPayload); + } else { + responseWriter.withOnlyPayload(rawOutput.get(), 0, rawOutput.len()); + } + nextPhase = Phase.WRITING_RESPONSE; + } catch (TTransportException e) { + failIO(e); + } catch (Exception e) { + fail(e); + } + } + + // Write handlings + + private void handleWritingSaslChallenge() { + try { + saslChallenge.write(underlyingTransport); + if (saslChallenge.isComplete()) { + saslChallenge.clear(); + nextPhase = Phase.READING_SASL_RESPONSE; + } + } catch (TTransportException e) { + fail(e); + } + } + + private void handleWritingSuccessMessage() { + try { + saslChallenge.write(underlyingTransport); + if (saslChallenge.isComplete()) { + LOGGER.debug("Authentication is done."); + saslChallenge = null; + saslResponse = null; + nextPhase = Phase.READING_REQUEST; + } + } catch (TTransportException e) { + fail(e); + } + } + + private void handleWritingFailureMessage() { + try { + saslChallenge.write(underlyingTransport); + if (saslChallenge.isComplete()) { + nextPhase = Phase.CLOSING; + } + } catch (TTransportException e) { + fail(e); + } + } + + private void handleWritingResponse() { + try { + responseWriter.write(underlyingTransport); + if (responseWriter.isComplete()) { + responseWriter.clear(); + nextPhase = Phase.READING_REQUEST; + } + } catch (TTransportException e) { + fail(e); + } + } + + /** + * Release all the resources managed by this state machine (connection, selection and sasl + * server). To avoid being blocked, this should be invoked in the network thread that manages the + * selector. + */ + public void close() { + try { + if (serverContextCreated) { + eventHandler.deleteContext( + serverContext, + inputProtocolFactory.getProtocol(underlyingTransport), + outputProtocolFactory.getProtocol(underlyingTransport)); + } + } finally { + selectionKey.cancel(); + if (saslPeer != null) { + saslPeer.dispose(); + } + nextPhase = Phase.CLOSED; + currentPhase = Phase.CLOSED; + underlyingTransport.close(); + LOGGER.trace("Connection closed: {}", underlyingTransport); + } + } + + public enum Phase { + INITIIALIIZING(INTEREST_READ) { + @Override + void unsafeRun(NonblockingSaslHandler statemachine) { + statemachine.handleInitializing(); + } + }, + READING_SASL_RESPONSE(INTEREST_READ) { + @Override + void unsafeRun(NonblockingSaslHandler statemachine) { + statemachine.handleReadingSaslResponse(); + } + }, + EVALUATING_SASL_RESPONSE(INTEREST_NONE) { + @Override + void unsafeRun(NonblockingSaslHandler statemachine) { + statemachine.executeEvaluatingSaslResponse(); + } + }, + WRITING_SASL_CHALLENGE(INTEREST_WRITE) { + @Override + void unsafeRun(NonblockingSaslHandler statemachine) { + statemachine.handleWritingSaslChallenge(); + } + }, + WRITING_SUCCESS_MESSAGE(INTEREST_WRITE) { + @Override + void unsafeRun(NonblockingSaslHandler statemachine) { + statemachine.handleWritingSuccessMessage(); + } + }, + WRITING_FAILURE_MESSAGE(INTEREST_WRITE) { + @Override + void unsafeRun(NonblockingSaslHandler statemachine) { + statemachine.handleWritingFailureMessage(); + } + }, + READING_REQUEST(INTEREST_READ) { + @Override + void unsafeRun(NonblockingSaslHandler statemachine) { + statemachine.handleReadingRequest(); + } + }, + PROCESSING(INTEREST_NONE) { + @Override + void unsafeRun(NonblockingSaslHandler statemachine) { + statemachine.executeProcessing(); + } + }, + WRITING_RESPONSE(INTEREST_WRITE) { + @Override + void unsafeRun(NonblockingSaslHandler statemachine) { + statemachine.handleWritingResponse(); + } + }, + CLOSING(INTEREST_NONE) { + @Override + void unsafeRun(NonblockingSaslHandler statemachine) { + statemachine.close(); + } + }, + CLOSED(INTEREST_NONE) { + @Override + void unsafeRun(NonblockingSaslHandler statemachine) { + // Do nothing. + } + }; + + // The interest on the selection key during the phase + private final int selectionInterest; + + Phase(int selectionInterest) { + this.selectionInterest = selectionInterest; + } + + /** + * Provide the execution to run for the state machine in current phase. The execution should + * return the next phase after running on the state machine. + * + * @param statemachine The state machine to run. + * @throws IllegalArgumentException if the state machine's current phase is different. + * @throws IllegalStateException if the state machine' current phase is already done. + */ + void runStateMachine(NonblockingSaslHandler statemachine) { + if (statemachine.currentPhase != this) { + throw new IllegalArgumentException( + "State machine is " + statemachine.currentPhase + " but is expected to be " + this); + } + if (statemachine.isCurrentPhaseDone()) { + throw new IllegalStateException("State machine should step into " + statemachine.nextPhase); + } + unsafeRun(statemachine); + } + + // Run the state machine without checkiing its own phase + // It should not be called direcly by users. + abstract void unsafeRun(NonblockingSaslHandler statemachine); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/SaslNegotiationFrameReader.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/SaslNegotiationFrameReader.java new file mode 100644 index 0000000..ede2177 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/SaslNegotiationFrameReader.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport.sasl; + +/** Read frames for sasl negotiatiions. */ +public class SaslNegotiationFrameReader extends FrameReader { + + public SaslNegotiationFrameReader() { + super(new SaslNegotiationHeaderReader()); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/SaslNegotiationFrameWriter.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/SaslNegotiationFrameWriter.java new file mode 100644 index 0000000..186d6d0 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/SaslNegotiationFrameWriter.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport.sasl; + +import static org.apache.thrift.transport.sasl.SaslNegotiationHeaderReader.PAYLOAD_LENGTH_BYTES; +import static org.apache.thrift.transport.sasl.SaslNegotiationHeaderReader.STATUS_BYTES; + +import java.nio.ByteBuffer; +import org.apache.thrift.EncodingUtils; +import org.apache.thrift.utils.StringUtils; + +/** + * Writer for sasl negotiation frames. It expect a status byte as header with a payload to be + * written out (any header whose size is not equal to 1 would be considered as error). + */ +public class SaslNegotiationFrameWriter extends FrameWriter { + + public static final int HEADER_BYTES = STATUS_BYTES + PAYLOAD_LENGTH_BYTES; + + @Override + public void withOnlyPayload(byte[] payload, int offset, int length) { + throw new UnsupportedOperationException("Status byte is expected for sasl frame header."); + } + + @Override + protected ByteBuffer buildFrame( + byte[] header, + int headerOffset, + int headerLength, + byte[] payload, + int payloadOffset, + int payloadLength) { + if (header == null || headerLength != STATUS_BYTES) { + throw new IllegalArgumentException( + "Header " + + StringUtils.bytesToHexString(header) + + " does not have expected length " + + STATUS_BYTES); + } + byte[] bytes = new byte[HEADER_BYTES + payloadLength]; + System.arraycopy(header, headerOffset, bytes, 0, STATUS_BYTES); + EncodingUtils.encodeBigEndian(payloadLength, bytes, STATUS_BYTES); + System.arraycopy(payload, payloadOffset, bytes, HEADER_BYTES, payloadLength); + return ByteBuffer.wrap(bytes); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/SaslNegotiationHeaderReader.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/SaslNegotiationHeaderReader.java new file mode 100644 index 0000000..5547585 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/SaslNegotiationHeaderReader.java @@ -0,0 +1,58 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport.sasl; + +import static org.apache.thrift.transport.sasl.TSaslNegotiationException.ErrorType.PROTOCOL_ERROR; + +/** + * Header for sasl negotiation frames. It contains status byte of negotiation and a 4-byte integer + * (payload size). + */ +public class SaslNegotiationHeaderReader extends FixedSizeHeaderReader { + public static final int STATUS_BYTES = 1; + public static final int PAYLOAD_LENGTH_BYTES = 4; + + private NegotiationStatus negotiationStatus; + private int payloadSize; + + @Override + protected int headerSize() { + return STATUS_BYTES + PAYLOAD_LENGTH_BYTES; + } + + @Override + protected void onComplete() throws TSaslNegotiationException { + negotiationStatus = NegotiationStatus.byValue(byteBuffer.get(0)); + payloadSize = byteBuffer.getInt(1); + if (payloadSize < 0) { + throw new TSaslNegotiationException( + PROTOCOL_ERROR, "Payload size is negative: " + payloadSize); + } + } + + @Override + public int payloadSize() { + return payloadSize; + } + + public NegotiationStatus getStatus() { + return negotiationStatus; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/SaslPeer.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/SaslPeer.java new file mode 100644 index 0000000..83515b8 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/SaslPeer.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport.sasl; + +import org.apache.thrift.transport.TTransportException; + +/** A peer in a sasl negotiation. */ +public interface SaslPeer { + + /** + * Evaluate and validate the negotiation message (response/challenge) received from peer. + * + * @param negotiationMessage response/challenge received from peer. + * @return new response/challenge to send to peer, can be null if authentication becomes success. + * @throws TSaslNegotiationException if sasl authentication fails. + */ + byte[] evaluate(byte[] negotiationMessage) throws TSaslNegotiationException; + + /** + * @return true if authentication is done. + */ + boolean isAuthenticated(); + + /** + * This method can only be called when the negotiation is complete (isAuthenticated returns true). + * Otherwise it will throw IllegalStateExceptiion. + * + * @return if the qop requires some integrity/confidential protection. + * @throws IllegalStateException if negotiation is not yet complete. + */ + boolean isDataProtected(); + + /** + * Wrap raw bytes to protect it. + * + * @param data raw bytes. + * @param offset the start position of the content to wrap. + * @param length the length of the content to wrap. + * @return bytes with protection to send to peer. + * @throws TTransportException if failure. + */ + byte[] wrap(byte[] data, int offset, int length) throws TTransportException; + + /** + * Wrap the whole byte array. + * + * @param data raw bytes. + * @return wrapped bytes. + * @throws TTransportException if failure. + */ + default byte[] wrap(byte[] data) throws TTransportException { + return wrap(data, 0, data.length); + } + + /** + * Unwrap protected data to raw bytes. + * + * @param data protected data received from peer. + * @param offset the start position of the content to unwrap. + * @param length the length of the content to unwrap. + * @return raw bytes. + * @throws TTransportException if failed. + */ + byte[] unwrap(byte[] data, int offset, int length) throws TTransportException; + + /** + * Unwrap the whole byte array. + * + * @param data wrapped bytes. + * @return raw bytes. + * @throws TTransportException if failure. + */ + default byte[] unwrap(byte[] data) throws TTransportException { + return unwrap(data, 0, data.length); + } + + /** Close this peer and release resources. */ + void dispose(); +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/ServerSaslPeer.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/ServerSaslPeer.java new file mode 100644 index 0000000..0264531 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/ServerSaslPeer.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport.sasl; + +import static org.apache.thrift.transport.sasl.TSaslNegotiationException.ErrorType.AUTHENTICATION_FAILURE; + +import javax.security.sasl.Sasl; +import javax.security.sasl.SaslException; +import javax.security.sasl.SaslServer; +import org.apache.thrift.transport.TTransportException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Server side sasl peer, a wrapper around SaslServer to provide some handy methods. */ +public class ServerSaslPeer implements SaslPeer { + private static final Logger LOGGER = LoggerFactory.getLogger(ServerSaslPeer.class); + + private static final String QOP_AUTH_INT = "auth-int"; + private static final String QOP_AUTH_CONF = "auth-conf"; + + private final SaslServer saslServer; + + public ServerSaslPeer(SaslServer saslServer) { + this.saslServer = saslServer; + } + + @Override + public byte[] evaluate(byte[] negotiationMessage) throws TSaslNegotiationException { + try { + return saslServer.evaluateResponse(negotiationMessage); + } catch (SaslException e) { + throw new TSaslNegotiationException( + AUTHENTICATION_FAILURE, "Authentication failed with " + saslServer.getMechanismName(), e); + } + } + + @Override + public boolean isAuthenticated() { + return saslServer.isComplete(); + } + + @Override + public boolean isDataProtected() { + Object qop = saslServer.getNegotiatedProperty(Sasl.QOP); + if (qop == null) { + return false; + } + for (String word : qop.toString().split("\\s*,\\s*")) { + String lowerCaseWord = word.toLowerCase(); + if (QOP_AUTH_INT.equals(lowerCaseWord) || QOP_AUTH_CONF.equals(lowerCaseWord)) { + return true; + } + } + return false; + } + + @Override + public byte[] wrap(byte[] data, int offset, int length) throws TTransportException { + try { + return saslServer.wrap(data, offset, length); + } catch (SaslException e) { + throw new TTransportException("Failed to wrap data", e); + } + } + + @Override + public byte[] unwrap(byte[] data, int offset, int length) throws TTransportException { + try { + return saslServer.unwrap(data, offset, length); + } catch (SaslException e) { + throw new TTransportException(TTransportException.CORRUPTED_DATA, "Failed to unwrap data", e); + } + } + + @Override + public void dispose() { + try { + saslServer.dispose(); + } catch (Exception e) { + LOGGER.warn("Failed to close sasl server " + saslServer.getMechanismName(), e); + } + } + + SaslServer getSaslServer() { + return saslServer; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/TBaseSaslProcessorFactory.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/TBaseSaslProcessorFactory.java new file mode 100644 index 0000000..c08884c --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/TBaseSaslProcessorFactory.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport.sasl; + +import org.apache.thrift.TProcessor; + +public class TBaseSaslProcessorFactory implements TSaslProcessorFactory { + + private final TProcessor processor; + + public TBaseSaslProcessorFactory(TProcessor processor) { + this.processor = processor; + } + + @Override + public TProcessor getProcessor(NonblockingSaslHandler saslHandler) { + return processor; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/TInvalidSaslFrameException.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/TInvalidSaslFrameException.java new file mode 100644 index 0000000..3ec8816 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/TInvalidSaslFrameException.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport.sasl; + +/** Got an invalid frame that does not respect the thrift sasl protocol. */ +public class TInvalidSaslFrameException extends TSaslNegotiationException { + + public TInvalidSaslFrameException(String message) { + super(ErrorType.PROTOCOL_ERROR, message); + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/TSaslNegotiationException.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/TSaslNegotiationException.java new file mode 100644 index 0000000..9018e3a --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/TSaslNegotiationException.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport.sasl; + +import org.apache.thrift.transport.TTransportException; + +/** Exception for sasl negotiation errors. */ +public class TSaslNegotiationException extends TTransportException { + + private final ErrorType error; + + public TSaslNegotiationException(ErrorType error, String summary) { + super(summary); + this.error = error; + } + + public TSaslNegotiationException(ErrorType error, String summary, Throwable cause) { + super(summary, cause); + this.error = error; + } + + public ErrorType getErrorType() { + return error; + } + + /** + * @return Errory type plus the message. + */ + public String getSummary() { + return error.name() + ": " + getMessage(); + } + + /** + * @return Summary and eventually the cause's message. + */ + public String getDetails() { + return getCause() == null + ? getSummary() + : getSummary() + "\nReason: " + getCause().getMessage(); + } + + public enum ErrorType { + // Unexpected system internal error during negotiation (e.g. sasl initialization failure) + INTERNAL_ERROR(NegotiationStatus.ERROR), + // Cannot read correct sasl frames from the connection => Send "ERROR" status byte to peer + PROTOCOL_ERROR(NegotiationStatus.ERROR), + // Peer is using unsupported sasl mechanisms => Send "BAD" status byte to peer + MECHANISME_MISMATCH(NegotiationStatus.BAD), + // Sasl authentication failure => Send "BAD" status byte to peer + AUTHENTICATION_FAILURE(NegotiationStatus.BAD), + ; + + public final NegotiationStatus code; + + ErrorType(NegotiationStatus code) { + this.code = code; + } + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/TSaslProcessorFactory.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/TSaslProcessorFactory.java new file mode 100644 index 0000000..877d049 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/TSaslProcessorFactory.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport.sasl; + +import org.apache.thrift.TException; +import org.apache.thrift.TProcessor; + +/** + * Get processor for a given state machine, so that users can customize the behavior of a TProcessor + * by interacting with the state machine. + */ +public interface TSaslProcessorFactory { + + TProcessor getProcessor(NonblockingSaslHandler saslHandler) throws TException; +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/TSaslServerDefinition.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/TSaslServerDefinition.java new file mode 100644 index 0000000..6e8d9ef --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/TSaslServerDefinition.java @@ -0,0 +1,45 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport.sasl; + +import java.util.Map; +import javax.security.auth.callback.CallbackHandler; + +/** Contains all the parameters used to define a SASL server implementation. */ +public class TSaslServerDefinition { + public final String mechanism; + public final String protocol; + public final String serverName; + public final Map props; + public final CallbackHandler cbh; + + public TSaslServerDefinition( + String mechanism, + String protocol, + String serverName, + Map props, + CallbackHandler cbh) { + this.mechanism = mechanism; + this.protocol = protocol; + this.serverName = serverName; + this.props = props; + this.cbh = cbh; + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/TSaslServerFactory.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/TSaslServerFactory.java new file mode 100644 index 0000000..ab0e245 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/TSaslServerFactory.java @@ -0,0 +1,70 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.transport.sasl; + +import static org.apache.thrift.transport.sasl.TSaslNegotiationException.ErrorType.MECHANISME_MISMATCH; +import static org.apache.thrift.transport.sasl.TSaslNegotiationException.ErrorType.PROTOCOL_ERROR; + +import java.util.HashMap; +import java.util.Map; +import javax.security.auth.callback.CallbackHandler; +import javax.security.sasl.Sasl; +import javax.security.sasl.SaslException; +import javax.security.sasl.SaslServer; + +/** + * Factory to create sasl server. Users can extend this class to customize the SaslServer creation. + */ +public class TSaslServerFactory { + + private final Map saslMechanisms; + + public TSaslServerFactory() { + this.saslMechanisms = new HashMap<>(); + } + + public void addSaslMechanism( + String mechanism, + String protocol, + String serverName, + Map props, + CallbackHandler cbh) { + TSaslServerDefinition definition = + new TSaslServerDefinition(mechanism, protocol, serverName, props, cbh); + saslMechanisms.put(definition.mechanism, definition); + } + + public ServerSaslPeer getSaslPeer(String mechanism) throws TSaslNegotiationException { + if (!saslMechanisms.containsKey(mechanism)) { + throw new TSaslNegotiationException( + MECHANISME_MISMATCH, "Unsupported mechanism " + mechanism); + } + TSaslServerDefinition saslDef = saslMechanisms.get(mechanism); + try { + SaslServer saslServer = + Sasl.createSaslServer( + saslDef.mechanism, saslDef.protocol, saslDef.serverName, saslDef.props, saslDef.cbh); + return new ServerSaslPeer(saslServer); + } catch (SaslException e) { + throw new TSaslNegotiationException( + PROTOCOL_ERROR, "Fail to create sasl server " + mechanism, e); + } + } +} diff --git a/hbase-shaded-thrift/src/main/java/org/apache/thrift/utils/StringUtils.java b/hbase-shaded-thrift/src/main/java/org/apache/thrift/utils/StringUtils.java new file mode 100644 index 0000000..6cb5070 --- /dev/null +++ b/hbase-shaded-thrift/src/main/java/org/apache/thrift/utils/StringUtils.java @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.thrift.utils; + +public final class StringUtils { + + private StringUtils() { + // Utility class. + } + + private static final char[] HEX_CHARS = { + '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F' + }; + + /** + * Stringify a byte array to the hex representation for each byte. + * + * @param bytes the byte array to convert to hex string. + * @return hex string. + */ + public static String bytesToHexString(byte[] bytes) { + if (bytes == null) { + return null; + } + return bytesToHexString(bytes, 0, bytes.length); + } + + /** + * Stringify a portion of the byte array. + * + * @param bytes byte array. + * @param offset portion start. + * @param length portion length. + * @return hex string. + */ + public static String bytesToHexString(byte[] bytes, int offset, int length) { + if (length < 0) { + throw new IllegalArgumentException("Negative length " + length); + } + if (offset < 0) { + throw new IndexOutOfBoundsException("Negative start offset " + offset); + } + if (length > bytes.length - offset) { + throw new IndexOutOfBoundsException( + "Invalid range, bytes.length: " + + bytes.length + + " offset: " + + offset + + " length: " + + length); + } + char[] chars = new char[length * 2]; + for (int i = 0; i < length; i++) { + int unsignedInt = bytes[i + offset] & 0xFF; + chars[2 * i] = HEX_CHARS[unsignedInt >>> 4]; + chars[2 * i + 1] = HEX_CHARS[unsignedInt & 0x0F]; + } + return new String(chars); + } +} diff --git a/hbase-shaded-thrift/src/main/patches/HBASE-30194-01-servlet-javax.patch b/hbase-shaded-thrift/src/main/patches/HBASE-30194-01-servlet-javax.patch new file mode 100644 index 0000000..16e0be7 --- /dev/null +++ b/hbase-shaded-thrift/src/main/patches/HBASE-30194-01-servlet-javax.patch @@ -0,0 +1,39 @@ +diff --git a/src/main/java/org/apache/thrift/server/TServlet.java b/src/main/java/org/apache/thrift/server/TServlet.java +--- a/src/main/java/org/apache/thrift/server/TServlet.java ++++ b/src/main/java/org/apache/thrift/server/TServlet.java +@@ -1,9 +1,9 @@ + package org.apache.thrift.server; + +-import jakarta.servlet.ServletException; +-import jakarta.servlet.http.HttpServlet; +-import jakarta.servlet.http.HttpServletRequest; +-import jakarta.servlet.http.HttpServletResponse; ++import javax.servlet.ServletException; ++import javax.servlet.http.HttpServlet; ++import javax.servlet.http.HttpServletRequest; ++import javax.servlet.http.HttpServletResponse; + import java.io.IOException; + import java.io.InputStream; + import java.io.OutputStream; +diff --git a/src/main/java/org/apache/thrift/server/TExtensibleServlet.java b/src/main/java/org/apache/thrift/server/TExtensibleServlet.java +--- a/src/main/java/org/apache/thrift/server/TExtensibleServlet.java ++++ b/src/main/java/org/apache/thrift/server/TExtensibleServlet.java +@@ -19,12 +19,12 @@ + + package org.apache.thrift.server; + +-import jakarta.servlet.ServletConfig; +-import jakarta.servlet.ServletContext; +-import jakarta.servlet.ServletException; +-import jakarta.servlet.http.HttpServlet; +-import jakarta.servlet.http.HttpServletRequest; +-import jakarta.servlet.http.HttpServletResponse; ++import javax.servlet.ServletConfig; ++import javax.servlet.ServletContext; ++import javax.servlet.ServletException; ++import javax.servlet.http.HttpServlet; ++import javax.servlet.http.HttpServletRequest; ++import javax.servlet.http.HttpServletResponse; + import java.io.IOException; + import java.io.InputStream; + import java.io.OutputStream; diff --git a/hbase-shaded-thrift/src/main/patches/HBASE-30194-02-httpclient4.patch b/hbase-shaded-thrift/src/main/patches/HBASE-30194-02-httpclient4.patch new file mode 100644 index 0000000..e76ac3e --- /dev/null +++ b/hbase-shaded-thrift/src/main/patches/HBASE-30194-02-httpclient4.patch @@ -0,0 +1,129 @@ +diff --git a/src/main/java/org/apache/thrift/THttpClientResponseHandler.java b/src/main/java/org/apache/thrift/THttpClientResponseHandler.java +--- a/src/main/java/org/apache/thrift/THttpClientResponseHandler.java ++++ b/src/main/java/org/apache/thrift/THttpClientResponseHandler.java +@@ -4,18 +4,18 @@ + import java.io.ByteArrayOutputStream; + import java.io.IOException; + import java.io.InputStream; +-import org.apache.hc.core5.http.ClassicHttpResponse; +-import org.apache.hc.core5.http.HttpEntity; +-import org.apache.hc.core5.http.HttpException; +-import org.apache.hc.core5.http.HttpStatus; +-import org.apache.hc.core5.http.io.HttpClientResponseHandler; ++import org.apache.http.HttpEntity; ++import org.apache.http.HttpResponse; ++import org.apache.http.HttpStatus; ++import org.apache.http.client.ResponseHandler; ++import org.apache.http.client.ClientProtocolException; + +-public class THttpClientResponseHandler implements HttpClientResponseHandler { ++public class THttpClientResponseHandler implements ResponseHandler { + @Override +- public InputStream handleResponse(ClassicHttpResponse response) +- throws HttpException, IOException { ++ public InputStream handleResponse(HttpResponse response) ++ throws ClientProtocolException, IOException { + try (InputStream is = response.getEntity().getContent()) { +- int responseCode = response.getCode(); ++ int responseCode = response.getStatusLine().getStatusCode(); + if (responseCode != HttpStatus.SC_OK) { + throw new IOException("HTTP Response code: " + responseCode); + } +diff --git a/src/main/java/org/apache/thrift/transport/THttpClient.java b/src/main/java/org/apache/thrift/transport/THttpClient.java +--- a/src/main/java/org/apache/thrift/transport/THttpClient.java ++++ b/src/main/java/org/apache/thrift/transport/THttpClient.java +@@ -27,13 +27,11 @@ + import java.util.Collections; + import java.util.HashMap; + import java.util.Map; +-import org.apache.hc.client5.http.classic.HttpClient; +-import org.apache.hc.client5.http.classic.methods.HttpPost; +-import org.apache.hc.client5.http.config.ConnectionConfig; +-import org.apache.hc.client5.http.config.RequestConfig; +-import org.apache.hc.core5.http.HttpHost; +-import org.apache.hc.core5.http.io.entity.ByteArrayEntity; +-import org.apache.hc.core5.util.Timeout; ++import org.apache.http.HttpHost; ++import org.apache.http.client.HttpClient; ++import org.apache.http.client.config.RequestConfig; ++import org.apache.http.client.methods.HttpPost; ++import org.apache.http.entity.ByteArrayEntity; + import org.apache.thrift.TConfiguration; + import org.apache.thrift.THttpClientResponseHandler; + +@@ -138,9 +136,9 @@ + this.client = client; + this.host = + new HttpHost( +- url_.getProtocol(), + url_.getHost(), +- -1 == url_.getPort() ? url_.getDefaultPort() : url_.getPort()); ++ -1 == url_.getPort() ? url_.getDefaultPort() : url_.getPort(), ++ url_.getProtocol()); + } catch (IOException iox) { + throw new TTransportException(iox); + } +@@ -153,9 +151,9 @@ + this.client = client; + this.host = + new HttpHost( +- url_.getProtocol(), + url_.getHost(), +- -1 == url_.getPort() ? url_.getDefaultPort() : url_.getPort()); ++ -1 == url_.getPort() ? url_.getDefaultPort() : url_.getPort(), ++ url_.getProtocol()); + } catch (IOException iox) { + throw new TTransportException(iox); + } +@@ -166,10 +164,8 @@ + } + + /** +- * Use instead {@link +- * org.apache.hc.client5.http.impl.io.BasicHttpClientConnectionManager#setConnectionConfig} or +- * {@link +- * org.apache.hc.client5.http.impl.io.PoolingHttpClientConnectionManager#setDefaultConnectionConfig} ++ * Use instead a {@link RequestConfig} pre-configured on the underlying Apache HttpClient (e.g. ++ * via {@code HttpClientBuilder#setDefaultRequestConfig}). + */ + @Deprecated + public void setReadTimeout(int timeout) { +@@ -233,25 +229,14 @@ + } + + private RequestConfig getRequestConfig() { +- RequestConfig requestConfig = RequestConfig.DEFAULT; ++ RequestConfig.Builder builder = RequestConfig.custom(); + if (connectTimeout_ > 0) { +- requestConfig = +- RequestConfig.copy(requestConfig) +- .setConnectionRequestTimeout(Timeout.ofMilliseconds(connectTimeout_)) +- .build(); ++ builder.setConnectTimeout(connectTimeout_).setConnectionRequestTimeout(connectTimeout_); + } +- return requestConfig; +- } +- +- private ConnectionConfig getConnectionConfig() { +- ConnectionConfig connectionConfig = ConnectionConfig.DEFAULT; + if (readTimeout_ > 0) { +- connectionConfig = +- ConnectionConfig.copy(connectionConfig) +- .setSocketTimeout(Timeout.ofMilliseconds(readTimeout_)) +- .build(); ++ builder.setSocketTimeout(readTimeout_); + } +- return connectionConfig; ++ return builder.build(); + } + + private static Map getDefaultHeaders() { +@@ -279,7 +264,7 @@ + if (null != customHeaders_) { + customHeaders_.forEach(post::addHeader); + } +- post.setEntity(new ByteArrayEntity(data, null)); ++ post.setEntity(new ByteArrayEntity(data)); + inputStream_ = client.execute(this.host, post, new THttpClientResponseHandler()); + } catch (IOException ioe) { + // Abort method so the connection gets released back to the connection manager diff --git a/hbase-shaded-thrift/src/main/resources/META-INF/LICENSE.txt b/hbase-shaded-thrift/src/main/resources/META-INF/LICENSE.txt new file mode 100644 index 0000000..f308fa8 --- /dev/null +++ b/hbase-shaded-thrift/src/main/resources/META-INF/LICENSE.txt @@ -0,0 +1,315 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +-------------------------------------------------- +SOFTWARE DISTRIBUTED WITH THRIFT: + +The Apache Thrift software includes a number of subcomponents with +separate copyright notices and license terms. Your use of the source +code for the these subcomponents is subject to the terms and +conditions of the following licenses. + +-------------------------------------------------- +Portions of the following files are licensed under the MIT License: + + lib/erl/src/Makefile.am + +Please see doc/otp-base-license.txt for the full terms of this license. + +-------------------------------------------------- +For the aclocal/ax_boost_base.m4 and contrib/fb303/aclocal/ax_boost_base.m4 components: + +# Copyright (c) 2007 Thomas Porschberg +# +# Copying and distribution of this file, with or without +# modification, are permitted in any medium without royalty provided +# the copyright notice and this notice are preserved. + +-------------------------------------------------- +For the lib/nodejs/lib/thrift/json_parse.js: + +/* + json_parse.js + 2015-05-02 + Public Domain. + NO WARRANTY EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. + +*/ +(By Douglas Crockford ) + +-------------------------------------------------- +For lib/cpp/src/thrift/windows/SocketPair.cpp + +/* socketpair.c + * Copyright 2007 by Nathan C. Myers ; some rights reserved. + * This code is Free Software. It may be copied freely, in original or + * modified form, subject only to the restrictions that (1) the author is + * relieved from all responsibilities for any use for any purpose, and (2) + * this copyright notice must be retained, unchanged, in its entirety. If + * for any reason the author might be held responsible for any consequences + * of copying or use, license is withheld. + */ + + +-------------------------------------------------- +For lib/py/compat/win32/stdint.h + +// ISO C9x compliant stdint.h for Microsoft Visual Studio +// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 +// +// Copyright (c) 2006-2008 Alexander Chemeris +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. The name of the author may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED +// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +/////////////////////////////////////////////////////////////////////////////// + + +-------------------------------------------------- +Codegen template in t_html_generator.h + +* Bootstrap v2.0.3 +* +* Copyright 2012 Twitter, Inc +* Licensed under the Apache License v2.0 +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Designed and built with all the love in the world @twitter by @mdo and @fat. + +--------------------------------------------------- +For t_cl_generator.cc + + * Copyright (c) 2008- Patrick Collison + * Copyright (c) 2006- Facebook + +--------------------------------------------------- + +--------------------------------------------------- +For compiler/cpp/src/thrift/generate/sha256.h + +SHA-256 implementation by Brad Conte (brad AT bradconte.com). +Source: https://github.com/B-Con/crypto-algorithms +The author has placed this code in the public domain (no copyright claimed). +No algorithmic changes were made; the file was adapted to a C++ header-only +form for inclusion in the Thrift compiler. diff --git a/hbase-shaded-thrift/src/main/resources/META-INF/NOTICE.txt b/hbase-shaded-thrift/src/main/resources/META-INF/NOTICE.txt new file mode 100644 index 0000000..37824e7 --- /dev/null +++ b/hbase-shaded-thrift/src/main/resources/META-INF/NOTICE.txt @@ -0,0 +1,5 @@ +Apache Thrift +Copyright (C) 2006 - 2019, The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). diff --git a/pom.xml b/pom.xml index 5b2d441..595b0b5 100644 --- a/pom.xml +++ b/pom.xml @@ -95,6 +95,7 @@ hbase-shaded-protobuf + hbase-shaded-thrift hbase-shaded-netty hbase-shaded-netty-tcnative hbase-shaded-gson @@ -245,6 +246,10 @@ **/src/main/java/com/google/protobuf/** **/src/main/resources/google/** **/src/main/java/META-INF/** + + **/hbase-shaded-thrift/src/main/java/org/apache/thrift/** + **/hbase-shaded-thrift/src/main/resources/META-INF/LICENSE.txt + **/hbase-shaded-thrift/src/main/resources/META-INF/NOTICE.txt From b1b7485747a8fdbff722aa55b5c1a86028f47be6 Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Tue, 2 Jun 2026 08:42:55 -0700 Subject: [PATCH 2/3] Review feedback --- .gitignore | 2 + hbase-shaded-thrift/pom.xml | 5 +- .../src/main/java/META-INF/MANIFEST.MF | 2 - .../apache/thrift/AsyncProcessFunction.java | 64 -- .../java/org/apache/thrift/EncodingUtils.java | 148 --- .../main/java/org/apache/thrift/Option.java | 143 --- .../org/apache/thrift/ProcessFunction.java | 96 -- .../apache/thrift/TApplicationException.java | 147 --- .../org/apache/thrift/TAsyncProcessor.java | 33 - .../main/java/org/apache/thrift/TBase.java | 74 -- .../apache/thrift/TBaseAsyncProcessor.java | 115 -- .../java/org/apache/thrift/TBaseHelper.java | 298 ----- .../org/apache/thrift/TBaseProcessor.java | 44 - .../apache/thrift/TByteArrayOutputStream.java | 57 - .../org/apache/thrift/TConfiguration.java | 102 -- .../java/org/apache/thrift/TDeserializer.java | 730 ------------ .../main/java/org/apache/thrift/TEnum.java | 24 - .../java/org/apache/thrift/TEnumHelper.java | 47 - .../java/org/apache/thrift/TException.java | 42 - .../java/org/apache/thrift/TFieldIdEnum.java | 36 - .../apache/thrift/TFieldRequirementType.java | 31 - .../thrift/THttpClientResponseHandler.java | 74 -- .../apache/thrift/TMultiplexedProcessor.java | 168 --- .../thrift/TNonblockingMultiFetchClient.java | 374 ------ .../thrift/TNonblockingMultiFetchStats.java | 132 --- .../java/org/apache/thrift/TProcessor.java | 30 - .../org/apache/thrift/TProcessorFactory.java | 40 - .../java/org/apache/thrift/TSerializable.java | 42 - .../java/org/apache/thrift/TSerializer.java | 83 -- .../org/apache/thrift/TServiceClient.java | 96 -- .../apache/thrift/TServiceClientFactory.java | 48 - .../main/java/org/apache/thrift/TUnion.java | 293 ----- .../apache/thrift/annotation/Nullable.java | 30 - .../thrift/async/AsyncMethodCallback.java | 47 - .../async/AsyncMethodFutureAdapter.java | 35 - .../org/apache/thrift/async/TAsyncClient.java | 113 -- .../thrift/async/TAsyncClientFactory.java | 25 - .../thrift/async/TAsyncClientManager.java | 209 ---- .../apache/thrift/async/TAsyncMethodCall.java | 294 ----- .../apache/thrift/meta_data/EnumMetaData.java | 31 - .../thrift/meta_data/FieldMetaData.java | 104 -- .../thrift/meta_data/FieldValueMetaData.java | 71 -- .../apache/thrift/meta_data/ListMetaData.java | 29 - .../apache/thrift/meta_data/MapMetaData.java | 31 - .../apache/thrift/meta_data/SetMetaData.java | 29 - .../thrift/meta_data/StructMetaData.java | 31 - .../org/apache/thrift/partial/EnumCache.java | 88 -- .../thrift/partial/PartialThriftComparer.java | 342 ------ .../java/org/apache/thrift/partial/README.md | 112 -- .../org/apache/thrift/partial/TFieldData.java | 44 - .../apache/thrift/partial/ThriftField.java | 193 ---- .../partial/ThriftFieldValueProcessor.java | 93 -- .../apache/thrift/partial/ThriftMetadata.java | 552 --------- .../thrift/partial/ThriftStructProcessor.java | 181 --- .../org/apache/thrift/partial/Validate.java | 233 ---- .../apache/thrift/protocol/ShortStack.java | 77 -- .../apache/thrift/protocol/TBase64Utils.java | 118 -- .../thrift/protocol/TBinaryProtocol.java | 572 ---------- .../thrift/protocol/TCompactProtocol.java | 968 ---------------- .../org/apache/thrift/protocol/TField.java | 63 -- .../apache/thrift/protocol/TJSONProtocol.java | 1004 ----------------- .../TLegacyUuidProtocolDecorator.java | 98 -- .../org/apache/thrift/protocol/TList.java | 43 - .../java/org/apache/thrift/protocol/TMap.java | 49 - .../org/apache/thrift/protocol/TMessage.java | 78 -- .../apache/thrift/protocol/TMessageType.java | 28 - .../thrift/protocol/TMultiplexedProtocol.java | 88 -- .../org/apache/thrift/protocol/TProtocol.java | 577 ---------- .../thrift/protocol/TProtocolDecorator.java | 320 ------ .../thrift/protocol/TProtocolException.java | 77 -- .../thrift/protocol/TProtocolFactory.java | 28 - .../apache/thrift/protocol/TProtocolUtil.java | 214 ---- .../apache/thrift/protocol/TReadProtocol.java | 50 - .../java/org/apache/thrift/protocol/TSet.java | 47 - .../thrift/protocol/TSimpleJSONProtocol.java | 509 --------- .../org/apache/thrift/protocol/TStruct.java | 37 - .../thrift/protocol/TTupleProtocol.java | 122 -- .../org/apache/thrift/protocol/TType.java | 41 - .../thrift/protocol/TWriteProtocol.java | 52 - .../org/apache/thrift/scheme/IScheme.java | 30 - .../apache/thrift/scheme/SchemeFactory.java | 24 - .../apache/thrift/scheme/StandardScheme.java | 23 - .../org/apache/thrift/scheme/TupleScheme.java | 23 - .../server/AbstractNonblockingServer.java | 626 ---------- .../org/apache/thrift/server/Invocation.java | 21 - .../apache/thrift/server/ServerContext.java | 55 - .../thrift/server/TExtensibleServlet.java | 168 --- .../org/apache/thrift/server/THsHaServer.java | 194 ---- .../thrift/server/TNonblockingServer.java | 234 ---- .../thrift/server/TSaslNonblockingServer.java | 488 -------- .../org/apache/thrift/server/TServer.java | 160 --- .../thrift/server/TServerEventHandler.java | 49 - .../org/apache/thrift/server/TServlet.java | 120 -- .../apache/thrift/server/TSimpleServer.java | 120 -- .../thrift/server/TThreadPoolServer.java | 316 ------ .../server/TThreadedSelectorServer.java | 730 ------------ .../thrift/transport/AutoExpandingBuffer.java | 49 - .../AutoExpandingBufferReadTransport.java | 89 -- .../AutoExpandingBufferWriteTransport.java | 93 -- .../transport/SocketAddressProvider.java | 30 - .../apache/thrift/transport/TByteBuffer.java | 111 -- .../thrift/transport/TEOFException.java | 28 - .../thrift/transport/TEndpointTransport.java | 120 -- .../thrift/transport/TFileProcessor.java | 122 -- .../thrift/transport/TFileTransport.java | 612 ---------- .../apache/thrift/transport/THttpClient.java | 329 ------ .../thrift/transport/TIOStreamTransport.java | 209 ---- .../thrift/transport/TMemoryBuffer.java | 120 -- .../transport/TMemoryInputTransport.java | 125 -- .../thrift/transport/TMemoryTransport.java | 88 -- .../transport/TNonblockingSSLSocket.java | 263 ----- .../transport/TNonblockingServerSocket.java | 186 --- .../TNonblockingServerTransport.java | 35 - .../thrift/transport/TNonblockingSocket.java | 214 ---- .../transport/TNonblockingTransport.java | 50 - .../transport/TSSLTransportFactory.java | 528 --------- .../transport/TSaslClientTransport.java | 109 -- .../transport/TSaslServerTransport.java | 224 ---- .../thrift/transport/TSaslTransport.java | 510 --------- .../thrift/transport/TSeekableFile.java | 37 - .../thrift/transport/TServerSocket.java | 154 --- .../thrift/transport/TServerTransport.java | 89 -- .../transport/TSimpleFileTransport.java | 225 ---- .../org/apache/thrift/transport/TSocket.java | 261 ----- .../thrift/transport/TStandardFile.java | 60 - .../apache/thrift/transport/TTransport.java | 202 ---- .../thrift/transport/TTransportException.java | 78 -- .../thrift/transport/TTransportFactory.java | 38 - .../thrift/transport/TZlibTransport.java | 154 --- .../layered/TFastFramedTransport.java | 201 ---- .../transport/layered/TFramedTransport.java | 186 --- .../transport/layered/TLayeredTransport.java | 53 - .../transport/sasl/DataFrameHeaderReader.java | 45 - .../transport/sasl/DataFrameReader.java | 28 - .../transport/sasl/DataFrameWriter.java | 69 -- .../transport/sasl/FixedSizeHeaderReader.java | 73 -- .../transport/sasl/FrameHeaderReader.java | 61 - .../thrift/transport/sasl/FrameReader.java | 155 --- .../thrift/transport/sasl/FrameWriter.java | 127 --- .../transport/sasl/NegotiationStatus.java | 59 - .../sasl/NonblockingSaslHandler.java | 537 --------- .../sasl/SaslNegotiationFrameReader.java | 28 - .../sasl/SaslNegotiationFrameWriter.java | 63 -- .../sasl/SaslNegotiationHeaderReader.java | 58 - .../thrift/transport/sasl/SaslPeer.java | 96 -- .../thrift/transport/sasl/ServerSaslPeer.java | 104 -- .../sasl/TBaseSaslProcessorFactory.java | 36 - .../sasl/TInvalidSaslFrameException.java | 28 - .../sasl/TSaslNegotiationException.java | 76 -- .../transport/sasl/TSaslProcessorFactory.java | 32 - .../transport/sasl/TSaslServerDefinition.java | 45 - .../transport/sasl/TSaslServerFactory.java | 70 -- .../org/apache/thrift/utils/StringUtils.java | 77 -- .../src/main/resources/META-INF/LICENSE.txt | 315 ------ .../src/main/resources/META-INF/NOTICE.txt | 5 - pom.xml | 4 + 156 files changed, 8 insertions(+), 23343 deletions(-) delete mode 100644 hbase-shaded-thrift/src/main/java/META-INF/MANIFEST.MF delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/AsyncProcessFunction.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/EncodingUtils.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/Option.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/ProcessFunction.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TApplicationException.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TAsyncProcessor.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TBase.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TBaseAsyncProcessor.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TBaseHelper.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TBaseProcessor.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TByteArrayOutputStream.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TConfiguration.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TDeserializer.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TEnum.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TEnumHelper.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TException.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TFieldIdEnum.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TFieldRequirementType.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/THttpClientResponseHandler.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TMultiplexedProcessor.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TNonblockingMultiFetchClient.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TNonblockingMultiFetchStats.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TProcessor.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TProcessorFactory.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TSerializable.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TSerializer.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TServiceClient.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TServiceClientFactory.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/TUnion.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/annotation/Nullable.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/async/AsyncMethodCallback.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/async/AsyncMethodFutureAdapter.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/async/TAsyncClient.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/async/TAsyncClientFactory.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/async/TAsyncClientManager.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/async/TAsyncMethodCall.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/meta_data/EnumMetaData.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/meta_data/FieldMetaData.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/meta_data/FieldValueMetaData.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/meta_data/ListMetaData.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/meta_data/MapMetaData.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/meta_data/SetMetaData.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/meta_data/StructMetaData.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/EnumCache.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/PartialThriftComparer.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/README.md delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/TFieldData.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/ThriftField.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/ThriftFieldValueProcessor.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/ThriftMetadata.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/ThriftStructProcessor.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/partial/Validate.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/ShortStack.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TBase64Utils.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TBinaryProtocol.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TCompactProtocol.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TField.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TJSONProtocol.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TLegacyUuidProtocolDecorator.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TList.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TMap.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TMessage.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TMessageType.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TMultiplexedProtocol.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TProtocol.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TProtocolDecorator.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TProtocolException.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TProtocolFactory.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TProtocolUtil.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TReadProtocol.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TSet.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TSimpleJSONProtocol.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TStruct.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TTupleProtocol.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TType.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/protocol/TWriteProtocol.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/scheme/IScheme.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/scheme/SchemeFactory.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/scheme/StandardScheme.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/scheme/TupleScheme.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/server/AbstractNonblockingServer.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/server/Invocation.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/server/ServerContext.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TExtensibleServlet.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/server/THsHaServer.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TNonblockingServer.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TSaslNonblockingServer.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TServer.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TServerEventHandler.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TServlet.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TSimpleServer.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TThreadPoolServer.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/server/TThreadedSelectorServer.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/AutoExpandingBuffer.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/AutoExpandingBufferReadTransport.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/AutoExpandingBufferWriteTransport.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/SocketAddressProvider.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TByteBuffer.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TEOFException.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TEndpointTransport.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TFileProcessor.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TFileTransport.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/THttpClient.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TIOStreamTransport.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TMemoryBuffer.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TMemoryInputTransport.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TMemoryTransport.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TNonblockingSSLSocket.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TNonblockingServerSocket.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TNonblockingServerTransport.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TNonblockingSocket.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TNonblockingTransport.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TSSLTransportFactory.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TSaslClientTransport.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TSaslServerTransport.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TSaslTransport.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TSeekableFile.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TServerSocket.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TServerTransport.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TSimpleFileTransport.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TSocket.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TStandardFile.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TTransport.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TTransportException.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TTransportFactory.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/TZlibTransport.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/layered/TFastFramedTransport.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/layered/TFramedTransport.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/layered/TLayeredTransport.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/DataFrameHeaderReader.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/DataFrameReader.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/DataFrameWriter.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/FixedSizeHeaderReader.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/FrameHeaderReader.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/FrameReader.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/FrameWriter.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/NegotiationStatus.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/NonblockingSaslHandler.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/SaslNegotiationFrameReader.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/SaslNegotiationFrameWriter.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/SaslNegotiationHeaderReader.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/SaslPeer.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/ServerSaslPeer.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/TBaseSaslProcessorFactory.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/TInvalidSaslFrameException.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/TSaslNegotiationException.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/TSaslProcessorFactory.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/TSaslServerDefinition.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/transport/sasl/TSaslServerFactory.java delete mode 100644 hbase-shaded-thrift/src/main/java/org/apache/thrift/utils/StringUtils.java delete mode 100644 hbase-shaded-thrift/src/main/resources/META-INF/LICENSE.txt delete mode 100644 hbase-shaded-thrift/src/main/resources/META-INF/NOTICE.txt diff --git a/.gitignore b/.gitignore index f4b640b..53cd238 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ **/dependency-reduced-pom.xml hbase-shaded-protobuf/src/main/java hbase-shaded-protobuf/src/main/resources +hbase-shaded-thrift/src/main/java +hbase-shaded-thrift/src/main/resources .project .settings .classpath diff --git a/hbase-shaded-thrift/pom.xml b/hbase-shaded-thrift/pom.xml index b4e7de2..f27cfab 100644 --- a/hbase-shaded-thrift/pom.xml +++ b/hbase-shaded-thrift/pom.xml @@ -37,7 +37,6 @@ ${hbase.unsafe.and.protobuf.java.version} ${java.release.version} ${java.release.version} - 0.23.0 + 0.23.0 4.1.131.Final 2.0.75.Final 33.4.8-jre From 809eac64fac6e7efef2ca839577ff4c894a1b6cc Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Wed, 3 Jun 2026 14:30:42 -0700 Subject: [PATCH 3/3] More review feedback --- hbase-shaded-thrift/pom.xml | 24 ++++++++++++------------ pom.xml | 9 +++++++++ 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/hbase-shaded-thrift/pom.xml b/hbase-shaded-thrift/pom.xml index f27cfab..521904e 100644 --- a/hbase-shaded-thrift/pom.xml +++ b/hbase-shaded-thrift/pom.xml @@ -42,49 +42,49 @@ The patched libthrift sources reference slf4j-api, javax.servlet-api, Apache HttpClient/Core 4.x, and commons-lang3 at compile time. These are NOT shaded; downstream consumers must provide their own runtime versions (hbase-thrift already does). We declare them as provided - scope so they participate in compilation but never get bundled into the shaded jar. + scope so they participate in compilation but never get bundled into the shaded jar. All + versions come from properties in the parent pom so they stay aligned with what the rest of + HBase expects at runtime. --> org.slf4j slf4j-api - 1.7.36 + ${slf4j.version} provided javax.servlet javax.servlet-api - 3.1.0 + ${servlet-api.version} provided org.apache.httpcomponents httpclient - 4.5.13 + ${httpclient.version} provided org.apache.httpcomponents httpcore - 4.4.13 + ${httpcore.version} provided org.apache.commons commons-lang3 - 3.12.0 + ${commons-lang3.version} provided - + maven-clean-plugin diff --git a/pom.xml b/pom.xml index 1417ebe..cdd517c 100644 --- a/pom.xml +++ b/pom.xml @@ -160,6 +160,15 @@ 3.30.2-GA 2.21.1 2.30.0 + + 1.7.36 + 4.5.13 + 4.4.13 + 3.12.0 -Dorg.apache.hbase.thirdparty.io.netty.tryReflectionSetAccessible=true