public class KafkaETLContext
extends java.lang.Object
Modifier and Type | Field and Description |
---|---|
protected int |
_bufferSize |
protected kafka.javaapi.consumer.SimpleConsumer |
_consumer |
protected long |
_count |
protected int |
_index |
protected java.lang.String |
_input |
protected java.util.Iterator<kafka.message.MessageAndOffset> |
_messageIt |
protected org.apache.hadoop.mapred.lib.MultipleOutputs |
_mos |
protected long |
_offset |
protected org.apache.hadoop.mapred.OutputCollector<KafkaETLKey,org.apache.hadoop.io.BytesWritable> |
_offsetOut |
protected long[] |
_offsetRange |
protected org.apache.hadoop.mapred.Reporter |
_reporter |
protected KafkaETLRequest |
_request |
protected long |
_requestTime |
protected java.util.Iterator<kafka.javaapi.message.ByteBufferMessageSet> |
_respIterator |
protected kafka.javaapi.FetchResponse |
_response |
protected int |
_retry |
protected long |
_startTime |
protected int |
_timeout |
protected kafka.api.FetchRequestBuilder |
builder |
protected static int |
MAX_RETRY_TIME |
Constructor and Description |
---|
KafkaETLContext(org.apache.hadoop.mapred.JobConf job,
Props props,
org.apache.hadoop.mapred.Reporter reporter,
org.apache.hadoop.mapred.lib.MultipleOutputs mos,
int index,
java.lang.String input)
construct using input string
|
Modifier and Type | Method and Description |
---|---|
void |
close() |
boolean |
fetchMore() |
protected boolean |
get(KafkaETLKey key,
org.apache.hadoop.io.BytesWritable value) |
static int |
getClientBufferSize(Props props) |
static int |
getClientTimeout(Props props) |
long |
getCount() |
boolean |
getNext(KafkaETLKey key,
org.apache.hadoop.io.BytesWritable value) |
protected long[] |
getOffsetRange()
Get offset ranges
|
long |
getReadBytes() |
long |
getTotalBytes() |
boolean |
hasMore() |
void |
output(java.lang.String fileprefix) |
protected static int MAX_RETRY_TIME
protected int _index
protected java.lang.String _input
protected KafkaETLRequest _request
protected kafka.javaapi.consumer.SimpleConsumer _consumer
protected long[] _offsetRange
protected long _offset
protected long _count
protected kafka.javaapi.FetchResponse _response
protected java.util.Iterator<kafka.message.MessageAndOffset> _messageIt
protected java.util.Iterator<kafka.javaapi.message.ByteBufferMessageSet> _respIterator
protected int _retry
protected long _requestTime
protected long _startTime
protected int _bufferSize
protected int _timeout
protected org.apache.hadoop.mapred.Reporter _reporter
protected org.apache.hadoop.mapred.lib.MultipleOutputs _mos
protected org.apache.hadoop.mapred.OutputCollector<KafkaETLKey,org.apache.hadoop.io.BytesWritable> _offsetOut
protected kafka.api.FetchRequestBuilder builder
public KafkaETLContext(org.apache.hadoop.mapred.JobConf job, Props props, org.apache.hadoop.mapred.Reporter reporter, org.apache.hadoop.mapred.lib.MultipleOutputs mos, int index, java.lang.String input) throws java.lang.Exception
java.lang.Exception
public long getTotalBytes()
public long getReadBytes()
public long getCount()
public boolean hasMore()
public boolean getNext(KafkaETLKey key, org.apache.hadoop.io.BytesWritable value) throws java.io.IOException
java.io.IOException
public boolean fetchMore() throws java.io.IOException
java.io.IOException
public void output(java.lang.String fileprefix) throws java.io.IOException
java.io.IOException
public void close() throws java.io.IOException
java.io.IOException
protected boolean get(KafkaETLKey key, org.apache.hadoop.io.BytesWritable value) throws java.io.IOException
java.io.IOException
protected long[] getOffsetRange() throws java.io.IOException
java.io.IOException
public static int getClientBufferSize(Props props) throws java.lang.Exception
java.lang.Exception
public static int getClientTimeout(Props props) throws java.lang.Exception
java.lang.Exception