org.apache.flink.connectors.hive.read

Class HiveTableInputFormat

java.lang.Object

org.apache.flink.api.common.io.RichInputFormat<T,SPITTYPE>

org.apache.flink.api.java.hadoop.common.HadoopInputFormatCommonBase<org.apache.flink.table.data.RowData,HiveTableInputSplit>

org.apache.flink.connectors.hive.read.HiveTableInputFormat

All Implemented Interfaces:

Serializable, org.apache.flink.api.common.io.CheckpointableInputFormat<HiveTableInputSplit,Long>, org.apache.flink.api.common.io.InputFormat<org.apache.flink.table.data.RowData,HiveTableInputSplit>, org.apache.flink.core.io.InputSplitSource<HiveTableInputSplit>

public class HiveTableInputFormat
extends org.apache.flink.api.java.hadoop.common.HadoopInputFormatCommonBase<org.apache.flink.table.data.RowData,HiveTableInputSplit>
implements org.apache.flink.api.common.io.CheckpointableInputFormat<HiveTableInputSplit,Long>

The HiveTableInputFormat are inspired by the HCatInputFormat and HadoopInputFormatBase. It's used to read from hive partition/non-partition table.

See Also:

Serialized Form

Field Summary

Fields

Modifier and Type	Field and Description
protected SplitReader	reader

Fields inherited from class org.apache.flink.api.java.hadoop.common.HadoopInputFormatCommonBase

credentials

Constructor Summary

Constructors

Constructor and Description
HiveTableInputFormat(org.apache.hadoop.mapred.JobConf jobConf, List<String> partitionKeys, org.apache.flink.table.types.DataType[] fieldTypes, String[] fieldNames, int[] projectedFields, Long limit, String hiveVersion, boolean useMapRedReader, List<HiveTablePartition> partitions)

Method Summary

Modifier and Type	Method and Description
void	close()
void	configure(org.apache.flink.configuration.Configuration parameters)
HiveTableInputSplit[]	createInputSplits(int minNumSplits)
static HiveTableInputSplit[]	createInputSplits(int minNumSplits, List<HiveTablePartition> partitions, org.apache.hadoop.mapred.JobConf jobConf)
Long	getCurrentState()
org.apache.flink.core.io.InputSplitAssigner	getInputSplitAssigner(HiveTableInputSplit[] inputSplits)
org.apache.hadoop.mapred.JobConf	getJobConf()
int	getNumFiles()
org.apache.flink.api.common.io.statistics.BaseStatistics	getStatistics(org.apache.flink.api.common.io.statistics.BaseStatistics cachedStats)
org.apache.flink.table.data.RowData	nextRecord(org.apache.flink.table.data.RowData reuse)
void	open(HiveTableInputSplit split)
boolean	reachedEnd()
void	reopen(HiveTableInputSplit split, Long state)

Methods inherited from class org.apache.flink.api.java.hadoop.common.HadoopInputFormatCommonBase

getCredentialsFromUGI, read, write

Methods inherited from class org.apache.flink.api.common.io.RichInputFormat

closeInputFormat, getRuntimeContext, openInputFormat, setRuntimeContext

Methods inherited from class java.lang.Object

clone, equals, finalize, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait

Field Detail

reader

@VisibleForTesting
protected transient SplitReader reader

Constructor Detail

HiveTableInputFormat

public HiveTableInputFormat(org.apache.hadoop.mapred.JobConf jobConf,
                            List<String> partitionKeys,
                            org.apache.flink.table.types.DataType[] fieldTypes,
                            String[] fieldNames,
                            int[] projectedFields,
                            Long limit,
                            String hiveVersion,
                            boolean useMapRedReader,
                            List<HiveTablePartition> partitions)

Method Detail

getJobConf

public org.apache.hadoop.mapred.JobConf getJobConf()

configure

public void configure(org.apache.flink.configuration.Configuration parameters)

Specified by:

configure in interface org.apache.flink.api.common.io.InputFormat<org.apache.flink.table.data.RowData,HiveTableInputSplit>

open

public void open(HiveTableInputSplit split)
          throws IOException

Specified by:

open in interface org.apache.flink.api.common.io.InputFormat<org.apache.flink.table.data.RowData,HiveTableInputSplit>

Throws:

IOException

reopen

public void reopen(HiveTableInputSplit split,
                   Long state)
            throws IOException

Specified by:

reopen in interface org.apache.flink.api.common.io.CheckpointableInputFormat<HiveTableInputSplit,Long>

Throws:

IOException

getCurrentState

public Long getCurrentState()

Specified by:

getCurrentState in interface org.apache.flink.api.common.io.CheckpointableInputFormat<HiveTableInputSplit,Long>

reachedEnd

public boolean reachedEnd()
                   throws IOException

Specified by:

reachedEnd in interface org.apache.flink.api.common.io.InputFormat<org.apache.flink.table.data.RowData,HiveTableInputSplit>

Throws:

IOException

nextRecord

public org.apache.flink.table.data.RowData nextRecord(org.apache.flink.table.data.RowData reuse)
                                               throws IOException

Specified by:

nextRecord in interface org.apache.flink.api.common.io.InputFormat<org.apache.flink.table.data.RowData,HiveTableInputSplit>

Throws:

IOException

close

public void close()
           throws IOException

Specified by:

close in interface org.apache.flink.api.common.io.InputFormat<org.apache.flink.table.data.RowData,HiveTableInputSplit>

Throws:

IOException

createInputSplits

public HiveTableInputSplit[] createInputSplits(int minNumSplits)
                                        throws IOException

Specified by:

createInputSplits in interface org.apache.flink.api.common.io.InputFormat<org.apache.flink.table.data.RowData,HiveTableInputSplit>

Specified by:

createInputSplits in interface org.apache.flink.core.io.InputSplitSource<HiveTableInputSplit>

Throws:

IOException

createInputSplits

public static HiveTableInputSplit[] createInputSplits(int minNumSplits,
                                                      List<HiveTablePartition> partitions,
                                                      org.apache.hadoop.mapred.JobConf jobConf)
                                               throws IOException

Throws:

IOException

getStatistics

public org.apache.flink.api.common.io.statistics.BaseStatistics getStatistics(org.apache.flink.api.common.io.statistics.BaseStatistics cachedStats)

Specified by:

getStatistics in interface org.apache.flink.api.common.io.InputFormat<org.apache.flink.table.data.RowData,HiveTableInputSplit>

getInputSplitAssigner

public org.apache.flink.core.io.InputSplitAssigner getInputSplitAssigner(HiveTableInputSplit[] inputSplits)

Specified by:

getInputSplitAssigner in interface org.apache.flink.api.common.io.InputFormat<org.apache.flink.table.data.RowData,HiveTableInputSplit>

Specified by:

getInputSplitAssigner in interface org.apache.flink.core.io.InputSplitSource<HiveTableInputSplit>

getNumFiles

public int getNumFiles()
                throws IOException

Throws:

IOException