public class HadoopUtil
extends java.lang.Object
Constructor and Description |
---|
HadoopUtil() |
Modifier and Type | Method and Description |
---|---|
static java.util.Map<org.apache.hadoop.fs.Path,org.apache.hadoop.fs.Path> |
addToClassPath(org.apache.hadoop.mapred.JobConf config,
java.util.List<java.lang.String> classpath)
Add to class path.
|
static org.apache.hadoop.mapred.JobConf |
copyJobConf(org.apache.hadoop.mapred.JobConf parentJobConf) |
static org.apache.hadoop.mapred.JobConf |
createJobConf(java.util.Map<java.lang.Object,java.lang.Object> properties,
org.apache.hadoop.mapred.JobConf defaultJobconf) |
static java.util.Map<java.lang.Object,java.lang.Object> |
createProperties(org.apache.hadoop.conf.Configuration jobConf) |
static byte[] |
decodeBytes(java.lang.String string) |
static <T> T |
deserializeBase64(java.lang.String string,
org.apache.hadoop.conf.Configuration conf,
java.lang.Class<T> type)
This method deserializes the Base64 encoded String into an Object instance.
|
static <T> T |
deserializeBase64(java.lang.String string,
org.apache.hadoop.conf.Configuration conf,
java.lang.Class<T> type,
boolean decompress) |
static java.lang.String |
encodeBytes(byte[] bytes) |
static java.lang.Class |
findMainClass(java.lang.Class defaultType) |
static java.util.Map<java.lang.String,java.lang.String> |
getConfig(org.apache.hadoop.mapred.JobConf defaultConf,
org.apache.hadoop.mapred.JobConf updatedConf) |
static java.lang.Thread |
getHDFSShutdownHook() |
static org.apache.hadoop.mapred.JobConf[] |
getJobConfs(org.apache.hadoop.mapred.JobConf job,
java.util.List<java.util.Map<java.lang.String,java.lang.String>> configs) |
static PlatformInfo |
getPlatformInfo() |
static void |
initLog4j(org.apache.hadoop.mapred.JobConf jobConf) |
static <T> ObjectSerializer |
instantiateSerializer(org.apache.hadoop.conf.Configuration conf,
java.lang.Class<T> type) |
static boolean |
isLocal(org.apache.hadoop.mapred.JobConf conf) |
static org.apache.hadoop.mapred.JobConf |
mergeConf(org.apache.hadoop.mapred.JobConf job,
java.util.Map<java.lang.String,java.lang.String> config,
boolean directly) |
static java.lang.String |
readStateFromDistCache(org.apache.hadoop.mapred.JobConf jobConf,
java.lang.String id) |
static org.apache.hadoop.mapred.JobConf |
removePropertiesFrom(org.apache.hadoop.mapred.JobConf jobConf,
java.lang.String... keys) |
static boolean |
removeStateFromDistCache(org.apache.hadoop.mapred.JobConf conf,
java.lang.String path) |
static <T> java.lang.String |
serializeBase64(T object,
org.apache.hadoop.mapred.JobConf conf) |
static <T> java.lang.String |
serializeBase64(T object,
org.apache.hadoop.mapred.JobConf conf,
boolean compress) |
static void |
setLocal(org.apache.hadoop.mapred.JobConf conf) |
static void |
syncPaths(org.apache.hadoop.mapred.JobConf config,
java.util.Map<org.apache.hadoop.fs.Path,org.apache.hadoop.fs.Path> commonPaths) |
static java.lang.String |
writeStateToDistCache(org.apache.hadoop.mapred.JobConf conf,
java.lang.String id,
java.lang.String stepState) |
public static void initLog4j(org.apache.hadoop.mapred.JobConf jobConf)
public static org.apache.hadoop.mapred.JobConf copyJobConf(org.apache.hadoop.mapred.JobConf parentJobConf)
public static org.apache.hadoop.mapred.JobConf createJobConf(java.util.Map<java.lang.Object,java.lang.Object> properties, org.apache.hadoop.mapred.JobConf defaultJobconf)
public static java.util.Map<java.lang.Object,java.lang.Object> createProperties(org.apache.hadoop.conf.Configuration jobConf)
public static java.lang.Thread getHDFSShutdownHook()
public static java.lang.String encodeBytes(byte[] bytes)
public static byte[] decodeBytes(java.lang.String string)
public static <T> ObjectSerializer instantiateSerializer(org.apache.hadoop.conf.Configuration conf, java.lang.Class<T> type) throws java.lang.ClassNotFoundException
java.lang.ClassNotFoundException
public static <T> java.lang.String serializeBase64(T object, org.apache.hadoop.mapred.JobConf conf) throws java.io.IOException
java.io.IOException
public static <T> java.lang.String serializeBase64(T object, org.apache.hadoop.mapred.JobConf conf, boolean compress) throws java.io.IOException
java.io.IOException
public static <T> T deserializeBase64(java.lang.String string, org.apache.hadoop.conf.Configuration conf, java.lang.Class<T> type) throws java.io.IOException
string
- java.io.IOException
public static <T> T deserializeBase64(java.lang.String string, org.apache.hadoop.conf.Configuration conf, java.lang.Class<T> type, boolean decompress) throws java.io.IOException
java.io.IOException
public static java.lang.Class findMainClass(java.lang.Class defaultType)
public static java.util.Map<java.lang.String,java.lang.String> getConfig(org.apache.hadoop.mapred.JobConf defaultConf, org.apache.hadoop.mapred.JobConf updatedConf)
public static org.apache.hadoop.mapred.JobConf[] getJobConfs(org.apache.hadoop.mapred.JobConf job, java.util.List<java.util.Map<java.lang.String,java.lang.String>> configs)
public static org.apache.hadoop.mapred.JobConf mergeConf(org.apache.hadoop.mapred.JobConf job, java.util.Map<java.lang.String,java.lang.String> config, boolean directly)
public static org.apache.hadoop.mapred.JobConf removePropertiesFrom(org.apache.hadoop.mapred.JobConf jobConf, java.lang.String... keys)
public static boolean removeStateFromDistCache(org.apache.hadoop.mapred.JobConf conf, java.lang.String path) throws java.io.IOException
java.io.IOException
public static java.lang.String writeStateToDistCache(org.apache.hadoop.mapred.JobConf conf, java.lang.String id, java.lang.String stepState)
public static java.lang.String readStateFromDistCache(org.apache.hadoop.mapred.JobConf jobConf, java.lang.String id) throws java.io.IOException
java.io.IOException
public static PlatformInfo getPlatformInfo()
public static java.util.Map<org.apache.hadoop.fs.Path,org.apache.hadoop.fs.Path> addToClassPath(org.apache.hadoop.mapred.JobConf config, java.util.List<java.lang.String> classpath)
config
- the configclasspath
- the classpathpublic static void syncPaths(org.apache.hadoop.mapred.JobConf config, java.util.Map<org.apache.hadoop.fs.Path,org.apache.hadoop.fs.Path> commonPaths)
public static boolean isLocal(org.apache.hadoop.mapred.JobConf conf)
public static void setLocal(org.apache.hadoop.mapred.JobConf conf)