搭建与理解Hadoop集群：大数据处理实战

需积分: 9 40 浏览量更新于2024-07-24 收藏 7.17MB PDF 举报

"《Pro Hadoop》是一本由Jason Venner编著的专业书籍，深入探讨了Hadoop技术，包括如何构建可扩展的分布式云应用程序。这本书面向那些希望理解和掌握Hadoop集群搭建及应用的读者，内容涵盖了Hadoop的核心组件、MapReduce等关键概念。" 在当前的数字化时代，Hadoop已经成为大数据处理领域的重要工具。它是一个开源框架，设计用于存储和处理海量数据，尤其适合处理和分析非结构化和半结构化的数据。Hadoop的核心包括两个主要组件：Hadoop Distributed File System (HDFS) 和 MapReduce。 HDFS是Hadoop的基础，是一个分布式文件系统，旨在跨多台机器（节点）分散存储数据，确保高可用性和容错性。通过复制数据块，HDFS能够在节点故障时自动恢复，保证数据的完整性。这种设计使得Hadoop能在低成本硬件上处理PB级别的数据。 MapReduce则是Hadoop进行大规模并行计算的编程模型。它将复杂的大数据处理任务分解为两个阶段：Map阶段和Reduce阶段。Map阶段将原始数据切分成小块，然后在各个节点上并行处理；Reduce阶段则将Map阶段的结果聚合，生成最终的输出。这种并行处理机制极大地提高了数据处理速度。在《Pro Hadoop》中，作者Jason Venner详细介绍了如何设置和管理Hadoop集群，包括安装、配置、优化和维护。此外，书中的实例和最佳实践可以帮助读者更好地理解和应用Hadoop技术。书中还可能涵盖了其他相关主题，如YARN（Yet Another Resource Negotiator），它是Hadoop 2.x版本中的资源管理和调度器，以及Hadoop生态系统中的其他工具，如Hive（用于数据仓库和SQL查询）、Pig（数据分析平台）和Spark（更快的批处理和实时处理框架）。《Pro Hadoop》是学习和掌握Hadoop的理想资源，无论你是初学者还是寻求深化理解的专业人士，都能从中受益。通过阅读本书，你可以了解Hadoop如何在云环境中运行大规模数据挖掘任务，并利用MapReduce进行高效的数据处理，从而开启你的大数据旅程。

■CONTENTS

xiv

public long getLong(String name, long defaultValue) . . . . . . . . . . . 351

public void setLong(String name, long value) ..................351

public float getFloat(String name, float defaultValue) . . . . . . . . . . . 351

public boolean getBoolean(String name, boolean

defaultValue)

. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 352

public void setBoolean(String name, boolean value) ............352

public Configuration.IntegerRanges getRange(String name,

String defaultValue)

. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 352

public Collection<String> getStringCollection(String name) . . . . . 353

public String[ ] getStrings(String name) .......................353

public String[ ] getStrings(String name, String... defaultValue) ...354

public void setStrings(String name, String... values) ............354

public Class<?> getClassByName(String name) throws

ClassNotFoundException

.................................355

public Class<?>[ ] getClasses(String name, Class<?>...

defaultValue)

. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 355

public Class<?> getClass(String name, Class<?>

defaultValue)

. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 355

public <U> Class<? extends U> getClass(String name,

Class<? extends U> defaultValue, Class<U> xface)

. . . . . . . . . 356

public void setClass(String name, Class<?> theClass,

Class<?> xface)

. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 356

Getters for Localized and Load Balanced Paths .....................356

public Path getLocalPath(String dirsProp, String pathTrailer)

throws IOException

......................................357

public File getFile(String dirsProp, String pathTrailer) throws

IOException

. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 357

public String[ ] getLocalDirs() throws IOException ..............357

public void deleteLocalFiles() throws IOException . . . . . . . . . . . . . . 358

public void deleteLocalFiles(String subdir)throws IOException . . . 358

public Path getLocalPath(String pathString) throws

IOException

. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 358

public String getJobLocalDir() . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 358

Methods for Accessing Classpath Resources .......................359

public URL getResource(String name) . . . . . . . . . . . . . . . . . . . . . . . . 359

public InputStream getConfResourceAsInputStream

(String name)

...........................................359

public Reader getConfResourceAsReader(String name) .........359

■CONTENTS

Methods for Controlling the Task Classpath ........................360

public String getJar() .......................................360

public void setJar(String jar) . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 360

public void setJarByClass(Class cls) ..........................360

Methods for Controlling the Task Execution Environment . . . . . . . . . . . . 360

public String getUser() ......................................360

public void setUser(String user) ..............................361

public void setKeepFailedTaskFiles(boolean keep) . . . . . . . . . . . . . 361

public boolean getKeepFailedTaskFiles() ......................361

public void setKeepTaskFilesPattern(String pattern) ............361

public String getKeepTaskFilesPattern() . . . . . . . . . . . . . . . . . . . . . . 361

public void setWorkingDirectory(Path dir) . . . . . . . . . . . . . . . . . . . . . 361

public Path getWorkingDirectory() ............................362

public void setNumTasksToExecutePerJvm(int numTasks) . . . . . . 362

public int getNumTasksToExecutePerJvm() . . . . . . . . . . . . . . . . . . . 362

Methods for Controlling the Input and Output of the Job . . . . . . . . . . . . . 362

public InputFormat getInputFormat() . . . . . . . . . . . . . . . . . . . . . . . . . 363

public void setInputFormat(Class<? extends InputFormat>

theClass)

...............................................363

public OutputFormat getOutputFormat() . . . . . . . . . . . . . . . . . . . . . . 363

public void setOutputFormat(Class<? extends OutputFormat>

theClass)

...............................................363

public OutputCommitter getOutputCommitter() . . . . . . . . . . . . . . . . 363

public void setOutputCommitter(Class<? extends

OutputCommitter> theClass)

. . . . . . . . . . . . . . . . . . . . . . . . . . . . . 364

public void setCompressMapOutput(boolean compress) . . . . . . . . 364

public boolean getCompressMapOutput() . . . . . . . . . . . . . . . . . . . . . 364

public void setMapOutputCompressorClass(Class<? extends

CompressionCodec> codecClass)

.........................365

public Class<? extends CompressionCodec>

getMapOutputCompressorClass(Class<? extends

CompressionCodec> defaultValue)

........................365

public void setMapOutputKeyClass(Class<?> theClass) .........366

public Class<?> getMapOutputKeyClass() . . . . . . . . . . . . . . . . . . . . 366

public Class<?> getMapOutputValueClass() ...................366

public void setMapOutputValueClass(Class<?> theClass) .......366

public Class<?> getOutputKeyClass() . . . . . . . . . . . . . . . . . . . . . . . . 367

public void setOutputKeyClass(Class<?> theClass) . . . . . . . . . . . . 367

public Class<?> getOutputValueClass() .......................367

public void setOutputValueClass(Class<?> theClass) ...........367

■CONTENTS

xvi

Methods for Controlling Output Partitioning and Sorting for

the Reduce

. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 367

public RawComparator getOutputKeyComparator() .............368

public void setOutputKeyComparatorClass(Class<? extends

RawComparator> theClass)

. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 368

public void setKeyFieldComparatorOptions(String keySpec) . . . . . 368

public String getKeyFieldComparatorOption() ..................369

public Class<? extends Partitioner> getPartitionerClass() .......370

public void setPartitionerClass(Class<? extends Partitioner>

theClass)

...............................................370

public void setKeyFieldPartitionerOptions(String keySpec) .......370

public String getKeyFieldPartitionerOption() . . . . . . . . . . . . . . . . . . . 371

public RawComparator getOutputValueGroupingComparator() . . . 371

public void setOutputValueGroupingComparator(Class<?

extends RawComparator> theClass)

.......................371

Methods that Control Map and Reduce Tasks . . . . . . . . . . . . . . . . . . . . . . 372

public Class<? extends Mapper> getMapperClass() ............373

public void setMapperClass(Class<? extends Mapper>

theClass)

...............................................373

public Class<? extends MapRunnable> getMapRunnerClass() ...373

public void setMapRunnerClass(Class<? extends

MapRunnable> theClass)

. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 373

public Class<? extends Reducer> getReducerClass() . . . . . . . . . . 374

public void setReducerClass(Class<? extends Reducer>

theClass)

...............................................374

public Class<? extends Reducer> getCombinerClass() . . . . . . . . . 374

public void setCombinerClass(Class<? extends Reducer>

theClass)

...............................................374

public boolean getSpeculativeExecution() .....................375

public void setSpeculativeExecution(boolean

speculativeExecution)

. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 375

public boolean getMapSpeculativeExecution() . . . . . . . . . . . . . . . . . 375

public void setMapSpeculativeExecution(boolean

speculativeExecution)

. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 375

public boolean getReduceSpeculativeExecution() . . . . . . . . . . . . . . 376

public void setReduceSpeculativeExecution(boolean

speculativeExecution)

. . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 376

public int getNumMapTasks() . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 376

public void setNumMapTasks(int n) . . . . . . . . . . . . . . . . . . . . . . . . . . 376

■CONTENTS

xvii

public int getNumReduceTasks() .............................376

public void setNumReduceTasks(int n) . . . . . . . . . . . . . . . . . . . . . . . 376

public int getMaxMapAttempts() . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 377

public void setMaxMapAttempts(int n) ........................377

public int getMaxReduceAttempts() . . . . . . . . . . . . . . . . . . . . . . . . . . 377

public void setMaxReduceAttempts(int n) . . . . . . . . . . . . . . . . . . . . . 377

public void setMaxTaskFailuresPerTracker(int noFailures) .......377

public int getMaxTaskFailuresPerTracker() ....................377

public int getMaxMapTaskFailuresPercent() ...................378

public void setMaxMapTaskFailuresPercent(int percent) . . . . . . . . 378

public int getMaxReduceTaskFailuresPercent() . . . . . . . . . . . . . . . . 378

public void setMaxReduceTaskFailuresPercent(int percent) . . . . . 378

Methods Providing Control Over Job Execution and Naming ..........379

public String getJobName() . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 379

public void setJobName(String name) . . . . . . . . . . . . . . . . . . . . . . . . 379

public String getSessionId() . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 379

public void setSessionId(String sessionId) .....................380

public JobPriority getJobPriority() ............................380

public void setJobPriority(JobPriority prio) . . . . . . . . . . . . . . . . . . . . 380

public boolean getProfileEnabled() . . . . . . . . . . . . . . . . . . . . . . . . . . . 380

public void setProfileEnabled(boolean newValue) . . . . . . . . . . . . . . 381

public String getProfileParams() . . . . . . . . . . . . . . . . . . . . . . . . . . . . . 381

public void setProfileParams(String value) . . . . . . . . . . . . . . . . . . . . 381

public Configuration.IntegerRanges getProfileTaskRange

(boolean isMap)

.........................................381

public void setProfileTaskRange(boolean isMap, String

newValue)

..............................................382

public String getMapDebugScript() ...........................382

public void setMapDebugScript(String mDbgScript) . . . . . . . . . . . . 383

public String getReduceDebugScript() . . . . . . . . . . . . . . . . . . . . . . . . 383

public void setReduceDebugScript(String rDbgScript) . . . . . . . . . . 383

public String getJobEndNotificationURI() ......................384

public void setJobEndNotificationURI(String uri) ................384

public String getQueueName() ...............................384

public void setQueueName(String queueName) ................384

long getMaxVirtualMemoryForTask() { . . . . . . . . . . . . . . . . . . . . . . . . 385

void setMaxVirtualMemoryForTask(long vmem) { . . . . . . . . . . . . . . 385

剩余441页未读，继续阅读

codeplayfish

粉丝: 0
资源: 1

搭建与理解Hadoop集群：大数据处理实战

《Pro Hadoop》：快速掌握云计算分布式应用指南

Pro Hadoop：打造云计算中的可扩展应用指南

Pro Hadoop电子书：构建可扩展的分布式云应用

Pro Hadoop

Pro hadoop

Apress Pro Hadoop

Hadoop开发者全集和proHadoop

Apress - Pro Hadoop

Pro Hadoop Data Analytics

pro-hadoop-data-analytics:Kerry Koitzsch的“ Pro Hadoop数据分析”源代码

最新资源