spark集群设置s3a配置

1. 在spark-env.sh文件中添加以下配置：

export HADOOP_CONF_DIR=/etc/hadoop/conf
export HADOOP_OPTS="$HADOOP_OPTS -Dcom.amazonaws.services.s3.enableV4=true"

2. 在core-site.xml文件中添加以下配置：

<property>
  <name>fs.s3a.impl</name>
  <value>org.apache.hadoop.fs.s3a.S3AFileSystem</value>
</property>

<property>
  <name>fs.s3a.access.key</name>
  <value>ACCESS_KEY</value>
</property>

<property>
  <name>fs.s3a.secret.key</name>
  <value>SECRET_KEY</value>
</property>

<property>
  <name>fs.s3a.endpoint</name>
  <value>s3.amazonaws.com</value>
</property>

<property>
  <name>fs.s3a.connection.ssl.enabled</name>
  <value>true</value>
</property>

<property>
  <name>fs.s3a.path.style.access</name>
  <value>true</value>
</property>

其中ACCESS_KEY和SECRET_KEY需要替换为你的AWS访问密钥。

3. 在hdfs-site.xml文件中添加以下配置：

<property>
  <name>dfs.client.use.datanode.hostname</name>
  <value>true</value>
</property>

<property>
  <name>dfs.datanode.use.datanode.hostname</name>
  <value>true</value>
</property>

<property>
  <name>dfs.client.read.shortcircuit</name>
  <value>true</value>
</property>

<property>
  <name>dfs.client.file-block-storage-locations.timeout.millis</name>
  <value>60000</value>
</property>

<property>
  <name>dfs.client.domain.socket.data.traffic</name>
  <value>false</value>
</property>

<property>
  <name>dfs.client.domain.socket.disabled.protocols</name>
  <value>SSLv3, TLSv1, TLSv1.1</value>
</property>

<property>
  <name>dfs.client.read.shortcircuit.skip.checksum</name>
  <value>true</value>
</property>

<property>
  <name>dfs.client.read.shortcircuit.buffer.size</name>
  <value>131072</value>
</property>

<property>
  <name>dfs.domain.socket.path</name>
  <value>/var/lib/hadoop-hdfs/dn_socket</value>
</property>

4. 在yarn-site.xml文件中添加以下配置：

<property>
  <name>yarn.nodemanager.local-dirs</name>
  <value>/var/lib/hadoop-yarn/cache/${USER}/nm-local-dir</value>
</property>

<property>
  <name>yarn.nodemanager.log-dirs</name>
  <value>/var/log/hadoop-yarn/containers</value>
</property>

<property>
  <name>yarn.nodemanager.remote-app-log-dir</name>
  <value>/var/log/hadoop-yarn/apps</value>
</property>

<property>
  <name>yarn.nodemanager.remote-app-log-dir-suffix</name>
  <value>/logs</value>
</property>

<property>
  <name>yarn.log-aggregation-enable</name>
  <value>true</value>
</property>

<property>
  <name>yarn.log-aggregation.retain-seconds</name>
  <value>2592000</value>
</property>

<property>
  <name>yarn.log-aggregation.retain-check-interval-seconds</name>
  <value>3600</value>
</property>

<property>
  <name>yarn.resourcemanager.fs.state-store.uri</name>
  <value>file:///var/lib/hadoop-yarn/system/rmstore</value>
</property>

<property>
  <name>yarn.nodemanager.aux-services</name>
  <value>mapreduce_shuffle</value>
</property>

<property>
  <name>yarn.nodemanager.aux-services.mapreduce_shuffle.class</name>
  <value>org.apache.hadoop.mapred.ShuffleHandler</value>
</property>

<property>
  <name>yarn.timeline-service.enabled</name>
  <value>true</value>
</property>

<property>
  <name>yarn.timeline-service.generic-application-history.store-class</name>
  <value>org.apache.hadoop.yarn.server.timeline.recovery.FileSystemTimelineStateStore</value>
</property>

<property>
  <name>yarn.timeline-service.store-class</name>
  <value>org.apache.hadoop.yarn.server.timeline.recovery.FileSystemTimelineStateStore</value>
</property>

5. 重启Hadoop和Spark服务。