macos - Hadoop 在启动时崩溃 - docker-compose (MacOS)

Question

我们正在使用 docker-compose 在本地开发机器（Mac）上启动 Hadoop 实例。

最近我们开始在 hadoop 容器中看到以下错误，在 /var/log/hadoop-yarn-resource-manager.log 文件中：

1-08-31 03:07:02,108 INFO  [main] ipc.CallQueueManager (CallQueueManager.java:<init>(75)) - Using callQueue: class java.util.concurrent.LinkedBlockingQueue scheduler: class org.apache.hadoop.ipc.DefaultRpcScheduler
2021-08-31 03:07:02,198 INFO  [main] service.AbstractService (AbstractService.java:noteFailure(272)) - Service org.apache.hadoop.yarn.server.resourcemanager.ResourceTrackerService failed in state STARTED; cause: org.apache.hadoop.yarn.exceptions.YarnRuntimeException: java.net.SocketException: Call From hadoop-master to null:0 failed on socket exception: java.net.SocketException: Unresolved address; For more details see:  http://wiki.apache.org/hadoop/SocketException
org.apache.hadoop.yarn.exceptions.YarnRuntimeException: java.net.SocketException: Call From hadoop-master to null:0 failed on socket exception: java.net.SocketException: Unresolved address; For more details see:  http://wiki.apache.org/hadoop/SocketException
        at org.apache.hadoop.yarn.factories.impl.pb.RpcServerFactoryPBImpl.getServer(RpcServerFactoryPBImpl.java:139)

我们在 /var/log/hadoop-yarn-nodemanager.log 中看到以下内容

2021-08-31 03:41:44,718 FATAL [main] nodemanager.NodeManager (NodeManager.java:initAndStartNodeManager(549)) - Error starting NodeManager
org.apache.hadoop.yarn.exceptions.YarnRuntimeException: java.net.UnknownHostException: Invalid host name: local host is: (unknown); destination host is: "hadoop-master":8031; java.net.UnknownHostException; For more det
ails see:  http://wiki.apache.org/hadoop/UnknownHost
        at org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdaterImpl.serviceStart(NodeStatusUpdaterImpl.java:230)
        at org.apache.hadoop.service.AbstractService.start(AbstractService.java:193)
        at org.apache.hadoop.service.CompositeService.serviceStart(CompositeService.java:120)
        at org.apache.hadoop.yarn.server.nodemanager.NodeManager.serviceStart(NodeManager.java:302)
        at org.apache.hadoop.service.AbstractService.start(AbstractService.java:193)
        at org.apache.hadoop.yarn.server.nodemanager.NodeManager.initAndStartNodeManager(NodeManager.java:547)
        at org.apache.hadoop.yarn.server.nodemanager.NodeManager.main(NodeManager.java:594)
Caused by: java.net.UnknownHostException: Invalid host name: local host is: (unknown); destination host is: "hadoop-master":8031; java.net.UnknownHostException; For more details see:  http://wiki.apache.org/hadoop/Unkn
ownHost
        at sun.reflect.GeneratedConstructorAccessor30.newInstance(Unknown Source)

我们正在调查目标主机是：“hadoop-master”：8031 - 我们不希望在任何地方出现“hadoop-master”或端口 8031。

码头工人撰写：

  version: '2.1' # We're using mem_limit so it stays version 2

services:
  
  zookeeper:
    image: confluentinc/cp-zookeeper:latest
    container_name: zookeeper
    ports:
      - 2181:2181
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
    mem_limit: 256m

  kafka:
    image: confluentinc/cp-kafka:latest
    container_name: kafka
    depends_on:
      - zookeeper
    ports:
      - 9092:9092
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_LISTENERS: INTERNAL://:29092,EXTERNAL://:9092
      KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka:29092,EXTERNAL://localhost:9092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT
      KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
    mem_limit: 768m

  kafka_admin:
    image: obsidiandynamics/kafdrop
    container_name: kafka-admin
    depends_on:
      - kafka
    ports:
      - 9093:9000
    environment: 
      KAFKA_BROKERCONNECT: kafka:29092
    mem_limit: 256m

  minio:
    image: minio/minio:RELEASE.2021-08-31T05-46-54Z
    container_name: minio
    ports:
      - 9094:9000
      - 9001:9001
    environment:
      MINIO_ACCESS_KEY: minio
      MINIO_SECRET_KEY: minio123
      MINIO_REGION: us-east-1
    command: server /data --console-address ":9001"
    mem_limit: 512m

  presto:
    image: ahanaio/prestodb-sandbox:0.250
    container_name: presto
    ports:
      - 9095:8080
    volumes: 
      - ./presto/minio.properties:/opt/presto-server/etc/catalog/minio.properties
      - ./presto/create-table.sql:/opt/create-table.sql
      - ./presto/sync-partitions.sql:/opt/sync-partitions.sql
    mem_limit: 1024m

  hadoop:
    image: prestodb/hdp2.6-hive
    container_name: hadoop
    volumes:
      - ./hadoop/core-site.xml:/etc/hadoop/conf/core-site.xml
    mem_limit: 2048m

和我们的 core-site.xml：

<configuration>

    <!-- OOZIE proxy user setting -->
    <property>
        <name>hadoop.proxyuser.oozie.hosts</name>
        <value>*</value>
    </property>
    <property>
        <name>hadoop.proxyuser.oozie.groups</name>
        <value>*</value>
    </property>

    <!-- HTTPFS proxy user setting -->
    <property>
        <name>hadoop.proxyuser.httpfs.hosts</name>
        <value>*</value>
    </property>
    <property>
        <name>hadoop.proxyuser.httpfs.groups</name>
        <value>*</value>
    </property>

    <!-- Llama proxy user setting -->
    <property>
        <name>hadoop.proxyuser.llama.hosts</name>
        <value>*</value>
    </property>
    <property>
        <name>hadoop.proxyuser.llama.groups</name>
        <value>*</value>
    </property>

    <!-- Hue proxy user setting -->
    <property>
        <name>hadoop.proxyuser.hue.hosts</name>
        <value>*</value>
    </property>
    <property>
        <name>hadoop.proxyuser.hue.groups</name>
        <value>*</value>
    </property>

    <!-- Mapred proxy user setting -->
    <property>
        <name>hadoop.proxyuser.mapred.hosts</name>
        <value>*</value>
    </property>
    <property>
        <name>hadoop.proxyuser.mapred.groups</name>
        <value>*</value>
    </property>

    <!-- Hive impersonation -->
    <property>
        <name>hadoop.proxyuser.hive.hosts</name>
        <value>*</value>
    </property>
  
    <property>
        <name>hadoop.proxyuser.hive.groups</name>
        <value>*</value>
    </property>

    <!-- Hdfs impersonation -->
    <property>
        <name>hadoop.proxyuser.hdfs.groups</name>
        <value>*</value>
    </property>

    <property>
        <name>hadoop.proxyuser.hdfs.hosts</name>
        <value>*</value>
    </property>

    <!-- Minio properties -->
    <property>
        <name>fs.s3a.connection.ssl.enabled</name>
        <value>false</value>
    </property>

    <property>
        <name>fs.s3a.endpoint</name>
        <value>http://minio:9000</value>
    </property>

    <property>
        <name>fs.s3a.access.key</name>
        <value>minio</value>
    </property>

    <property>
        <name>fs.s3a.secret.key</name>
        <value>minio123</value>
    </property>

    <property>
        <name>fs.s3a.path.style.access</name>
        <value>true</value>
    </property>

    <property>
        <name>fs.s3a.impl</name>
        <value>org.apache.hadoop.fs.s3a.S3AFileSystem</value>
    </property>

</configuration>

有人在使用新版本的 Docker 时遇到过这个问题吗？

标签： macosdockerhadoopdocker-compose