hadoop组映射从配置文件读取

hadoop实现类似linux系统的文件权限，需要知道某个用户是属于哪个组。系统默认是使用ShellBasedUnixGroupsMapping 类来获取某个用户属于哪个组，实际是调用id命令来获取用户组信息。为了方便管理，也可以自己实现这个类，通过读取文件的方式来获得组信息。只要实现org.apache.hadoop.security.GroupMappingServiceProvider接口即可，具体可以参考ShellBasedUnixGroupsMapping的实现。  

代码

package cn.uc.hadoop.security;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.security.GroupMappingServiceProvider;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;

/**
 * Created by fatkun on 2016/7/31.
 */
public class FileBasedUnixGroupsMapping implements GroupMappingServiceProvider {
    private ConcurrentHashMap<String, List<String>> groups = null;
    private static final Log LOG = LogFactory.getLog(FileBasedUnixGroupsMapping.class);
    private String confFilePath = "/home/cloudera/var/group/groups.txt";
    // 格式： user1=group1,group2

    public void setConfFilePath(String confFilePath) {
        this.confFilePath = confFilePath;
    }

    private synchronized void loadConf(boolean refresh) {

        if (refresh || groups == null) {
            ConcurrentHashMap<String, List<String>> tmpGroups = new ConcurrentHashMap<>();

            try {
                BufferedReader reader = new BufferedReader(new FileReader(confFilePath));
                String line = null;
                while ((line = reader.readLine()) != null) {
                    line = line.trim();
                    if (line.startsWith("#")) continue;
                    String[] ugArray = line.split("=");
                    if (ugArray.length == 2) {
                        String user = ugArray[0].trim();
                        List<String> groupList = new ArrayList<>();
                        Set<String> groupSet = new HashSet<String>();
                        groupSet.add(user);
                        for (String group : ugArray[1].split(",")) {
                            groupSet.add(group.trim());
                        }
                        groupList.addAll(groupSet);
                        LOG.info("add user=" + user + " groups=" + groupList);
                        tmpGroups.put(user, groupList);
                    }
                }
                reader.close();
            } catch (IOException e) {
                LOG.error("LOAD GROUP MAPPING ERROR", e);
            }
            groups = tmpGroups;
        }
    }

    @Override
    public List<String> getGroups(String user) {
        if (groups == null) {
            loadConf(false);
        }

        List<String> groupList = groups.get(user);

        // 如果找不到组配置，返回一个和用户名一样的组名称
        if (groupList == null) {
            groupList = new ArrayList<>();
            groupList.add(user);
            LOG.error("CAN'T NOT FIND GROUPS FOR USER " + user);
        }

        return groupList;
    }

    @Override
    public void cacheGroupsRefresh() {
        loadConf(true);
    }

    @Override
    public void cacheGroupsAdd(List<String> groups) throws IOException {
        // does nothing in this provider of user to groups mapping
    }

    public static void main(String[] args) throws IOException {
        FileBasedUnixGroupsMapping f = new FileBasedUnixGroupsMapping();
        f.setConfFilePath("d:/tmp/groups.txt");
        System.out.println(f.getGroups("impala"));
    }
}

在cloudera如何使用

在namenode的配置“hdfs-site.xml 的 NameNode 高级配置代码段”加入以下配置，注意不要改“服务级别的hadoop.security.group.mapping”，我们只打算改掉namenode获取组映射的方式

<property>
    <name>hadoop.security.group.mapping</name>
    <value>cn.uc.hadoop.security.FileBasedUnixGroupsMapping</value>
  </property>

在 “NameNode 环境高级配置代码段（安全阀）”加入（把jar加入namenode的classpath里面）

HADOOP_CLASSPATH=$HADOOP_CLASSPATH:/home/cloudera/var/group/hadoopgroup.jar

生成groups.txt见 http://fatkun.com/2016/08/get-user-group-mapping.html