MiniYARNCluster   MiniDFSCluster Kerberos

import static org.apache.hadoop.fs.CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SASL_KEY;

import static org.apache.hadoop.hdfs.DFSConfigKeys.*;

import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY;

import static org.junit.Assert.*;

import java.io.*;

import java.util.ArrayList;

import java.util.List;

import java.util.Properties;

import org.apache.commons.io.FileUtils;

import org.apache.hadoop.conf.Configuration;

import org.apache.hadoop.fs.*;

import org.apache.hadoop.hdfs.HdfsConfiguration;

import org.apache.hadoop.hdfs.MiniDFSCluster;

import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.SaslDataTransferTestCase;

import org.apache.hadoop.http.HttpConfig;

import org.apache.hadoop.io.IntWritable;

import org.apache.hadoop.io.Text;

import org.apache.hadoop.mapreduce.Job;

import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;

import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;

import org.apache.hadoop.minikdc.MiniKdc;

import org.apache.hadoop.security.SecurityUtil;

import org.apache.hadoop.security.UserGroupInformation;

import org.apache.hadoop.security.ssl.KeyStoreTestUtil;

import org.apache.hadoop.yarn.conf.YarnConfiguration;

import org.apache.hadoop.yarn.server.MiniYARNCluster;

import org.junit.*;

public class TestClusterWithKerberos {

private static File baseDir;

private static String hdfsPrincipal;

private static MiniKdc kdc;

private static String keytab;

private static String spnegoPrincipal;

private MiniYARNCluster yarnCluster;

private MiniDFSCluster cluster;

@BeforeClass

public static void initKdc() throws Exception {

baseDir = new File(System.getProperty("test.build.dir", "target/test-dir"),

SaslDataTransferTestCase.class.getSimpleName());

FileUtil.fullyDelete(baseDir);

assertTrue(baseDir.mkdirs());

Properties kdcConf = MiniKdc.createConf();

kdc = new MiniKdc(kdcConf, baseDir);

kdc.start();

UserGroupInformation ugi = UserGroupInformation.createRemoteUser("tjj");

UserGroupInformation.setLoginUser(ugi);

String userName = UserGroupInformation.getLoginUser().getShortUserName();

File keytabFile = new File(baseDir, userName + ".keytab");

keytab = keytabFile.getAbsolutePath();

kdc.createPrincipal(keytabFile, userName + "/localhost", "HTTP/localhost");

hdfsPrincipal = userName + "/[email protected]" + kdc.getRealm();

spnegoPrincipal = "HTTP/[email protected]" + kdc.getRealm();

System.out.println("keytab "+keytab+"hdfsPrincipal "+hdfsPrincipal);

}

@AfterClass

public static void shutdownKdc() {

if (kdc != null) {

kdc.stop();

}

FileUtil.fullyDelete(baseDir);

}

private void startCluster(HdfsConfiguration conf) throws IOException {

cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();//

cluster.waitActive();

yarnCluster = new MiniYARNCluster("MiniClusterStartsWithCountJobTest", // testName

1, // number of node managers

1, // number of local log dirs per node manager

1); // number of hdfs dirs per node manager

yarnCluster.init(conf);

yarnCluster.start();

yarnCluster.getConfig().writeXml(new FileOutputStream(new File("conf.Xml")));

}

@Test

public void testWithMiniCluster() throws Exception {

HdfsConfiguration clusterConf = createSecureConfig("authentication,integrity,privacy");

YarnConfiguration yarnConf =  createYarnSecureConfig();

clusterConf.addResource(yarnConf);

startCluster(clusterConf);

Configuration conf = new Configuration();

conf.addResource(FileUtils.openInputStream(new File("conf.Xml")));

String IN_DIR = "testing/wordcount/input";

String OUT_DIR = "testing/wordcount/output";

String DATA_FILE = "sample.txt";

FileSystem fs = FileSystem.get(conf);

Path inDir = new Path(IN_DIR);

Path outDir = new Path(OUT_DIR);

fs.delete(inDir, true);

fs.delete(outDir, true);

// create the input data files

List<String> content = new ArrayList<String>();

content.add("She sells seashells at the seashore, and she sells nuts in the mountain.");

writeHDFSContent(fs, inDir, DATA_FILE, content);

// set up the job, submit the job and wait for it complete

Job job = Job.getInstance(conf);

job.setOutputKeyClass(Text.class);

job.setOutputValueClass(IntWritable.class);

job.setMapperClass(BasicWordCount.TokenizerMapper.class);

job.setReducerClass(BasicWordCount.IntSumReducer.class);

FileInputFormat.addInputPath(job, inDir);

FileOutputFormat.setOutputPath(job, outDir);

job.waitForCompletion(true);

assertTrue(job.isSuccessful());

// now check that the output is as expected

List<String> results = getJobResults(fs, outDir, 11);

assertTrue(results.contains("She\t1"));

assertTrue(results.contains("sells\t2"));

// clean up after test case

fs.delete(inDir, true);

fs.delete(outDir, true);

}

/* @Test

public void wordcount() throws Exception {

HdfsConfiguration clusterConf = createSecureConfig("authentication,integrity,privacy");

YarnConfiguration yarnConf =  createYarnSecureConfig();

clusterConf.addResource(yarnConf);

startCluster(clusterConf);

Configuration conf = new Configuration();

conf.addResource(FileUtils.openInputStream(new File("conf.Xml")));

String IN_DIR = "testing/wordcount/input";

String OUT_DIR = "testing/wordcount/output";

String DATA_FILE = "sample.txt";

FileSystem fs = FileSystem.get(conf);

Path inDir = new Path(IN_DIR);

Path outDir = new Path(OUT_DIR);

fs.delete(inDir, true);

fs.delete(outDir, true);

// create the input data files

List<String> content = new ArrayList<String>();

content.add("She sells seashells at the seashore, and she sells nuts in the mountain.");

writeHDFSContent(fs, inDir, DATA_FILE, content);

String[] args = new String[]{IN_DIR,OUT_DIR};

int exitCode = ToolRunner.run(conf,new WordCount(), args);

fs.delete(inDir, true);

fs.delete(outDir, true);

}*/

private void writeHDFSContent(FileSystem fs, Path dir, String fileName, List<String> content) throws IOException {

Path newFilePath = new Path(dir, fileName);

FSDataOutputStream out = fs.create(newFilePath);

for (String line : content) {

out.writeBytes(line);

}

out.close();

}

protected List<String> getJobResults(FileSystem fs, Path outDir, int numLines) throws Exception {

List<String> results = new ArrayList<String>();

FileStatus[] fileStatus = fs.listStatus(outDir);

for (FileStatus file : fileStatus) {

String name = file.getPath().getName();

if (name.contains("part-r-00000")) {

Path filePath = new Path(outDir + "/" + name);

BufferedReader reader = new BufferedReader(new InputStreamReader(fs.open(filePath)));

for (int i = 0; i < numLines; i++) {

String line = reader.readLine();

if (line == null) {

fail("Results are not what was expected");

}

System.out.println("line info: "+line);

results.add(line);

}

assertNull(reader.readLine());

reader.close();

}

}

return results;

}

private HdfsConfiguration createSecureConfig(String dataTransferProtection) throws Exception {

HdfsConfiguration conf = new HdfsConfiguration();

SecurityUtil.setAuthenticationMethod(UserGroupInformation.AuthenticationMethod.KERBEROS, conf);

conf.set(DFS_NAMENODE_KERBEROS_PRINCIPAL_KEY, hdfsPrincipal);

conf.set(DFS_NAMENODE_KEYTAB_FILE_KEY, keytab);

conf.set(DFS_DATANODE_KERBEROS_PRINCIPAL_KEY, hdfsPrincipal);

conf.set(DFS_DATANODE_KEYTAB_FILE_KEY, keytab);

conf.set(DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY, spnegoPrincipal);

conf.setBoolean(DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, true);

conf.set(DFS_DATA_TRANSFER_PROTECTION_KEY, dataTransferProtection);

conf.set(DFS_HTTP_POLICY_KEY, HttpConfig.Policy.HTTPS_ONLY.name());

conf.set(DFS_NAMENODE_HTTPS_ADDRESS_KEY, "localhost:0");

conf.set(DFS_DATANODE_HTTPS_ADDRESS_KEY, "localhost:0");

conf.setInt(IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SASL_KEY, 10);

conf.set(DFS_ENCRYPT_DATA_TRANSFER_KEY, "true");//https://issues.apache.org/jira/browse/HDFS-7431

String keystoresDir = baseDir.getAbsolutePath();

String sslConfDir = KeyStoreTestUtil.getClasspathDir(this.getClass());

KeyStoreTestUtil.setupSSLConfig(keystoresDir, sslConfDir, conf, false);

return conf;

}

private YarnConfiguration createYarnSecureConfig(){

YarnConfiguration conf = new YarnConfiguration();

//yarn secure config

conf.set("yarn.resourcemanager.keytab", keytab);

conf.set("yarn.resourcemanager.principal", hdfsPrincipal);

conf.set("yarn.nodemanager.keytab", keytab);

conf.set("yarn.nodemanager.principal", hdfsPrincipal);

//   conf.set("yarn.nodemanager.container-executor.class", "org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor");

conf.set("yarn.nodemanager.container-executor.class", "org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor");

conf.set("yarn.nodemanager.linux-container-executor.path", "/container/container-executor");

conf.set("mapreduce.jobhistory.keytab", keytab);

conf.set("mapreduce.jobhistory.principal", hdfsPrincipal);

conf.set("yarn.nodemanager.aux-services", "mapreduce_shuffle");//https://issues.apache.org/jira/browse/YARN-1289

//enable security

conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHORIZATION, "true");

//yarn

conf.set("mapreduce.framework.name", "yarn");  //http://stackoverflow.com/questions/26567223/java-io-ioexception-cannot-initialize-cluster-in-hadoop2-with-yarn   use Yarn runner

return conf;

}

}

down vote

I have run into similar issues today. In my case I was building an über jar, where some dependency (I have not found the culprit yet) was bringing in a META-INF/services/org.apache.hadoop.mapreduce.protocol.ClientProtocolProvider with the contents:

org.apache.hadoop.mapred.LocalClientProtocolProvider

I provided my own in the project (e.g. put it on the classpath) with the following:

org.apache.hadoop.mapred.YarnClientProtocolProvider

and the correct one is picked up. I suspect you are seeing similar. To fix, please create the file described above, and put it on the classpath. If I find the culprit Jar, I will update the answer.

http://stackoverflow.com/questions/26567223/java-io-ioexception-cannot-initialize-cluster-in-hadoop2-with-yarn

hadoop-mapreduce-client-common-2.6.0.jar

#

#   Licensed under the Apache License, Version 2.0 (the "License");

#   you may not use this file except in compliance with the License.

#   You may obtain a copy of the License at

#

#       http://www.apache.org/licenses/LICENSE-2.0

#

#   Unless required by applicable law or agreed to in writing, software

#   distributed under the License is distributed on an "AS IS" BASIS,

#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

#   See the License for the specific language governing permissions and

#   limitations under the License.

#

#org.apache.hadoop.mapred.LocalClientProtocolProvider

org.apache.hadoop.mapred.YarnClientProtocolProvider

时间: 2024-11-25 23:38:25

MiniYARNCluster   MiniDFSCluster Kerberos的相关文章

Kerberos 互信免登陆

第一步:机器加互信 将机器A的Kerberos name加到机器B的~/.k5login中,同时将机器B的Kerberos name加到机器A的~/.k5login中 例如:host/[email protected] 第二步:使用默认的Kerberos账号登录(先注销再用krb5.keytab登录) kdestroy kinit -kt /etc/krb5.keytab 然后就可以免登陆,ssh IP登录到对方机器了 (如果互信用的不是机器的Kerberos账号,就kinit互信的Kerber

在Windows Server 2008 R2上启用Kerberos事件日志

在Windows Server 2008 R2上启用Kerberos事件日志 一.点击"开始"."运行",输入"REGEDIT"开始注册表编辑器. 二.展开到以下目录 HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\Lsa\Kerberos\Parameters 添加注册表值LogLevel,类型为REG_DWORD,值为0x1 如果Parameters下没有该子键,创建它. 注意:当定位完

Kerberos认证原理简介

1.1 What is Kerberos 1.1.1 简单介绍 Kerberos是一个用于鉴定身份(authentication)的协议, 它采取对称密钥加密(symmetric-key cryptography),这意味着密钥不会在网络上传输.在Kerberos中,未加密的密码(unencrypted password)不会在网络上传输,因此攻击者无法通过嗅探网络来偷取用户的密码. Kerberos利用对称加密和受信任的第三方(即KDC, key distribution center)来鉴别

Kerberos认证协议中TGS服务器可以去掉吗?

Kerberos协议最早是由MIT提出的,是一种身份认证协议. 应用场景:在一个开放环境中,一个工作站用户想通过网络对分布在网络中的各种服务提出请求,那么希望服务器能够只对授权用户提供服务,并能够鉴别服务请求的种类. Kerberos协议的原理:Kerberos通过提供一个集中的授权服务器来负责用户对服务器的认证和服务器对用户的认证,而不是为每个服务器提供详细的认证协议. Kerberos名词: Client:用户. AS:认证服务器,可以通过查询数据库,判断用户的口令,从而为用户颁发票据授权票

【原创】kerberos无密码登录

通常在远程客户端上登录kerbros都需要密码,在学习hadoop的时候提到了ktutil这个工具,这里将使用方法贴出来. 用到的命令如下: 1.ktutil 2.add_entry -password -p hadoop/[email protected] -k 3 -e aes256-cts-hmac-sha1-96 解释:-k 指编号 -e指加密方式 -password 指使用密码的方式 例子: add_entry -password -p host/[email protected] -

无线端安全登录与鉴权一之Kerberos

无线端登录与鉴权是安全登录以及保证用户数据安全的第一步,也是最重要的一步.之前做过一个安全登录与鉴权的方案,借这个机会,系统的思考一下,与大家交流交流 先介绍一下TX系统使用的Kerberos方案,参考了 http://blog.csdn.net/wulantian/article/details/42418231 的文章 一.概念介绍 Kerberos:起源于希腊神话,是一支守护着冥界长着3个头颅的神犬,在keberos Authentication中,Kerberos的3个头颅代表中认证过程

Kerberos安装及使用

2. 安装 Kerberos2.1. 环境配置 安装kerberos前,要确保主机名可以被解析. 主机名 内网IP 角色 Vmw201 172.16.18.201 Master KDC Vmw202 172.16.18.202 Kerberos client Vmw203 172.16.18.203 Kerberos client 2.2 Configuring a Kerberos Server2.2.1 确保环境可用 确保所有的clients与servers之间的时间同步以及DNS正确解析2

Kerberos

一.Kerberos Concept Kerberos是一种网络认证协议,其设计目标是通过密钥系统为客户机/服务器应用程序提供强大的认证服务,为通信双方提供双向身份认证. Kerberos关键术语: KDC提供两大主要功能:认证服务器(Authentication Service,AS)和票据授权服务(Ticket Granting Service,TGS).AS负责对用户和服务进行认证,TGS负责生成由时效的密码消息组成的票据.票据用于客户端向服务器进行认证. 通信双方成为标识(Princip

kafka kerberos 认证访问与非认证访问共存下的ACL问题

在一个正在运行的kafka集群中添加kerberos认证和ACL权限控制,同时保证以前所有的producer\consumer服务不中断 解决方式: 使kafka集群监听两个端口,一个为无认证连接,另一个为kerberos的认证连接 这时候在配置ACL的时候出了问题: 假如我以kerberos认证的方式连接kafka集群,那么我的用户名是principal的primary部分.例如principal是  kafka/[email protected] ,那么我的用户名就是kafka. 这时候我只