示例工程开发

环境准备

参考Spark环境准备。

代码逻辑说明

1. 功能说明。
   本示例演示获取yarn app列表的功能。

2. POM依赖。

           <dependency>
              <groupId>org.apache.hadoop</groupId>
              <artifactId>hadoop-common</artifactId>
              <version>3.2.2-${TBDS_VERSION}</version>
           </dependency>
           <dependency>
              <groupId>org.apache.hadoop</groupId>
              <artifactId>hadoop-auth</artifactId>
              <version>3.2.2-${TBDS_VERSION}</version>
           </dependency>
           <dependency>
              <groupId>org.apache.hadoop</groupId>
              <artifactId>tq-security</artifactId>
              <version>3.2.2-${TBDS_VERSION}</version>
           </dependency>
           <dependency>
               <groupId>org.apache.hadoop</groupId>
              <artifactId>hadoop-yarn-common</artifactId>
              <version>3.2.2-${TBDS_VERSION}</version>
           </dependency>
           <dependency>
              <groupId>org.apache.hadoop</groupId>
              <artifactId>hadoop-yarn-api</artifactId>
              <version>3.2.2-${TBDS_VERSION}</version>
           </dependency>
           <dependency>
              <groupId>org.apache.hadoop</groupId>
              <artifactId>hadoop-yarn-client</artifactId>
              <version>3.2.2-${TBDS_VERSION}</version>
           </dependency>  
   

3. 代码示例。

   import org.apache.hadoop.conf.Configuration;
   import org.apache.hadoop.fs.Path;
   import org.apache.hadoop.security.UserGroupInformation;
   import org.apache.hadoop.yarn.api.records.ApplicationReport;
   import org.apache.hadoop.yarn.api.records.YarnApplicationState;
   import org.apache.hadoop.yarn.client.api.YarnClient;
   import org.apache.hadoop.yarn.exceptions.YarnException;
    
   import java.io.File;
   import java.io.IOException;
   import java.util.EnumSet;
   import java.util.List;
    
   public class YarnClientExample {
    
       public static void main(String[] args) throws IOException{
           
           StringconfPath = "/usr/local/service/hadoop/etc/hadoop";
           Configurationconf = new Configuration();
           System.out.println(confPath + File.separator + "core-site.xml");
           conf.addResource(new Path(confPath + File.separator + "core-site.xml"));
           conf.addResource(new Path(confPath + File.separator + "hdfs-site.xml"));
           conf.addResource(new Path(confPath + File.separator + "yarn-site.xml"));
           YarnClientyarnClient = YarnClient.createYarnClient();
           //kerberos认证
           System.setProperty("java.security.krb5.conf", "/etc/krb5.conf");
          conf.setBoolean("hadoop.security.authorization", true);
           conf.set("hadoop.security.authentication", "Kerberos");
           UserGroupInformation.setConfiguration(conf);
           UserGroupInformation.loginUserFromKeytab("xxxx/x.x.x.x@TBDS-XXXXXX", "/var/krb5kdc/emr.keytab");
    
           yarnClient.init(conf);
           yarnClient.start();
           try {
               List<ApplicationReport>applications = yarnClient.getApplications(EnumSet.of(YarnApplicationState.RUNNING, YarnApplicationState.FINISHED));
               System.out.println("getApplications..." + applications.size());
               for (ApplicationReportapplication : applications) {
                   System.out.println("\nApplicationId ============> " + application.getApplicationId());
                   System.out.println("name ============> " + application.getName());
                   System.out.println("queue ============> " + application.getQueue());
                   System.out.println("user ============> " + application.getUser());
                   System.out.println("type ============> " + application.getApplicationType());
                   System.out.println("status ============> " + application.getFinalApplicationStatus());
               }
               System.out.println("end...");
           } catch (YarnException| IOExceptione) {
               e.printStackTrace();
           }
           yarnClient.stop();
       }
   }
   

代码调试

1. IntelliJ IDEA调试，参考Spark代码调试。

2. Linux环境编译参考：

     //编译
   /usr/local/jdk/bin/javac -classpath hadoop-common-3.2.2-${TBDS_VERSION}.jar:hadoop-yarn-api-3.2.2-${TBDS_VERSION}.jar:hadoop-yarn-client-3.2.2-${TBDS_VERSION}.jar:security-common-1.1.9.1.jar YarnClientExample.java
   //打包
   /usr/local/jdk/bin/jar -cvf YarnClientExample.jar YarnClientExample.class
    
   //建议使用idea编译打包fat jar
   //参考示例工程 yarn-examples/yarn-example 打包出 com.tencent.tbds.yarn.YarnClientExample
   mvn clean package
   

   

3. 测试运行。

     //运行
   /usr/local/jdk/bin/java -classpath yarn-example-1.0-SNAPSHOT.jar com.tencent.tbds.yarn.YarnClientExample