Spark教程——（11）Spark程序本地执行和集群执行的差异

本地执行Spark SQL程序：

package com.fc
//import common.util.{phoenixConnectMode, timeUtil}
import org.apache.spark.sql.SQLContext
import org.apache.spark.sql.functions.col
import org.apache.spark.{SparkConf, SparkContext}

/*
每天执行
 */
object costDay {
  def main(args: Array[String]): Unit = {

    val conf = new SparkConf()
          .setAppName("fdsf")
       .setMaster("local")
    val sc = new SparkContext(conf)
    val sqlContext = new SQLContext(sc)

//    val df = sqlContext.load(
//      "org.apache.phoenix.spark"
//      , Map("table" -> "ASSET_NORMAL"
//        , "zkUrl" -> "node3,node4,node5:2181")
//    )
val tableName = "ASSET_NORMAL"
    val columes = Array(
      "ID",
      "ASSET_ID",
      "ASSET_NAME",
      "ASSET_FIRST_DEGREE_ID",
      "ASSET_FIRST_DEGREE_NAME",
      "ASSET_SECOND_DEGREE_ID",
      "ASSET_SECOND_DEGREE_NAME",
      "GB_DEGREE_ID",
      "GB_DEGREE_NAME",
      "ASSET_USE_FIRST_DEGREE_ID",
      "ASSET_USE_FIRST_DEGREE_NAME",
      "ASSET_USE_SECOND_DEGREE_ID",
      "ASSET_USE_SECOND_DEGREE_NAME",
      "MANAGEMENT_TYPE_ID",
      "MANAGEMENT_TYPE_NAME",
      "ASSET_MODEL",
      "FACTORY_NUMBER",
      "ASSET_COUNTRY_ID",
      "ASSET_COUNTRY_NAME",
      "MANUFACTURER",
      "SUPPLIER",
      "SUPPLIER_TEL",
      "ORIGINAL_VALUE",
      "USE_DEPARTMENT_ID",
      "USE_DEPARTMENT_NAME",
      "USER_ID",
      "USER_NAME",
      "ASSET_LOCATION_OF_PARK_ID",
      "ASSET_LOCATION_OF_PARK_NAME",
      "ASSET_LOCATION_OF_BUILDING_ID",
      "ASSET_LOCATION_OF_BUILDING_NAME",
      "ASSET_LOCATION_OF_ROOM_ID",
      "ASSET_LOCATION_OF_ROOM_NUMBER",
      "PRODUCTION_DATE",
      "ACCEPTANCE_DATE",
      "REQUISITION_DATE",
      "PERFORMANCE_INDEX",
      "ASSET_STATE_ID",
      "ASSET_STATE_NAME",
      "INSPECTION_TYPE_ID",
      "INSPECTION_TYPE_NAME",
      "SEAL_DATE",
      "SEAL_CAUSE",
      "COST_ITEM_ID",
      "COST_ITEM_NAME",
      "ITEM_COMMENTS",
      "UNSEAL_DATE",
      "SCRAP_DATE",
      "PURCHASE_NUMBER",
      "WARRANTY_PERIOD",
      "DEPRECIABLE_LIVES_ID",
      "DEPRECIABLE_LIVES_NAME",
      "MEASUREMENT_UNITS_ID",
      "MEASUREMENT_UNITS_NAME",
      "ANNEX",
      "REMARK",
      "ACCOUNTING_TYPE_ID",
      "ACCOUNTING_TYPE_NAME",
      "SYSTEM_TYPE_ID",
      "SYSTEM_TYPE_NAME",
      "ASSET_ID_PARENT",
      "CLASSIFIED_LEVEL_ID",
      "CLASSIFIED_LEVEL_NAME",
      "ASSET_PICTURE",
      "MILITARY_SPECIAL_CODE",
      "CHECK_CYCLE_ID",
      "CHECK_CYCLE_NAME",
      "CHECK_DATE",
      "CHECK_EFFECTIVE_DATE",
      "CHECK_MODE_ID",
      "CHECK_MODE_NAME",
      "CHECK_DEPARTMENT_ID",
      "CHECK_DEPARTMENT_NAME",
      "RENT_STATUS_ID",
      "RENT_STATUS_NAME",
      "STORAGE_TIME",
      "UPDATE_USER",
      "UPDATE_TIME",
      "IS_ON_PROCESS",
      "IS_DELETED",
      "FIRST_DEPARTMENT_ID",
      "FIRST_DEPARTMENT_NAME",
      "SECOND_DEPARTMENT_ID",
      "SECOND_DEPARTMENT_NAME",
      "CREATE_USER",
      "CREATE_TIME"
    )
    val df = phoenixConnectMode.getMode1(sqlContext, tableName, columes)
      .filter(col("USE_DEPARTMENT_ID") isNotNull)
    df.registerTempTable("asset_normal")
    //    df.show(false)

    def costingWithin(originalValue: Double, years: Int): Double =  (originalValue*0.95)/(years*365)
    sqlContext.udf.register("costingWithin", costingWithin _)

    def costingBeyond(originalValue: Double): Double = originalValue*0.05/365
    sqlContext.udf.register("costingBeyond", costingBeyond _)

    def expire(acceptanceDate: String, years: Int): Boolean = timeUtil.dateStrAddYears2TimeStamp(acceptanceDate, timeUtil.SECOND_TIME_FORMAT, years) > System.currentTimeMillis()
    sqlContext.udf.register("expire", expire _)

    val costDay = sqlContext
      .sql(
        "select " +
          "ID" +
          ",USE_DEPARTMENT_ID as FIRST_DEPARTMENT_ID" +
          ",case when expire(ACCEPTANCE_DATE, DEPRECIABLE_LIVES_NAME) then costingWithin(ORIGINAL_VALUE, DEPRECIABLE_LIVES_NAME) else costingBeyond(ORIGINAL_VALUE) end as ACTUAL_COST" +
          ",ORIGINAL_VALUE" +
          ",current_timestamp() as GENERATION_TIME" +
          " from asset_normal"
      )

    costDay.printSchema()
    println(costDay.count())
    costDay.col("ORIGINAL_VALUE")
    costDay.describe("ORIGINAL_VALUE").show()
    costDay.show(false)
//    costDay.write
//      .format("org.apache.phoenix.spark")
//      .mode("overwrite")
//      .option("table", "ASSET_FINANCIAL_DETAIL_DAY")
//      .option("zkUrl", "node3,node4,node5:2181")
//      .save()
  }
}

执行结果参考《Spark教程——（10）Spark SQL读取Phoenix数据本地执行计算》，如下：

19/09/24 00:52:14 INFO spark.SparkContext: Running Spark version 1.6.0
19/09/24 00:52:16 INFO spark.SecurityManager: Changing view acls to: cf_pc
19/09/24 00:52:16 INFO spark.SecurityManager: Changing modify acls to: cf_pc
19/09/24 00:52:16 INFO spark.SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(cf_pc); users with modify permissions: Set(cf_pc)
19/09/24 00:52:16 INFO util.Utils: Successfully started service ‘sparkDriver‘ on port 54691.
19/09/24 00:52:17 INFO slf4j.Slf4jLogger: Slf4jLogger started
19/09/24 00:52:17 INFO Remoting: Starting remoting
19/09/24 00:52:17 INFO Remoting: Remoting started; listening on addresses :[akka.tcp://[email protected]:54708]
19/09/24 00:52:17 INFO Remoting: Remoting now listens on addresses: [akka.tcp://[email protected]:54708]
19/09/24 00:52:17 INFO util.Utils: Successfully started service ‘sparkDriverActorSystem‘ on port 54708.
19/09/24 00:52:17 INFO spark.SparkEnv: Registering MapOutputTracker
19/09/24 00:52:17 INFO spark.SparkEnv: Registering BlockManagerMaster
19/09/24 00:52:17 INFO storage.DiskBlockManager: Created local directory at C:\Users\cf_pc\AppData\Local\Temp\blockmgr-9012d6b9-eab0-41f0-8e9e-b63c3fe7fb09
19/09/24 00:52:17 INFO storage.MemoryStore: MemoryStore started with capacity 478.2 MB
19/09/24 00:52:17 INFO spark.SparkEnv: Registering OutputCommitCoordinator
19/09/24 00:52:18 INFO server.Server: jetty-8.y.z-SNAPSHOT
19/09/24 00:52:18 INFO server.AbstractConnector: Started [email protected]0.0.0.0:4040
19/09/24 00:52:18 INFO util.Utils: Successfully started service ‘SparkUI‘ on port 4040.
19/09/24 00:52:18 INFO ui.SparkUI: Started SparkUI at http://10.200.74.155:4040
19/09/24 00:52:18 INFO executor.Executor: Starting executor ID driver on host localhost
19/09/24 00:52:18 INFO util.Utils: Successfully started service ‘org.apache.spark.network.netty.NettyBlockTransferService‘ on port 54720.
19/09/24 00:52:18 INFO netty.NettyBlockTransferService: Server created on 54720
19/09/24 00:52:18 INFO storage.BlockManagerMaster: Trying to register BlockManager
19/09/24 00:52:18 INFO storage.BlockManagerMasterEndpoint: Registering block manager localhost:54720 with 478.2 MB RAM, BlockManagerId(driver, localhost, 54720)
19/09/24 00:52:18 INFO storage.BlockManagerMaster: Registered BlockManager
19/09/24 00:52:21 INFO storage.MemoryStore: Block broadcast_0 stored as values in memory (estimated size 247.5 KB, free 477.9 MB)
19/09/24 00:52:22 INFO storage.MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 15.4 KB, free 477.9 MB)
19/09/24 00:52:22 INFO storage.BlockManagerInfo: Added broadcast_0_piece0 in memory on localhost:54720 (size: 15.4 KB, free: 478.2 MB)
19/09/24 00:52:22 INFO spark.SparkContext: Created broadcast 0 from newAPIHadoopRDD at PhoenixRDD.scala:49
19/09/24 00:52:22 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Trying to connect to a secure cluster as 2181 with keytab /hbase
19/09/24 00:52:22 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Successful login to secure cluster
19/09/24 00:52:23 INFO log.QueryLoggerDisruptor: Starting  QueryLoggerDisruptor for with ringbufferSize=8192, waitStrategy=BlockingWaitStrategy, exceptionHandler=[email protected]09...
19/09/24 00:52:23 INFO query.ConnectionQueryServicesImpl: An instance of ConnectionQueryServices was created.
19/09/24 00:52:23 INFO zookeeper.RecoverableZooKeeper: Process identifier=hconnection-0x4bc59b27 connecting to ZooKeeper ensemble=node3:2181
19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:zookeeper.version=3.4.6-1569965, built on 02/20/2014 09:09 GMT
19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:host.name=DESKTOP-0CDQ4PM
19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:java.version=1.8.0_212
19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:java.vendor=Oracle Corporation
19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:java.home=C:\3rd\Java\jdk1.8.0_212\jre
19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:java.class.path=C:\3rd\Java\jdk1.8.0_212\jre\lib\charsets.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\deploy.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\ext\access-bridge-64.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\ext\cldrdata.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\ext\dnsns.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\ext\jaccess.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\ext\jfxrt.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\ext\localedata.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\ext\nashorn.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\ext\sunec.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\ext\sunjce_provider.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\ext\sunmscapi.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\ext\sunpkcs11.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\ext\zipfs.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\javaws.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\jce.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\jfr.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\jfxswt.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\jsse.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\management-agent.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\plugin.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\resources.jar;C:\3rd\Java\jdk1.8.0_212\jre\lib\rt.jar;C:\development\cf\scalapp\target\classes;C:\3rd\scala\scala-2.10.5\lib\scala-actors-migration.jar;C:\3rd\scala\scala-2.10.5\lib\scala-actors.jar;C:\3rd\scala\scala-2.10.5\lib\scala-library.jar;C:\3rd\scala\scala-2.10.5\lib\scala-reflect.jar;C:\3rd\scala\scala-2.10.5\lib\scala-swing.jar;C:\3rd\scala\scala-2.10.5\src\scala-actors-src.jar;C:\3rd\scala\scala-2.10.5\src\scala-library-src.jar;C:\3rd\scala\scala-2.10.5\src\scala-reflect-src.jar;C:\3rd\scala\scala-2.10.5\src\scala-swing-src.jar;C:\development\MavenRepository\org\apache\spark\spark-core_2.10\1.6.0-cdh5.14.2\spark-core_2.10-1.6.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\avro\avro-mapred\1.7.6-cdh5.14.2\avro-mapred-1.7.6-cdh5.14.2-hadoop2.jar;C:\development\MavenRepository\org\apache\avro\avro-ipc\1.7.6-cdh5.14.2\avro-ipc-1.7.6-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\avro\avro-ipc\1.7.6-cdh5.14.2\avro-ipc-1.7.6-cdh5.14.2-tests.jar;C:\development\MavenRepository\com\twitter\chill_2.10\0.5.0\chill_2.10-0.5.0.jar;C:\development\MavenRepository\com\esotericsoftware\kryo\kryo\2.21\kryo-2.21.jar;C:\development\MavenRepository\com\esotericsoftware\reflectasm\reflectasm\1.07\reflectasm-1.07-shaded.jar;C:\development\MavenRepository\com\esotericsoftware\minlog\minlog\1.2\minlog-1.2.jar;C:\development\MavenRepository\org\objenesis\objenesis\1.2\objenesis-1.2.jar;C:\development\MavenRepository\com\twitter\chill-java\0.5.0\chill-java-0.5.0.jar;C:\development\MavenRepository\org\apache\xbean\xbean-asm5-shaded\4.4\xbean-asm5-shaded-4.4.jar;C:\development\MavenRepository\org\apache\hadoop\hadoop-client\2.6.0-cdh5.14.2\hadoop-client-2.6.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\hadoop\hadoop-hdfs\2.6.0-cdh5.14.2\hadoop-hdfs-2.6.0-cdh5.14.2.jar;C:\development\MavenRepository\xerces\xercesImpl\2.9.1\xercesImpl-2.9.1.jar;C:\development\MavenRepository\xml-apis\xml-apis\1.3.04\xml-apis-1.3.04.jar;C:\development\MavenRepository\org\apache\hadoop\hadoop-mapreduce-client-app\2.6.0-cdh5.14.2\hadoop-mapreduce-client-app-2.6.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\hadoop\hadoop-mapreduce-client-common\2.6.0-cdh5.14.2\hadoop-mapreduce-client-common-2.6.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\hadoop\hadoop-yarn-client\2.6.0-cdh5.14.2\hadoop-yarn-client-2.6.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\hadoop\hadoop-yarn-server-common\2.6.0-cdh5.14.2\hadoop-yarn-server-common-2.6.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\hadoop\hadoop-mapreduce-client-shuffle\2.6.0-cdh5.14.2\hadoop-mapreduce-client-shuffle-2.6.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\hadoop\hadoop-yarn-api\2.6.0-cdh5.14.2\hadoop-yarn-api-2.6.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\hadoop\hadoop-mapreduce-client-jobclient\2.6.0-cdh5.14.2\hadoop-mapreduce-client-jobclient-2.6.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\hadoop\hadoop-aws\2.6.0-cdh5.14.2\hadoop-aws-2.6.0-cdh5.14.2.jar;C:\development\MavenRepository\com\amazonaws\aws-java-sdk-bundle\1.11.134\aws-java-sdk-bundle-1.11.134.jar;C:\development\MavenRepository\org\apache\spark\spark-launcher_2.10\1.6.0-cdh5.14.2\spark-launcher_2.10-1.6.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\spark\spark-network-common_2.10\1.6.0-cdh5.14.2\spark-network-common_2.10-1.6.0-cdh5.14.2.jar;C:\development\MavenRepository\org\fusesource\leveldbjni\leveldbjni-all\1.8\leveldbjni-all-1.8.jar;C:\development\MavenRepository\com\fasterxml\jackson\core\jackson-annotations\2.2.3\jackson-annotations-2.2.3.jar;C:\development\MavenRepository\org\apache\spark\spark-network-shuffle_2.10\1.6.0-cdh5.14.2\spark-network-shuffle_2.10-1.6.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\spark\spark-unsafe_2.10\1.6.0-cdh5.14.2\spark-unsafe_2.10-1.6.0-cdh5.14.2.jar;C:\development\MavenRepository\net\java\dev\jets3t\jets3t\0.9.0\jets3t-0.9.0.jar;C:\development\MavenRepository\org\apache\httpcomponents\httpcore\4.1.2\httpcore-4.1.2.jar;C:\development\MavenRepository\com\jamesmurty\utils\java-xmlbuilder\0.4\java-xmlbuilder-0.4.jar;C:\development\MavenRepository\org\apache\curator\curator-recipes\2.7.1\curator-recipes-2.7.1.jar;C:\development\MavenRepository\org\apache\curator\curator-framework\2.7.1\curator-framework-2.7.1.jar;C:\development\MavenRepository\org\apache\zookeeper\zookeeper\3.4.6\zookeeper-3.4.6.jar;C:\development\MavenRepository\org\eclipse\jetty\orbit\javax.servlet\3.0.0.v201112011016\javax.servlet-3.0.0.v201112011016.jar;C:\development\MavenRepository\org\apache\commons\commons-lang3\3.3.2\commons-lang3-3.3.2.jar;C:\development\MavenRepository\org\apache\commons\commons-math3\3.4.1\commons-math3-3.4.1.jar;C:\development\MavenRepository\com\google\code\findbugs\jsr305\1.3.9\jsr305-1.3.9.jar;C:\development\MavenRepository\org\slf4j\slf4j-api\1.7.5\slf4j-api-1.7.5.jar;C:\development\MavenRepository\org\slf4j\jul-to-slf4j\1.7.5\jul-to-slf4j-1.7.5.jar;C:\development\MavenRepository\org\slf4j\jcl-over-slf4j\1.7.5\jcl-over-slf4j-1.7.5.jar;C:\development\MavenRepository\log4j\log4j\1.2.17\log4j-1.2.17.jar;C:\development\MavenRepository\org\slf4j\slf4j-log4j12\1.7.5\slf4j-log4j12-1.7.5.jar;C:\development\MavenRepository\com\ning\compress-lzf\1.0.3\compress-lzf-1.0.3.jar;C:\development\MavenRepository\org\xerial\snappy\snappy-java\1.0.4.1\snappy-java-1.0.4.1.jar;C:\development\MavenRepository\net\jpountz\lz4\lz4\1.3.0\lz4-1.3.0.jar;C:\development\MavenRepository\org\roaringbitmap\RoaringBitmap\0.5.11\RoaringBitmap-0.5.11.jar;C:\development\MavenRepository\commons-net\commons-net\2.2\commons-net-2.2.jar;C:\development\MavenRepository\org\spark-project\akka\akka-remote_2.10\2.2.3-shaded-protobuf\akka-remote_2.10-2.2.3-shaded-protobuf.jar;C:\development\MavenRepository\org\spark-project\akka\akka-actor_2.10\2.2.3-shaded-protobuf\akka-actor_2.10-2.2.3-shaded-protobuf.jar;C:\development\MavenRepository\com\typesafe\config\1.0.2\config-1.0.2.jar;C:\development\MavenRepository\org\spark-project\protobuf\protobuf-java\2.4.1-shaded\protobuf-java-2.4.1-shaded.jar;C:\development\MavenRepository\org\uncommons\maths\uncommons-maths\1.2.2a\uncommons-maths-1.2.2a.jar;C:\development\MavenRepository\org\spark-project\akka\akka-slf4j_2.10\2.2.3-shaded-protobuf\akka-slf4j_2.10-2.2.3-shaded-protobuf.jar;C:\development\MavenRepository\org\scala-lang\scala-library\2.10.5\scala-library-2.10.5.jar;C:\development\MavenRepository\org\json4s\json4s-jackson_2.10\3.2.10\json4s-jackson_2.10-3.2.10.jar;C:\development\MavenRepository\org\json4s\json4s-core_2.10\3.2.10\json4s-core_2.10-3.2.10.jar;C:\development\MavenRepository\org\json4s\json4s-ast_2.10\3.2.10\json4s-ast_2.10-3.2.10.jar;C:\development\MavenRepository\org\scala-lang\scalap\2.10.0\scalap-2.10.0.jar;C:\development\MavenRepository\org\scala-lang\scala-compiler\2.10.0\scala-compiler-2.10.0.jar;C:\development\MavenRepository\com\sun\jersey\jersey-server\1.9\jersey-server-1.9.jar;C:\development\MavenRepository\asm\asm\3.1\asm-3.1.jar;C:\development\MavenRepository\com\sun\jersey\jersey-core\1.9\jersey-core-1.9.jar;C:\development\MavenRepository\org\apache\mesos\mesos\0.21.1\mesos-0.21.1-shaded-protobuf.jar;C:\development\MavenRepository\io\netty\netty-all\4.0.29.Final\netty-all-4.0.29.Final.jar;C:\development\MavenRepository\com\clearspring\analytics\stream\2.7.0\stream-2.7.0.jar;C:\development\MavenRepository\io\dropwizard\metrics\metrics-core\3.1.2\metrics-core-3.1.2.jar;C:\development\MavenRepository\io\dropwizard\metrics\metrics-jvm\3.1.2\metrics-jvm-3.1.2.jar;C:\development\MavenRepository\io\dropwizard\metrics\metrics-json\3.1.2\metrics-json-3.1.2.jar;C:\development\MavenRepository\io\dropwizard\metrics\metrics-graphite\3.1.2\metrics-graphite-3.1.2.jar;C:\development\MavenRepository\com\fasterxml\jackson\core\jackson-databind\2.2.3\jackson-databind-2.2.3.jar;C:\development\MavenRepository\com\fasterxml\jackson\core\jackson-core\2.2.3\jackson-core-2.2.3.jar;C:\development\MavenRepository\com\fasterxml\jackson\module\jackson-module-scala_2.10\2.2.3\jackson-module-scala_2.10-2.2.3.jar;C:\development\MavenRepository\com\thoughtworks\paranamer\paranamer\2.3\paranamer-2.3.jar;C:\development\MavenRepository\org\apache\ivy\ivy\2.4.0\ivy-2.4.0.jar;C:\development\MavenRepository\oro\oro\2.0.8\oro-2.0.8.jar;C:\development\MavenRepository\org\tachyonproject\tachyon-client\0.8.2\tachyon-client-0.8.2.jar;C:\development\MavenRepository\commons-lang\commons-lang\2.4\commons-lang-2.4.jar;C:\development\MavenRepository\commons-io\commons-io\2.4\commons-io-2.4.jar;C:\development\MavenRepository\org\tachyonproject\tachyon-underfs-hdfs\0.8.2\tachyon-underfs-hdfs-0.8.2.jar;C:\development\MavenRepository\org\tachyonproject\tachyon-underfs-s3\0.8.2\tachyon-underfs-s3-0.8.2.jar;C:\development\MavenRepository\org\tachyonproject\tachyon-underfs-local\0.8.2\tachyon-underfs-local-0.8.2.jar;C:\development\MavenRepository\net\razorvine\pyrolite\4.9\pyrolite-4.9.jar;C:\development\MavenRepository\net\sf\py4j\py4j\0.9\py4j-0.9.jar;C:\development\MavenRepository\com\intel\chimera\chimera\0.9.2\chimera-0.9.2.jar;C:\development\MavenRepository\org\spark-project\spark\unused\1.0.0\unused-1.0.0.jar;C:\development\MavenRepository\org\apache\spark\spark-sql_2.10\1.6.0-cdh5.14.2\spark-sql_2.10-1.6.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\spark\spark-catalyst_2.10\1.6.0-cdh5.14.2\spark-catalyst_2.10-1.6.0-cdh5.14.2.jar;C:\development\MavenRepository\org\scala-lang\scala-reflect\2.10.5\scala-reflect-2.10.5.jar;C:\development\MavenRepository\org\codehaus\janino\janino\2.7.8\janino-2.7.8.jar;C:\development\MavenRepository\org\codehaus\janino\commons-compiler\2.7.8\commons-compiler-2.7.8.jar;C:\development\MavenRepository\com\twitter\parquet-column\1.5.0-cdh5.14.2\parquet-column-1.5.0-cdh5.14.2.jar;C:\development\MavenRepository\com\twitter\parquet-common\1.5.0-cdh5.14.2\parquet-common-1.5.0-cdh5.14.2.jar;C:\development\MavenRepository\com\twitter\parquet-encoding\1.5.0-cdh5.14.2\parquet-encoding-1.5.0-cdh5.14.2.jar;C:\development\MavenRepository\com\twitter\parquet-hadoop\1.5.0-cdh5.14.2\parquet-hadoop-1.5.0-cdh5.14.2.jar;C:\development\MavenRepository\com\twitter\parquet-format\2.1.0-cdh5.14.2\parquet-format-2.1.0-cdh5.14.2.jar;C:\development\MavenRepository\com\twitter\parquet-jackson\1.5.0-cdh5.14.2\parquet-jackson-1.5.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\hbase\hbase-spark\1.2.0-cdh5.14.2\hbase-spark-1.2.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\hadoop\hadoop-common\2.6.0-cdh5.14.2\hadoop-common-2.6.0-cdh5.14.2.jar;C:\development\MavenRepository\xmlenc\xmlenc\0.52\xmlenc-0.52.jar;C:\development\MavenRepository\commons-httpclient\commons-httpclient\3.1\commons-httpclient-3.1.jar;C:\development\MavenRepository\org\mortbay\jetty\jetty\6.1.26.cloudera.4\jetty-6.1.26.cloudera.4.jar;C:\development\MavenRepository\org\mortbay\jetty\jetty-util\6.1.26.cloudera.4\jetty-util-6.1.26.cloudera.4.jar;C:\development\MavenRepository\com\sun\jersey\jersey-json\1.9\jersey-json-1.9.jar;C:\development\MavenRepository\org\codehaus\jettison\jettison\1.1\jettison-1.1.jar;C:\development\MavenRepository\com\sun\xml\bind\jaxb-impl\2.2.3-1\jaxb-impl-2.2.3-1.jar;C:\development\MavenRepository\tomcat\jasper-compiler\5.5.23\jasper-compiler-5.5.23.jar;C:\development\MavenRepository\tomcat\jasper-runtime\5.5.23\jasper-runtime-5.5.23.jar;C:\development\MavenRepository\commons-el\commons-el\1.0\commons-el-1.0.jar;C:\development\MavenRepository\commons-configuration\commons-configuration\1.6\commons-configuration-1.6.jar;C:\development\MavenRepository\commons-digester\commons-digester\1.8\commons-digester-1.8.jar;C:\development\MavenRepository\commons-beanutils\commons-beanutils\1.7.0\commons-beanutils-1.7.0.jar;C:\development\MavenRepository\commons-beanutils\commons-beanutils-core\1.8.0\commons-beanutils-core-1.8.0.jar;C:\development\MavenRepository\com\google\code\gson\gson\2.2.4\gson-2.2.4.jar;C:\development\MavenRepository\org\apache\hadoop\hadoop-auth\2.6.0-cdh5.14.2\hadoop-auth-2.6.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\directory\server\apacheds-kerberos-codec\2.0.0-M15\apacheds-kerberos-codec-2.0.0-M15.jar;C:\development\MavenRepository\org\apache\directory\server\apacheds-i18n\2.0.0-M15\apacheds-i18n-2.0.0-M15.jar;C:\development\MavenRepository\org\apache\directory\api\api-asn1-api\1.0.0-M20\api-asn1-api-1.0.0-M20.jar;C:\development\MavenRepository\org\apache\directory\api\api-util\1.0.0-M20\api-util-1.0.0-M20.jar;C:\development\MavenRepository\com\jcraft\jsch\0.1.42\jsch-0.1.42.jar;C:\development\MavenRepository\org\apache\curator\curator-client\2.7.1\curator-client-2.7.1.jar;C:\development\MavenRepository\org\apache\htrace\htrace-core4\4.0.1-incubating\htrace-core4-4.0.1-incubating.jar;C:\development\MavenRepository\org\apache\commons\commons-compress\1.4.1\commons-compress-1.4.1.jar;C:\development\MavenRepository\org\tukaani\xz\1.0\xz-1.0.jar;C:\development\MavenRepository\org\apache\hbase\hbase-client\1.2.0-cdh5.14.2\hbase-client-1.2.0-cdh5.14.2.jar;C:\development\MavenRepository\org\jruby\jcodings\jcodings\1.0.8\jcodings-1.0.8.jar;C:\development\MavenRepository\com\yammer\metrics\metrics-core\2.2.0\metrics-core-2.2.0.jar;C:\development\MavenRepository\org\apache\hbase\hbase-protocol\1.2.0-cdh5.14.2\hbase-protocol-1.2.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\hbase\hbase-server\1.2.0-cdh5.14.2\hbase-server-1.2.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\hbase\hbase-procedure\1.2.0-cdh5.14.2\hbase-procedure-1.2.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\hbase\hbase-common\1.2.0-cdh5.14.2\hbase-common-1.2.0-cdh5.14.2-tests.jar;C:\development\MavenRepository\org\apache\hbase\hbase-prefix-tree\1.2.0-cdh5.14.2\hbase-prefix-tree-1.2.0-cdh5.14.2.jar;C:\development\MavenRepository\com\github\stephenc\high-scale-lib\high-scale-lib\1.1.1\high-scale-lib-1.1.1.jar;C:\development\MavenRepository\org\apache\commons\commons-math\2.1\commons-math-2.1.jar;C:\development\MavenRepository\org\mortbay\jetty\jetty-sslengine\6.1.26.cloudera.4\jetty-sslengine-6.1.26.cloudera.4.jar;C:\development\MavenRepository\org\mortbay\jetty\jsp-2.1\6.1.14\jsp-2.1-6.1.14.jar;C:\development\MavenRepository\org\mortbay\jetty\jsp-api-2.1\6.1.14\jsp-api-2.1-6.1.14.jar;C:\development\MavenRepository\org\mortbay\jetty\servlet-api-2.5\6.1.14\servlet-api-2.5-6.1.14.jar;C:\development\MavenRepository\org\codehaus\jackson\jackson-jaxrs\1.8.8\jackson-jaxrs-1.8.8.jar;C:\development\MavenRepository\org\jamon\jamon-runtime\2.4.1\jamon-runtime-2.4.1.jar;C:\development\MavenRepository\org\hamcrest\hamcrest-core\1.3\hamcrest-core-1.3.jar;C:\development\MavenRepository\org\apache\hadoop\hadoop-core\2.6.0-mr1-cdh5.14.2\hadoop-core-2.6.0-mr1-cdh5.14.2.jar;C:\development\MavenRepository\javax\servlet\jsp\jsp-api\2.1\jsp-api-2.1.jar;C:\development\MavenRepository\hsqldb\hsqldb\1.8.0.10\hsqldb-1.8.0.10.jar;C:\development\MavenRepository\org\eclipse\jdt\core\3.1.1\core-3.1.1.jar;C:\development\MavenRepository\org\apache\hadoop\hadoop-hdfs\2.6.0-cdh5.14.2\hadoop-hdfs-2.6.0-cdh5.14.2-tests.jar;C:\development\MavenRepository\commons-daemon\commons-daemon\1.0.13\commons-daemon-1.0.13.jar;C:\development\MavenRepository\com\google\protobuf\protobuf-java\2.5.0\protobuf-java-2.5.0.jar;C:\development\MavenRepository\commons-logging\commons-logging\1.2\commons-logging-1.2.jar;C:\development\MavenRepository\com\github\stephenc\findbugs\findbugs-annotations\1.3.9-1\findbugs-annotations-1.3.9-1.jar;C:\development\MavenRepository\org\apache\phoenix\phoenix-spark\4.14.0-cdh5.14.2\phoenix-spark-4.14.0-cdh5.14.2.jar;C:\development\MavenRepository\com\lmax\disruptor\3.3.8\disruptor-3.3.8.jar;C:\development\MavenRepository\org\apache\phoenix\phoenix-core\4.14.0-cdh5.14.2\phoenix-core-4.14.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\tephra\tephra-api\0.14.0-incubating\tephra-api-0.14.0-incubating.jar;C:\development\MavenRepository\org\apache\tephra\tephra-core\0.14.0-incubating\tephra-core-0.14.0-incubating.jar;C:\development\MavenRepository\com\google\inject\guice\3.0\guice-3.0.jar;C:\development\MavenRepository\javax\inject\javax.inject\1\javax.inject-1.jar;C:\development\MavenRepository\aopalliance\aopalliance\1.0\aopalliance-1.0.jar;C:\development\MavenRepository\com\google\inject\extensions\guice-assistedinject\3.0\guice-assistedinject-3.0.jar;C:\development\MavenRepository\org\apache\thrift\libthrift\0.9.0\libthrift-0.9.0.jar;C:\development\MavenRepository\it\unimi\dsi\fastutil\6.5.6\fastutil-6.5.6.jar;C:\development\MavenRepository\org\apache\twill\twill-common\0.8.0\twill-common-0.8.0.jar;C:\development\MavenRepository\org\apache\twill\twill-core\0.8.0\twill-core-0.8.0.jar;C:\development\MavenRepository\org\apache\twill\twill-api\0.8.0\twill-api-0.8.0.jar;C:\development\MavenRepository\org\ow2\asm\asm-all\5.0.2\asm-all-5.0.2.jar;C:\development\MavenRepository\org\apache\twill\twill-discovery-api\0.8.0\twill-discovery-api-0.8.0.jar;C:\development\MavenRepository\org\apache\twill\twill-discovery-core\0.8.0\twill-discovery-core-0.8.0.jar;C:\development\MavenRepository\org\apache\twill\twill-zookeeper\0.8.0\twill-zookeeper-0.8.0.jar;C:\development\MavenRepository\org\apache\tephra\tephra-hbase-compat-1.2-cdh\0.14.0-incubating\tephra-hbase-compat-1.2-cdh-0.14.0-incubating.jar;C:\development\MavenRepository\org\antlr\antlr-runtime\3.5.2\antlr-runtime-3.5.2.jar;C:\development\MavenRepository\jline\jline\2.11\jline-2.11.jar;C:\development\MavenRepository\sqlline\sqlline\1.2.0\sqlline-1.2.0.jar;C:\development\MavenRepository\com\google\guava\guava\13.0.1\guava-13.0.1.jar;C:\development\MavenRepository\joda-time\joda-time\1.6\joda-time-1.6.jar;C:\development\MavenRepository\com\github\stephenc\jcip\jcip-annotations\1.0-1\jcip-annotations-1.0-1.jar;C:\development\MavenRepository\org\codehaus\jackson\jackson-core-asl\1.8.8\jackson-core-asl-1.8.8.jar;C:\development\MavenRepository\org\codehaus\jackson\jackson-mapper-asl\1.8.8\jackson-mapper-asl-1.8.8.jar;C:\development\MavenRepository\junit\junit\4.12\junit-4.12.jar;C:\development\MavenRepository\org\apache\httpcomponents\httpclient\4.0.1\httpclient-4.0.1.jar;C:\development\MavenRepository\org\iq80\snappy\snappy\0.3\snappy-0.3.jar;C:\development\MavenRepository\org\apache\htrace\htrace-core\3.2.0-incubating\htrace-core-3.2.0-incubating.jar;C:\development\MavenRepository\commons-cli\commons-cli\1.2\commons-cli-1.2.jar;C:\development\MavenRepository\commons-codec\commons-codec\1.7\commons-codec-1.7.jar;C:\development\MavenRepository\commons-collections\commons-collections\3.2.2\commons-collections-3.2.2.jar;C:\development\MavenRepository\org\apache\commons\commons-csv\1.0\commons-csv-1.0.jar;C:\development\MavenRepository\org\apache\hbase\hbase-annotations\1.2.0-cdh5.14.2\hbase-annotations-1.2.0-cdh5.14.2.jar;C:\3rd\Java\jdk1.8.0_212\lib\tools.jar;C:\development\MavenRepository\org\apache\hbase\hbase-common\1.2.0-cdh5.14.2\hbase-common-1.2.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\hbase\hbase-hadoop-compat\1.2.0-cdh5.14.2\hbase-hadoop-compat-1.2.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\hbase\hbase-hadoop2-compat\1.2.0-cdh5.14.2\hbase-hadoop2-compat-1.2.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\hadoop\hadoop-annotations\2.6.0-cdh5.14.2\hadoop-annotations-2.6.0-cdh5.14.2.jar;C:\development\MavenRepository\org\jruby\joni\joni\2.1.2\joni-2.1.2.jar;C:\development\MavenRepository\com\salesforce\i18n\i18n-util\1.0.4\i18n-util-1.0.4.jar;C:\development\MavenRepository\com\ibm\icu\icu4j\60.2\icu4j-60.2.jar;C:\development\MavenRepository\com\ibm\icu\icu4j-localespi\60.2\icu4j-localespi-60.2.jar;C:\development\MavenRepository\com\ibm\icu\icu4j-charset\60.2\icu4j-charset-60.2.jar;C:\development\MavenRepository\org\apache\hadoop\hadoop-mapreduce-client-core\2.6.0-cdh5.14.2\hadoop-mapreduce-client-core-2.6.0-cdh5.14.2.jar;C:\development\MavenRepository\org\apache\hadoop\hadoop-yarn-common\2.6.0-cdh5.14.2\hadoop-yarn-common-2.6.0-cdh5.14.2.jar;C:\development\MavenRepository\javax\xml\bind\jaxb-api\2.2.2\jaxb-api-2.2.2.jar;C:\development\MavenRepository\javax\xml\stream\stax-api\1.0-2\stax-api-1.0-2.jar;C:\development\MavenRepository\javax\activation\activation\1.1\activation-1.1.jar;C:\development\MavenRepository\javax\servlet\servlet-api\2.5\servlet-api-2.5.jar;C:\development\MavenRepository\com\sun\jersey\jersey-client\1.9\jersey-client-1.9.jar;C:\development\MavenRepository\org\codehaus\jackson\jackson-xc\1.8.8\jackson-xc-1.8.8.jar;C:\development\MavenRepository\com\sun\jersey\contribs\jersey-guice\1.9\jersey-guice-1.9.jar;C:\development\MavenRepository\org\apache\avro\avro\1.7.6-cdh5.14.2\avro-1.7.6-cdh5.14.2.jar;C:\development\MavenRepository\com\google\inject\extensions\guice-servlet\3.0\guice-servlet-3.0.jar;C:\development\MavenRepository\io\netty\netty\3.10.5.Final\netty-3.10.5.Final.jar;C:\development\MavenRepository\mysql\mysql-connector-java\5.1.43\mysql-connector-java-5.1.43.jar;C:\3rd\JetBrains\IntelliJ IDEA 2019.1.3\lib\idea_rt.jar
19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:java.library.path=C:\3rd\Java\jdk1.8.0_212\bin;C:\WINDOWS\Sun\Java\bin;C:\WINDOWS\system32;C:\WINDOWS;C:\3rd\Anaconda2;C:\3rd\Anaconda2\Library\mingw-w64\bin;C:\3rd\Anaconda2\Library\usr\bin;C:\3rd\Anaconda2\Library\bin;C:\3rd\Anaconda2\Scripts;C:\Program Files (x86)\Intel\Intel(R) Management Engine Components\iCLS\;C:\Program Files\Intel\Intel(R) Management Engine Components\iCLS\;C:\WINDOWS\system32;C:\WINDOWS;C:\WINDOWS\System32\Wbem;C:\WINDOWS\System32\WindowsPowerShell\v1.0\;C:\Program Files (x86)\Intel\Intel(R) Management Engine Components\DAL;C:\Program Files\Intel\Intel(R) Management Engine Components\DAL;C:\Program Files (x86)\Intel\Intel(R) Management Engine Components\IPT;C:\Program Files\Intel\Intel(R) Management Engine Components\IPT;C:\3rd\MATLAB\R2016a\runtime\win64;C:\3rd\MATLAB\R2016a\bin;C:\3rd\MATLAB\R2016a\polyspace\bin;C:\WINDOWS\System32\OpenSSH\;C:\Program Files\TortoiseSVN\bin;C:\Users\cf_pc\Documents\Caffe\Release;C:\3rd\scala\scala-2.10.5\bin;C:\3rd\Java\jdk1.8.0_212\bin;C:\development\apache-maven-3.6.1\bin;C:\3rd\mysql-8.0.16-winx64\bin;C:\Program Files\Intel\WiFi\bin\;C:\Program Files\Common Files\Intel\WirelessCommon\;C:\3rd\hadoop-common-2.6.0\bin;C:\Users\cf_pc\AppData\Local\Microsoft\WindowsApps;;C:\Program Files\Microsoft VS Code\bin;.
19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:java.io.tmpdir=C:\Users\cf_pc\AppData\Local\Temp19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:java.compiler=<NA>
19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:os.name=Windows 10
19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:os.arch=amd64
19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:os.version=10.0
19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:user.name=cf_pc
19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:user.home=C:\Users\cf_pc
19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Client environment:user.dir=C:\development\cf\scalapp
19/09/24 00:52:23 INFO zookeeper.ZooKeeper: Initiating client connection, connectString=node3:2181 sessionTimeout=90000 watcher=hconnection-0x4bc59b270x0, quorum=node3:2181, baseZNode=/hbase
19/09/24 00:52:23 INFO zookeeper.ClientCnxn: Opening socket connection to server node3/10.200.101.133:2181. Will not attempt to authenticate using SASL (unknown error)
19/09/24 00:52:23 INFO zookeeper.ClientCnxn: Socket connection established to node3/10.200.101.133:2181, initiating session
19/09/24 00:52:23 INFO zookeeper.ClientCnxn: Session establishment complete on server node3/10.200.101.133:2181, sessionid = 0x36ca2ccfed69558, negotiated timeout = 60000
19/09/24 00:52:24 INFO query.ConnectionQueryServicesImpl: HConnection established. Stacktrace for informational purposes: hconnection-0x4bc59b27 java.lang.Thread.getStackTrace(Thread.java:1559)
org.apache.phoenix.util.LogUtil.getCallerStackTrace(LogUtil.java:55)
org.apache.phoenix.query.ConnectionQueryServicesImpl.openConnection(ConnectionQueryServicesImpl.java:427)
org.apache.phoenix.query.ConnectionQueryServicesImpl.access$400(ConnectionQueryServicesImpl.java:267)
org.apache.phoenix.query.ConnectionQueryServicesImpl$12.call(ConnectionQueryServicesImpl.java:2515)
org.apache.phoenix.query.ConnectionQueryServicesImpl$12.call(ConnectionQueryServicesImpl.java:2491)
org.apache.phoenix.util.PhoenixContextExecutor.call(PhoenixContextExecutor.java:76)
org.apache.phoenix.query.ConnectionQueryServicesImpl.init(ConnectionQueryServicesImpl.java:2491)
org.apache.phoenix.jdbc.PhoenixDriver.getConnectionQueryServices(PhoenixDriver.java:255)
org.apache.phoenix.jdbc.PhoenixEmbeddedDriver.createConnection(PhoenixEmbeddedDriver.java:150)
org.apache.phoenix.jdbc.PhoenixDriver.connect(PhoenixDriver.java:221)
java.sql.DriverManager.getConnection(DriverManager.java:664)
java.sql.DriverManager.getConnection(DriverManager.java:208)
org.apache.phoenix.mapreduce.util.ConnectionUtil.getConnection(ConnectionUtil.java:113)
org.apache.phoenix.mapreduce.util.ConnectionUtil.getInputConnection(ConnectionUtil.java:58)
org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil.getSelectColumnMetadataList(PhoenixConfigurationUtil.java:354)
org.apache.phoenix.spark.PhoenixRDD.toDataFrame(PhoenixRDD.scala:118)
org.apache.phoenix.spark.SparkSqlContextFunctions.phoenixTableAsDataFrame(SparkSqlContextFunctions.scala:39)
com.fc.phoenixConnectMode$.getMode1(phoenixConnectMode.scala:16)
com.fc.costDay$.main(costDay.scala:113)
com.fc.costDay.main(costDay.scala)

19/09/24 00:52:25 INFO Configuration.deprecation: hadoop.native.lib is deprecated. Instead, use io.native.lib.available
19/09/24 00:52:27 INFO mapreduce.PhoenixInputFormat: UseSelectColumns=true, selectColumnList.size()=86, selectColumnList=ID,ASSET_ID,ASSET_NAME,ASSET_FIRST_DEGREE_ID,ASSET_FIRST_DEGREE_NAME,ASSET_SECOND_DEGREE_ID,ASSET_SECOND_DEGREE_NAME,GB_DEGREE_ID,GB_DEGREE_NAME,ASSET_USE_FIRST_DEGREE_ID,ASSET_USE_FIRST_DEGREE_NAME,ASSET_USE_SECOND_DEGREE_ID,ASSET_USE_SECOND_DEGREE_NAME,MANAGEMENT_TYPE_ID,MANAGEMENT_TYPE_NAME,ASSET_MODEL,FACTORY_NUMBER,ASSET_COUNTRY_ID,ASSET_COUNTRY_NAME,MANUFACTURER,SUPPLIER,SUPPLIER_TEL,ORIGINAL_VALUE,USE_DEPARTMENT_ID,USE_DEPARTMENT_NAME,USER_ID,USER_NAME,ASSET_LOCATION_OF_PARK_ID,ASSET_LOCATION_OF_PARK_NAME,ASSET_LOCATION_OF_BUILDING_ID,ASSET_LOCATION_OF_BUILDING_NAME,ASSET_LOCATION_OF_ROOM_ID,ASSET_LOCATION_OF_ROOM_NUMBER,PRODUCTION_DATE,ACCEPTANCE_DATE,REQUISITION_DATE,PERFORMANCE_INDEX,ASSET_STATE_ID,ASSET_STATE_NAME,INSPECTION_TYPE_ID,INSPECTION_TYPE_NAME,SEAL_DATE,SEAL_CAUSE,COST_ITEM_ID,COST_ITEM_NAME,ITEM_COMMENTS,UNSEAL_DATE,SCRAP_DATE,PURCHASE_NUMBER,WARRANTY_PERIOD,DEPRECIABLE_LIVES_ID,DEPRECIABLE_LIVES_NAME,MEASUREMENT_UNITS_ID,MEASUREMENT_UNITS_NAME,ANNEX,REMARK,ACCOUNTING_TYPE_ID,ACCOUNTING_TYPE_NAME,SYSTEM_TYPE_ID,SYSTEM_TYPE_NAME,ASSET_ID_PARENT,CLASSIFIED_LEVEL_ID,CLASSIFIED_LEVEL_NAME,ASSET_PICTURE,MILITARY_SPECIAL_CODE,CHECK_CYCLE_ID,CHECK_CYCLE_NAME,CHECK_DATE,CHECK_EFFECTIVE_DATE,CHECK_MODE_ID,CHECK_MODE_NAME,CHECK_DEPARTMENT_ID,CHECK_DEPARTMENT_NAME,RENT_STATUS_ID,RENT_STATUS_NAME,STORAGE_TIME,UPDATE_USER,UPDATE_TIME,IS_ON_PROCESS,IS_DELETED,FIRST_DEPARTMENT_ID,FIRST_DEPARTMENT_NAME,SECOND_DEPARTMENT_ID,SECOND_DEPARTMENT_NAME,CREATE_USER,CREATE_TIME
root
 |-- ID: string (nullable = true)
 |-- FIRST_DEPARTMENT_ID: string (nullable = true)
 |-- ACTUAL_COST: double (nullable = true)
 |-- ORIGINAL_VALUE: double (nullable = true)
 |-- GENERATION_TIME: timestamp (nullable = false)

19/09/24 00:52:32 INFO Configuration.deprecation: io.bytes.per.checksum is deprecated. Instead, use dfs.bytes-per-checksum
19/09/24 00:52:32 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Trying to connect to a secure cluster as 2181 with keytab /hbase
19/09/24 00:52:32 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Successful login to secure cluster
19/09/24 00:52:32 INFO Configuration.deprecation: io.bytes.per.checksum is deprecated. Instead, use dfs.bytes-per-checksum
19/09/24 00:52:32 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Trying to connect to a secure cluster as 2181 with keytab /hbase
19/09/24 00:52:32 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Successful login to secure cluster
19/09/24 00:52:32 INFO mapreduce.PhoenixInputFormat: UseSelectColumns=true, selectColumnList.size()=86, selectColumnList=ID,ASSET_ID,ASSET_NAME,ASSET_FIRST_DEGREE_ID,ASSET_FIRST_DEGREE_NAME,ASSET_SECOND_DEGREE_ID,ASSET_SECOND_DEGREE_NAME,GB_DEGREE_ID,GB_DEGREE_NAME,ASSET_USE_FIRST_DEGREE_ID,ASSET_USE_FIRST_DEGREE_NAME,ASSET_USE_SECOND_DEGREE_ID,ASSET_USE_SECOND_DEGREE_NAME,MANAGEMENT_TYPE_ID,MANAGEMENT_TYPE_NAME,ASSET_MODEL,FACTORY_NUMBER,ASSET_COUNTRY_ID,ASSET_COUNTRY_NAME,MANUFACTURER,SUPPLIER,SUPPLIER_TEL,ORIGINAL_VALUE,USE_DEPARTMENT_ID,USE_DEPARTMENT_NAME,USER_ID,USER_NAME,ASSET_LOCATION_OF_PARK_ID,ASSET_LOCATION_OF_PARK_NAME,ASSET_LOCATION_OF_BUILDING_ID,ASSET_LOCATION_OF_BUILDING_NAME,ASSET_LOCATION_OF_ROOM_ID,ASSET_LOCATION_OF_ROOM_NUMBER,PRODUCTION_DATE,ACCEPTANCE_DATE,REQUISITION_DATE,PERFORMANCE_INDEX,ASSET_STATE_ID,ASSET_STATE_NAME,INSPECTION_TYPE_ID,INSPECTION_TYPE_NAME,SEAL_DATE,SEAL_CAUSE,COST_ITEM_ID,COST_ITEM_NAME,ITEM_COMMENTS,UNSEAL_DATE,SCRAP_DATE,PURCHASE_NUMBER,WARRANTY_PERIOD,DEPRECIABLE_LIVES_ID,DEPRECIABLE_LIVES_NAME,MEASUREMENT_UNITS_ID,MEASUREMENT_UNITS_NAME,ANNEX,REMARK,ACCOUNTING_TYPE_ID,ACCOUNTING_TYPE_NAME,SYSTEM_TYPE_ID,SYSTEM_TYPE_NAME,ASSET_ID_PARENT,CLASSIFIED_LEVEL_ID,CLASSIFIED_LEVEL_NAME,ASSET_PICTURE,MILITARY_SPECIAL_CODE,CHECK_CYCLE_ID,CHECK_CYCLE_NAME,CHECK_DATE,CHECK_EFFECTIVE_DATE,CHECK_MODE_ID,CHECK_MODE_NAME,CHECK_DEPARTMENT_ID,CHECK_DEPARTMENT_NAME,RENT_STATUS_ID,RENT_STATUS_NAME,STORAGE_TIME,UPDATE_USER,UPDATE_TIME,IS_ON_PROCESS,IS_DELETED,FIRST_DEPARTMENT_ID,FIRST_DEPARTMENT_NAME,SECOND_DEPARTMENT_ID,SECOND_DEPARTMENT_NAME,CREATE_USER,CREATE_TIME
19/09/24 00:52:32 INFO mapreduce.PhoenixInputFormat: Select Statement: SELECT "ID","0"."ASSET_ID","0"."ASSET_NAME","0"."ASSET_FIRST_DEGREE_ID","0"."ASSET_FIRST_DEGREE_NAME","0"."ASSET_SECOND_DEGREE_ID","0"."ASSET_SECOND_DEGREE_NAME","0"."GB_DEGREE_ID","0"."GB_DEGREE_NAME","0"."ASSET_USE_FIRST_DEGREE_ID","0"."ASSET_USE_FIRST_DEGREE_NAME","0"."ASSET_USE_SECOND_DEGREE_ID","0"."ASSET_USE_SECOND_DEGREE_NAME","0"."MANAGEMENT_TYPE_ID","0"."MANAGEMENT_TYPE_NAME","0"."ASSET_MODEL","0"."FACTORY_NUMBER","0"."ASSET_COUNTRY_ID","0"."ASSET_COUNTRY_NAME","0"."MANUFACTURER","0"."SUPPLIER","0"."SUPPLIER_TEL","0"."ORIGINAL_VALUE","0"."USE_DEPARTMENT_ID","0"."USE_DEPARTMENT_NAME","0"."USER_ID","0"."USER_NAME","0"."ASSET_LOCATION_OF_PARK_ID","0"."ASSET_LOCATION_OF_PARK_NAME","0"."ASSET_LOCATION_OF_BUILDING_ID","0"."ASSET_LOCATION_OF_BUILDING_NAME","0"."ASSET_LOCATION_OF_ROOM_ID","0"."ASSET_LOCATION_OF_ROOM_NUMBER","0"."PRODUCTION_DATE","0"."ACCEPTANCE_DATE","0"."REQUISITION_DATE","0"."PERFORMANCE_INDEX","0"."ASSET_STATE_ID","0"."ASSET_STATE_NAME","0"."INSPECTION_TYPE_ID","0"."INSPECTION_TYPE_NAME","0"."SEAL_DATE","0"."SEAL_CAUSE","0"."COST_ITEM_ID","0"."COST_ITEM_NAME","0"."ITEM_COMMENTS","0"."UNSEAL_DATE","0"."SCRAP_DATE","0"."PURCHASE_NUMBER","0"."WARRANTY_PERIOD","0"."DEPRECIABLE_LIVES_ID","0"."DEPRECIABLE_LIVES_NAME","0"."MEASUREMENT_UNITS_ID","0"."MEASUREMENT_UNITS_NAME","0"."ANNEX","0"."REMARK","0"."ACCOUNTING_TYPE_ID","0"."ACCOUNTING_TYPE_NAME","0"."SYSTEM_TYPE_ID","0"."SYSTEM_TYPE_NAME","0"."ASSET_ID_PARENT","0"."CLASSIFIED_LEVEL_ID","0"."CLASSIFIED_LEVEL_NAME","0"."ASSET_PICTURE","0"."MILITARY_SPECIAL_CODE","0"."CHECK_CYCLE_ID","0"."CHECK_CYCLE_NAME","0"."CHECK_DATE","0"."CHECK_EFFECTIVE_DATE","0"."CHECK_MODE_ID","0"."CHECK_MODE_NAME","0"."CHECK_DEPARTMENT_ID","0"."CHECK_DEPARTMENT_NAME","0"."RENT_STATUS_ID","0"."RENT_STATUS_NAME","0"."STORAGE_TIME","0"."UPDATE_USER","0"."UPDATE_TIME","0"."IS_ON_PROCESS","0"."IS_DELETED","0"."FIRST_DEPARTMENT_ID","0"."FIRST_DEPARTMENT_NAME","0"."SECOND_DEPARTMENT_ID","0"."SECOND_DEPARTMENT_NAME","0"."CREATE_USER","0"."CREATE_TIME" FROM ASSET_NORMAL
19/09/24 00:52:32 INFO zookeeper.RecoverableZooKeeper: Process identifier=hconnection-0x22617270 connecting to ZooKeeper ensemble=node3:2181
19/09/24 00:52:32 INFO zookeeper.ZooKeeper: Initiating client connection, connectString=node3:2181 sessionTimeout=90000 watcher=hconnection-0x226172700x0, quorum=node3:2181, baseZNode=/hbase
19/09/24 00:52:32 INFO zookeeper.ClientCnxn: Opening socket connection to server node3/10.200.101.133:2181. Will not attempt to authenticate using SASL (unknown error)
19/09/24 00:52:32 INFO zookeeper.ClientCnxn: Socket connection established to node3/10.200.101.133:2181, initiating session
19/09/24 00:52:32 INFO zookeeper.ClientCnxn: Session establishment complete on server node3/10.200.101.133:2181, sessionid = 0x36ca2ccfed69559, negotiated timeout = 60000
19/09/24 00:52:32 INFO util.RegionSizeCalculator: Calculating region sizes for table "IDX_ASSET_NORMAL".
19/09/24 00:52:32 INFO client.ConnectionManager$HConnectionImplementation: Closing master protocol: MasterService
19/09/24 00:52:32 INFO client.ConnectionManager$HConnectionImplementation: Closing zookeeper sessionid=0x36ca2ccfed69559
19/09/24 00:52:32 INFO zookeeper.ZooKeeper: Session: 0x36ca2ccfed69559 closed
19/09/24 00:52:32 INFO zookeeper.ClientCnxn: EventThread shut down
19/09/24 00:52:32 INFO spark.SparkContext: Starting job: count at costDay.scala:139
19/09/24 00:52:33 INFO scheduler.DAGScheduler: Registering RDD 7 (count at costDay.scala:139)
19/09/24 00:52:33 INFO scheduler.DAGScheduler: Got job 0 (count at costDay.scala:139) with 1 output partitions
19/09/24 00:52:33 INFO scheduler.DAGScheduler: Final stage: ResultStage 1 (count at costDay.scala:139)
19/09/24 00:52:33 INFO scheduler.DAGScheduler: Parents of final stage: List(ShuffleMapStage 0)
19/09/24 00:52:33 INFO scheduler.DAGScheduler: Missing parents: List(ShuffleMapStage 0)
19/09/24 00:52:33 INFO scheduler.DAGScheduler: Submitting ShuffleMapStage 0 (MapPartitionsRDD[7] at count at costDay.scala:139), which has no missing parents
19/09/24 00:52:33 INFO storage.MemoryStore: Block broadcast_1 stored as values in memory (estimated size 24.8 KB, free 477.9 MB)
19/09/24 00:52:33 INFO storage.MemoryStore: Block broadcast_1_piece0 stored as bytes in memory (estimated size 9.5 KB, free 477.9 MB)
19/09/24 00:52:33 INFO storage.BlockManagerInfo: Added broadcast_1_piece0 in memory on localhost:54720 (size: 9.5 KB, free: 478.1 MB)
19/09/24 00:52:33 INFO spark.SparkContext: Created broadcast 1 from broadcast at DAGScheduler.scala:1004
19/09/24 00:52:33 INFO scheduler.DAGScheduler: Submitting 1 missing tasks from ShuffleMapStage 0 (MapPartitionsRDD[7] at count at costDay.scala:139) (first 15 tasks are for partitions Vector(0))
19/09/24 00:52:33 INFO scheduler.TaskSchedulerImpl: Adding task set 0.0 with 1 tasks
19/09/24 00:52:33 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 0.0 (TID 0, localhost, executor driver, partition 0, ANY, 2538 bytes)
19/09/24 00:52:33 INFO executor.Executor: Running task 0.0 in stage 0.0 (TID 0)
19/09/24 00:52:33 INFO rdd.NewHadoopRDD: Input split: [email protected]
19/09/24 00:52:33 INFO Configuration.deprecation: io.bytes.per.checksum is deprecated. Instead, use dfs.bytes-per-checksum
19/09/24 00:52:33 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Trying to connect to a secure cluster as 2181 with keytab /hbase
19/09/24 00:52:33 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Successful login to secure cluster
19/09/24 00:52:33 INFO codegen.GeneratePredicate: Code generated in 309.4486 ms
19/09/24 00:52:33 INFO codegen.GenerateUnsafeProjection: Code generated in 20.3535 ms
19/09/24 00:52:34 INFO codegen.GenerateMutableProjection: Code generated in 10.5156 ms
19/09/24 00:52:34 INFO codegen.GenerateMutableProjection: Code generated in 10.4614 ms
19/09/24 00:52:34 INFO codegen.GenerateUnsafeRowJoiner: Code generated in 6.8774 ms
19/09/24 00:52:34 INFO codegen.GenerateUnsafeProjection: Code generated in 6.7907 ms
19/09/24 00:52:43 INFO executor.Executor: Finished task 0.0 in stage 0.0 (TID 0). 2629 bytes result sent to driver
19/09/24 00:52:43 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 0.0 (TID 0) in 10699 ms on localhost (executor driver) (1/1)
19/09/24 00:52:43 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed, from pool
19/09/24 00:52:43 INFO scheduler.DAGScheduler: ShuffleMapStage 0 (count at costDay.scala:139) finished in 10.745 s
19/09/24 00:52:43 INFO scheduler.DAGScheduler: looking for newly runnable stages
19/09/24 00:52:43 INFO scheduler.DAGScheduler: running: Set()
19/09/24 00:52:43 INFO scheduler.DAGScheduler: waiting: Set(ResultStage 1)
19/09/24 00:52:43 INFO scheduler.DAGScheduler: failed: Set()
19/09/24 00:52:43 INFO scheduler.DAGScheduler: Submitting ResultStage 1 (MapPartitionsRDD[10] at count at costDay.scala:139), which has no missing parents
19/09/24 00:52:43 INFO storage.MemoryStore: Block broadcast_2 stored as values in memory (estimated size 25.3 KB, free 477.9 MB)
19/09/24 00:52:43 INFO storage.MemoryStore: Block broadcast_2_piece0 stored as bytes in memory (estimated size 9.8 KB, free 477.8 MB)
19/09/24 00:52:43 INFO storage.BlockManagerInfo: Added broadcast_2_piece0 in memory on localhost:54720 (size: 9.8 KB, free: 478.1 MB)
19/09/24 00:52:43 INFO spark.SparkContext: Created broadcast 2 from broadcast at DAGScheduler.scala:1004
19/09/24 00:52:43 INFO scheduler.DAGScheduler: Submitting 1 missing tasks from ResultStage 1 (MapPartitionsRDD[10] at count at costDay.scala:139) (first 15 tasks are for partitions Vector(0))
19/09/24 00:52:43 INFO scheduler.TaskSchedulerImpl: Adding task set 1.0 with 1 tasks
19/09/24 00:52:43 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 1.0 (TID 1, localhost, executor driver, partition 0, NODE_LOCAL, 1999 bytes)
19/09/24 00:52:43 INFO executor.Executor: Running task 0.0 in stage 1.0 (TID 1)
19/09/24 00:52:43 INFO storage.ShuffleBlockFetcherIterator: Getting 1 non-empty blocks out of 1 blocks
19/09/24 00:52:44 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 20 ms
19/09/24 00:52:44 INFO codegen.GenerateMutableProjection: Code generated in 22.44 ms
19/09/24 00:52:44 INFO codegen.GenerateMutableProjection: Code generated in 13.992 ms
19/09/24 00:52:44 INFO executor.Executor: Finished task 0.0 in stage 1.0 (TID 1). 1959 bytes result sent to driver
19/09/24 00:52:44 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 1.0 (TID 1) in 199 ms on localhost (executor driver) (1/1)
19/09/24 00:52:44 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 1.0, whose tasks have all completed, from pool
19/09/24 00:52:44 INFO scheduler.DAGScheduler: ResultStage 1 (count at costDay.scala:139) finished in 0.205 s
19/09/24 00:52:44 INFO scheduler.DAGScheduler: Job 0 finished: count at costDay.scala:139, took 11.157661 s
50858
19/09/24 00:52:44 INFO spark.SparkContext: Starting job: describe at costDay.scala:141
19/09/24 00:52:44 INFO scheduler.DAGScheduler: Registering RDD 14 (describe at costDay.scala:141)
19/09/24 00:52:44 INFO scheduler.DAGScheduler: Got job 1 (describe at costDay.scala:141) with 1 output partitions
19/09/24 00:52:44 INFO scheduler.DAGScheduler: Final stage: ResultStage 3 (describe at costDay.scala:141)
19/09/24 00:52:44 INFO scheduler.DAGScheduler: Parents of final stage: List(ShuffleMapStage 2)
19/09/24 00:52:44 INFO scheduler.DAGScheduler: Missing parents: List(ShuffleMapStage 2)
19/09/24 00:52:44 INFO scheduler.DAGScheduler: Submitting ShuffleMapStage 2 (MapPartitionsRDD[14] at describe at costDay.scala:141), which has no missing parents
19/09/24 00:52:44 INFO storage.MemoryStore: Block broadcast_3 stored as values in memory (estimated size 27.2 KB, free 477.8 MB)
19/09/24 00:52:44 INFO storage.MemoryStore: Block broadcast_3_piece0 stored as bytes in memory (estimated size 10.4 KB, free 477.8 MB)
19/09/24 00:52:44 INFO storage.BlockManagerInfo: Added broadcast_3_piece0 in memory on localhost:54720 (size: 10.4 KB, free: 478.1 MB)
19/09/24 00:52:44 INFO spark.SparkContext: Created broadcast 3 from broadcast at DAGScheduler.scala:1004
19/09/24 00:52:44 INFO scheduler.DAGScheduler: Submitting 1 missing tasks from ShuffleMapStage 2 (MapPartitionsRDD[14] at describe at costDay.scala:141) (first 15 tasks are for partitions Vector(0))
19/09/24 00:52:44 INFO scheduler.TaskSchedulerImpl: Adding task set 2.0 with 1 tasks
19/09/24 00:52:44 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 2.0 (TID 2, localhost, executor driver, partition 0, ANY, 2538 bytes)
19/09/24 00:52:44 INFO executor.Executor: Running task 0.0 in stage 2.0 (TID 2)
19/09/24 00:52:44 INFO rdd.NewHadoopRDD: Input split: [email protected]
19/09/24 00:52:44 INFO Configuration.deprecation: io.bytes.per.checksum is deprecated. Instead, use dfs.bytes-per-checksum
19/09/24 00:52:44 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Trying to connect to a secure cluster as 2181 with keytab /hbase
19/09/24 00:52:44 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Successful login to secure cluster
19/09/24 00:52:44 INFO codegen.GenerateUnsafeProjection: Code generated in 11.405 ms
19/09/24 00:52:44 INFO codegen.GenerateMutableProjection: Code generated in 10.5886 ms
19/09/24 00:52:44 INFO codegen.GenerateMutableProjection: Code generated in 39.2201 ms
19/09/24 00:52:44 INFO codegen.GenerateUnsafeRowJoiner: Code generated in 8.3737 ms
19/09/24 00:52:44 INFO codegen.GenerateUnsafeProjection: Code generated in 22.542 ms
19/09/24 00:52:45 INFO storage.BlockManagerInfo: Removed broadcast_2_piece0 on localhost:54720 in memory (size: 9.8 KB, free: 478.1 MB)
19/09/24 00:52:53 INFO executor.Executor: Finished task 0.0 in stage 2.0 (TID 2). 2629 bytes result sent to driver
19/09/24 00:52:53 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 2.0 (TID 2) in 8824 ms on localhost (executor driver) (1/1)
19/09/24 00:52:53 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 2.0, whose tasks have all completed, from pool
19/09/24 00:52:53 INFO scheduler.DAGScheduler: ShuffleMapStage 2 (describe at costDay.scala:141) finished in 8.825 s
19/09/24 00:52:53 INFO scheduler.DAGScheduler: looking for newly runnable stages
19/09/24 00:52:53 INFO scheduler.DAGScheduler: running: Set()
19/09/24 00:52:53 INFO scheduler.DAGScheduler: waiting: Set(ResultStage 3)
19/09/24 00:52:53 INFO scheduler.DAGScheduler: failed: Set()
19/09/24 00:52:53 INFO scheduler.DAGScheduler: Submitting ResultStage 3 (MapPartitionsRDD[18] at describe at costDay.scala:141), which has no missing parents
19/09/24 00:52:53 INFO storage.MemoryStore: Block broadcast_4 stored as values in memory (estimated size 28.7 KB, free 477.8 MB)
19/09/24 00:52:53 INFO storage.MemoryStore: Block broadcast_4_piece0 stored as bytes in memory (estimated size 11.0 KB, free 477.8 MB)
19/09/24 00:52:53 INFO storage.BlockManagerInfo: Added broadcast_4_piece0 in memory on localhost:54720 (size: 11.0 KB, free: 478.1 MB)
19/09/24 00:52:53 INFO spark.SparkContext: Created broadcast 4 from broadcast at DAGScheduler.scala:1004
19/09/24 00:52:53 INFO scheduler.DAGScheduler: Submitting 1 missing tasks from ResultStage 3 (MapPartitionsRDD[18] at describe at costDay.scala:141) (first 15 tasks are for partitions Vector(0))
19/09/24 00:52:53 INFO scheduler.TaskSchedulerImpl: Adding task set 3.0 with 1 tasks
19/09/24 00:52:53 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 3.0 (TID 3, localhost, executor driver, partition 0, NODE_LOCAL, 1999 bytes)
19/09/24 00:52:53 INFO executor.Executor: Running task 0.0 in stage 3.0 (TID 3)
19/09/24 00:52:53 INFO storage.ShuffleBlockFetcherIterator: Getting 1 non-empty blocks out of 1 blocks
19/09/24 00:52:53 INFO storage.ShuffleBlockFetcherIterator: Started 0 remote fetches in 1 ms
19/09/24 00:52:53 INFO codegen.GenerateMutableProjection: Code generated in 21.5454 ms
19/09/24 00:52:53 INFO codegen.GenerateMutableProjection: Code generated in 10.7314 ms
19/09/24 00:52:53 INFO codegen.GenerateUnsafeProjection: Code generated in 13.0969 ms
19/09/24 00:52:53 INFO codegen.GenerateSafeProjection: Code generated in 8.2397 ms
19/09/24 00:52:53 INFO executor.Executor: Finished task 0.0 in stage 3.0 (TID 3). 2138 bytes result sent to driver
19/09/24 00:52:53 INFO scheduler.DAGScheduler: ResultStage 3 (describe at costDay.scala:141) finished in 0.097 s
19/09/24 00:52:53 INFO scheduler.DAGScheduler: Job 1 finished: describe at costDay.scala:141, took 8.948356 s
19/09/24 00:52:53 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 3.0 (TID 3) in 97 ms on localhost (executor driver) (1/1)
19/09/24 00:52:53 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 3.0, whose tasks have all completed, from pool
+-------+--------------------+
|summary|      ORIGINAL_VALUE|
+-------+--------------------+
|  count|               50858|
|   mean|2.427485546653306E12|
| stddev|5.474385018305400...|
|    min|          -7970934.0|
|    max|1.234567890123456...|
+-------+--------------------+

19/09/24 00:52:53 INFO spark.SparkContext: Starting job: show at costDay.scala:142
19/09/24 00:52:53 INFO scheduler.DAGScheduler: Got job 2 (show at costDay.scala:142) with 1 output partitions
19/09/24 00:52:53 INFO scheduler.DAGScheduler: Final stage: ResultStage 4 (show at costDay.scala:142)
19/09/24 00:52:53 INFO scheduler.DAGScheduler: Parents of final stage: List()
19/09/24 00:52:53 INFO scheduler.DAGScheduler: Missing parents: List()
19/09/24 00:52:53 INFO scheduler.DAGScheduler: Submitting ResultStage 4 (MapPartitionsRDD[22] at show at costDay.scala:142), which has no missing parents
19/09/24 00:52:53 INFO storage.MemoryStore: Block broadcast_5 stored as values in memory (estimated size 23.6 KB, free 477.8 MB)
19/09/24 00:52:53 INFO storage.MemoryStore: Block broadcast_5_piece0 stored as bytes in memory (estimated size 9.0 KB, free 477.8 MB)
19/09/24 00:52:53 INFO storage.BlockManagerInfo: Added broadcast_5_piece0 in memory on localhost:54720 (size: 9.0 KB, free: 478.1 MB)
19/09/24 00:52:53 INFO spark.SparkContext: Created broadcast 5 from broadcast at DAGScheduler.scala:1004
19/09/24 00:52:53 INFO scheduler.DAGScheduler: Submitting 1 missing tasks from ResultStage 4 (MapPartitionsRDD[22] at show at costDay.scala:142) (first 15 tasks are for partitions Vector(0))
19/09/24 00:52:53 INFO scheduler.TaskSchedulerImpl: Adding task set 4.0 with 1 tasks
19/09/24 00:52:53 INFO scheduler.TaskSetManager: Starting task 0.0 in stage 4.0 (TID 4, localhost, executor driver, partition 0, ANY, 2549 bytes)
19/09/24 00:52:53 INFO executor.Executor: Running task 0.0 in stage 4.0 (TID 4)
19/09/24 00:52:53 INFO rdd.NewHadoopRDD: Input split: [email protected]
19/09/24 00:52:53 INFO Configuration.deprecation: io.bytes.per.checksum is deprecated. Instead, use dfs.bytes-per-checksum
19/09/24 00:52:53 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Trying to connect to a secure cluster as 2181 with keytab /hbase
19/09/24 00:52:53 INFO jdbc.PhoenixEmbeddedDriver$ConnectionInfo: Successful login to secure cluster
19/09/24 00:52:53 INFO codegen.GenerateUnsafeProjection: Code generated in 33.6563 ms
19/09/24 00:52:53 INFO codegen.GenerateSafeProjection: Code generated in 7.0589 ms
19/09/24 00:52:54 INFO executor.Executor: Finished task 0.0 in stage 4.0 (TID 4). 5544 bytes result sent to driver
19/09/24 00:52:54 INFO scheduler.TaskSetManager: Finished task 0.0 in stage 4.0 (TID 4) in 476 ms on localhost (executor driver) (1/1)
19/09/24 00:52:54 INFO scheduler.DAGScheduler: ResultStage 4 (show at costDay.scala:142) finished in 0.466 s
19/09/24 00:52:54 INFO scheduler.TaskSchedulerImpl: Removed TaskSet 4.0, whose tasks have all completed, from pool
19/09/24 00:52:54 INFO scheduler.DAGScheduler: Job 2 finished: show at costDay.scala:142, took 0.489113 s
+--------------------------------+-------------------+---------------------+--------------+-----------------------+
|ID                              |FIRST_DEPARTMENT_ID|ACTUAL_COST          |ORIGINAL_VALUE|GENERATION_TIME        |
+--------------------------------+-------------------+---------------------+--------------+-----------------------+
|d25bb550a290457382c175b0e57c0982|1149564326588321792|0.0010410958904109589|2.0           |2019-09-24 00:52:31.864|
|492016e2f7ec4cd18615c164c92c6c6d|1149564326584127489|5.205479452054794E-4 |1.0           |2019-09-24 00:52:31.864|
|1d138a7401bd493da12f8f8323e9dee0|1149564326588321710|0.00353972602739726  |34.0          |2019-09-24 00:52:31.864|
|09718925d34b4a099e09a30a0621ded8|1149564326588321710|0.002891643835616438 |11.11         |2019-09-24 00:52:31.864|
|d5cfd5e898464130b71530d74b43e9d1|1149564326584127489|3135.8751712328767   |9638690.0     |2019-09-24 00:52:31.864|
|6b39ac96b8734103b2413520d3195ee6|1149564326584127489|1744.2413835616437   |6701559.0     |2019-09-24 00:52:31.864|
|8d20d0abd04d49cea3e52d9ca67e39da|1149569393202696195|0.22175342465753425  |852.0         |2019-09-24 00:52:31.864|
|66ae7e7c7a104cea99615358e12c03b0|1149569393202696195|1.0410958904109588   |2000.0        |2019-09-24 00:52:31.864|
|d49b0324bbf14b70adefe8b1d9163db2|1149569393202696195|1.0410958904109588   |2000.0        |2019-09-24 00:52:31.864|
|d4d701514a2a425e8192acf47bb57f9b|1149569393202696195|0.032534246575342464 |100.0         |2019-09-24 00:52:31.864|
|d6a016c618c1455ca0e2c7d73ba947ac|1149569393202696195|0.6506849315068494   |2000.0        |2019-09-24 00:52:31.864|
|5dfa3be825464ddd98764b2790720fae|1149569393202696195|147.91532534246576   |454645.0      |2019-09-24 00:52:31.864|
|6e5653ef4aaa4c03bcd00fbeb1e6811d|1149569393202696195|0.1952054794520548   |600.0         |2019-09-24 00:52:31.864|
|32bd2654082645cba35527d50e0d52f9|1149569393202696195|0.6506849315068494   |2000.0        |2019-09-24 00:52:31.864|
|8ed4424408bc458dbe200acffe5733bf|1149564326584127488|5.205479452054794E-4 |1.0           |2019-09-24 00:52:31.864|
|1b2faa31f139461488847e77eacd794a|1149564326584127488|33499.042109589034   |6.4353423E7   |2019-09-24 00:52:31.864|
|f398245c9ccc4760a5eb3251db3680bf|1149564326584127488|33499.042109589034   |6.4353423E7   |2019-09-24 00:52:31.864|
|2696de9733d247e5bf88573244f36ba2|1149564326584127488|0.011452054794520548 |22.0          |2019-09-24 00:52:31.864|
|9c8cfad3d4334b37a7b9beb56b528c22|1149569976173203457|0.06506849315068493  |200.0         |2019-09-24 00:52:31.864|
|3e2721b79e754a798d0be940ae011d72|1149569976173203457|0.004001712328767123 |12.3          |2019-09-24 00:52:31.864|
+--------------------------------+-------------------+---------------------+--------------+-----------------------+
only showing top 20 rows

19/09/24 00:52:54 INFO spark.SparkContext: Invoking stop() from shutdown hook
19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/static/sql,null}
19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/SQL/execution/json,null}
19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/SQL/execution,null}
19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/SQL/json,null}
19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/SQL,null}
19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/metrics/json,null}
19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/stage/kill,null}
19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/api,null}
19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/,null}
19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/static,null}
19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/executors/threadDump/json,null}
19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/executors/threadDump,null}
19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/executors/json,null}
19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/executors,null}
19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/environment/json,null}
19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/environment,null}
19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/storage/rdd/json,null}
19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/storage/rdd,null}
19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/storage/json,null}
19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/storage,null}
19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/pool/json,null}
19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/pool,null}
19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/stage/json,null}
19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/stage,null}
19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages/json,null}
19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/stages,null}
19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/jobs/job/json,null}
19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/jobs/job,null}
19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/jobs/json,null}
19/09/24 00:52:54 INFO handler.ContextHandler: stopped o.s.j.s.ServletContextHandler{/jobs,null}
19/09/24 00:52:54 INFO ui.SparkUI: Stopped Spark web UI at http://10.200.74.155:4040
19/09/24 00:52:54 INFO spark.MapOutputTrackerMasterEndpoint: MapOutputTrackerMasterEndpoint stopped!
19/09/24 00:52:54 INFO storage.MemoryStore: MemoryStore cleared
19/09/24 00:52:54 INFO storage.BlockManager: BlockManager stopped
19/09/24 00:52:54 INFO storage.BlockManagerMaster: BlockManagerMaster stopped
19/09/24 00:52:54 INFO scheduler.OutputCommitCoordinator$OutputCommitCoordinatorEndpoint: OutputCommitCoordinator stopped!
19/09/24 00:52:54 INFO spark.SparkContext: Successfully stopped SparkContext
19/09/24 00:52:54 INFO util.ShutdownHookManager: Shutdown hook called
19/09/24 00:52:54 INFO util.ShutdownHookManager: Deleting directory C:\Users\cf_pc\AppData\Local\Temp\spark-4fcfa10d-c258-46b7-b4f5-ee977276fa00

Process finished with exit code 0

执行打包操作，返回信息：

C:\3rd\Java\jdk1.8.0_212\bin\java.exe -Dmaven.multiModuleProjectDirectory=C:\development\cf\scalapp -Dmaven.home=C:\development\apache-maven-3.6.1 -Dclassworlds.conf=C:\development\apache-maven-3.6.1\bin\m2.conf "-javaagent:C:\3rd\JetBrains\IntelliJ IDEA 2019.1.3\lib\idea_rt.jar=49792:C:\3rd\JetBrains\IntelliJ IDEA 2019.1.3\bin" -Dfile.encoding=UTF-8 -classpath C:\development\apache-maven-3.6.1\boot\plexus-classworlds-2.6.0.jar org.codehaus.classworlds.Launcher -Didea.version2019.1.3 -s C:\development\apache-maven-3.6.1\conf\settings.xml -Dmaven.repo.local=C:\development\MavenRepository package
[INFO] Scanning for projects...
[INFO]
[INFO] ---------------------------< com.fc:scalapp >---------------------------
[INFO] Building scalapp 1.0-SNAPSHOT
[INFO] --------------------------------[ jar ]---------------------------------
[INFO]
[INFO] --- maven-resources-plugin:2.6:resources (default-resources) @ scalapp ---
[WARNING] Using platform encoding (UTF-8 actually) to copy filtered resources, i.e. build is platform dependent!
[INFO] Copying 0 resource
[INFO]
[INFO] --- maven-compiler-plugin:3.1:compile (default-compile) @ scalapp ---
[INFO] Nothing to compile - all classes are up to date
[INFO]
[INFO] --- maven-scala-plugin:2.15.2:compile (default) @ scalapp ---
[INFO] Checking for multiple versions of scala
[WARNING]  Expected all dependencies to require Scala version: 2.10.5
[WARNING]  com.twitter:chill_2.10:0.5.0 requires scala version: 2.10.4
[WARNING] Multiple versions of scala libraries detected!
[INFO] includes = [**/*.java,**/*.scala,]
[INFO] excludes = []
[INFO] C:\development\cf\scalapp\src\main\scala:-1: info: compiling
[INFO] Compiling 4 source files to C:\development\cf\scalapp\target\classes at 1569255082006
[WARNING] warning: there were 1 feature warning(s); re-run with -feature for details
[WARNING] one warning found
[INFO] prepare-compile in 0 s
[INFO] compile in 7 s
[INFO]
[INFO] --- maven-resources-plugin:2.6:testResources (default-testResources) @ scalapp ---
[WARNING] Using platform encoding (UTF-8 actually) to copy filtered resources, i.e. build is platform dependent!
[INFO] skip non existing resourceDirectory C:\development\cf\scalapp\src\test\resources
[INFO]
[INFO] --- maven-compiler-plugin:3.1:testCompile (default-testCompile) @ scalapp ---
[INFO] Nothing to compile - all classes are up to date
[INFO]
[INFO] --- maven-scala-plugin:2.15.2:testCompile (default) @ scalapp ---
[INFO] Checking for multiple versions of scala
[WARNING]  Expected all dependencies to require Scala version: 2.10.5
[WARNING]  com.twitter:chill_2.10:0.5.0 requires scala version: 2.10.4
[WARNING] Multiple versions of scala libraries detected!
[INFO] includes = [**/*.java,**/*.scala,]
[INFO] excludes = []
[WARNING] No source files found.
[INFO]
[INFO] --- maven-surefire-plugin:2.12.4:test (default-test) @ scalapp ---
[INFO]
[INFO] --- maven-jar-plugin:2.4:jar (default-jar) @ scalapp ---
[INFO] Building jar: C:\development\cf\scalapp\target\scalapp-1.0-SNAPSHOT.jar
[INFO]
[INFO] --- maven-shade-plugin:3.1.0:shade (default) @ scalapp ---
[INFO] Including org.apache.spark:spark-core_2.10:jar:1.6.0-cdh5.14.2 in the shaded jar.
[INFO] Including org.apache.avro:avro-mapred:jar:hadoop2:1.7.6-cdh5.14.2 in the shaded jar.
……
[INFO] Including mysql:mysql-connector-java:jar:5.1.43 in the shaded jar.
[WARNING] commons-collections-3.2.2.jar, commons-beanutils-1.7.0.jar, commons-beanutils-core-1.8.0.jar define 10 overlapping classes:
[WARNING]   - org.apache.commons.collections.FastHashMap$EntrySet
[WARNING]   - org.apache.commons.collections.FastHashMap$KeySet
[WARNING]   - org.apache.commons.collections.FastHashMap$CollectionView$CollectionViewIterator
[WARNING]   - org.apache.commons.collections.ArrayStack
[WARNING]   - org.apache.commons.collections.FastHashMap$Values
[WARNING]   - org.apache.commons.collections.FastHashMap$CollectionView
[WARNING]   - org.apache.commons.collections.FastHashMap$1
[WARNING]   - org.apache.commons.collections.Buffer
[WARNING]   - org.apache.commons.collections.FastHashMap
[WARNING]   - org.apache.commons.collections.BufferUnderflowException
……
[WARNING] mvn dependency:tree -Ddetail=true and the above output.
[WARNING] See http://maven.apache.org/plugins/maven-shade-plugin/
[INFO] Replacing original artifact with shaded artifact.
[INFO] Replacing C:\development\cf\scalapp\target\scalapp-1.0-SNAPSHOT.jar with C:\development\cf\scalapp\target\scalapp-1.0-SNAPSHOT-shaded.jar
[INFO] ------------------------------------------------------------------------
[INFO] BUILD SUCCESS
[INFO] ------------------------------------------------------------------------
[INFO] Total time:  01:00 min
[INFO] Finished at: 2019-09-24T00:12:08+08:00
[INFO] ------------------------------------------------------------------------

Process finished with exit code 0

上传到服务器上，执行命令：

spark-submit --class com.fc.costDay --executor-memory 500m --total-executor-cores 1 /home/cf/scalapp-1.0-SNAPSHOT.jar

报错：

java.lang.ClassNotFoundException: Class org.apache.phoenix.spark.PhoenixRecordWritable not found

参考：https://www.jianshu.com/p/f336f7e5f31b，更改执行命令为：

spark-submit  --master yarn-cluster --driver-memory 4g --num-executors 2 --executor-memory 2g --executor-cores 2  --class com.fc.costDay   --conf spark.driver.extraClassPath=/opt/cloudera/parcels/APACHE_PHOENIX-4.14.0-cdh5.14.2.p0.3/lib/phoenix/lib/*  --conf spark.executor.extraClassPath=/opt/cloudera/parcels/APACHE_PHOENIX-4.14.0-cdh5.14.2.p0.3/lib/phoenix/lib/* /home/cf/scalapp-1.0-SNAPSHOT.jar

仍然报错：

File does not exist: hdfs://node1:8020/user/root/.sparkStaging/application_1566100765602_0105/……

完整错误信息如下：

[[email protected] ~]# spark-submit  --master yarn-cluster --driver-memory 4g --num-executors 2 --executor-memory 2g --executor-cores 2  --class com.fc.costDay   --conf spark.driver.extraClassPath=/opt/cloudera/parcels/APACHE_PHOENIX-4.14.0-cdh5.14.2.p0.3/lib/phoenix/lib/*  --conf spark.executor.extraClassPath=/opt/cloudera/parcels/APACHE_PHOENIX-4.14.0-cdh5.14.2.p0.3/lib/phoenix/lib/* /home/cf/scalapp-1.0-SNAPSHOT.jar
19/09/24 00:01:36 INFO client.RMProxy: Connecting to ResourceManager at node1/10.200.101.131:8032
19/09/24 00:01:36 INFO yarn.Client: Requesting a new application from cluster with 3 NodeManagers
19/09/24 00:01:36 INFO yarn.Client: Verifying our application has not requested more than the maximum memory capability of the cluster (40874 MB per container)
19/09/24 00:01:36 INFO yarn.Client: Will allocate AM container, with 4505 MB memory including 409 MB overhead
19/09/24 00:01:36 INFO yarn.Client: Setting up container launch context for our AM
19/09/24 00:01:36 INFO yarn.Client: Setting up the launch environment for our AM container
19/09/24 00:01:37 INFO yarn.Client: Preparing resources for our AM container
19/09/24 00:01:37 INFO yarn.Client: Uploading resource file:/home/cf/scalapp-1.0-SNAPSHOT.jar -> hdfs://node1:8020/user/root/.sparkStaging/application_1566100765602_0105/scalapp-1.0-SNAPSHOT.jar
19/09/24 00:01:38 INFO yarn.Client: Uploading resource file:/tmp/spark-2c548285-8012-414b-ab4e-797a164e38bc/__spark_conf__8997139291240118668.zip -> hdfs://node1:8020/user/root/.sparkStaging/application_1566100765602_0105/__spark_conf__8997139291240118668.zip
19/09/24 00:01:38 INFO spark.SecurityManager: Changing view acls to: root
19/09/24 00:01:38 INFO spark.SecurityManager: Changing modify acls to: root
19/09/24 00:01:38 INFO spark.SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(root); users with modify permissions: Set(root)
19/09/24 00:01:38 INFO yarn.Client: Submitting application 105 to ResourceManager
19/09/24 00:01:38 INFO impl.YarnClientImpl: Submitted application application_1566100765602_0105
19/09/24 00:01:39 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED)
19/09/24 00:01:39 INFO yarn.Client:
     client token: N/A
     diagnostics: N/A
     ApplicationMaster host: N/A
     ApplicationMaster RPC port: -1
     queue: root.users.root
     start time: 1569254498669
     final status: UNDEFINED
     tracking URL: http://node1:8088/proxy/application_1566100765602_0105/
     user: root
19/09/24 00:01:40 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED)
19/09/24 00:01:41 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED)
19/09/24 00:01:42 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED)
19/09/24 00:01:43 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED)
19/09/24 00:01:44 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED)
19/09/24 00:01:45 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED)
19/09/24 00:01:46 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED)
19/09/24 00:01:47 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED)
19/09/24 00:01:48 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED)
19/09/24 00:01:49 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED)
19/09/24 00:01:50 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED)
19/09/24 00:01:51 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED)
19/09/24 00:01:52 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED)
19/09/24 00:01:53 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED)
19/09/24 00:01:54 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED)
19/09/24 00:01:55 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED)
19/09/24 00:01:56 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED)
19/09/24 00:01:57 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED)
19/09/24 00:01:58 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED)
19/09/24 00:01:59 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED)
19/09/24 00:02:00 INFO yarn.Client: Application report for application_1566100765602_0105 (state: ACCEPTED)
19/09/24 00:02:01 INFO yarn.Client: Application report for application_1566100765602_0105 (state: FAILED)
19/09/24 00:02:01 INFO yarn.Client:
     client token: N/A
     diagnostics: Application application_1566100765602_0105 failed 2 times due to AM Container for appattempt_1566100765602_0105_000002 exited with  exitCode: -1000
For more detailed output, check application tracking page:http://node1:8088/proxy/application_1566100765602_0105/Then, click on links to logs of each attempt.
Diagnostics: File does not exist: hdfs://node1:8020/user/root/.sparkStaging/application_1566100765602_0105/__spark_conf__8997139291240118668.zip
java.io.FileNotFoundException: File does not exist: hdfs://node1:8020/user/root/.sparkStaging/application_1566100765602_0105/__spark_conf__8997139291240118668.zip
    at org.apache.hadoop.hdfs.DistributedFileSystem$20.doCall(DistributedFileSystem.java:1269)
    at org.apache.hadoop.hdfs.DistributedFileSystem$20.doCall(DistributedFileSystem.java:1261)
    at org.apache.hadoop.fs.FileSystemLinkResolver.resolve(FileSystemLinkResolver.java:81)
    at org.apache.hadoop.hdfs.DistributedFileSystem.getFileStatus(DistributedFileSystem.java:1261)
    at org.apache.hadoop.yarn.util.FSDownload.copy(FSDownload.java:251)
    at org.apache.hadoop.yarn.util.FSDownload.access$000(FSDownload.java:61)
    at org.apache.hadoop.yarn.util.FSDownload$2.run(FSDownload.java:364)
    at org.apache.hadoop.yarn.util.FSDownload$2.run(FSDownload.java:362)
    at java.security.AccessController.doPrivileged(Native Method)
    at javax.security.auth.Subject.doAs(Subject.java:422)
    at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1920)
    at org.apache.hadoop.yarn.util.FSDownload.call(FSDownload.java:361)
    at org.apache.hadoop.yarn.util.FSDownload.call(FSDownload.java:60)
    at java.util.concurrent.FutureTask.run(FutureTask.java:266)
    at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
    at java.util.concurrent.FutureTask.run(FutureTask.java:266)
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
    at java.lang.Thread.run(Thread.java:748)

Failing this attempt. Failing the application.
     ApplicationMaster host: N/A
     ApplicationMaster RPC port: -1
     queue: root.users.root
     start time: 1569254498669
     final status: FAILED
     tracking URL: http://node1:8088/cluster/app/application_1566100765602_0105
     user: root
Exception in thread "main" org.apache.spark.SparkException: Application application_1566100765602_0105 finished with failed status
    at org.apache.spark.deploy.yarn.Client.run(Client.scala:1025)
    at org.apache.spark.deploy.yarn.Client$.main(Client.scala:1072)
    at org.apache.spark.deploy.yarn.Client.main(Client.scala)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
    at java.lang.reflect.Method.invoke(Method.java:498)
    at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:730)
    at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:181)
    at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:206)
    at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:121)
    at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
19/09/24 00:02:01 INFO util.ShutdownHookManager: Shutdown hook called
19/09/24 00:02:01 INFO util.ShutdownHookManager: Deleting directory /tmp/spark-2c548285-8012-414b-ab4e-797a164e38bc

参考：https://blog.csdn.net/adorechen/article/details/78746363，将源代码中设定本地执行的语句注释掉：

    val conf = new SparkConf()
          .setAppName("fdsf")
       // .setMaster("local") //本地执行

重新打包，执行以上语句，成功！返回如下信息：

[[email protected] ~]# spark-submit  --master yarn-cluster --driver-memory 4g --num-executors 2 --executor-memory 2g --executor-cores 2  --class com.fc.costDay   --conf spark.driver.extraClassPath=/opt/cloudera/parcels/APACHE_PHOENIX-4.14.0-cdh5.14.2.p0.3/lib/phoenix/lib/*  --conf spark.executor.extraClassPath=/opt/cloudera/parcels/APACHE_PHOENIX-4.14.0-cdh5.14.2.p0.3/lib/phoenix/lib/* /home/cf/scalapp-1.0-SNAPSHOT.jar
19/09/24 00:13:58 INFO client.RMProxy: Connecting to ResourceManager at node1/10.200.101.131:8032
19/09/24 00:13:58 INFO yarn.Client: Requesting a new application from cluster with 3 NodeManagers
19/09/24 00:13:58 INFO yarn.Client: Verifying our application has not requested more than the maximum memory capability of the cluster (40874 MB per container)
19/09/24 00:13:58 INFO yarn.Client: Will allocate AM container, with 4505 MB memory including 409 MB overhead
19/09/24 00:13:58 INFO yarn.Client: Setting up container launch context for our AM
19/09/24 00:13:58 INFO yarn.Client: Setting up the launch environment for our AM container
19/09/24 00:13:58 INFO yarn.Client: Preparing resources for our AM container
19/09/24 00:13:59 INFO yarn.Client: Uploading resource file:/home/cf/scalapp-1.0-SNAPSHOT.jar -> hdfs://node1:8020/user/root/.sparkStaging/application_1566100765602_0106/scalapp-1.0-SNAPSHOT.jar
19/09/24 00:14:00 INFO yarn.Client: Uploading resource file:/tmp/spark-875aaa2a-1c5d-4c6a-95a6-12f276a80054/__spark_conf__4530217919668141816.zip -> hdfs://node1:8020/user/root/.sparkStaging/application_1566100765602_0106/__spark_conf__4530217919668141816.zip
19/09/24 00:14:00 INFO spark.SecurityManager: Changing view acls to: root
19/09/24 00:14:00 INFO spark.SecurityManager: Changing modify acls to: root
19/09/24 00:14:00 INFO spark.SecurityManager: SecurityManager: authentication disabled; ui acls disabled; users with view permissions: Set(root); users with modify permissions: Set(root)
19/09/24 00:14:00 INFO yarn.Client: Submitting application 106 to ResourceManager
19/09/24 00:14:00 INFO impl.YarnClientImpl: Submitted application application_1566100765602_0106
19/09/24 00:14:01 INFO yarn.Client: Application report for application_1566100765602_0106 (state: ACCEPTED)
19/09/24 00:14:01 INFO yarn.Client:
     client token: N/A
     diagnostics: N/A
     ApplicationMaster host: N/A
     ApplicationMaster RPC port: -1
     queue: root.users.root
     start time: 1569255240698
     final status: UNDEFINED
     tracking URL: http://node1:8088/proxy/application_1566100765602_0106/
     user: root
19/09/24 00:14:02 INFO yarn.Client: Application report for application_1566100765602_0106 (state: ACCEPTED)
19/09/24 00:14:03 INFO yarn.Client: Application report for application_1566100765602_0106 (state: ACCEPTED)
19/09/24 00:14:04 INFO yarn.Client: Application report for application_1566100765602_0106 (state: ACCEPTED)
19/09/24 00:14:05 INFO yarn.Client: Application report for application_1566100765602_0106 (state: ACCEPTED)
19/09/24 00:14:06 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING)
19/09/24 00:14:06 INFO yarn.Client:
     client token: N/A
     diagnostics: N/A
     ApplicationMaster host: 10.200.101.135
     ApplicationMaster RPC port: 0
     queue: root.users.root
     start time: 1569255240698
     final status: UNDEFINED
     tracking URL: http://node1:8088/proxy/application_1566100765602_0106/
     user: root
19/09/24 00:14:07 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING)
19/09/24 00:14:08 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING)
19/09/24 00:14:09 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING)
19/09/24 00:14:10 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING)
19/09/24 00:14:11 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING)
19/09/24 00:14:12 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING)
19/09/24 00:14:13 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING)
19/09/24 00:14:14 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING)
19/09/24 00:14:15 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING)
19/09/24 00:14:16 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING)
19/09/24 00:14:17 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING)
19/09/24 00:14:18 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING)
19/09/24 00:14:19 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING)
19/09/24 00:14:20 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING)
19/09/24 00:14:21 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING)
19/09/24 00:14:22 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING)
19/09/24 00:14:23 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING)
19/09/24 00:14:24 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING)
19/09/24 00:14:25 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING)
19/09/24 00:14:26 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING)
19/09/24 00:14:27 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING)
19/09/24 00:14:28 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING)
19/09/24 00:14:29 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING)
19/09/24 00:14:30 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING)
19/09/24 00:14:31 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING)
19/09/24 00:14:32 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING)
19/09/24 00:14:33 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING)
19/09/24 00:14:34 INFO yarn.Client: Application report for application_1566100765602_0106 (state: RUNNING)
19/09/24 00:14:35 INFO yarn.Client: Application report for application_1566100765602_0106 (state: FINISHED)
19/09/24 00:14:35 INFO yarn.Client:
     client token: N/A
     diagnostics: N/A
     ApplicationMaster host: 10.200.101.135
     ApplicationMaster RPC port: 0
     queue: root.users.root
     start time: 1569255240698
     final status: SUCCEEDED
     tracking URL: http://node1:8088/proxy/application_1566100765602_0106/
     user: root
19/09/24 00:14:35 INFO util.ShutdownHookManager: Shutdown hook called
19/09/24 00:14:35 INFO util.ShutdownHookManager: Deleting directory /tmp/spark-875aaa2a-1c5d-4c6a-95a6-12f276a80054

编排Spark任务：

选择jar包：

添加jar包：

填写jar包和主类等信息：

填写执行模式信息：

保存任务：

执行，返回执行信息： 

错误详细信息为：

2019-09-24 01:30:02,556 ERROR org.apache.oozie.command.wf.SignalXCommand: SERVER[node1] USER[admin] GROUP[-] TOKEN[] APP[Cf0924] JOB[0000000-190922201507304-oozie-oozi-W] ACTION[0000000-190922201507304[email protected]:start:] Workflow action failed : E0700: XML error, For input string: ""
org.apache.oozie.workflow.WorkflowException: E0700: XML error, For input string: ""
    at org.apache.oozie.service.LiteWorkflowStoreService.getUserRetryMax(LiteWorkflowStoreService.java:171)
    at org.apache.oozie.service.LiteWorkflowStoreService.liteExecute(LiteWorkflowStoreService.java:122)
    at org.apache.oozie.service.LiteWorkflowStoreService$LiteActionHandler.start(LiteWorkflowStoreService.java:252)
    at org.apache.oozie.workflow.lite.ActionNodeHandler.enter(ActionNodeHandler.java:33)
    at org.apache.oozie.workflow.lite.LiteWorkflowInstance.signal(LiteWorkflowInstance.java:214)
    at org.apache.oozie.workflow.lite.LiteWorkflowInstance.signal(LiteWorkflowInstance.java:290)
    at org.apache.oozie.command.wf.SignalXCommand.execute(SignalXCommand.java:212)
    at org.apache.oozie.command.wf.SignalXCommand.execute(SignalXCommand.java:82)
    at org.apache.oozie.command.XCommand.call(XCommand.java:286)
    at org.apache.oozie.command.XCommand.call(XCommand.java:356)
    at org.apache.oozie.command.wf.ActionEndXCommand.execute(ActionEndXCommand.java:280)
    at org.apache.oozie.command.wf.ActionEndXCommand.execute(ActionEndXCommand.java:61)
    at org.apache.oozie.command.XCommand.call(XCommand.java:286)
    at org.apache.oozie.command.XCommand.call(XCommand.java:356)
    at org.apache.oozie.command.wf.ActionStartXCommand.callActionEnd(ActionStartXCommand.java:340)
    at org.apache.oozie.command.wf.ActionStartXCommand.execute(ActionStartXCommand.java:326)
    at org.apache.oozie.command.wf.ActionStartXCommand.execute(ActionStartXCommand.java:63)
    at org.apache.oozie.command.XCommand.call(XCommand.java:286)
    at org.apache.oozie.command.XCommand.call(XCommand.java:356)
    at org.apache.oozie.command.wf.SignalXCommand.execute(SignalXCommand.java:459)
    at org.apache.oozie.command.wf.SignalXCommand.execute(SignalXCommand.java:82)
    at org.apache.oozie.command.XCommand.call(XCommand.java:286)
    at org.apache.oozie.DagEngine.start(DagEngine.java:202)
    at org.apache.oozie.servlet.V1JobServlet.startWorkflowJob(V1JobServlet.java:332)
    at org.apache.oozie.servlet.V1JobServlet.startJob(V1JobServlet.java:76)
    at org.apache.oozie.servlet.BaseJobServlet.doPut(BaseJobServlet.java:80)
    at javax.servlet.http.HttpServlet.service(HttpServlet.java:730)
    at org.apache.oozie.servlet.JsonRestServlet.service(JsonRestServlet.java:289)
    at javax.servlet.http.HttpServlet.service(HttpServlet.java:820)
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:290)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
    at org.apache.oozie.servlet.AuthFilter$2.doFilter(AuthFilter.java:171)
    at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:631)
    at org.apache.hadoop.security.authentication.server.AuthenticationFilter.doFilter(AuthenticationFilter.java:579)
    at org.apache.oozie.servlet.AuthFilter.doFilter(AuthFilter.java:176)
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
    at org.apache.oozie.servlet.HostnameFilter.doFilter(HostnameFilter.java:86)
    at org.apache.catalina.core.ApplicationFilterChain.internalDoFilter(ApplicationFilterChain.java:235)
    at org.apache.catalina.core.ApplicationFilterChain.doFilter(ApplicationFilterChain.java:206)
    at org.apache.catalina.core.StandardWrapperValve.invoke(StandardWrapperValve.java:233)
    at org.apache.catalina.core.StandardContextValve.invoke(StandardContextValve.java:191)
    at org.apache.catalina.core.StandardHostValve.invoke(StandardHostValve.java:127)
    at org.apache.catalina.valves.ErrorReportValve.invoke(ErrorReportValve.java:103)
    at org.apache.catalina.core.StandardEngineValve.invoke(StandardEngineValve.java:109)
    at org.apache.catalina.connector.CoyoteAdapter.service(CoyoteAdapter.java:293)
    at org.apache.coyote.http11.Http11Processor.process(Http11Processor.java:859)
    at org.apache.coyote.http11.Http11Protocol$Http11ConnectionHandler.process(Http11Protocol.java:610)
    at org.apache.tomcat.util.net.JIoEndpoint$Worker.run(JIoEndpoint.java:503)
    at java.lang.Thread.run(Thread.java:748)
Caused by: java.lang.NumberFormatException: For input string: ""
    at java.lang.NumberFormatException.forInputString(NumberFormatException.java:65)
    at java.lang.Integer.parseInt(Integer.java:592)
    at java.lang.Integer.parseInt(Integer.java:615)
    at org.apache.oozie.service.LiteWorkflowStoreService.getUserRetryMax(LiteWorkflowStoreService.java:164)
    ... 49 more

关联到oozie任务信息：

点开链接，查看任务详情：

附依赖文件：

phoenixConnectMode.scala ：

package com.fc

import org.apache.hadoop.conf.Configuration
import org.apache.spark.sql.{DataFrame, SQLContext}
import org.apache.phoenix.spark._

object phoenixConnectMode {

  private val zookeeper = "node3:2181"

  def getMode1(sqlContext: SQLContext, tableName: String, columns: Array[String]): DataFrame = {
    val configuration = new Configuration()
    configuration.set("phoenix.schema.isNamespaceMappingEnabled", "true")
    configuration.set("phoenix.schema.mapSystemTablesToNamespace", "true")
    configuration.set("hbase.zookeeper.quorum", zookeeper)
    val df = sqlContext.phoenixTableAsDataFrame(tableName, columns, conf = configuration)
    df
  }
}

其中 configuration.set("phoenix.schema.isNamespaceMappingEnabled", "true") 需要根据具体环境修改为 true 或者 false

timeUtil.scala ：

package com.fc

import java.text.SimpleDateFormat
import java.util.Calendar

import org.joda.time.DateTime

object timeUtil {

  final val ONE_DAY_Millis = 3600*24*1000

  final val SECOND_TIME_FORMAT = "yyyy-MM-dd HH:mm:ss"

  final val MILLISECOND_TIME_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS"

  final val DAY_DATE_FORMAT_ONE = "yyyy-MM-dd"

  final val DAY_DATE_FORMAT_TWO = "yyyyMMdd"

  final val MONTH_DATE_FORMAT = "yyyy-MM"

  /**
    * 时间字符串to时间戳
    * @param dateStr 时间字符串
    * @param pattern 时间格式
    * @return
    */
  def convertDateStr2TimeStamp(dateStr: String, pattern: String): Long = {
    new SimpleDateFormat(pattern).parse(dateStr).getTime
  }

  /**
    *时间戳to字符串
    * @param timestamp 时间戳
    * @param pattern 字符串
    * @return
    */
  def convertTimeStamp2DateStr(timestamp: Long, pattern: String): String = {
    new DateTime(timestamp).toString(pattern)
  }

  /**
    * 时间字符串+天数to时间戳
    * @param dateStr 时间字符串
    * @param pattern 时间格式
    * @param days 增加天数
    * @return
    */
  def dateStrAddDays2TimeStamp(dateStr: String, pattern: String, days: Int): Long = {
    convertDateStr2Date(dateStr, pattern).plusDays(days).toDate.getTime
  }

  /**
    * 时间字符串+年数to时间戳
    * @param dateStr 时间字符串
    * @param pattern 时间格式
    * @param years 增加年数
    * @return
    */
  def dateStrAddYears2TimeStamp(dateStr: String, pattern: String, years: Int): Long = {
    convertDateStr2Date(dateStr, pattern).plusYears(years).toDate.getTime
  }

  def dateStrAddYears2Str(dateStr: String, pattern: String, years: Int): String = {
    val t = convertDateStr2Date(dateStr, pattern).plusYears(years).toDate.getTime
    convertTimeStamp2DateStr(t, pattern)
  }

  /**
    * 时间字符串to日期
    * @param dateStr 时间字符串
    * @param pattern 时间格式
    * @return
    */
  def convertDateStr2Date(dateStr: String, pattern: String): DateTime = {
    new DateTime(new SimpleDateFormat(pattern).parse(dateStr))
  }

  /**
    * 月差
    * @param stDate 开始时间
    * @param endDate 结束时间
    * @return
    */
  def getMonthSpace(stDate: String, endDate: String): Int = {
    val c1 = Calendar.getInstance()
    val c2 = Calendar.getInstance()
    c1.setTime(new SimpleDateFormat(MONTH_DATE_FORMAT).parse(stDate))
    c2.setTime(new SimpleDateFormat(MONTH_DATE_FORMAT).parse(endDate))
    val month1 = c2.get(Calendar.MONTH) - c1.get(Calendar.MONTH)
    val month2 = (c2.get(Calendar.YEAR) - c1.get(Calendar.YEAR)) * 12
    Math.abs(month1 + month2)
  }

  /**
    * 日差
    * @param stDate 开始时间
    * @param endDate 结束时间
    * @return
    */
  def getDaySpace(stDate: String, endDate: String): Long = {
    val c1 = Calendar.getInstance()
    val c2 = Calendar.getInstance()
    c1.setTime(new SimpleDateFormat(SECOND_TIME_FORMAT).parse(stDate))
    c2.setTime(new SimpleDateFormat(SECOND_TIME_FORMAT).parse(endDate))
    val difference=c2.getTimeInMillis -c1.getTimeInMillis
    val days = difference/ONE_DAY_Millis
    Math.abs(days)
  }

  def getLastYearDateStr(dateStr: String): String = {
    val e = timeUtil.dateStrAddYears2TimeStamp(dateStr, "yyyy", -1)
    val f = timeUtil.convertTimeStamp2DateStr(e, "yyyy")
    f+"-12-31 23:59:59.999"
  }

  def getCurDateStr: String = {
    convertTimeStamp2DateStr(System.currentTimeMillis(), timeUtil.SECOND_TIME_FORMAT)
  }

  def lastYearEnd: String = {
    timeUtil.getLastYearDateStr(getCurDateStr)
  }

  def getCurMonthDays: Int = {
    val a = Calendar.getInstance()
    a.set(Calendar.DATE, 1)
    a.roll(Calendar.DATE, -1)
    val days = a.get(Calendar.DATE)
    days
  }

}

pom.xml ：

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.fc</groupId>
    <artifactId>scalapp</artifactId>
    <version>1.0-SNAPSHOT</version>

    <dependencies>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-core_2.10</artifactId>
            <version>1.6.0-cdh5.14.2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.spark</groupId>
            <artifactId>spark-sql_2.10</artifactId>
            <version>1.6.0-cdh5.14.2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hbase</groupId>
            <artifactId>hbase-spark</artifactId>
            <version>1.2.0-cdh5.14.2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.phoenix</groupId>
            <artifactId>phoenix-spark</artifactId>
            <version>4.14.0-cdh5.14.2</version>
        </dependency>
        <dependency>
            <groupId>com.lmax</groupId>
            <artifactId>disruptor</artifactId>
            <version>3.3.8</version>
        </dependency>
        <dependency>
            <groupId>org.apache.phoenix</groupId>
            <artifactId>phoenix-core</artifactId>
            <version>4.14.0-cdh5.14.2</version>
        </dependency>
        <dependency>
            <groupId>org.apache.hadoop</groupId>
            <artifactId>hadoop-mapreduce-client-core</artifactId>
            <version>2.6.0-cdh5.14.2</version>
        </dependency>
        <!--MySQL驱动-->
        <dependency>
            <groupId>mysql</groupId>
            <artifactId>mysql-connector-java</artifactId>
            <version>5.1.43</version>
        </dependency>
    </dependencies>

    <repositories>
        <repository>
            <id>mvnrepository</id>
            <name>mvnrepository</name>
            <url>http://mvnrepository.com/</url>
            <layout>default</layout>
            <releases>
                <enabled>true</enabled>
            </releases>
            <snapshots>
                <enabled>false</enabled>
            </snapshots>
        </repository>

        <repository>
            <id>cloudera</id>
            <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
        </repository>

        <repository>
            <id>hortonworks</id>
            <name>hortonworks</name>
            <url>http://repo.hortonworks.com/content/repositories/releases/</url>
        </repository>
    </repositories>

    <pluginRepositories>
        <pluginRepository>
            <id>hortonworks</id>
            <name>hortonworks</name>
            <url>http://repo.hortonworks.com/content/repositories/releases/</url>
        </pluginRepository>
    </pluginRepositories>

    <build>
        <sourceDirectory>src/main/scala</sourceDirectory>
        <plugins>
            <plugin>
                <groupId>org.scala-tools</groupId>
                <artifactId>maven-scala-plugin</artifactId>
                <version>2.15.2</version>
                <executions>
                    <execution>
                        <goals>
                            <goal>compile</goal>
                            <goal>testCompile</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-shade-plugin</artifactId>
                <version>3.1.0</version>
                <executions>
                    <execution>
                        <phase>package</phase>
                        <goals>
                            <goal>shade</goal>
                        </goals>
                        <configuration>
                            <createDependencyReducedPom>false</createDependencyReducedPom>
                            <filters>
                                <filter>
                                    <artifact>*:*</artifact>
                                    <excludes>
                                        <exclude>META-INF/*.SF</exclude>
                                        <exclude>META-INF/*.DSA</exclude>
                                        <exclude>META-INF/*.RSA</exclude>
                                    </excludes>
                                </filter>
                            </filters>
                        </configuration>
                    </execution>
                </executions>
            </plugin>
        </plugins>
    </build>
</project>

参考：

https://www.jianshu.com/p/f336f7e5f31b

https://blog.csdn.net/adorechen/article/details/78746363

原文地址：https://www.cnblogs.com/ratels/p/11576073.html