Scala异常获取一例
在处理第11行读文件时,由于数据文件出现的不规律,在指定日期内可能存在日志文件不存在的情况,这里需要处理下异常:
123456789101112131415161718192021def readLog(sc: SparkContext, startDate: String, endDate: String, logNames: List[String]): RDD[String] = { val dateLst = DateUtils.getDateListBetweenTwoDate(startDate, endDate) var logRdd = sc.makeRDD(List[String]()) for (date <- dateLst) { val year = date.substring(0, 4) val month = date.substring(4, 6) val day = date.substring(6, 8) for (logName <- logNames) { val logRdd = logRdd.union( try {sc.textFile(s"cosn://fuge/mid-data/fuge/ssp/bid-log/$year/$month/$day/${logName}*") .map(x => x.split("\\|", -1)) .filter(x => x.length >= 2 && (x(1).trim == "6" || x(1).trim == "0")).map(_.toString) // 0和6为请求成功的状态码 } catch { case _: Exception => sc.makeRDD(List[String]()) } ) } } logRdd}
Coding