使用一个SparkContext时，可以针对不同Job进行分组提交和取消：
1 | // 提交任务 |
获取任务执行进度信息：1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65String jobGroup = getJobGroup(context);
SQLContext sqlc = getSparkInterpreter().getSQLContext();
SparkContext sc = sqlc.sparkContext();
int completedTasks = 0;
int totalTasks = 0;
JobProgressListener sparkListener = new JobProgressListener(context.getConf());
DAGScheduler scheduler = sc.dagScheduler();
HashSet<ActiveJob> jobs = scheduler.activeJobs();
Iterator<ActiveJob> it = jobs.iterator();
while (it.hasNext()) {
ActiveJob job = it.next();
String g = (String) job.properties().get("spark.jobGroup.id");
if (jobGroup.equals(g)) {
int[] progressInfo = null;
progressInfo = getProgressFromStage_1_1x(sparkListener, job.finalStage());
totalTasks += progressInfo[0];
completedTasks += progressInfo[1];
}
}
if (totalTasks == 0) {
return 0;
}
return completedTasks * 100 / totalTasks;
}
private int[] getProgressFromStage_1_1x(JobProgressListener sparkListener, Stage stage) {
int numTasks = stage.numTasks();
int completedTasks = 0;
try {
Method stageIdToData = sparkListener.getClass().getMethod("stageIdToData");
HashMap<Tuple2<Object, Object>, Object> stageIdData =
(HashMap<Tuple2<Object, Object>, Object>) stageIdToData.invoke(sparkListener);
Class<?> stageUIDataClass =
this.getClass().forName("org.apache.spark.ui.jobs.UIData$StageUIData");
Method numCompletedTasks = stageUIDataClass.getMethod("numCompleteTasks");
Set<Tuple2<Object, Object>> keys =
JavaConverters.asJavaSetConverter(stageIdData.keySet()).asJava();
for (Tuple2<Object, Object> k : keys) {
if (stage.id() == (int) k._1()) {
Object uiData = stageIdData.get(k).get();
completedTasks += (int) numCompletedTasks.invoke(uiData);
}
}
} catch (Exception e) {
logger.error("Error on getting progress information", e);
}
List<Stage> parents = JavaConversions.asJavaList(stage.parents());
if (parents != null) {
for (Stage s : parents) {
int[] p = getProgressFromStage_1_1x(sparkListener, s);
numTasks += p[0];
completedTasks += p[1];
}
}
return new int[] {numTasks, completedTasks};
}
