AdsHub Analysis Processor - Details for Job 288

Status: SUCCEEDED
Submitted: 2026/09/11 08:43:01
Duration: 16 s
Associated SQL Query: 18
Job Group: 906fc2c7-4e02-4dd8-b6bd-e1479cd0eeb1
Completed Stages: 3

Enable zooming

digraph G { subgraph clusterstage_864 { label="Stage 864"; subgraph cluster481 { label="MicroBatchScan"; 432 [labelType="html" label="DataSourceRDD [432] run at <unknown>:0"]; 433 [labelType="html" label="MapPartitionsRDD [433] run at <unknown>:0"]; } subgraph cluster480 { label="Filter"; 434 [labelType="html" label="MapPartitionsRDD [434] run at <unknown>:0"]; } subgraph cluster479 { label="Project"; 435 [labelType="html" label="MapPartitionsRDD [435] run at <unknown>:0"]; } subgraph cluster476 { label="WholeStageCodegen (1)"; 436 [labelType="html" label="MapPartitionsRDD [436] run at <unknown>:0"]; } subgraph cluster475 { label="EventTimeWatermark"; 437 [labelType="html" label="MapPartitionsRDD [437] run at <unknown>:0"]; } subgraph cluster472 { label="WholeStageCodegen (2)"; 438 [labelType="html" label="MapPartitionsRDD [438] run at <unknown>:0"]; } subgraph cluster471 { label="ObjectHashAggregate"; 439 [labelType="html" label="MapPartitionsRDD [439] run at <unknown>:0"]; } subgraph cluster470 { label="Exchange"; 440 [labelType="html" label="MapPartitionsRDD [440] run at <unknown>:0"]; } } 432->433; 433->434; 434->435; 435->436; 436->437; 437->438; 438->439; 439->440; }

digraph G { subgraph clusterstage_865 { label="Stage 865"; subgraph cluster470 { label="Exchange"; 441 [labelType="html" label="ShuffledRowRDD [441] [Unordered] run at <unknown>:0"]; } subgraph cluster469 { label="ObjectHashAggregate"; 442 [labelType="html" label="MapPartitionsRDD [442] [Unordered] run at <unknown>:0"]; } subgraph cluster468 { label="StateStoreRestore"; 443 [labelType="html" label="ReadStateStoreRDD [443] [Unordered] run at <unknown>:0"]; } subgraph cluster467 { label="ObjectHashAggregate"; 444 [labelType="html" label="MapPartitionsRDD [444] [Unordered] run at <unknown>:0"]; } subgraph cluster466 { label="StateStoreSave"; 445 [labelType="html" label="StateStoreRDD [445] [Unordered] run at <unknown>:0"]; } subgraph cluster465 { label="ObjectHashAggregate"; 446 [labelType="html" label="MapPartitionsRDD [446] [Unordered] run at <unknown>:0"]; } subgraph cluster464 { label="Exchange"; 447 [labelType="html" label="MapPartitionsRDD [447] [Unordered] run at <unknown>:0"]; 448 [labelType="html" label="MapPartitionsRDD [448] [Unordered] run at <unknown>:0"]; } } 441->442; 442->443; 443->444; 444->445; 445->446; 446->447; 447->448; }

440,441

digraph G { subgraph clusterstage_866 { label="Stage 866"; 450 [labelType="html" label="SQLExecutionRDD [450] [Unordered] run at <unknown>:0"]; 452 [labelType="html" label="SQLExecutionRDD [452] [Unordered] javaToPython at <unknown>:0"]; subgraph cluster464 { label="Exchange"; 449 [labelType="html" label="ShuffledRowRDD [449] [Unordered] run at <unknown>:0"]; } subgraph cluster482 { label="WholeStageCodegen (1)"; 451 [labelType="html" label="MapPartitionsRDD [451] [Unordered] javaToPython at <unknown>:0"]; } subgraph cluster484 { label="map"; 453 [labelType="html" label="MapPartitionsRDD [453] [Unordered] javaToPython at <unknown>:0"]; } subgraph cluster485 { label="mapPartitions"; 454 [labelType="html" label="MapPartitionsRDD [454] [Unordered] javaToPython at <unknown>:0"]; } } 449->450; 450->451; 451->452; 452->453; 453->454; }

448,449

Completed Stages (3)

Page:

Stage Id ▾	Description	Submitted	Duration	Tasks: Succeeded/Total	Shuffle Read	Shuffle Write
866	REFERRAL id = 844d91ec-b3ed-4d48-80ba-59fb76b066a7 runId = 906fc2c7-4e02-4dd8-b6bd-e1479cd0eeb1 batch = 7 call at /opt/spark/python/lib/py4j-0.10.9.7-src.zip/py4j/clientserver.py:617 +details org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source)	2026/09/11 08:43:17	85 ms	1/1
865	REFERRAL id = 844d91ec-b3ed-4d48-80ba-59fb76b066a7 runId = 906fc2c7-4e02-4dd8-b6bd-e1479cd0eeb1 batch = 7 run at <unknown>:0 +details org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source)	2026/09/11 08:43:07	10 s	50/50	3.4 KiB	110.0 B
864	REFERRAL id = 844d91ec-b3ed-4d48-80ba-59fb76b066a7 runId = 906fc2c7-4e02-4dd8-b6bd-e1479cd0eeb1 batch = 7 run at <unknown>:0 +details org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source) java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source) java.base/java.lang.reflect.Method.invoke(Unknown Source) py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244) py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374) py4j.Gateway.invoke(Gateway.java:282) py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132) py4j.commands.CallCommand.execute(CallCommand.java:79) py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182) py4j.ClientServerConnection.run(ClientServerConnection.java:106) java.base/java.lang.Thread.run(Unknown Source)	2026/09/11 08:43:01	6 s	28/28		3.4 KiB

Stage Id ▾

Description

Submitted

Duration

Tasks: Succeeded/Total

Input

Output

Shuffle Read

Shuffle Write

866

REFERRAL id = 844d91ec-b3ed-4d48-80ba-59fb76b066a7 runId = 906fc2c7-4e02-4dd8-b6bd-e1479cd0eeb1 batch = 7 call at /opt/spark/python/lib/py4j-0.10.9.7-src.zip/py4j/clientserver.py:617 +details

org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.base/java.lang.Thread.run(Unknown Source)

2026/09/11 08:43:17

85 ms

1/1

865

REFERRAL id = 844d91ec-b3ed-4d48-80ba-59fb76b066a7 runId = 906fc2c7-4e02-4dd8-b6bd-e1479cd0eeb1 batch = 7 run at <unknown>:0 +details

org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.base/java.lang.Thread.run(Unknown Source)

2026/09/11 08:43:07

10 s

50/50

3.4 KiB

110.0 B

864

REFERRAL id = 844d91ec-b3ed-4d48-80ba-59fb76b066a7 runId = 906fc2c7-4e02-4dd8-b6bd-e1479cd0eeb1 batch = 7 run at <unknown>:0 +details

org.apache.spark.sql.streaming.DataStreamWriter.start(DataStreamWriter.scala:251)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(Unknown Source)
java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(Unknown Source)
java.base/java.lang.reflect.Method.invoke(Unknown Source)
py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:374)
py4j.Gateway.invoke(Gateway.java:282)
py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
py4j.commands.CallCommand.execute(CallCommand.java:79)
py4j.ClientServerConnection.waitForCommands(ClientServerConnection.java:182)
py4j.ClientServerConnection.run(ClientServerConnection.java:106)
java.base/java.lang.Thread.run(Unknown Source)

2026/09/11 08:43:01

6 s

28/28

3.4 KiB

Page: