crp-wrkload-determination - Details for Query 36

Submitted Time: 2026/09/05 08:26:20
Duration: 24 s
Succeeded Jobs: 44 45 46 47

Plan Visualization

Show the Stage ID and Task ID that corresponds to the max metric

digraph G { 0 [id="node0" labelType="html" label=" AdaptiveSparkPlan " tooltip="AdaptiveSparkPlan isFinalPlan=true"]; subgraph cluster1 { isCluster="true"; id="cluster1"; label="WholeStageCodegen (2)\n \nduration: 28 ms"; tooltip="WholeStageCodegen (2)"; 2 [id="node2" labelType="html" label="HashAggregate time in aggregation build: 28 ms number of output rows: 1" tooltip="HashAggregate(keys=[], functions=[count(1)])"]; } 3 [id="node3" labelType="html" label="Exchange shuffle records written: 200 remote merged reqs duration: 0 ms remote merged blocks fetched: 0 records read: 200 local bytes read: 14.1 KiB merged fetch fallback count: 0 local blocks read: 200 remote merged chunks fetched: 0 remote blocks read: 0 data size total (min, med, max (stageId: taskId)) 3.1 KiB (16.0 B, 16.0 B, 16.0 B (stage 95.0: task 10864)) local merged bytes read: 0.0 B local merged chunks fetched: 0 shuffle write time total (min, med, max (stageId: taskId)) 34 ms (0 ms, 0 ms, 0 ms (stage 95.0: task 10996)) remote merged bytes read: 0.0 B local merged blocks fetched: 0 corrupt merged block chunks: 0 fetch wait time: 0 ms remote bytes read: 0.0 B number of partitions: 1 remote reqs duration: 0 ms remote bytes read to disk: 0.0 B shuffle bytes written total (min, med, max (stageId: taskId)) 14.1 KiB (72.0 B, 72.0 B, 72.0 B (stage 95.0: task 10864))" tooltip="Exchange SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2100]"]; subgraph cluster4 { isCluster="true"; id="cluster4"; label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n4 ms (0 ms, 0 ms, 1 ms (stage 95.0: task 10864))"; tooltip="WholeStageCodegen (1)"; 5 [id="node5" labelType="html" label="HashAggregate time in aggregation build total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 95.0: task 10864)) number of output rows: 200" tooltip="HashAggregate(keys=[], functions=[partial_count(1)])"]; 6 [id="node6" labelType="html" label=" Project " tooltip="Project"]; 7 [id="node7" labelType="html" label="Filter number of output rows: 0" tooltip="Filter isDir#1701: boolean"]; } 8 [id="node8" labelType="html" label="InMemoryTableScan number of output rows: 0" tooltip="InMemoryTableScan [isDir#1701], [isDir#1701]"]; 9 [id="node9" labelType="html" label=" AdaptiveSparkPlan " tooltip="AdaptiveSparkPlan isFinalPlan=true"]; subgraph cluster10 { isCluster="true"; id="cluster10"; label="WholeStageCodegen (2)\n \nduration: total (min, med, max (stageId: taskId))\n9 ms (0 ms, 0 ms, 3 ms (stage 92.0: task 10664))"; tooltip="WholeStageCodegen (2)"; 11 [id="node11" labelType="html" label=" SerializeFromObject " tooltip="SerializeFromObject [static_invoke(UTF8String.fromString(invoke(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.SerializableFileStatus, true])).path()))) AS path#1699, invoke(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.SerializableFileStatus, true])).length()) AS length#1700L, invoke(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.SerializableFileStatus, true])).isDir()) AS isDir#1701, invoke(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.SerializableFileStatus, true])).modificationTime()) AS modificationTime#1702L]"]; } 12 [id="node12" labelType="html" label=" MapGroups " tooltip="MapGroups org.apache.spark.sql.internal.UDFAdaptors$$$Lambda/0x000000080209d5c0@563ebb8d, invoke(value#1693.toString()), newInstance(class org.apache.spark.sql.delta.SerializableFileStatus), [value#1693], [path#1683, length#1684L, isDir#1685, modificationTime#1686L], obj#1698: org.apache.spark.sql.delta.SerializableFileStatus"]; subgraph cluster13 { isCluster="true"; id="cluster13"; label="WholeStageCodegen (1)\n \nduration: total (min, med, max (stageId: taskId))\n117 ms (0 ms, 0 ms, 39 ms (stage 92.0: task 10664))"; tooltip="WholeStageCodegen (1)"; 14 [id="node14" labelType="html" label="Sort sort time total (min, med, max (stageId: taskId)) 0 ms (0 ms, 0 ms, 0 ms (stage 92.0: task 10664)) peak memory total (min, med, max (stageId: taskId)) 12.5 MiB (64.0 KiB, 64.0 KiB, 64.0 KiB (stage 92.0: task 10664)) spill size total (min, med, max (stageId: taskId)) 0.0 B (0.0 B, 0.0 B, 0.0 B (stage 92.0: task 10664))" tooltip="Sort [value#1693 ASC NULLS FIRST], false, 0"]; } 15 [id="node15" labelType="html" label="Exchange shuffle records written: 0 remote merged reqs duration: 0 ms remote merged blocks fetched: 0 records read: 0 local bytes read: 0.0 B merged fetch fallback count: 0 local blocks read: 0 remote merged chunks fetched: 0 remote blocks read: 0 data size: 0.0 B local merged bytes read: 0.0 B local merged chunks fetched: 0 shuffle write time: 0 ms remote merged bytes read: 0.0 B local merged blocks fetched: 0 corrupt merged block chunks: 0 fetch wait time: 0 ms remote bytes read: 0.0 B number of partitions: 200 remote reqs duration: 0 ms remote bytes read to disk: 0.0 B shuffle bytes written: 0.0 B" tooltip="Exchange hashpartitioning(value#1693, 200), ENSURE_REQUIREMENTS, [plan_id=2018]"]; 16 [id="node16" labelType="html" label=" AppendColumnsWithObject " tooltip="AppendColumnsWithObject org.apache.spark.sql.delta.commands.VacuumCommand$$$Lambda/0x000000080209c000@3c79482b, [static_invoke(UTF8String.fromString(invoke(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.SerializableFileStatus, true])).path()))) AS path#1683, invoke(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.SerializableFileStatus, true])).length()) AS length#1684L, invoke(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.SerializableFileStatus, true])).isDir()) AS isDir#1685, invoke(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.SerializableFileStatus, true])).modificationTime()) AS modificationTime#1686L], [static_invoke(UTF8String.fromString(input[0, java.lang.String, true])) AS value#1693]"]; 17 [id="node17" labelType="html" label=" MapElements " tooltip="MapElements org.apache.spark.sql.delta.commands.VacuumCommand$$$Lambda/0x0000000802093950@5e5aa004, obj#1682: org.apache.spark.sql.delta.SerializableFileStatus"]; 18 [id="node18" labelType="html" label="Scan number of output rows: 0" tooltip="Scan[obj#1667]"]; 2->0; 3->2; 5->3; 6->5; 7->6; 8->7; 9->8; 11->9; 12->11; 14->12; 15->14; 16->15; 17->16; 18->17; }

Details

== Physical Plan ==
AdaptiveSparkPlan (29)
+- == Final Plan ==
   ResultQueryStage (23), Statistics(sizeInBytes=8.0 EiB)
   +- * HashAggregate (22)
      +- ShuffleQueryStage (21), Statistics(sizeInBytes=3.1 KiB, rowCount=200)
         +- Exchange (20)
            +- * HashAggregate (19)
               +- * Project (18)
                  +- * Filter (17)
                     +- TableCacheQueryStage (16), Statistics(sizeInBytes=0.0 B, rowCount=0)
                        +- InMemoryTableScan (1)
                              +- InMemoryRelation (2)
                                    +- AdaptiveSparkPlan (15)
                                    +- == Final Plan ==
                                       ResultQueryStage (11)
                                       +- * SerializeFromObject (10)
                                          +- MapGroups (9)
                                             +- * Sort (8)
                                                +- ShuffleQueryStage (7), Statistics(sizeInBytes=0.0 B, rowCount=0)
                                                   +- Exchange (6)
                                                      +- AppendColumnsWithObject (5)
                                                         +- MapElements (4)
                                                            +- Scan (3)
                                    +- == Initial Plan ==
                                       SerializeFromObject (14)
                                       +- MapGroups (13)
                                          +- Sort (12)
                                             +- Exchange (6)
                                                +- AppendColumnsWithObject (5)
                                                   +- MapElements (4)
                                                      +- Scan (3)
+- == Initial Plan ==
   HashAggregate (28)
   +- Exchange (27)
      +- HashAggregate (26)
         +- Project (25)
            +- Filter (24)
               +- InMemoryTableScan (1)
                     +- InMemoryRelation (2)
                           +- AdaptiveSparkPlan (15)
                           +- == Final Plan ==
                              ResultQueryStage (11)
                              +- * SerializeFromObject (10)
                                 +- MapGroups (9)
                                    +- * Sort (8)
                                       +- ShuffleQueryStage (7), Statistics(sizeInBytes=0.0 B, rowCount=0)
                                          +- Exchange (6)
                                             +- AppendColumnsWithObject (5)
                                                +- MapElements (4)
                                                   +- Scan (3)
                           +- == Initial Plan ==
                              SerializeFromObject (14)
                              +- MapGroups (13)
                                 +- Sort (12)
                                    +- Exchange (6)
                                       +- AppendColumnsWithObject (5)
                                          +- MapElements (4)
                                             +- Scan (3)


(1) InMemoryTableScan
Output [1]: [isDir#1701]
Arguments: [isDir#1701], [isDir#1701]

(2) InMemoryRelation
Arguments: [path#1699, length#1700L, isDir#1701, modificationTime#1702L], StorageLevel(disk, memory, deserialized, 1 replicas)

(3) Scan
Output [1]: [obj#1667]
Arguments: obj#1667: org.apache.spark.sql.delta.SerializableFileStatus, MapPartitionsRDD[189] at $anonfun$recordDeltaOperationInternal$1 at DatabricksLogging.scala:128

(4) MapElements
Input [1]: [obj#1667]
Arguments: org.apache.spark.sql.delta.commands.VacuumCommand$$$Lambda/0x0000000802093950@5e5aa004, obj#1682: org.apache.spark.sql.delta.SerializableFileStatus

(5) AppendColumnsWithObject
Input [1]: [obj#1682]
Arguments: org.apache.spark.sql.delta.commands.VacuumCommand$$$Lambda/0x000000080209c000@3c79482b, [static_invoke(UTF8String.fromString(invoke(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.SerializableFileStatus, true])).path()))) AS path#1683, invoke(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.SerializableFileStatus, true])).length()) AS length#1684L, invoke(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.SerializableFileStatus, true])).isDir()) AS isDir#1685, invoke(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.SerializableFileStatus, true])).modificationTime()) AS modificationTime#1686L], [static_invoke(UTF8String.fromString(input[0, java.lang.String, true])) AS value#1693]

(6) Exchange
Input [5]: [path#1683, length#1684L, isDir#1685, modificationTime#1686L, value#1693]
Arguments: hashpartitioning(value#1693, 200), ENSURE_REQUIREMENTS, [plan_id=2018]

(7) ShuffleQueryStage
Output [5]: [path#1683, length#1684L, isDir#1685, modificationTime#1686L, value#1693]
Arguments: 0

(8) Sort [codegen id : 1]
Input [5]: [path#1683, length#1684L, isDir#1685, modificationTime#1686L, value#1693]
Arguments: [value#1693 ASC NULLS FIRST], false, 0

(9) MapGroups
Input [5]: [path#1683, length#1684L, isDir#1685, modificationTime#1686L, value#1693]
Arguments: org.apache.spark.sql.internal.UDFAdaptors$$$Lambda/0x000000080209d5c0@563ebb8d, invoke(value#1693.toString()), newInstance(class org.apache.spark.sql.delta.SerializableFileStatus), [value#1693], [path#1683, length#1684L, isDir#1685, modificationTime#1686L], obj#1698: org.apache.spark.sql.delta.SerializableFileStatus

(10) SerializeFromObject [codegen id : 2]
Input [1]: [obj#1698]
Arguments: [static_invoke(UTF8String.fromString(invoke(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.SerializableFileStatus, true])).path()))) AS path#1699, invoke(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.SerializableFileStatus, true])).length()) AS length#1700L, invoke(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.SerializableFileStatus, true])).isDir()) AS isDir#1701, invoke(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.SerializableFileStatus, true])).modificationTime()) AS modificationTime#1702L]

(11) ResultQueryStage
Output [4]: [path#1699, length#1700L, isDir#1701, modificationTime#1702L]
Arguments: 1

(12) Sort
Input [5]: [path#1683, length#1684L, isDir#1685, modificationTime#1686L, value#1693]
Arguments: [value#1693 ASC NULLS FIRST], false, 0

(13) MapGroups
Input [5]: [path#1683, length#1684L, isDir#1685, modificationTime#1686L, value#1693]
Arguments: org.apache.spark.sql.internal.UDFAdaptors$$$Lambda/0x000000080209d5c0@563ebb8d, invoke(value#1693.toString()), newInstance(class org.apache.spark.sql.delta.SerializableFileStatus), [value#1693], [path#1683, length#1684L, isDir#1685, modificationTime#1686L], obj#1698: org.apache.spark.sql.delta.SerializableFileStatus

(14) SerializeFromObject
Input [1]: [obj#1698]
Arguments: [static_invoke(UTF8String.fromString(invoke(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.SerializableFileStatus, true])).path()))) AS path#1699, invoke(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.SerializableFileStatus, true])).length()) AS length#1700L, invoke(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.SerializableFileStatus, true])).isDir()) AS isDir#1701, invoke(knownnotnull(assertnotnull(input[0, org.apache.spark.sql.delta.SerializableFileStatus, true])).modificationTime()) AS modificationTime#1702L]

(15) AdaptiveSparkPlan
Output [4]: [path#1699, length#1700L, isDir#1701, modificationTime#1702L]
Arguments: isFinalPlan=true

(16) TableCacheQueryStage
Output [1]: [isDir#1701]
Arguments: 0

(17) Filter [codegen id : 1]
Input [1]: [isDir#1701]
Condition : isDir#1701

(18) Project [codegen id : 1]
Output: []
Input [1]: [isDir#1701]

(19) HashAggregate [codegen id : 1]
Input: []
Keys: []
Functions [1]: [partial_count(1)]
Aggregate Attributes [1]: [count#1799L]
Results [1]: [count#1800L]

(20) Exchange
Input [1]: [count#1800L]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2100]

(21) ShuffleQueryStage
Output [1]: [count#1800L]
Arguments: 1

(22) HashAggregate [codegen id : 2]
Input [1]: [count#1800L]
Keys: []
Functions [1]: [count(1)]
Aggregate Attributes [1]: [count(1)#1738L]
Results [1]: [count(1)#1738L AS count#1733L]

(23) ResultQueryStage
Output [1]: [count#1733L]
Arguments: 2

(24) Filter
Input [1]: [isDir#1701]
Condition : isDir#1701

(25) Project
Output: []
Input [1]: [isDir#1701]

(26) HashAggregate
Input: []
Keys: []
Functions [1]: [partial_count(1)]
Aggregate Attributes [1]: [count#1799L]
Results [1]: [count#1800L]

(27) Exchange
Input [1]: [count#1800L]
Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2037]

(28) HashAggregate
Input [1]: [count#1800L]
Keys: []
Functions [1]: [count(1)]
Aggregate Attributes [1]: [count(1)#1738L]
Results [1]: [count(1)#1738L AS count#1733L]

(29) AdaptiveSparkPlan
Output [1]: [count#1733L]
Arguments: isFinalPlan=true

SQL / DataFrame Properties

Name	Value
spark.sql.parquet.fieldId.read.enabled	true
spark.sql.parquet.fieldId.write.enabled	true