Skip to content
share

Spark Monitoring Integration

Monitor your Hadoop ecosystem with Sematext Infrastructure and Application Monitoring SaaS, namely HDFS, MapReduce, YARN, and Spark. Visualize Apache Spark Manager/Master, Worker, Executor, and Driver componenets, and other key metrics reports. Create dashboards and alerts, and send notifications to your devops team through various messaging integrations

Integration

Metrics

Metric Name
Key (Type) (Unit)
Description
driver disk used
spark.driver.disk.used
(long gauge) (bytes)
driver memory max
spark.driver.memory.total
(long gauge) (bytes)
driver memory used
spark.driver.memory.used
(long gauge) (bytes)
driver memory free
spark.driver.memory.free
(long gauge) (bytes)
active driver jobs
spark.jobs.active
(long gauge)
all driver jobs
spark.jobs
(long gauge)
failed driver stages
spark.stages.failed
(long gauge)
running driver stages
spark.stages.running
(long gauge)
waiting driver stages
spark.stages.waiting
(long gauge)
executor large file reads
spark.executor.file.reads.large
(long gauge)
executor file reads bytes
spark.executor.file.reads.bytes
(long gauge) (bytes)
executor file reads
spark.executor.file.reads
(long gauge)
executor file writes bytes
spark.executor.file.writes.bytes
(long gauge) (bytes)
executor file writes
spark.executor.file.writes
(long gauge)
executor large hdfs reads
spark.executor.hdfs.reads.large
(long gauge)
executor hdfs reads bytes
spark.executor.hdfs.reads.bytes
(long gauge) (bytes)
executor hdfs reads
spark.executor.hdfs.reads
(long gauge)
executor hdfs writes bytes
spark.executor.hdfs.writes.bytes
(long gauge) (bytes)
executor hdfs writes
spark.executor.hdfs.writes
(long gauge)
executor active tasks
spark.executor.tasks.active
(long gauge)
executor completed tasks
spark.executor.tasks.completed
(long gauge)
executor used pool size
spark.executor.used.pool.size
(long gauge) (bytes)
executor max pool size
spark.executor.max.pool.size
(long gauge) (bytes)
spark apps
spark.apps
(long gauge)
waiting apps
spark.apps.waiting
(long gauge)
spark workers
spark.workers
(long gauge)
live spark workers
spark.workers.alive
(long gauge)
spark receivers
spark.receivers
(long gauge)
total completed batches
spark.batches.completed
(long gauge)
unprocessed batches
spark.batches.unprocessed
(long gauge)
waiting batches
spark.batches.waiting
(long gauge)
running batches
spark.batches.running
(long gauge)
retained completed batches
spark.batches.completed.retained
(long gauge)
total received records
spark.records.received
(long gauge)
total processed records
spark.records.processed
(long gauge)
last completed batch scheduling time
spark.batches.completed.scheduling.delay
(long gauge) (ms)
last completed batch processing time
spark.batches.completed.processing.delay
(long gauge) (ms)
last received batch scheduling time
spark.batches.received.scheduling.delay
(long gauge) (ms)
last received batch processing time
spark.batches.received.processing.delay
(long gauge) (ms)
last received batch records
spark.batches.received.records
(long gauge)
worker executors
spark.executors
(long gauge)
worker cores used
spark.cores.used
(long gauge)
worker cores free
spark.cores.free
(long gauge)
worker used memory
spark.worker.memory.used
(long gauge) (bytes)
worker free memory
spark.worker.memory.free
(long gauge) (bytes)
uncompleted apps
spark.applications.uncompleted
(long gauge)