- Download Spark from here
- Download spark-rapids from here
- Base image: nvidia/cuda:12.2.2-base-ubuntu22.04
docker build -t <name>:<version> .
docker run --gpus all --rm -it --shm-size=1g --ulimit memlock=-1 --ulimit stack=67108864 <name>:<version>
Run with Local Mode
$SPARK_HOME/bin/spark-shell \
--master local \
--num-executors 1 \
--conf spark.executor.cores=1 \
--conf spark.rapids.sql.concurrentGpuTasks=1 \
--driver-memory 10g \
--conf spark.rapids.memory.pinnedPool.size=2G \
--conf spark.sql.files.maxPartitionBytes=512m \
--conf spark.plugins=com.nvidia.spark.SQLPlugin \
--jars ${SPARK_RAPIDS_PLUGIN_JAR}