Store a Spark 3.1.1 DataFrame into an ElasticSearch 7.17.1 index from docker-elk

• Download file elasticsearch-spark-30_2.12-7.17.6.jar from https://mvnrepository.com/artifact/org.elasticsearch/elasticsearch-spark-30_2.12/7.17.6 in the folder spark-3.1.1-bin-hadoop2.7/jars

• Open the spark shell

spark-3.1.1-bin-hadoop2.7/bin/spark-shell --driver-memory 16g

• Run this script

import org.elasticsearch.spark.sql._      

// case class used to define the DataFrame
case class Person(Name:String, Age:Int, Address: String)

// create DataFrame
val indexDocuments = Seq(
     Person("John Doe",45,"2690 Cessna Drive, Fort Wayne, 46802, IN"),
     Person("Jane Doe",57,"4620 Radford Street, Louisville, 40242, KY"),
     Person("Harry Potter", 112,"4, Privet Drive, Little Whinging, Surrey")
).toDF


indexDocuments.saveToEs("demoindex/personindex",  Map("es.nodes" -> "127.0.0.1", "es.nodes.wan.only" -> "true", "es.net.http.auth.user" -> "elastic", "es.net.http.auth.pass" -> "changeme"))