From 02a388459bec095aa299f8bd60d9404c7296da23 Mon Sep 17 00:00:00 2001 From: jcjimenez Date: Fri, 18 Aug 2017 16:50:12 -0500 Subject: [PATCH 1/3] Proposal for sbt-based project --- .gitignore | 4 + build.sbt | 67 +++++ pom.xml | 249 ------------------ project/build.properties | 1 + project/plugins.sbt | 4 + .../gremlin/AbstractGremlinSparkTest.java | 2 - version.sbt | 2 + 7 files changed, 78 insertions(+), 251 deletions(-) create mode 100644 build.sbt delete mode 100644 pom.xml create mode 100644 project/build.properties create mode 100644 project/plugins.sbt create mode 100644 version.sbt diff --git a/.gitignore b/.gitignore index a8bd942c..756b7d53 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,7 @@ *.iml target/ *.DS_Store + +project/project/ +project/target/ + diff --git a/build.sbt b/build.sbt new file mode 100644 index 00000000..f2895cfe --- /dev/null +++ b/build.sbt @@ -0,0 +1,67 @@ +organization := "com.microsoft.azure" +name := "azure-cosmosdb-spark" +description := "Spark Connector for Microsoft Azure CosmosDB." + +scalaVersion := "2.11.7" + +scalacOptions ++= Seq( + "-unchecked", + "-deprecation", + "-feature" +) + +val sparkVersion = "2.1.0" +val slf4jVersion = "1.7.6" +val log4jVersion = "1.2.17" +val tinkerpopVersion = "3.2.5" + +libraryDependencies ++= Seq( + "com.microsoft.azure" % "azure-documentdb" % "1.12.0", + "com.microsoft.azure" % "azure-documentdb-rx" % "0.9.0-rc1", + + "org.apache.spark" %% "spark-core" % sparkVersion, + "org.apache.spark" %% "spark-launcher" % sparkVersion, + "org.apache.spark" %% "spark-network-common" % sparkVersion, + "org.apache.spark" %% "spark-network-shuffle" % sparkVersion, + "org.apache.spark" %% "spark-sql" % sparkVersion, + "org.apache.spark" %% "spark-streaming" % sparkVersion, + "org.apache.spark" %% "spark-unsafe" % sparkVersion, + + "io.reactivex" % "rxjava" % "1.3.0", + + "org.slf4j" % "slf4j-api" % slf4jVersion, + + "org.apache.tinkerpop" % "spark-gremlin" % tinkerpopVersion intransitive(), + "org.apache.tinkerpop" % "tinkergraph-gremlin" % tinkerpopVersion, + + "com.fasterxml.jackson.core" % "jackson-databind" % "2.6.5" +).map(_ % "compile") + +// The following dependencies are from spark-gremlin, which had to be included with intransitive() because of scala +// version 2.10 vs 2.11 conflicts. +libraryDependencies ++= Seq( + "com.thoughtworks.paranamer" % "paranamer" % "2.6", + "org.apache.tinkerpop" % "gremlin-core" % tinkerpopVersion, + "org.apache.tinkerpop" % "gremlin-groovy" % tinkerpopVersion, + "org.apache.tinkerpop" % "hadoop-gremlin" % tinkerpopVersion, + "org.xerial.snappy" % "snappy-java" % "1.1.1.7" +).map(_ % "compile") + +libraryDependencies ++= Seq( + "junit" % "junit" % "4.11" % "test", + "org.slf4j" % "slf4j-log4j12" % slf4jVersion % "test", + "org.scalatest" %% "scalatest" % "3.0.3" % "test" +) + +assemblyMergeStrategy in assembly := { + case PathList("javax", "inject", xs @ _*) => MergeStrategy.last + case PathList("javax", "servlet", xs @ _*) => MergeStrategy.last + case PathList("javax", "activation", xs @ _*) => MergeStrategy.last + case PathList("org", "apache", xs @ _*) => MergeStrategy.last + case "about.html" => MergeStrategy.rename + case "plugin.properties" => MergeStrategy.last + case "log4j.properties" => MergeStrategy.last + case PathList("META-INF", xs @ _*) => MergeStrategy.discard + case x => MergeStrategy.first +} + diff --git a/pom.xml b/pom.xml deleted file mode 100644 index af35170e..00000000 --- a/pom.xml +++ /dev/null @@ -1,249 +0,0 @@ - - - - 4.0.0 - com.microsoft.azure - azure-cosmosdb-spark_2.0.2_2.11 - jar - 0.0.4-SNAPSHOT - ${project.groupId}:${project.artifactId} - Spark Connector for Microsoft Azure CosmosDB - http://azure.microsoft.com/en-us/services/documentdb/ - - - MIT License - http://www.opensource.org/licenses/mit-license.php - - - - 2.11.8 - 2.11 - azure-cosmosdb-spark - 3.0.1 - 2.1.0 - 1.7.6 - 1.2.17 - 3.2.5 - UTF-8 - - - - org.scala-lang - scala-library - ${scala.version} - - - org.apache.spark - spark-sql_${scala.binary.version} - ${spark.version} - provided - - - com.microsoft.azure - azure-documentdb - 1.12.0 - - - io.reactivex - rxjava - 1.3.0 - - - com.microsoft.azure - azure-documentdb-rx - 0.9.0-rc1 - - - junit - junit - 4.11 - test - - - org.scalatest - scalatest_${scala.binary.version} - ${scala.test.version} - test - - - org.slf4j - slf4j-api - ${slf4j.version} - - - org.slf4j - slf4j-log4j12 - ${slf4j.version} - test - - - log4j - log4j - ${log4j.version} - test - - - org.apache.tinkerpop - spark-gremlin - ${tinkerpop.version} - - - org.apache.tinkerpop - tinkergraph-gremlin - ${tinkerpop.version} - - - com.fasterxml.jackson.core - jackson-databind - 2.6.5 - - - - - - ./src/test/resources - true - - - ./src/main/java - ./src/test/scala - - - maven-assembly-plugin - - - jar-with-dependencies - - - - - make-assembly - package - - single - - - - - - org.apache.maven.plugins - maven-dependency-plugin - 2.8 - - - copy-dependencies - package - - copy-dependencies - - - ${project.build.directory}/alternateLocation - false - false - true - ::* - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.5 - - 1.8 - 1.8 - - - - default-compile - none - - - - - net.alchim31.maven - scala-maven-plugin - 3.2.2 - - false - ${scala.version} - incremental - - - - scala-compile-first - process-resources - - add-source - - - - scala-compile - compile - - compile - - - - scala-testCompile - test-compile - - testCompile - - - - scala-doc - prepare-package - - doc - doc-jar - - - - - - org.apache.maven.plugins - maven-source-plugin - 2.2.1 - - - attach-sources - - jar-no-fork - - - - - - - - - CosmosDB Programmability Devs - askcosmosdb@microsoft.com - Microsoft - http://www.microsoft.com/ - - - - scm:git:git@github.com:Azure/azure-cosmosdb-spark.git - scm:git:git@github.com:Azure/azure-cosmosdb-spark.git - git@github.com:Azure/azure-cosmosdb-spark.git - - diff --git a/project/build.properties b/project/build.properties new file mode 100644 index 00000000..6be49589 --- /dev/null +++ b/project/build.properties @@ -0,0 +1 @@ +sbt.version = 0.13.15 \ No newline at end of file diff --git a/project/plugins.sbt b/project/plugins.sbt new file mode 100644 index 00000000..d3fbde94 --- /dev/null +++ b/project/plugins.sbt @@ -0,0 +1,4 @@ +addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "1.1") +addSbtPlugin("com.jsuereth" % "sbt-pgp" % "1.0.0") +addSbtPlugin("com.github.gseitz" % "sbt-release" % "1.0.4") +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.3") diff --git a/src/test/java/com/microsoft/azure/cosmosdb/spark/gremlin/AbstractGremlinSparkTest.java b/src/test/java/com/microsoft/azure/cosmosdb/spark/gremlin/AbstractGremlinSparkTest.java index b912ccce..60346a59 100644 --- a/src/test/java/com/microsoft/azure/cosmosdb/spark/gremlin/AbstractGremlinSparkTest.java +++ b/src/test/java/com/microsoft/azure/cosmosdb/spark/gremlin/AbstractGremlinSparkTest.java @@ -69,8 +69,6 @@ public void setupTest() { HttpClientFactory.DISABLE_HOST_NAME_VERIFICATION = true; // needed to run on localhost Spark.close(); - HadoopPools.close(); - KryoShimServiceLoader.close(); logger.info("SparkContext has been closed for " + this.getClass().getCanonicalName() + "-setupTest"); } diff --git a/version.sbt b/version.sbt new file mode 100644 index 00000000..94290d68 --- /dev/null +++ b/version.sbt @@ -0,0 +1,2 @@ +version := "0.0.4" + From 3a863f3a0472862c7644220e7de25b5f4b6e9050 Mon Sep 17 00:00:00 2001 From: jcjimenez Date: Fri, 18 Aug 2017 17:16:19 -0500 Subject: [PATCH 2/3] Added .travis.yml and using excludeAll instead of intransitive() in build.sbt. --- .travis.yml | 14 ++++++++++++++ build.sbt | 18 +++++++----------- 2 files changed, 21 insertions(+), 11 deletions(-) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..9179bd34 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,14 @@ +language: + scala + +scala: + - 2.11.7 + +env: + - TARGET=test + - TARGET=packageBin + - TARGET=packageDoc + - TARGET=packageSrc + +script: + - sbt ++$TRAVIS_SCALA_VERSION $TARGET diff --git a/build.sbt b/build.sbt index f2895cfe..3379f3c8 100644 --- a/build.sbt +++ b/build.sbt @@ -31,22 +31,18 @@ libraryDependencies ++= Seq( "org.slf4j" % "slf4j-api" % slf4jVersion, - "org.apache.tinkerpop" % "spark-gremlin" % tinkerpopVersion intransitive(), + "org.apache.tinkerpop" % "spark-gremlin" % tinkerpopVersion excludeAll( +// ExclusionRule(organization = "com.fasterxml.jackson.core"), +// ExclusionRule(organization = "com.thoughtworks.paranamer"), +// ExclusionRule(organization = "commons-lang"), + ExclusionRule(organization = "org.apache.spark"), + ExclusionRule(organization = "org.scala-lang") + ), "org.apache.tinkerpop" % "tinkergraph-gremlin" % tinkerpopVersion, "com.fasterxml.jackson.core" % "jackson-databind" % "2.6.5" ).map(_ % "compile") -// The following dependencies are from spark-gremlin, which had to be included with intransitive() because of scala -// version 2.10 vs 2.11 conflicts. -libraryDependencies ++= Seq( - "com.thoughtworks.paranamer" % "paranamer" % "2.6", - "org.apache.tinkerpop" % "gremlin-core" % tinkerpopVersion, - "org.apache.tinkerpop" % "gremlin-groovy" % tinkerpopVersion, - "org.apache.tinkerpop" % "hadoop-gremlin" % tinkerpopVersion, - "org.xerial.snappy" % "snappy-java" % "1.1.1.7" -).map(_ % "compile") - libraryDependencies ++= Seq( "junit" % "junit" % "4.11" % "test", "org.slf4j" % "slf4j-log4j12" % slf4jVersion % "test", From 04939702aac63e2d9c61a4b26538aee0c1a8cf0c Mon Sep 17 00:00:00 2001 From: JC Jimenez Date: Sat, 19 Aug 2017 11:16:49 -0500 Subject: [PATCH 3/3] Removing test task since it requires the cosmos emulator and I don't think Travis would be able to provide that --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 9179bd34..fb19b51e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,6 @@ scala: - 2.11.7 env: - - TARGET=test - TARGET=packageBin - TARGET=packageDoc - TARGET=packageSrc