build.gradle

/*
 * Copyright 2017 Netflix, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

buildscript {
  repositories { jcenter() }
  dependencies {
    classpath 'com.github.jengelman.gradle.plugins:shadow:2.0.0'
    classpath 'com.netflix.nebula:gradle-aggregate-javadocs-plugin:2.2.+'
    classpath 'com.netflix.nebula:nebula-publishing-plugin:5.1.5'
  }
}

plugins {
  id 'nebula.netflixoss' version '4.1.0'
  id 'com.palantir.git-version' version '0.9.1'
}

if (JavaVersion.current() != JavaVersion.VERSION_1_8) {
  throw new GradleException("This build must be run with Java 8")
}

allprojects {
  group = "com.netflix.iceberg"
  apply plugin: 'idea'
  version = gitVersion()
}

apply plugin: 'nebula-aggregate-javadocs'

subprojects {
  apply plugin: 'nebula.javadoc-jar'
  apply plugin: 'nebula.source-jar'
  apply plugin: 'java'
  apply plugin: 'maven' // make pom files for deployment
  apply plugin: 'nebula.maven-base-publish'

  repositories {
    mavenCentral()
    mavenLocal()
  }

  configurations {
    testCompile.extendsFrom compileOnly
    all {
      resolutionStrategy {
        force 'org.apache.httpcomponents:httpclient:4.4.1'
      }
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
    }
  }

  ext {
    hadoopVersion = '2.7.3'
    avroVersion = '1.8.2'
    orcVersion = '1.4.2'
    parquetVersion = '1.10.0'
    hiveVersion = '3.1.0'

    jacksonVersion = '2.6.7'

    scalaVersion = '2.11'
    sparkVersion = '2.3.2'
  }

  sourceCompatibility = '1.8'
  targetCompatibility = '1.8'

  dependencies {
    compileOnly 'org.slf4j:slf4j-api:1.7.5'
    compileOnly 'com.google.guava:guava:16.0.1'

    testCompile 'junit:junit:4.12'
    testCompile 'org.slf4j:slf4j-simple:1.7.5'
    testCompile 'org.mockito:mockito-all:1.10.19'
  }
  publishing {
    publications {
      nebula(MavenPublication) {
        from components.java
      }
    }
  }
}

project(':iceberg-api') {
  dependencies {
    testCompile "org.apache.avro:avro:$avroVersion"
    testCompile 'joda-time:joda-time:2.9.9'
  }
}

project(':iceberg-common') {
}

project(':iceberg-core') {
  dependencies {
    compile project(':iceberg-api')
    compile project(':iceberg-common')

    compile "org.apache.avro:avro:$avroVersion"

    compile "com.fasterxml.jackson.core:jackson-databind:$jacksonVersion"
    compile "com.fasterxml.jackson.core:jackson-core:$jacksonVersion"
    compileOnly("org.apache.hadoop:hadoop-client:$hadoopVersion") {
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
    }
  }
}

project(':iceberg-data') {
  dependencies {
    compile project(':iceberg-api')
    compile project(':iceberg-core')
    compileOnly project(':iceberg-parquet')

    testCompile("org.apache.hadoop:hadoop-client:$hadoopVersion") {
      exclude group: 'org.apache.avro', module: 'avro'
      exclude group: 'org.slf4j', module: 'slf4j-log4j12'
    }
  }
}

project(':iceberg-hive') {
    dependencies {
      compile project(':iceberg-core')

      compileOnly "org.apache.hive:hive-standalone-metastore:$hiveVersion"

      testCompile "org.apache.hive:hive-exec:3.1.0"

      compileOnly('org.apache.hadoop:hadoop-client:2.7.3') {
        exclude group: 'org.apache.avro', module: 'avro'
        exclude group: 'org.slf4j', module: 'slf4j-log4j12'
      }
    }
}

project(':iceberg-orc') {
  dependencies {
    compile project(':iceberg-api')
    compile project(':iceberg-core')

    compile("org.apache.orc:orc-core:$orcVersion:nohive") {
      exclude group: 'org.apache.hadoop', module: 'hadoop-common'
    }
    
    
    compileOnly("org.apache.hadoop:hadoop-client:$hadoopVersion") {
      exclude group: 'org.apache.avro', module: 'avro'
    }

  }
}

project(':iceberg-parquet') {
  dependencies {
    compile project(':iceberg-api')
    compile project(':iceberg-core')

    compile "org.apache.parquet:parquet-avro:$parquetVersion"

    compileOnly "org.apache.avro:avro:$avroVersion"
    compileOnly("org.apache.hadoop:hadoop-client:$hadoopVersion") {
      exclude group: 'org.apache.avro', module: 'avro'
    }
  }
}

project(':iceberg-spark') {
  apply plugin: 'scala'

  dependencies {
    compile project(':iceberg-api')
    compile project(':iceberg-common')
    compile project(':iceberg-core')
    compile project(':iceberg-orc')
    compile project(':iceberg-parquet')

    compileOnly "org.apache.avro:avro:$avroVersion"
    compileOnly("org.apache.spark:spark-hive_$scalaVersion:$sparkVersion") {
      exclude group: 'org.apache.avro', module: 'avro'
    }

    testCompile "org.apache.hadoop:hadoop-hdfs:$hadoopVersion:tests"
    testCompile "org.apache.hadoop:hadoop-common:$hadoopVersion:tests"
    testCompile("org.apache.hadoop:hadoop-minicluster:$hadoopVersion") {
      exclude group: 'org.apache.avro', module: 'avro'
    }
  }
}

project(':iceberg-pig') {
  dependencies {
    compile project(':iceberg-api')
    compile project(':iceberg-common')
    compile project(':iceberg-core')
    compile project(':iceberg-parquet')

    compileOnly("org.apache.pig:pig:0.14.0")
    compileOnly("org.apache.hadoop:hadoop-mapreduce-client-core:$hadoopVersion")
    compileOnly("org.apache.hadoop:hadoop-client:$hadoopVersion") {
      exclude group: 'org.apache.avro', module: 'avro'
    }

    testCompile "org.apache.hadoop:hadoop-hdfs:$hadoopVersion:tests"
    testCompile "org.apache.hadoop:hadoop-common:$hadoopVersion:tests"
    testCompile("org.apache.hadoop:hadoop-minicluster:$hadoopVersion") {
      exclude group: 'org.apache.avro', module: 'avro'
    }
  }
}

// the runtime jar is a self-contained artifact for testing in a notebook
project(':iceberg-runtime') {
  apply plugin: 'com.github.johnrengelman.shadow'

  configurations {
    shadow
    compileOnly.extendsFrom shadow
  }

  shadowJar {
    configurations = [project.configurations.shadow]
    from(project.sourceSets.main.output)
  }

  tasks.build.dependsOn tasks.shadowJar
  tasks.install.dependsOn tasks.shadowJar
  tasks.javadocJar.dependsOn tasks.shadowJar

  dependencies {
    shadow project(':iceberg-api')
    shadow project(':iceberg-common')
    shadow project(':iceberg-core')
    shadow project(':iceberg-orc')
    shadow project(':iceberg-parquet')
    shadow project(':iceberg-spark')
    shadow project(':iceberg-pig')
    shadow project(':iceberg-hive')

    shadow "org.apache.avro:avro:$avroVersion"
    shadow "org.apache.parquet:parquet-avro:$parquetVersion"
  }

  publishing {
    publications {
      withType(IvyPublication) {
        configurations {
          'shadow' { extend 'runtime' }
        }
      }
    }
  }

  shadowJar {
    zip64 true

    // Relocate dependencies to avoid conflicts
    // relocate 'com.google.common', 'com.netflix.bdp.shaded.com.google.common'
    relocate 'com.fasterxml', 'com.netflix.iceberg.shaded.com.fasterxml'
    relocate 'org.apache.avro', 'com.netflix.iceberg.shaded.org.apache.avro'
    relocate 'org.apache.parquet.avro', 'com.netflix.iceberg.shaded.org.apache.parquet.avro'

    archiveName = "iceberg-runtime-${version}.${extension}"
  }
}

// Runtime jar for presto. Presto depends on hive fat jar that does not shade avro and bunch
// of other dependencies. This artifact is used by presto-iceberg plugin.
project(':iceberg-presto-runtime') {
    apply plugin: 'com.github.johnrengelman.shadow'

    configurations {
        shadow
        compileOnly.extendsFrom shadow
    }

    dependencies {
        shadow project(':iceberg-api')
        shadow project(':iceberg-core')
        shadow project(':iceberg-parquet')
        shadow project(':iceberg-hive')

        shadow "org.apache.parquet:parquet-avro:$parquetVersion"
        shadow "org.apache.avro:avro:$avroVersion"
        shadow ("org.apache.hive:hive-standalone-metastore:$hiveVersion") {
            exclude group: 'org.apache.hadoop', module: 'hadoop-common'
            exclude group: 'org.apache.orc', module: 'orc-core'
        }
    }

    tasks.build.dependsOn tasks.shadowJar
    tasks.install.dependsOn tasks.shadowJar
    tasks.javadocJar.dependsOn tasks.shadowJar

    shadowJar {
        zip64 true
        exclude 'org/htrace/**'
        exclude 'org/objectweb/**'
        exclude 'org/apache/thrift/**'
        exclude 'META-INF/services/org.apache.hadoop**'
        exclude 'com/thoughtworks/**'
        exclude 'org/slf4j/**'
        exclude 'org/znerd/**'
        exclude 'org/xerial/**'
        exclude 'org/tukaani/**'
        exclude 'org/objectweb/**'
        exclude 'org/apache/commons/**'
        exclude 'it/unimi/**'
        exclude 'io/airlift/**'
        exclude 'com/google/**'
        exclude 'com/fasterxml/**'
        exclude 'org/apache/log4j/**'
        exclude 'org/apache/http/**'
        exclude 'org/apache/http/version.properties'
        exclude 'mozilla/public-suffix-list.txt'
        exclude 'org/apache/http/client/version.properties'
        exclude 'common-version-info.properties'
        exclude 'core-default.xml'
        exclude 'org/apache/hadoop/application-classloader.properties'
        exclude '**application-classloader.properties'

        configurations = [project.configurations.shadow]
        from(project.sourceSets.main.output)

        relocate 'org.apache.avro', 'com.netflix.iceberg.shaded.org.apache.avro'
        relocate 'org.apache.parquet.avro', 'com.netflix.iceberg.shaded.org.apache.parquet.avro'
        relocate 'org.apache.hadoop.hive', 'com.netflix.iceberg.shaded.org.apache.hadoop.hive'
        relocate 'org.apache.hive', 'com.netflix.iceberg.shaded.org.apache.hive'
        archiveName = "iceberg-presto-runtime-${version}.${extension}"
    }
}