Skip to content

Commit

Permalink
Support for Oozie and Ambari
Browse files Browse the repository at this point in the history
  • Loading branch information
@kosinsky
kosinsky committed Mar 23, 2020
1 parent b230942 commit 2437ccd
Show file tree
Hide file tree
Showing 6 changed files with 281 additions and 23 deletions.
7 changes: 7 additions & 0 deletions .env-dev
View file
@@ -0,0 +1,7 @@
export ARM_CLIENT_ID=9be63db0-c96b-4581-948a-5fd48b235fef
export ARM_CLIENT_SECRET=86331dba-ef2e-4571-8293-4d7755c552e1
export ARM_SUBSCRIPTION_ID=e2d17d7c-32e7-4236-9a1e-078f732f3048
export ARM_TENANT_ID=72f988bf-86f1-41af-91ab-2d7cd011db47
export ARM_TEST_LOCATION=eastus
export ARM_TEST_LOCATION_ALT=eastus2
export ARM_TEST_LOCATION_ALT2=westus2
119 changes: 117 additions & 2 deletions azurerm/helpers/azure/hdinsight.go
View file
Expand Up @@ -105,7 +105,7 @@ func SchemaHDInsightsGateway() *schema.Schema {
}
}

func SchemaHDInsightsHiveMetastore() *schema.Schema {
func SchemaHDInsightsExternalMetastore() *schema.Schema {
return &schema.Schema{
Type: schema.TypeList,
Optional: true,
Expand Down Expand Up @@ -159,7 +159,7 @@ func ExpandHDInsightsConfigurations(input []interface{}) map[string]interface{}
}
}

func ExpandHDInsightsMetastore(input []interface{}) map[string]interface{} {
func ExpandHDInsightsHiveMetastore(input []interface{}) map[string]interface{} {
vs := input[0].(map[string]interface{})

server := vs["server"].(string)
Expand All @@ -185,6 +185,51 @@ func ExpandHDInsightsMetastore(input []interface{}) map[string]interface{} {
}
}

func ExpandHDInsightsOozieMetastore(input []interface{}) map[string]interface{} {
vs := input[0].(map[string]interface{})

server := vs["server"].(string)
database := vs["database_name"].(string)
username := vs["username"].(string)
password := vs["password"].(string)

return map[string]interface{}{
"oozie-site": map[string]interface{}{
"oozie.service.JPAService.jdbc.driver": "com.microsoft.sqlserver.jdbc.SQLServerDriver",
"oozie.service.JPAService.jdbc.url": fmt.Sprintf("jdbc:sqlserver://%s;database=%s;encrypt=true;trustServerCertificate=true;create=false;loginTimeout=300", server, database),
"oozie.service.JPAService.jdbc.username": username,
"oozie.service.JPAService.jdbc.password": password,
"oozie.db.schema.name": "oozie",
},
"oozie-env": map[string]interface{}{
"oozie_database": "Existing MSSQL Server database with SQL authentication",
"oozie_database_name": database,
"oozie_database_type": "mssql",
"oozie_existing_mssql_server_database": database,
"oozie_existing_mssql_server_host": server,
"oozie_hostname": server,
},
}
}

func ExpandHDInsightsAmbariMetastore(input []interface{}) map[string]interface{} {
vs := input[0].(map[string]interface{})

server := vs["server"].(string)
database := vs["database_name"].(string)
username := vs["username"].(string)
password := vs["password"].(string)

return map[string]interface{}{
"ambari-conf": map[string]interface{}{
"database-server": server,
"database-name": database,
"database-user-name": username,
"database-user-password": password,
},
}
}

func FlattenHDInsightsConfigurations(input map[string]*string) []interface{} {
enabled := false
if v, exists := input["restAuthCredential.isEnabled"]; exists && v != nil {
Expand Down Expand Up @@ -248,6 +293,76 @@ func FlattenHDInsightsHiveMetastore(env map[string]*string, site map[string]*str
return nil
}

func FlattenHDInsightsOozieMetastore(env map[string]*string, site map[string]*string) []interface{} {
server := ""
if v, exists := env["oozie_hostname"]; exists && v != nil {
server = *v
}

database := ""
if v, exists := env["oozie_database_name"]; exists && v != nil {
database = *v
}

username := ""
if v, exists := site["oozie.service.JPAService.jdbc.username"]; exists && v != nil {
username = *v
}

password := ""
if v, exists := site["oozie.service.JPAService.jdbc.password"]; exists && v != nil {
password = *v
}

if server != "" && database != "" {
return []interface{}{
map[string]interface{}{
"server": server,
"database_name": database,
"username": username,
"password": password,
},
}
}

return nil
}

func FlattenHDInsightsAmbariMetastore(conf map[string]*string) []interface{} {
server := ""
if v, exists := conf["database-server"]; exists && v != nil {
server = *v
}

database := ""
if v, exists := conf["database-name"]; exists && v != nil {
database = *v
}

username := ""
if v, exists := conf["database-user-name"]; exists && v != nil {
username = *v
}

password := ""
if v, exists := conf["database-user-password"]; exists && v != nil {
password = *v
}

if server != "" && database != "" {
return []interface{}{
map[string]interface{}{
"server": server,
"database_name": database,
"username": username,
"password": password,
},
}
}

return nil
}

func SchemaHDInsightsStorageAccounts() *schema.Schema {
return &schema.Schema{
Type: schema.TypeList,
Expand Down
53 changes: 53 additions & 0 deletions azurerm/internal/services/hdinsight/common_hdinsight.go
View file
Expand Up @@ -272,3 +272,56 @@ func deleteHDInsightEdgeNodes(ctx context.Context, client *hdinsight.Application

return nil
}

func expandHDInsightsMetastore(input []interface{}) map[string]interface{} {
v := input[0].(map[string]interface{})

config := map[string]interface{}{}

if hiveRaw, ok := v["hive"]; ok {
for k, val := range azure.ExpandHDInsightsHiveMetastore(hiveRaw.([]interface{})) {
config[k] = val
}
}

if oozieRaw, ok := v["oozie"]; ok {
for k, val := range azure.ExpandHDInsightsOozieMetastore(oozieRaw.([]interface{})) {
config[k] = val
}
}

if ambariRaw, ok := v["ambari"]; ok {
for k, val := range azure.ExpandHDInsightsAmbariMetastore(ambariRaw.([]interface{})) {
config[k] = val
}
}

return config
}

func flattenHDInsightsMetastores(d *schema.ResourceData, configurations map[string]map[string]*string) {
result := map[string]interface{}{}

hiveEnv, envExists := configurations["hive-env"]
hiveSite, siteExists := configurations["hive-site"]
if envExists && siteExists {
result["hive"] = azure.FlattenHDInsightsHiveMetastore(hiveEnv, hiveSite)
}

oozieEnv, envExists := configurations["oozie-env"]
oozieSite, siteExists := configurations["oozie-site"]
if envExists && siteExists {
result["oozie"] = azure.FlattenHDInsightsOozieMetastore(oozieEnv, oozieSite)
}

ambari, ambariExists := configurations["ambari-conf"]
if ambariExists {
result["ambari"] = azure.FlattenHDInsightsAmbariMetastore(ambari)
}

if len(result) > 0 {
d.Set("metastores", []interface{}{
result,
})
}
}
30 changes: 19 additions & 11 deletions azurerm/internal/services/hdinsight/resource_arm_hdinsight_hadoop_cluster.go
View file
Expand Up @@ -91,7 +91,20 @@ func resourceArmHDInsightHadoopCluster() *schema.Resource {

"gateway": azure.SchemaHDInsightsGateway(),

"hive_metastore": azure.SchemaHDInsightsHiveMetastore(),
"metastores": {
Type: schema.TypeList,
Required: true,
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"hive": azure.SchemaHDInsightsExternalMetastore(),

"oozie": azure.SchemaHDInsightsExternalMetastore(),

"ambari": azure.SchemaHDInsightsExternalMetastore(),
},
},
},

"storage_account": azure.SchemaHDInsightsStorageAccounts(),

Expand Down Expand Up @@ -187,11 +200,10 @@ func resourceArmHDInsightHadoopClusterCreate(d *schema.ResourceData, meta interf
gatewayRaw := d.Get("gateway").([]interface{})
configurations := azure.ExpandHDInsightsConfigurations(gatewayRaw)

if metastoreRaw, ok := d.GetOkExists("hive_metastore"); ok {
metastore := azure.ExpandHDInsightsMetastore(metastoreRaw.([]interface{}))
for k, v := range metastore {
configurations[k] = v
}
metastoresRaw := d.Get("metastores").([]interface{})
metastores := expandHDInsightsMetastore(metastoresRaw)
for k, v := range metastores {
configurations[k] = v
}

storageAccountsRaw := d.Get("storage_account").([]interface{})
Expand Down Expand Up @@ -351,11 +363,7 @@ func resourceArmHDInsightHadoopClusterRead(d *schema.ResourceData, meta interfac
return fmt.Errorf("Error flattening `gateway`: %+v", err)
}

hiveEnv, envExists := configurations.Configurations["hive-env"]
hiveSite, siteExists := configurations.Configurations["hive-site"]
if envExists && siteExists {
d.Set("hive_metastore", azure.FlattenHDInsightsHiveMetastore(hiveEnv, hiveSite))
}
flattenHDInsightsMetastores(d, configurations.Configurations)
}

hadoopRoles := hdInsightRoleDefinition{
Expand Down
54 changes: 46 additions & 8 deletions azurerm/internal/services/hdinsight/tests/resource_arm_hdinsight_hadoop_cluster_test.go
View file
Expand Up @@ -341,7 +341,9 @@ func TestAccAzureRMHDInsightHadoopCluster_metastore(t *testing.T) {
"roles.0.zookeeper_node.0.password",
"roles.0.zookeeper_node.0.vm_size",
"storage_account",
"hive_metastore.0.password"),
"metastores.0.hive.0.password",
"metastores.0.oozie.0.password",
"metastores.0.ambari.0.password"),
},
})
}
Expand Down Expand Up @@ -1016,8 +1018,28 @@ resource "azurerm_sql_server" "test" {
version = "12.0"
}
resource "azurerm_sql_database" "test" {
name = "metastore"
resource "azurerm_sql_database" "hive" {
name = "hive"
resource_group_name = azurerm_resource_group.test.name
location = azurerm_resource_group.test.location
server_name = azurerm_sql_server.test.name
collation = "SQL_Latin1_General_CP1_CI_AS"
create_mode = "Default"
requested_service_objective_name = "GP_Gen5_2"
}
resource "azurerm_sql_database" "oozie" {
name = "oozie"
resource_group_name = azurerm_resource_group.test.name
location = azurerm_resource_group.test.location
server_name = azurerm_sql_server.test.name
collation = "SQL_Latin1_General_CP1_CI_AS"
create_mode = "Default"
requested_service_objective_name = "GP_Gen5_2"
}
resource "azurerm_sql_database" "ambari" {
name = "ambari"
resource_group_name = azurerm_resource_group.test.name
location = azurerm_resource_group.test.location
server_name = azurerm_sql_server.test.name
Expand Down Expand Up @@ -1078,11 +1100,27 @@ resource "azurerm_hdinsight_hadoop_cluster" "test" {
}
}
hive_metastore {
server = azurerm_sql_server.test.fully_qualified_domain_name
database_name = azurerm_sql_database.test.name
username = azurerm_sql_server.test.administrator_login
password = azurerm_sql_server.test.administrator_login_password
metastores {
hive {
server = azurerm_sql_server.test.fully_qualified_domain_name
database_name = azurerm_sql_database.hive.name
username = azurerm_sql_server.test.administrator_login
password = azurerm_sql_server.test.administrator_login_password
}
oozie {
server = azurerm_sql_server.test.fully_qualified_domain_name
database_name = azurerm_sql_database.oozie.name
username = azurerm_sql_server.test.administrator_login
password = azurerm_sql_server.test.administrator_login_password
}
ambari {
server = azurerm_sql_server.test.fully_qualified_domain_name
database_name = azurerm_sql_database.ambari.name
username = azurerm_sql_server.test.administrator_login
password = azurerm_sql_server.test.administrator_login_password
}
}
}
`, template, data.RandomInteger, data.RandomInteger)
Expand Down

0 comments on commit 2437ccd

Please sign in to comment.