Skip to content

Commit

Permalink
Support for Oozie and Ambari
Browse files Browse the repository at this point in the history
  • Loading branch information
@kosinsky
kosinsky committed Mar 23, 2020
1 parent b230942 commit ce341b9
Show file tree
Hide file tree
Showing 5 changed files with 274 additions and 23 deletions.
119 changes: 117 additions & 2 deletions azurerm/helpers/azure/hdinsight.go
View file
Expand Up @@ -105,7 +105,7 @@ func SchemaHDInsightsGateway() *schema.Schema {
}
}

func SchemaHDInsightsHiveMetastore() *schema.Schema {
func SchemaHDInsightsExternalMetastore() *schema.Schema {
return &schema.Schema{
Type: schema.TypeList,
Optional: true,
Expand Down Expand Up @@ -159,7 +159,7 @@ func ExpandHDInsightsConfigurations(input []interface{}) map[string]interface{}
}
}

func ExpandHDInsightsMetastore(input []interface{}) map[string]interface{} {
func ExpandHDInsightsHiveMetastore(input []interface{}) map[string]interface{} {
vs := input[0].(map[string]interface{})

server := vs["server"].(string)
Expand All @@ -185,6 +185,51 @@ func ExpandHDInsightsMetastore(input []interface{}) map[string]interface{} {
}
}

func ExpandHDInsightsOozieMetastore(input []interface{}) map[string]interface{} {
vs := input[0].(map[string]interface{})

server := vs["server"].(string)
database := vs["database_name"].(string)
username := vs["username"].(string)
password := vs["password"].(string)

return map[string]interface{}{
"oozie-site": map[string]interface{}{
"oozie.service.JPAService.jdbc.driver": "com.microsoft.sqlserver.jdbc.SQLServerDriver",
"oozie.service.JPAService.jdbc.url": fmt.Sprintf("jdbc:sqlserver://%s;database=%s;encrypt=true;trustServerCertificate=true;create=false;loginTimeout=300", server, database),
"oozie.service.JPAService.jdbc.username": username,
"oozie.service.JPAService.jdbc.password": password,
"oozie.db.schema.name": "oozie",
},
"oozie-env": map[string]interface{}{
"oozie_database": "Existing MSSQL Server database with SQL authentication",
"oozie_database_name": database,
"oozie_database_type": "mssql",
"oozie_existing_mssql_server_database": database,
"oozie_existing_mssql_server_host": server,
"oozie_hostname": server,
},
}
}

func ExpandHDInsightsAmbariMetastore(input []interface{}) map[string]interface{} {
vs := input[0].(map[string]interface{})

server := vs["server"].(string)
database := vs["database_name"].(string)
username := vs["username"].(string)
password := vs["password"].(string)

return map[string]interface{}{
"ambari-conf": map[string]interface{}{
"database-server": server,
"database-name": database,
"database-user-name": username,
"database-user-password": password,
},
}
}

func FlattenHDInsightsConfigurations(input map[string]*string) []interface{} {
enabled := false
if v, exists := input["restAuthCredential.isEnabled"]; exists && v != nil {
Expand Down Expand Up @@ -248,6 +293,76 @@ func FlattenHDInsightsHiveMetastore(env map[string]*string, site map[string]*str
return nil
}

func FlattenHDInsightsOozieMetastore(env map[string]*string, site map[string]*string) []interface{} {
server := ""
if v, exists := env["oozie_hostname"]; exists && v != nil {
server = *v
}

database := ""
if v, exists := env["oozie_database_name"]; exists && v != nil {
database = *v
}

username := ""
if v, exists := site["oozie.service.JPAService.jdbc.username"]; exists && v != nil {
username = *v
}

password := ""
if v, exists := site["oozie.service.JPAService.jdbc.password"]; exists && v != nil {
password = *v
}

if server != "" && database != "" {
return []interface{}{
map[string]interface{}{
"server": server,
"database_name": database,
"username": username,
"password": password,
},
}
}

return nil
}

func FlattenHDInsightsAmbariMetastore(conf map[string]*string) []interface{} {
server := ""
if v, exists := conf["database-server"]; exists && v != nil {
server = *v
}

database := ""
if v, exists := conf["database-name"]; exists && v != nil {
database = *v
}

username := ""
if v, exists := conf["database-user-name"]; exists && v != nil {
username = *v
}

password := ""
if v, exists := conf["database-user-password"]; exists && v != nil {
password = *v
}

if server != "" && database != "" {
return []interface{}{
map[string]interface{}{
"server": server,
"database_name": database,
"username": username,
"password": password,
},
}
}

return nil
}

func SchemaHDInsightsStorageAccounts() *schema.Schema {
return &schema.Schema{
Type: schema.TypeList,
Expand Down
53 changes: 53 additions & 0 deletions azurerm/internal/services/hdinsight/common_hdinsight.go
View file
Expand Up @@ -272,3 +272,56 @@ func deleteHDInsightEdgeNodes(ctx context.Context, client *hdinsight.Application

return nil
}

func expandHDInsightsMetastore(input []interface{}) map[string]interface{} {
v := input[0].(map[string]interface{})

config := map[string]interface{}{}

if hiveRaw, ok := v["hive"]; ok {
for k, val := range azure.ExpandHDInsightsHiveMetastore(hiveRaw.([]interface{})) {
config[k] = val
}
}

if oozieRaw, ok := v["oozie"]; ok {
for k, val := range azure.ExpandHDInsightsOozieMetastore(oozieRaw.([]interface{})) {
config[k] = val
}
}

if ambariRaw, ok := v["ambari"]; ok {
for k, val := range azure.ExpandHDInsightsAmbariMetastore(ambariRaw.([]interface{})) {
config[k] = val
}
}

return config
}

func flattenHDInsightsMetastores(d *schema.ResourceData, configurations map[string]map[string]*string) {
result := map[string]interface{}{}

hiveEnv, envExists := configurations["hive-env"]
hiveSite, siteExists := configurations["hive-site"]
if envExists && siteExists {
result["hive"] = azure.FlattenHDInsightsHiveMetastore(hiveEnv, hiveSite)
}

oozieEnv, envExists := configurations["oozie-env"]
oozieSite, siteExists := configurations["oozie-site"]
if envExists && siteExists {
result["oozie"] = azure.FlattenHDInsightsOozieMetastore(oozieEnv, oozieSite)
}

ambari, ambariExists := configurations["ambari-conf"]
if ambariExists {
result["ambari"] = azure.FlattenHDInsightsAmbariMetastore(ambari)
}

if len(result) > 0 {
d.Set("metastores", []interface{}{
result,
})
}
}
30 changes: 19 additions & 11 deletions azurerm/internal/services/hdinsight/resource_arm_hdinsight_hadoop_cluster.go
View file
Expand Up @@ -91,7 +91,20 @@ func resourceArmHDInsightHadoopCluster() *schema.Resource {

"gateway": azure.SchemaHDInsightsGateway(),

"hive_metastore": azure.SchemaHDInsightsHiveMetastore(),
"metastores": {
Type: schema.TypeList,
Required: true,
MaxItems: 1,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"hive": azure.SchemaHDInsightsExternalMetastore(),

"oozie": azure.SchemaHDInsightsExternalMetastore(),

"ambari": azure.SchemaHDInsightsExternalMetastore(),
},
},
},

"storage_account": azure.SchemaHDInsightsStorageAccounts(),

Expand Down Expand Up @@ -187,11 +200,10 @@ func resourceArmHDInsightHadoopClusterCreate(d *schema.ResourceData, meta interf
gatewayRaw := d.Get("gateway").([]interface{})
configurations := azure.ExpandHDInsightsConfigurations(gatewayRaw)

if metastoreRaw, ok := d.GetOkExists("hive_metastore"); ok {
metastore := azure.ExpandHDInsightsMetastore(metastoreRaw.([]interface{}))
for k, v := range metastore {
configurations[k] = v
}
metastoresRaw := d.Get("metastores").([]interface{})
metastores := expandHDInsightsMetastore(metastoresRaw)
for k, v := range metastores {
configurations[k] = v
}

storageAccountsRaw := d.Get("storage_account").([]interface{})
Expand Down Expand Up @@ -351,11 +363,7 @@ func resourceArmHDInsightHadoopClusterRead(d *schema.ResourceData, meta interfac
return fmt.Errorf("Error flattening `gateway`: %+v", err)
}

hiveEnv, envExists := configurations.Configurations["hive-env"]
hiveSite, siteExists := configurations.Configurations["hive-site"]
if envExists && siteExists {
d.Set("hive_metastore", azure.FlattenHDInsightsHiveMetastore(hiveEnv, hiveSite))
}
flattenHDInsightsMetastores(d, configurations.Configurations)
}

hadoopRoles := hdInsightRoleDefinition{
Expand Down
54 changes: 46 additions & 8 deletions azurerm/internal/services/hdinsight/tests/resource_arm_hdinsight_hadoop_cluster_test.go
View file
Expand Up @@ -341,7 +341,9 @@ func TestAccAzureRMHDInsightHadoopCluster_metastore(t *testing.T) {
"roles.0.zookeeper_node.0.password",
"roles.0.zookeeper_node.0.vm_size",
"storage_account",
"hive_metastore.0.password"),
"metastores.0.hive.0.password",
"metastores.0.oozie.0.password",
"metastores.0.ambari.0.password"),
},
})
}
Expand Down Expand Up @@ -1016,8 +1018,28 @@ resource "azurerm_sql_server" "test" {
version = "12.0"
}
resource "azurerm_sql_database" "test" {
name = "metastore"
resource "azurerm_sql_database" "hive" {
name = "hive"
resource_group_name = azurerm_resource_group.test.name
location = azurerm_resource_group.test.location
server_name = azurerm_sql_server.test.name
collation = "SQL_Latin1_General_CP1_CI_AS"
create_mode = "Default"
requested_service_objective_name = "GP_Gen5_2"
}
resource "azurerm_sql_database" "oozie" {
name = "oozie"
resource_group_name = azurerm_resource_group.test.name
location = azurerm_resource_group.test.location
server_name = azurerm_sql_server.test.name
collation = "SQL_Latin1_General_CP1_CI_AS"
create_mode = "Default"
requested_service_objective_name = "GP_Gen5_2"
}
resource "azurerm_sql_database" "ambari" {
name = "ambari"
resource_group_name = azurerm_resource_group.test.name
location = azurerm_resource_group.test.location
server_name = azurerm_sql_server.test.name
Expand Down Expand Up @@ -1078,11 +1100,27 @@ resource "azurerm_hdinsight_hadoop_cluster" "test" {
}
}
hive_metastore {
server = azurerm_sql_server.test.fully_qualified_domain_name
database_name = azurerm_sql_database.test.name
username = azurerm_sql_server.test.administrator_login
password = azurerm_sql_server.test.administrator_login_password
metastores {
hive {
server = azurerm_sql_server.test.fully_qualified_domain_name
database_name = azurerm_sql_database.hive.name
username = azurerm_sql_server.test.administrator_login
password = azurerm_sql_server.test.administrator_login_password
}
oozie {
server = azurerm_sql_server.test.fully_qualified_domain_name
database_name = azurerm_sql_database.oozie.name
username = azurerm_sql_server.test.administrator_login
password = azurerm_sql_server.test.administrator_login_password
}
ambari {
server = azurerm_sql_server.test.fully_qualified_domain_name
database_name = azurerm_sql_database.ambari.name
username = azurerm_sql_server.test.administrator_login
password = azurerm_sql_server.test.administrator_login_password
}
}
}
`, template, data.RandomInteger, data.RandomInteger)
Expand Down
41 changes: 39 additions & 2 deletions website/docs/r/hdinsight_hadoop_cluster.html.markdown
View file
Expand Up @@ -107,7 +107,7 @@ The following arguments are supported:

* `tags` - (Optional) A map of Tags which should be assigned to this HDInsight Hadoop Cluster.

* `hive_metastore` - (Optional) A `hive_metastore` block as defined below.
* `metastores` - (Optional) A `metastores` block as defined below.

---

Expand Down Expand Up @@ -249,8 +249,19 @@ A `install_script_action` block supports the following:

* `uri` - (Required) The URI pointing to the script to run during the installation of the edge node. Changing this forces a new resource to be created.

---

A `metastores` block supports the following:

* `hive` - (Required) A `hive` block as defined below.

* `oozie` - (Required) A `oozie` block as defined below.

* `ambari` - (Required) A `amabari` block as defined below.

---
A `hive_metastore` block supports the following:

A `hive` block supports the following:

* `server` - (Required) The fully-qualified domain name (FQDN) of the SQL server to use for the external Hive metastore.

Expand All @@ -260,6 +271,32 @@ A `hive_metastore` block supports the following:

* `password` - (Required) The external Hive metastore's existing SQL server admin password.


---

A `oozie` block supports the following:

* `server` - (Required) The fully-qualified domain name (FQDN) of the SQL server to use for the external Oozie metastore.

* `database_name` - (Required) The external Oozie metastore's existing SQL database.

* `username` - (Required) The external Oozie metastore's existing SQL server admin username.

* `password` - (Required) The external Oozie metastore's existing SQL server admin password.

---

A `ambari` block supports the following:

* `server` - (Required) The fully-qualified domain name (FQDN) of the SQL server to use for the external Ambari metastore.

* `database_name` - (Required) The external Hive metastore's existing SQL database.

* `username` - (Required) The external Ambari metastore's existing SQL server admin username.

* `password` - (Required) The external Ambari metastore's existing SQL server admin password.


## Attributes Reference

The following attributes are exported:
Expand Down

0 comments on commit ce341b9

Please sign in to comment.