feat: added support for IBM Cloud Logs (#536)

terraform-ibm-modules · Jul 31, 2024 · a42eb4a · a42eb4a
1 parent d8e91b1
commit a42eb4a
Show file tree

Hide file tree

Showing 30 changed files with 625 additions and 39 deletions.
diff --git a/.github/settings.yml b/.github/settings.yml
@@ -22,5 +22,5 @@ repository:
 
   # Uncomment this description property
   # and update the description to the current repo description.
-  description: "Deploys services for LogDNA, Activity Tracker, and SysDig"
-  topics: core-team, terraform, ibm-cloud, terraform-module, logdna, sysdig, activity-tracker, observability-instances, logging, monitoring, supported, graduated
+  description: "Deploys services for Log Analysis, Activity Tracker, Monitoring, and Cloud Logs"
+  topics: core-team, terraform, ibm-cloud, terraform-module, log-analysis, monitoring, activity-tracker, observability-instances, logging, monitoring, supported, graduated
diff --git a/.secrets.baseline b/.secrets.baseline
@@ -3,7 +3,7 @@
     "files": "go.sum|^.secrets.baseline$",
     "lines": null
   },
-  "generated_at": "2024-07-17T15:33:23Z",
+  "generated_at": "2024-07-29T04:20:03Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -82,15 +82,15 @@
         "hashed_secret": "91199272d5d6a574a51722ca6f3d1148edb1a0e7",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 110,
+        "line_number": 111,
         "type": "Secret Keyword",
         "verified_result": null
       },
       {
         "hashed_secret": "3bd02b996f65f3548c1a0b5d93b00bfa7c88341a",
         "is_secret": true,
         "is_verified": false,
-        "line_number": 220,
+        "line_number": 246,
         "type": "Secret Keyword",
         "verified_result": null
       }

diff --git a/README.md b/README.md
@@ -23,11 +23,12 @@ This module supports provisioning the following observability instances:
 * [terraform-ibm-observability-instances](#terraform-ibm-observability-instances)
 * [Submodules](./modules)
     * [activity_tracker](./modules/activity_tracker)
+    * [cloud_logs](./modules/cloud_logs)
     * [cloud_monitoring](./modules/cloud_monitoring)
     * [log_analysis](./modules/log_analysis)
 * [Examples](./examples)
-    * [Provision IBM Cloud Monitoring, Log Analysis and Activity Tracker with archiving and event routing](./examples/advanced)
-    * [Provision basic observability instances (Log Analysis, Cloud Monitoring, Activity Tracker)](./examples/basic)
+    * [Provision IBM Cloud Monitoring, Log Analysis, Cloud Logs and Activity Tracker with archiving and event routing](./examples/advanced)
+    * [Provision basic observability instances (Log Analysis, Cloud Monitoring, Activity Tracker, Cloud Logs)](./examples/basic)
 * [Contributing](#contributing)
 <!-- END OVERVIEW HOOK -->
 
@@ -143,6 +144,17 @@ module "cloud_monitoring" {
 }
 ```
 
+To provision IBM Cloud Logs only
+
+```hcl
+module "cloud_logs" {
+  source  = "terraform-ibm-modules/observability-instances/ibm//modules/cloud_logs"
+  version = "X.X.X" # Replace "X.X.X" with a release version to lock into a specific release
+  resource_group_id = module.resource_group.resource_group_id
+  region = var.region
+}
+```
+
 ### Required IAM access policies
 
 You need the following permissions to run this module.
@@ -160,6 +172,9 @@ You need the following permissions to run this module.
     - **IBM Log Analysis** service
         - `Editor` platform access
         - `Manager` service access
+    - **IBM Cloud Logs** service
+        - `Editor` platform access
+        - `Manager` service access
 
 To attach access management tags to resources in this module, you need the following permissions.
 
@@ -174,14 +189,15 @@ To attach access management tags to resources in this module, you need the follo
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.0.0 |
-| <a name="requirement_ibm"></a> [ibm](#requirement\_ibm) | >= 1.56.1, < 2.0.0 |
+| <a name="requirement_ibm"></a> [ibm](#requirement\_ibm) | >= 1.67.1, < 2.0.0 |
 | <a name="requirement_logdna"></a> [logdna](#requirement\_logdna) | >= 1.14.2, < 2.0.0 |
 
 ### Modules
 
 | Name | Source | Version |
 |------|--------|---------|
 | <a name="module_activity_tracker"></a> [activity\_tracker](#module\_activity\_tracker) | ./modules/activity_tracker | n/a |
+| <a name="module_cloud_logs"></a> [cloud\_logs](#module\_cloud\_logs) | ./modules/cloud_logs | n/a |
 | <a name="module_cloud_monitoring"></a> [cloud\_monitoring](#module\_cloud\_monitoring) | ./modules/cloud_monitoring | n/a |
 | <a name="module_log_analysis"></a> [log\_analysis](#module\_log\_analysis) | ./modules/log_analysis | n/a |
 
@@ -206,6 +222,16 @@ No resources.
 | <a name="input_at_cos_bucket_endpoint"></a> [at\_cos\_bucket\_endpoint](#input\_at\_cos\_bucket\_endpoint) | An endpoint for the COS bucket for the Activity Tracker archive. Pass either the public or private endpoint (Only required when var.activity\_tracker\_enable\_archive and var.activity\_tracker\_provision are true) | `string` | `null` | no |
 | <a name="input_at_cos_bucket_name"></a> [at\_cos\_bucket\_name](#input\_at\_cos\_bucket\_name) | The name of an existing COS bucket to be used for the Activity Tracker archive (Only required when var.activity\_tracker\_enable\_archive and var.activity\_tracker\_provision are true). | `string` | `null` | no |
 | <a name="input_at_cos_instance_id"></a> [at\_cos\_instance\_id](#input\_at\_cos\_instance\_id) | The ID of the cloud object storage instance containing the Activity Tracker archive bucket (Only required when var.activity\_tracker\_enable\_archive and var.activity\_tracker\_provision are true). | `string` | `null` | no |
+| <a name="input_cloud_logs_access_tags"></a> [cloud\_logs\_access\_tags](#input\_cloud\_logs\_access\_tags) | A list of access tags to apply to the IBM Cloud Logs instance created by the module. For more information, see https://cloud.ibm.com/docs/account?topic=account-access-tags-tutorial. | `list(string)` | `[]` | no |
+| <a name="input_cloud_logs_data_storage"></a> [cloud\_logs\_data\_storage](#input\_cloud\_logs\_data\_storage) | A logs data bucket and a metrics bucket in IBM Cloud Object Storage to store your IBM Cloud Logs data for long term storage, search, analysis and alerting. | <pre>object({<br>    logs_data = optional(object({<br>      enabled              = optional(bool, false)<br>      bucket_crn           = optional(string)<br>      bucket_endpoint      = optional(string)<br>      skip_cos_auth_policy = optional(bool, false)<br>    }), {})<br>    metrics_data = optional(object({<br>      enabled              = optional(bool, false)<br>      bucket_crn           = optional(string)<br>      bucket_endpoint      = optional(string)<br>      skip_cos_auth_policy = optional(bool, false)<br>    }), {})<br>    }<br>  )</pre> | <pre>{<br>  "logs_data": null,<br>  "metrics_data": null<br>}</pre> | no |
+| <a name="input_cloud_logs_existing_en_instances"></a> [cloud\_logs\_existing\_en\_instances](#input\_cloud\_logs\_existing\_en\_instances) | List of Event Notifications instance details for routing critical events that occur in your IBM Cloud Logs. | <pre>list(object({<br>    en_instance_id      = string<br>    en_region           = string<br>    en_instance_name    = optional(string)<br>    source_id           = optional(string)<br>    source_name         = optional(string)<br>    skip_en_auth_policy = optional(bool, false)<br>  }))</pre> | `[]` | no |
+| <a name="input_cloud_logs_instance_name"></a> [cloud\_logs\_instance\_name](#input\_cloud\_logs\_instance\_name) | The name of the IBM Cloud Logs instance to create. Defaults to 'cloud\_logs-<region>' | `string` | `null` | no |
+| <a name="input_cloud_logs_plan"></a> [cloud\_logs\_plan](#input\_cloud\_logs\_plan) | The IBM Cloud Logs plan to provision. Available: standard | `string` | `"standard"` | no |
+| <a name="input_cloud_logs_provision"></a> [cloud\_logs\_provision](#input\_cloud\_logs\_provision) | Provision a IBM Cloud Logs instance? | `bool` | `true` | no |
+| <a name="input_cloud_logs_region"></a> [cloud\_logs\_region](#input\_cloud\_logs\_region) | The IBM Cloud region where Cloud Logs instances will be created. | `string` | `null` | no |
+| <a name="input_cloud_logs_retention_period"></a> [cloud\_logs\_retention\_period](#input\_cloud\_logs\_retention\_period) | The number of days IBM Cloud Logs will retain the logs data in Priority insights. | `number` | `7` | no |
+| <a name="input_cloud_logs_service_endpoints"></a> [cloud\_logs\_service\_endpoints](#input\_cloud\_logs\_service\_endpoints) | The type of the service endpoint that will be set for the IBM Cloud Logs instance. | `string` | `"public-and-private"` | no |
+| <a name="input_cloud_logs_tags"></a> [cloud\_logs\_tags](#input\_cloud\_logs\_tags) | Tags associated with the IBM Cloud Logs instance (Optional, array of strings). | `list(string)` | `[]` | no |
 | <a name="input_cloud_monitoring_access_tags"></a> [cloud\_monitoring\_access\_tags](#input\_cloud\_monitoring\_access\_tags) | A list of access tags to apply to the Cloud Monitoring instance created by the module. For more information, see https://cloud.ibm.com/docs/account?topic=account-access-tags-tutorial. | `list(string)` | `[]` | no |
 | <a name="input_cloud_monitoring_instance_name"></a> [cloud\_monitoring\_instance\_name](#input\_cloud\_monitoring\_instance\_name) | The name of the IBM Cloud Monitoring instance to create. Defaults to 'cloud\_monitoring-<region>' | `string` | `null` | no |
 | <a name="input_cloud_monitoring_manager_key_name"></a> [cloud\_monitoring\_manager\_key\_name](#input\_cloud\_monitoring\_manager\_key\_name) | The name to give the IBM Cloud Monitoring manager key. | `string` | `"SysdigManagerKey"` | no |
@@ -235,7 +261,7 @@ No resources.
 | <a name="input_log_analysis_tags"></a> [log\_analysis\_tags](#input\_log\_analysis\_tags) | Tags associated with the IBM Cloud Logging instance (Optional, array of strings). | `list(string)` | `[]` | no |
 | <a name="input_log_analysis_targets"></a> [log\_analysis\_targets](#input\_log\_analysis\_targets) | List of log analysis target to be created | <pre>list(object({<br>    instance_id   = string<br>    ingestion_key = string<br>    target_region = optional(string)<br>    target_name   = string<br>  }))</pre> | `[]` | no |
 | <a name="input_region"></a> [region](#input\_region) | The IBM Cloud region where instances will be created. | `string` | `"us-south"` | no |
-| <a name="input_resource_group_id"></a> [resource\_group\_id](#input\_resource\_group\_id) | The id of the IBM Cloud resource group where the instance(s) will be created. | `string` | `null` | no |
+| <a name="input_resource_group_id"></a> [resource\_group\_id](#input\_resource\_group\_id) | The id of the IBM Cloud resource group where the instance(s) will be created. | `string` | n/a | yes |
 
 ### Outputs
 
@@ -249,6 +275,10 @@ No resources.
 | <a name="output_activity_tracker_resource_key"></a> [activity\_tracker\_resource\_key](#output\_activity\_tracker\_resource\_key) | The resource/service key for agents to use |
 | <a name="output_activity_tracker_routes"></a> [activity\_tracker\_routes](#output\_activity\_tracker\_routes) | The map of created routes |
 | <a name="output_activity_tracker_targets"></a> [activity\_tracker\_targets](#output\_activity\_tracker\_targets) | The map of created targets |
+| <a name="output_cloud_logs_crn"></a> [cloud\_logs\_crn](#output\_cloud\_logs\_crn) | The id of the provisioned Cloud Logs instance. |
+| <a name="output_cloud_logs_guid"></a> [cloud\_logs\_guid](#output\_cloud\_logs\_guid) | The guid of the provisioned Cloud Logs instance. |
+| <a name="output_cloud_logs_name"></a> [cloud\_logs\_name](#output\_cloud\_logs\_name) | The name of the provisioned Cloud Logs instance. |
+| <a name="output_cloud_logs_resource_group_id"></a> [cloud\_logs\_resource\_group\_id](#output\_cloud\_logs\_resource\_group\_id) | The resource group where Cloud Logs instance resides. |
 | <a name="output_cloud_monitoring_access_key"></a> [cloud\_monitoring\_access\_key](#output\_cloud\_monitoring\_access\_key) | IBM cloud monitoring access key for agents to use |
 | <a name="output_cloud_monitoring_crn"></a> [cloud\_monitoring\_crn](#output\_cloud\_monitoring\_crn) | The id of the provisioned IBM cloud monitoring instance. |
 | <a name="output_cloud_monitoring_guid"></a> [cloud\_monitoring\_guid](#output\_cloud\_monitoring\_guid) | The guid of the provisioned IBM cloud monitoring instance. |

diff --git a/common-dev-assets b/common-dev-assets
diff --git a/examples/advanced/README.md b/examples/advanced/README.md
@@ -1,8 +1,10 @@
-# Provision IBM Cloud Monitoring, Log Analysis and Activity Tracker with archiving and event routing
+# Provision IBM Cloud Monitoring, Log Analysis, Cloud Logs and Activity Tracker with archiving and event routing
 
 Example that deploys:
 
 - Log Analysis, Cloud Monitoring, and Activity Tracker instances
 - Key Protect instance and root key
 - COS instance and COS bucket for archiving Log Analysis and Activity Tracker logs into an encrypted bucket.
+- Additional logs data bucket and a metrics bucket in COS instance to store IBM Cloud Logs data
 - Activity Tracker instance with event routing to COS bucket, Event Streams, and Log Analysis
+- Cloud Logs instance with Event Notification integration.
diff --git a/examples/advanced/main.tf b/examples/advanced/main.tf
@@ -45,6 +45,22 @@ module "key_protect" {
   key_protect_instance_name = "${var.prefix}-kp"
 }
 
+##############################################################################
+# Event Notification
+##############################################################################
+
+module "event_notification" {
+  source            = "terraform-ibm-modules/event-notifications/ibm"
+  version           = "1.6.5"
+  resource_group_id = module.resource_group.resource_group_id
+  name              = "${var.prefix}-en"
+  tags              = var.resource_tags
+  plan              = "standard"
+  service_endpoints = "public"
+  region            = var.en_region
+}
+
+
 ##############################################################################
 # Event stream target
 ##############################################################################
@@ -94,6 +110,33 @@ module "cos" {
   kms_key_crn                = module.key_protect.keys["observability.observability-key"].crn
 }
 
+module "cloud_logs_buckets" {
+  source  = "terraform-ibm-modules/cos/ibm//modules/buckets"
+  version = "8.6.2"
+  bucket_configs = [
+    {
+      bucket_name                   = "${var.prefix}-logs-data"
+      kms_encryption_enabled        = true
+      region_location               = var.region
+      resource_instance_id          = module.cos.cos_instance_id
+      kms_encryption_enabled        = true
+      kms_guid                      = module.key_protect.kms_guid
+      kms_key_crn                   = module.key_protect.keys["observability.observability-key"].crn
+      skip_iam_authorization_policy = false
+    },
+    {
+      bucket_name                   = "${var.prefix}-metrics-data"
+      kms_encryption_enabled        = true
+      region_location               = var.region
+      resource_instance_id          = module.cos.cos_instance_id
+      kms_encryption_enabled        = true
+      kms_guid                      = module.key_protect.kms_guid
+      kms_key_crn                   = module.key_protect.keys["observability.observability-key"].crn
+      skip_iam_authorization_policy = true
+    }
+  ]
+}
+
 module "activity_tracker_event_routing_bucket" {
   source                     = "terraform-ibm-modules/cos/ibm"
   version                    = "8.6.2"
@@ -123,20 +166,24 @@ module "observability_instance_creation" {
   log_analysis_instance_name        = "${var.prefix}-log-analysis"
   cloud_monitoring_instance_name    = "${var.prefix}-cloud-monitoring"
   activity_tracker_instance_name    = "${var.prefix}-activity-tracker"
+  cloud_logs_instance_name          = "${var.prefix}-cloud-logs"
   enable_platform_metrics           = false
   enable_platform_logs              = false
   log_analysis_plan                 = "7-day"
   cloud_monitoring_plan             = "graduated-tier"
   activity_tracker_plan             = "7-day"
+  cloud_logs_plan                   = "standard"
   log_analysis_tags                 = var.resource_tags
   cloud_monitoring_tags             = var.resource_tags
   activity_tracker_tags             = var.resource_tags
   log_analysis_manager_key_tags     = var.resource_tags
   cloud_monitoring_manager_key_tags = var.resource_tags
   activity_tracker_manager_key_tags = var.resource_tags
+  cloud_logs_tags                   = var.resource_tags
   log_analysis_access_tags          = var.access_tags
   cloud_monitoring_access_tags      = var.access_tags
   activity_tracker_access_tags      = var.access_tags
+  cloud_logs_access_tags            = var.access_tags
   log_analysis_enable_archive       = true
   activity_tracker_enable_archive   = true
   ibmcloud_api_key                  = local.archive_api_key
@@ -197,4 +244,23 @@ module "observability_instance_creation" {
     metadata_region_backup    = var.metadata_region_backup
     private_api_endpoint_only = var.private_api_endpoint_only
   }
+
+  cloud_logs_retention_period = 14
+  cloud_logs_region           = "eu-es"
+  cloud_logs_data_storage = {
+    logs_data = {
+      enabled         = true
+      bucket_crn      = module.cloud_logs_buckets.buckets["${var.prefix}-logs-data"].bucket_crn
+      bucket_endpoint = module.cloud_logs_buckets.buckets["${var.prefix}-logs-data"].s3_endpoint_direct
+    },
+    metrics_data = {
+      enabled         = true
+      bucket_crn      = module.cloud_logs_buckets.buckets["${var.prefix}-metrics-data"].bucket_crn
+      bucket_endpoint = module.cloud_logs_buckets.buckets["${var.prefix}-metrics-data"].s3_endpoint_direct
+    }
+  }
+  cloud_logs_existing_en_instances = [{
+    en_instance_id = module.event_notification.guid
+    en_region      = var.en_region
+  }]
 }
diff --git a/examples/advanced/variables.tf b/examples/advanced/variables.tf
@@ -82,3 +82,9 @@ variable "private_api_endpoint_only" {
   description = "Set this true to restrict access only to private api endpoint."
   default     = false
 }
+
+variable "en_region" {
+  type        = string
+  description = "Region where event notification will be created"
+  default     = "au-syd"
+}
diff --git a/examples/advanced/version.tf b/examples/advanced/version.tf
@@ -5,7 +5,7 @@ terraform {
   required_providers {
     ibm = {
       source  = "ibm-cloud/ibm"
-      version = ">= 1.56.1"
+      version = ">= 1.67.1"
     }
     logdna = {
       source  = "logdna/logdna"

diff --git a/examples/basic/README.md b/examples/basic/README.md
@@ -1,3 +1,3 @@
-# Provision basic observability instances (Log Analysis, Cloud Monitoring, Activity Tracker)
+# Provision basic observability instances (Log Analysis, Cloud Monitoring, Activity Tracker, Cloud Logs)
 
-Example that deploys Log Analysis, Cloud Monitoring, and Activity Tracker instances with basic configuration.
+Example that deploys Log Analysis, Cloud Monitoring, Activity Tracker, and Cloud Logs instances with basic configuration.
diff --git a/examples/basic/main.tf b/examples/basic/main.tf
@@ -20,19 +20,24 @@ module "test_observability_instance_creation" {
   log_analysis_instance_name        = "${var.prefix}-log-analysis"
   cloud_monitoring_instance_name    = "${var.prefix}-cloud-monitoring"
   activity_tracker_instance_name    = "${var.prefix}-activity-tracker"
+  cloud_logs_instance_name          = "${var.prefix}-cloud-logs"
   resource_group_id                 = module.resource_group.resource_group_id
   log_analysis_plan                 = "7-day"
   cloud_monitoring_plan             = "graduated-tier"
   activity_tracker_plan             = "7-day"
+  cloud_logs_plan                   = "standard"
   enable_platform_logs              = false
   enable_platform_metrics           = false
   log_analysis_tags                 = var.resource_tags
   cloud_monitoring_tags             = var.resource_tags
   activity_tracker_tags             = var.resource_tags
+  cloud_logs_tags                   = var.resource_tags
   log_analysis_manager_key_tags     = var.resource_tags
   cloud_monitoring_manager_key_tags = var.resource_tags
   activity_tracker_manager_key_tags = var.resource_tags
   log_analysis_access_tags          = var.access_tags
   cloud_monitoring_access_tags      = var.access_tags
   activity_tracker_access_tags      = var.access_tags
+  cloud_logs_access_tags            = var.access_tags
+  cloud_logs_region                 = "eu-es"
 }