{"page":{"id":"01JWFTD1WG9PADY289K45S1WHV","name":"CASTAI","url":"https://status.cast.ai/","updated_at":"2026-11-07T04:32:39Z"},"incidents":[{"id":"01M6SNAHKPK4YTC0FNEW3SS0YW","name":"AWS EKS – Oregon (us-west-2) Service Disruption","status":"resolved","created_at":"2026-11-07T00:07:52Z","updated_at":"2026-11-07T04:32:39Z","resolved_at":"2026-11-07T04:32:39Z","impact":"minor","page_id":"01JWFTD1WG9PADY289K45S1WHV","incident_updates":[{"id":"01M6T4FCCADG61GXW84B2C5EE2","body":"AWS confirmed full recovery at **4:21 PM PST**. The issue was caused by an EBS subsystem failure affecting EC2 instance launches across all AZs.\n\n**Impact:** EKS node provisioning and autoscaling may have failed between **2:35–4:21 PM PST**. Existing instances were unaffected.\n\nAll services are operating normally.\n\nAWS Status: \u003chttps://health.aws.amazon.com/health/status\u003e","created_at":"2026-11-07T04:32:39Z","display_at":"2026-11-07T04:32:39Z","incident_id":"01M6SNAHKPK4YTC0FNEW3SS0YW","status":"resolved","updated_at":"2026-11-07T04:33:57Z"},{"id":"01M6SNAHKP0PSRRNRK0KDWN5DR","body":"AWS is currently experiencing elevated EC2 instance launch failures in the Oregon region (us-west-2) due to an underlying EBS service issue. This may impact EKS clusters in the region, including node provisioning, autoscaling, rebalancing and workloads requiring new instances.\n\nAWS is actively working on mitigation. We'll continue monitoring the situation and share updates as they become available.\n\nAWS Status: https://health.aws.amazon.com/health/status","created_at":"2026-11-07T00:07:52Z","display_at":"2026-11-07T00:07:52Z","incident_id":"01M6SNAHKPK4YTC0FNEW3SS0YW","status":"identified","updated_at":"2026-11-07T00:08:55Z"}]},{"id":"01KV8278D5YR6TNWYFPANJEFPQ","name":"Degraded: CAST AI node provisioning issues on AKS with Kubernetes 1.36","status":"resolved","created_at":"2026-06-16T11:14:16Z","updated_at":"2026-06-16T14:47:03Z","resolved_at":"2026-06-16T14:47:03Z","impact":"minor","page_id":"01JWFTD1WG9PADY289K45S1WHV","incident_updates":[{"id":"01KV8ECWG2EMDWF8G7J66RKF96","body":"We have implemented a workaround and successfully remediated all affected clusters. Node provisioning is now functioning normally, and node functionality is not impacted.\n\nIf you are running Kubernetes 1.36 and experience any node provisioning issues, please contact Support and we will be happy to assist. For any further questions, please reach out to Support or your Technical Account Manager.","created_at":"2026-06-16T14:47:03Z","display_at":"2026-06-16T14:47:03Z","incident_id":"01KV8278D5YR6TNWYFPANJEFPQ","status":"resolved","updated_at":"2026-06-16T14:47:03Z"},{"id":"01KV8C6QE8EDP1JR3HT82DRX19","body":"We have identified a workaround which allows nodes to continue provisioning normally for the affected clusters. Node functionality is not impacted.\n\nIf you have already upgraded to 1.36 and are experiencing provisioning failures, please reach out to support and we will assist in applying the workaround. A permanent fix with Ubuntu 24.04 support is in progress. We will notify you once available.","created_at":"2026-06-16T14:08:45Z","display_at":"2026-06-16T14:08:45Z","incident_id":"01KV8278D5YR6TNWYFPANJEFPQ","status":"monitoring","updated_at":"2026-06-16T14:08:45Z"},{"id":"01KV8278D50XDW3WY7XSF67VFV","body":"CAST AI node provisioning fails on AKS clusters running Kubernetes 1.36.0 or above. This is caused by Azure removing support for Ubuntu 22.04 (the OS currently used by CAST AI for AKS worker nodes) on Kubernetes 1.36+.\n\n**Impact:** Clusters upgraded to Kubernetes 1.36 on AKS will be unable to provision new nodes via CAST AI.\n**Workaround:** Do not upgrade AKS control plane to Kubernetes 1.36 until further notice.\n**Next update:** We are working on Ubuntu 24.04 support / temporary workaround and will provide an update once a fix is available.","created_at":"2026-06-16T11:14:16Z","display_at":"2026-06-16T11:14:16Z","incident_id":"01KV8278D5YR6TNWYFPANJEFPQ","status":"identified","updated_at":"2026-06-16T11:14:16Z"}]},{"id":"01KTSBQTEJXSH3M76G26AA9Z63","name":"Degraded Performance - api.cast.ai","status":"resolved","created_at":"2026-06-10T18:12:00Z","updated_at":"2026-06-10T19:27:12Z","resolved_at":"2026-06-10T19:27:12Z","impact":"minor","page_id":"01JWFTD1WG9PADY289K45S1WHV","incident_updates":[{"id":"01KTSG1GMS8EYHD2T4J1KTMYFF","body":"The issue affecting the availability of api.cast.ai has been resolved.","created_at":"2026-06-10T19:27:12Z","display_at":"2026-06-10T19:27:12Z","incident_id":"01KTSBQTEJXSH3M76G26AA9Z63","status":"resolved","updated_at":"2026-06-10T19:27:12Z"},{"id":"01KTSBQTEJZN3FSWSZR7AJ7VSW","body":"We are currently experiencing degraded performance with api.cast.ai, Autoscaling remains unaffected .\n\nOur engineering team is actively investigating the issue and working towards a resolution. We will provide updates as more information becomes available","created_at":"2026-06-10T18:12:00Z","display_at":"2026-06-10T18:12:00Z","incident_id":"01KTSBQTEJXSH3M76G26AA9Z63","status":"investigating","updated_at":"2026-06-10T18:12:00Z"}]},{"id":"01KT992WJ7Z08K40W96CB57564","name":"Intermittent Azure AKS Node Provisioning Failures in Select Regions","status":"resolved","created_at":"2026-06-04T12:17:46Z","updated_at":"2026-06-05T06:42:59Z","resolved_at":"2026-06-05T06:42:59Z","impact":"minor","page_id":"01JWFTD1WG9PADY289K45S1WHV","incident_updates":[{"id":"01KTB8AKC8QR4E6GAK2FEWWYVP","body":"A fix has been deployed to restore compatibility with recent Azure AKS node bootstrap changes. \n\nWe have not observed any additional affected customer clusters, and node provisioning is operating normally. \n\nIf you experience any issues with node provisioning, please contact our Support team for assistance.","created_at":"2026-06-05T06:42:59Z","display_at":"2026-06-05T06:42:59Z","incident_id":"01KT992WJ7Z08K40W96CB57564","status":"resolved","updated_at":"2026-06-05T06:42:59Z"},{"id":"01KTA2E2YD0EW2AVGTA83G8ZS7","body":"Azure AKS node provisioning is currently operating normally.\n\nWe will continue monitoring closely. ","created_at":"2026-06-04T19:40:47Z","display_at":"2026-06-04T19:40:47Z","incident_id":"01KT992WJ7Z08K40W96CB57564","status":"monitoring","updated_at":"2026-06-04T19:40:47Z"},{"id":"01KT9RH1PNTSRG7EAAGAAN92SW","body":"We continue to investigate intermittent Azure AKS node provisioning failures affecting a limited number of clusters in select regions.\n\nA mitigation has been applied to all currently affected clusters, and no additional customer clusters have been observed experiencing the issue at this time. We have identified the cause and are actively testing additional fixes to ensure compatibility with recent changes introduced by Azure's node bootstrap process.\n\nOur team is closely monitoring for any new occurrences and will provide further updates as progress continues.","created_at":"2026-06-04T16:47:38Z","display_at":"2026-06-04T16:47:38Z","incident_id":"01KT992WJ7Z08K40W96CB57564","status":"monitoring","updated_at":"2026-06-04T16:47:38Z"},{"id":"01KT992WJ78J7EQDZSKA48QYBG","body":"We are investigating an issue affecting a small number of Azure AKS clusters where node provisioning can fail.\n\n**Current impact:**\n\n• New node provisioning (\"Add Node\") may fail on affected Azure AKS clusters.\n\n• Existing workloads and running nodes are not impacted.\n\n• We are working on a permanent fix and monitoring all AKS clusters for further occurrences.\n\n\nWe will provide updates as more information becomes available.","created_at":"2026-06-04T12:17:46Z","display_at":"2026-06-04T12:17:46Z","incident_id":"01KT992WJ7Z08K40W96CB57564","status":"identified","updated_at":"2026-06-04T12:17:46Z"}]},{"id":"01KSN6RRD6F1CQ1NDTP909236J","name":"Documentation","status":"resolved","created_at":"2026-05-27T17:12:28Z","updated_at":"2026-05-27T17:38:35Z","resolved_at":"2026-05-27T17:38:35Z","impact":"none","page_id":"01JWFTD1WG9PADY289K45S1WHV","incident_updates":[{"id":"01KSN88K2XKDCMFJYBPG1WBZH7","body":"Cast AI documentation is now accessible. ReadMe reports their incident is resolved. \nhttps://www.readmestatus.com/","created_at":"2026-05-27T17:38:35Z","display_at":"2026-05-27T17:38:35Z","incident_id":"01KSN6RRD6F1CQ1NDTP909236J","status":"resolved","updated_at":"2026-05-27T17:38:35Z"},{"id":"01KSN6RRD6G3A3DK8SV12FV9J4","body":"Cast AI documentation is inaccessible as a result of the ReadMe outage. Please follow their status page for updates:\nhttps://www.readmestatus.com/","created_at":"2026-05-27T17:12:28Z","display_at":"2026-05-27T17:12:28Z","incident_id":"01KSN6RRD6F1CQ1NDTP909236J","status":"monitoring","updated_at":"2026-05-27T17:12:28Z"}]},{"id":"01KQF4ASMQYC1ZDMAZF6E2KES3","name":"Degraded Performance – CAST AI Autoscaler","status":"resolved","created_at":"2026-04-30T12:03:03Z","updated_at":"2026-04-30T12:43:20Z","resolved_at":"2026-04-30T12:43:20Z","impact":"minor","page_id":"01JWFTD1WG9PADY289K45S1WHV","incident_updates":[{"id":"01KQF6MHX8YR9YWYF2WPGVR8YA","body":"**Update:**\nThe issue affecting the CAST AI Autoscaler has been resolved. The autoscaler is now operating normally, and node provisioning has been fully restored.","created_at":"2026-04-30T12:43:20Z","display_at":"2026-04-30T12:43:20Z","incident_id":"01KQF4ASMQYC1ZDMAZF6E2KES3","status":"resolved","updated_at":"2026-04-30T12:43:20Z"},{"id":"01KQF4ASMQDE65N1YNW6P5AZY4","body":"We are currently experiencing degraded performance with the CAST AI Autoscaler. During this time, the autoscaler may be unable to provision new nodes, which can impact workload scheduling and scaling.\n\n \n\nOur engineering team is actively investigating the issue and working towards a resolution. We will provide updates as more information becomes available","created_at":"2026-04-30T12:03:03Z","display_at":"2026-04-30T12:03:03Z","incident_id":"01KQF4ASMQYC1ZDMAZF6E2KES3","status":"investigating","updated_at":"2026-04-30T12:03:03Z"}]},{"id":"01KQ05BGH9MYFQF60NPE90SV9W","name":"Microsoft Azure VM Provisioning Issues in East US Region","status":"resolved","created_at":"2026-04-24T16:32:18Z","updated_at":"2026-04-24T21:46:02Z","resolved_at":"2026-04-24T21:46:02Z","impact":"minor","page_id":"01JWFTD1WG9PADY289K45S1WHV","incident_updates":[{"id":"01KQ0Q9YW2MW3CC0Z7VZ8T6SJA","body":"Autoscaler node provisioning for AKS clusters in East US is working as expected from a Cast AI perspective. We are declaring our impact as resolved but will continue actively monitoring the situation.\n\nMicrosoft reports progress but the incident in Azure has not been declared as resolved. Please continue monitoring the Azure status page for future updates.\n\nhttps://azure.status.microsoft/en-us/status\n\n\n","created_at":"2026-04-24T21:46:02Z","display_at":"2026-04-24T21:46:02Z","incident_id":"01KQ05BGH9MYFQF60NPE90SV9W","status":"resolved","updated_at":"2026-04-24T21:46:02Z"},{"id":"01KQ05BGH9Z348AP8WDQA3DZDV","body":"Microsoft reports an active incident affecting VM provisioning in the Azure East US region. For updates please watch the Azure status page:\nhttps://azure.status.microsoft/en-us/status","created_at":"2026-04-24T16:32:19Z","display_at":"2026-04-24T16:32:18Z","incident_id":"01KQ05BGH9MYFQF60NPE90SV9W","status":"monitoring","updated_at":"2026-04-24T16:32:19Z"}]},{"id":"01KQG11Z63X8AJ5CGG4M1F3FGA","name":"Node Scaling Degradation for EKS Clusters in AWS UAE (me-central-1)","status":"resolved","created_at":"2026-04-30T20:25:02Z","updated_at":"2026-04-30T20:25:02Z","resolved_at":"2026-03-01T14:49:00Z","impact":"minor","page_id":"01JWFTD1WG9PADY289K45S1WHV","incident_updates":[{"id":"01KQG11Z63K5XWK8WEJ329XSQZ","body":"At this time, we have not received any further updates from AWS regarding the earlier disruption in the UAE (me-central-1) region. Based on the current information available, we are closing this incident.\n\nWe recommend continuing to monitor the official AWS Health Dashboard for any future updates or changes: https://health.aws.amazon.com/health/status\n\nIf you are still experiencing issues, please reach out to our support team.","created_at":"2026-04-30T20:25:02Z","display_at":"2026-03-01T14:49:00Z","incident_id":"01KQG11Z63X8AJ5CGG4M1F3FGA","status":"resolved","updated_at":"2026-04-30T20:25:02Z"},{"id":"01KQG11Z63ZF18KHCDPNYMCVFY","body":"Update\n\nThe AWS UAE (me-central-1) disruption has escalated into a regional outage. Multiple Availability Zones are impacted due to a power issue, and new EC2 instance launches are currently unavailable across the region.\n\nImpact: CAST AI node scaling and node replacement operations are currently unavailable in this region, and some EKS clusters may be partially or fully degraded. AWS expects recovery to take multiple hours and recommends failing over to another region where possible.\n\nWe are actively monitoring the situation and will provide further updates.","created_at":"2026-04-30T20:25:02Z","display_at":"2026-03-01T14:49:00Z","incident_id":"01KQG11Z63X8AJ5CGG4M1F3FGA","status":"identified","updated_at":"2026-04-30T20:25:02Z"}]},{"id":"01KGFZX3MFTDRM8PJTCZE0337K","name":"Ongoing issue with Azure AKS Clusters","status":"resolved","created_at":"2026-02-02T20:15:02Z","updated_at":"2026-03-11T15:56:02Z","resolved_at":"2026-02-03T06:54:00Z","impact":"major","page_id":"01JWFTD1WG9PADY289K45S1WHV","incident_updates":[{"id":"01KGH4F2SQQ76930NVTQA0PC74","body":"System should be fully recovered and node additional is operational. Resolution confirmed. ","created_at":"2026-02-03T06:54:00Z","display_at":"2026-02-03T06:54:00Z","incident_id":"01KGFZX3MFTDRM8PJTCZE0337K","status":"resolved","updated_at":"2026-02-03T06:54:00Z"},{"id":"01KGG9NV4016MV4CYJG52X1RKP","body":"**Update:** Azure has released a fix and is rolling it out across all impacted regions. On the CAST AI side, we are observing node operations completing successfully, with nodes being added and provisioning as expected.\n\nWe will provide another update once Azure confirms full resolution.\n\n**Reference:** https://azure.status.microsoft/en-us/status","created_at":"2026-02-02T23:05:49Z","display_at":"2026-02-02T23:05:49Z","incident_id":"01KGFZX3MFTDRM8PJTCZE0337K","status":"monitoring","updated_at":"2026-02-02T23:05:49Z"},{"id":"01KGG4RK8TWMCYXAMN7HFEKCGX","body":"**Update:** Azure has reported issues with virtual machine service management across multiple regions following a recent configuration change. We’re actively monitoring the situation and will continue to provide updates as more information becomes available.\n\nReference: https://azure.status.microsoft/en-us/status","created_at":"2026-02-02T21:39:57Z","display_at":"2026-02-02T21:39:57Z","incident_id":"01KGFZX3MFTDRM8PJTCZE0337K","status":"identified","updated_at":"2026-02-02T21:39:57Z"},{"id":"01KGG3E8J6KGA1BB7BMFK94MA9","body":"**Update:** Our investigation indicates this may be related to an Azure service outage impacting node creation and addition. We are awaiting further updates from Azure. As a precaution, we recommend disabling node deletion policies and scheduled rebalancing for critical clusters to avoid triggering downscaling. We’ll continue to share updates as more information becomes available.\n\n**Reference**: \u003chttps://downdetector.com/status/windows-azure/\u003e","created_at":"2026-02-02T21:16:50Z","display_at":"2026-02-02T21:16:50Z","incident_id":"01KGFZX3MFTDRM8PJTCZE0337K","status":"investigating","updated_at":"2026-02-02T21:16:50Z"},{"id":"01KGFZX3MFG7GRAE37SRYDWHQ5","body":"We’re currently observing an increased rate of node add failures and long add node operations, affecting Azure AKS clusters. Our engineering team is actively investigating and will provide updates as more information becomes available.","created_at":"2026-02-02T20:15:02Z","display_at":"2026-02-02T20:15:02Z","incident_id":"01KGFZX3MFTDRM8PJTCZE0337K","status":"investigating","updated_at":"2026-02-02T20:15:35Z"}]},{"id":"01KBPWG5SD21N5EBT8WE3VKVP8","name":"Cloudflare API Incident Impacting Cast AI Console","status":"resolved","created_at":"2025-12-05T09:11:44Z","updated_at":"2026-03-11T15:56:02Z","resolved_at":"2025-12-05T09:39:57Z","impact":"critical","page_id":"01JWFTD1WG9PADY289K45S1WHV","incident_updates":[{"id":"01KBPY3V3SB5NHCJ8H8743SZHN","body":"**C**onsole is now operating normally and is stable ","created_at":"2025-12-05T09:39:57Z","display_at":"2025-12-05T09:39:57Z","incident_id":"01KBPWG5SD21N5EBT8WE3VKVP8","status":"resolved","updated_at":"2025-12-05T09:39:57Z"},{"id":"01KBPX18EWV3JYJCBGHDT06JZQ","body":"Cloudflare has implemented a fix. We are monitoring stability of our console.","created_at":"2025-12-05T09:21:04Z","display_at":"2025-12-05T09:21:04Z","incident_id":"01KBPWG5SD21N5EBT8WE3VKVP8","status":"monitoring","updated_at":"2025-12-05T09:21:04Z"},{"id":"01KBPWG5SDGY4JM2V4CX6WJYZC","body":"Cloudflare is currently experiencing an API service fault which is impacting our console.\n\n\u003chttps://www.cloudflarestatus.com/\u003e\n\nCore Cast AI systems are still operational, including cluster and workload scaling, and we are working hard to restore console access.","created_at":"2025-12-05T09:11:44Z","display_at":"2025-12-05T09:11:44Z","incident_id":"01KBPWG5SD21N5EBT8WE3VKVP8","status":"identified","updated_at":"2025-12-05T09:11:44Z"}]},{"id":"01K8087GSECZ79GAHEVQJHK20Q","name":"Ongoing issue with AWS EKS Clusters ","status":"resolved","created_at":"2025-10-20T07:27:06Z","updated_at":"2026-03-11T15:56:02Z","resolved_at":"2025-10-20T23:05:14Z","impact":"major","page_id":"01JWFTD1WG9PADY289K45S1WHV","incident_updates":[{"id":"01K81XX9QN0QABVBX981CE13D7","body":"AWS has fully resolved the US-EAST-1 service degradation that began late yesterday. All affected AWS services have returned to normal operation, and CAST AI continues to run stable.","created_at":"2025-10-20T23:05:14Z","display_at":"2025-10-20T23:05:14Z","incident_id":"01K8087GSECZ79GAHEVQJHK20Q","status":"resolved","updated_at":"2025-10-20T23:09:22Z"},{"id":"01K80QXC9W3FN096556MRKSXEK","body":"EKS clusters are recovering to normal operations, however, delays can still occur in node provisioning due to ongoing EC2 capacity recovery in AWS.","created_at":"2025-10-20T12:01:11Z","display_at":"2025-10-20T12:01:11Z","incident_id":"01K8087GSECZ79GAHEVQJHK20Q","status":"monitoring","updated_at":"2025-10-20T12:01:11Z"},{"id":"01K809VP02EVQ1087Y6R5WTMRX","body":"We're currently observing increased 500 errors from the AWS EKS DescribeCluster API, which appear to be originating on AWS's side.","created_at":"2025-10-20T07:55:35Z","display_at":"2025-10-20T07:55:35Z","incident_id":"01K8087GSECZ79GAHEVQJHK20Q","status":"identified","updated_at":"2025-10-20T07:55:35Z"},{"id":"01K8087GSE6G66ACCJYBNF4K8R","body":"We’re currently observing increased 500 errors from the AWS EKS DescribeCluster API, which appear to be originating on AWS’s side.","created_at":"2025-10-20T07:27:06Z","display_at":"2025-10-20T07:27:06Z","incident_id":"01K8087GSECZ79GAHEVQJHK20Q","status":"identified","updated_at":"2025-10-20T07:27:06Z"}]},{"id":"01JXJRKZWW7VRKA52Q3WWSJC4P","name":"GCP Outage","status":"resolved","created_at":"2025-06-12T19:07:12Z","updated_at":"2026-03-11T15:56:02Z","resolved_at":"2025-06-12T18:53:17Z","impact":"major","page_id":"01JWFTD1WG9PADY289K45S1WHV","incident_updates":[{"id":"01JXJRKZWW7J2NG1CPSM1BPWWF","body":"Issue has been resolved and operations are back to normal.","created_at":"2025-06-12T19:07:12Z","display_at":"2025-06-12T18:53:17Z","incident_id":"01JXJRKZWW7VRKA52Q3WWSJC4P","status":"resolved","updated_at":"2025-06-19T10:06:07Z"},{"id":"01JXJRKZWWY7QMM0JEXR13NSMQ","body":"We are seeing an outage across Google Cloud. We are trying to ascertain the scope of the outage, we will provide an update within 15 minutes.","created_at":"2025-06-12T19:07:12Z","display_at":"2025-06-12T18:38:00Z","incident_id":"01JXJRKZWW7VRKA52Q3WWSJC4P","status":"investigating","updated_at":"2025-06-19T10:06:07Z"}]},{"id":"01JWFTD75MZ3EE50DRZ7M855TA","name":"Snapshots are not being processed","status":"resolved","created_at":"2025-05-30T05:25:02Z","updated_at":"2026-03-11T15:56:01Z","resolved_at":"2022-07-21T12:46:24Z","impact":"major","page_id":"01JWFTD1WG9PADY289K45S1WHV","incident_updates":[{"id":"01JWFTD75MW348G8VC1E9RWC3K","body":"This incident has been resolved.","created_at":"2025-05-30T05:25:02Z","display_at":"2022-07-21T12:46:24Z","incident_id":"01JWFTD75MZ3EE50DRZ7M855TA","status":"resolved","updated_at":"2025-06-19T10:06:08Z"},{"id":"01JWFTD75M8643MSV41QP8GZXH","body":"A fix has been implemented and we are monitoring the results.","created_at":"2025-05-30T05:25:02Z","display_at":"2022-07-21T12:30:03Z","incident_id":"01JWFTD75MZ3EE50DRZ7M855TA","status":"monitoring","updated_at":"2025-06-19T10:06:08Z"},{"id":"01JWFTD75MFTE2B415MCJEHV6W","body":"The issue has been identified and a fix is being implemented.","created_at":"2025-05-30T05:25:02Z","display_at":"2022-07-21T11:40:54Z","incident_id":"01JWFTD75MZ3EE50DRZ7M855TA","status":"identified","updated_at":"2025-06-19T10:06:08Z"}]},{"id":"01JWFTD7CNGZGF9TSZY4R8R5BM","name":"CAST AI is down due to Cloudflare outage","status":"resolved","created_at":"2025-05-30T05:25:02Z","updated_at":"2026-03-11T15:56:01Z","resolved_at":"2022-06-21T07:24:49Z","impact":"critical","page_id":"01JWFTD1WG9PADY289K45S1WHV","incident_updates":[{"id":"01JWFTD7CN90JPD84A3H8QJ6MK","body":"A fix has been implemented and we are monitoring the results.","created_at":"2025-05-30T05:25:02Z","display_at":"2022-06-21T07:24:49Z","incident_id":"01JWFTD7CNGZGF9TSZY4R8R5BM","status":"resolved","updated_at":"2025-06-19T10:06:08Z"},{"id":"01JWFTD7CNKD67RVCX28S4EDMQ","body":"Root cause identified: Cloudflare is experiencing an outage","created_at":"2025-05-30T05:25:02Z","display_at":"2022-06-21T06:50:43Z","incident_id":"01JWFTD7CNGZGF9TSZY4R8R5BM","status":"identified","updated_at":"2025-06-19T10:06:08Z"},{"id":"01JWFTD7CN741CSYN7TXH3HQSX","body":"Customers can't use CAST AI due to https://www.cloudflarestatus.com/ being down.","created_at":"2025-05-30T05:25:02Z","display_at":"2022-06-21T06:48:59Z","incident_id":"01JWFTD7CNGZGF9TSZY4R8R5BM","status":"investigating","updated_at":"2025-06-19T10:06:08Z"}]},{"id":"01JWFTD7K6WS11E0RCT1BYKE9N","name":"https://api.cast.ai status is Down","status":"resolved","created_at":"2025-05-30T05:25:02Z","updated_at":"2026-03-11T15:56:01Z","resolved_at":"2022-03-17T16:39:16Z","impact":"major","page_id":"01JWFTD1WG9PADY289K45S1WHV","incident_updates":[{"id":"01JWFTD7K6WXS465ZX54KD6SSR","body":"This incident has been resolved.","created_at":"2025-05-30T05:25:02Z","display_at":"2022-03-17T16:39:16Z","incident_id":"01JWFTD7K6WS11E0RCT1BYKE9N","status":"resolved","updated_at":"2025-06-19T10:06:08Z"},{"id":"01JWFTD7K6Y51P6Y52Q5NB4B0N","body":"We are continuing to investigate this issue.","created_at":"2025-05-30T05:25:02Z","display_at":"2022-03-17T16:16:48Z","incident_id":"01JWFTD7K6WS11E0RCT1BYKE9N","status":"investigating","updated_at":"2025-06-19T10:06:08Z"},{"id":"01JWFTD7K68VN6P2VCY3DWHA4T","body":"We are currently investigating this issue.","created_at":"2025-05-30T05:25:02Z","display_at":"2022-03-17T16:12:10Z","incident_id":"01JWFTD7K6WS11E0RCT1BYKE9N","status":"investigating","updated_at":"2025-06-19T10:06:08Z"}]},{"id":"01JWFTD7SAZZRZZ6E149CQ6GP0","name":"Autoscaler is down","status":"resolved","created_at":"2025-05-30T05:25:02Z","updated_at":"2026-03-11T15:56:01Z","resolved_at":"2022-03-04T12:11:31Z","impact":"critical","page_id":"01JWFTD1WG9PADY289K45S1WHV","incident_updates":[{"id":"01JWFTD7SAJZ7VEV98BKWA61W0","body":"Incident is closed. Total downtime: 11:00-11:51 (UTC)","created_at":"2025-05-30T05:25:03Z","display_at":"2022-03-04T12:11:31Z","incident_id":"01JWFTD7SAZZRZZ6E149CQ6GP0","status":"resolved","updated_at":"2025-06-19T10:06:08Z"},{"id":"01JWFTD7SAQW6H1DC0HJQX6JWW","body":"Monitoring the situation after the fix","created_at":"2025-05-30T05:25:03Z","display_at":"2022-03-04T12:07:08Z","incident_id":"01JWFTD7SAZZRZZ6E149CQ6GP0","status":"monitoring","updated_at":"2025-06-19T10:06:08Z"},{"id":"01JWFTD7SAVJRPJFMZQKZWY54F","body":"We are continuing to monitor for any further issues.","created_at":"2025-05-30T05:25:03Z","display_at":"2022-03-04T12:06:39Z","incident_id":"01JWFTD7SAZZRZZ6E149CQ6GP0","status":"monitoring","updated_at":"2025-06-19T10:06:08Z"},{"id":"01JWFTD7SAVZ6J1M1G1MD5CGGX","body":"A fix has been implemented and we are monitoring the results.","created_at":"2025-05-30T05:25:03Z","display_at":"2022-03-04T12:06:32Z","incident_id":"01JWFTD7SAZZRZZ6E149CQ6GP0","status":"monitoring","updated_at":"2025-06-19T10:06:08Z"},{"id":"01JWFTD7SABWTYD7D36SAXFJ6P","body":"The issue has been identified and fix is in progress","created_at":"2025-05-30T05:25:03Z","display_at":"2022-03-04T12:01:20Z","incident_id":"01JWFTD7SAZZRZZ6E149CQ6GP0","status":"identified","updated_at":"2025-06-19T10:06:08Z"},{"id":"01JWFTD7SA9VPV1JMK15C5WAC2","body":"We are continuing to investigate this issue.","created_at":"2025-05-30T05:25:02Z","display_at":"2022-03-04T11:49:24Z","incident_id":"01JWFTD7SAZZRZZ6E149CQ6GP0","status":"investigating","updated_at":"2025-06-19T10:06:08Z"},{"id":"01JWFTD7SAFB4B1A9YX4DSCQP8","body":"Investigation is in progress","created_at":"2025-05-30T05:25:02Z","display_at":"2022-03-04T11:00:59Z","incident_id":"01JWFTD7SAZZRZZ6E149CQ6GP0","status":"investigating","updated_at":"2025-06-19T10:06:08Z"}]},{"id":"01JWFTD85K51ZJYJ317Q7QW4QF","name":"Delayed generation of savings report and rebalancing plans","status":"resolved","created_at":"2025-05-30T05:25:03Z","updated_at":"2026-03-11T15:56:01Z","resolved_at":"2022-01-06T22:36:12Z","impact":"minor","page_id":"01JWFTD1WG9PADY289K45S1WHV","incident_updates":[{"id":"01JWFTD85KAX2G8RV6SZS1Q5DF","body":"Monitoring period is over, issue is fixed.","created_at":"2025-05-30T05:25:03Z","display_at":"2022-01-06T22:36:12Z","incident_id":"01JWFTD85K51ZJYJ317Q7QW4QF","status":"resolved","updated_at":"2025-06-19T10:06:08Z"},{"id":"01JWFTD85KV65AZVFGN4069ARC","body":"Issue has been fixed, we are monitoring the situation.","created_at":"2025-05-30T05:25:03Z","display_at":"2022-01-06T22:05:53Z","incident_id":"01JWFTD85K51ZJYJ317Q7QW4QF","status":"monitoring","updated_at":"2025-06-19T10:06:08Z"},{"id":"01JWFTD85K8JGCCEQMCSV0XPAS","body":"We are experiencing issue with messaging system that affects generation of savings reports and rebalancing plans.  Autoscaler is not affected.","created_at":"2025-05-30T05:25:03Z","display_at":"2022-01-06T21:52:48Z","incident_id":"01JWFTD85K51ZJYJ317Q7QW4QF","status":"investigating","updated_at":"2025-06-19T10:06:08Z"}]},{"id":"01JWFTD8BTN7QSDAPEVBVT603Y","name":"Degraded autoscaler service","status":"resolved","created_at":"2025-05-30T05:25:03Z","updated_at":"2026-03-11T15:56:01Z","resolved_at":"2022-01-06T06:30:00Z","impact":"none","page_id":"01JWFTD1WG9PADY289K45S1WHV","incident_updates":[{"id":"01JWFTD8BTNBV4YP916HG0ZBCZ","body":"06:30 (UTC) - Autoscaler stopped snapshot processing, due to out of memory issue. \n07:05 (UTC) - Service is back to normal","created_at":"2025-05-30T05:25:03Z","display_at":"2022-01-06T06:30:00Z","incident_id":"01JWFTD8BTN7QSDAPEVBVT603Y","status":"resolved","updated_at":"2025-06-19T10:06:08Z"}]},{"id":"01JWFTD8EVCENVK0M5K60VYM91","name":"Cluster snapshots not being processed","status":"resolved","created_at":"2025-05-30T05:25:03Z","updated_at":"2026-03-11T15:56:01Z","resolved_at":"2021-12-08T09:51:34Z","impact":"critical","page_id":"01JWFTD1WG9PADY289K45S1WHV","incident_updates":[{"id":"01JWFTD8EVHV5WG36GRYF94S59","body":"Incident has been resolved, monitoring period is now completed. \n\nPostmortem will be prepared asap.","created_at":"2025-05-30T05:25:03Z","display_at":"2021-12-08T09:51:34Z","incident_id":"01JWFTD8EVCENVK0M5K60VYM91","status":"resolved","updated_at":"2025-06-19T10:06:08Z"},{"id":"01JWFTD8EVDWAFXS11841QZMS8","body":"A fix has been implemented, autoscaling is working as expected, we are monitoring the situation.","created_at":"2025-05-30T05:25:03Z","display_at":"2021-12-08T08:47:11Z","incident_id":"01JWFTD8EVCENVK0M5K60VYM91","status":"monitoring","updated_at":"2025-06-19T10:06:08Z"},{"id":"01JWFTD8EVMPZ6M5RMPBFF4C32","body":"The issue has been found, fix is being applied.","created_at":"2025-05-30T05:25:03Z","display_at":"2021-12-08T07:55:26Z","incident_id":"01JWFTD8EVCENVK0M5K60VYM91","status":"identified","updated_at":"2025-06-19T10:06:08Z"},{"id":"01JWFTD8EV2YMDDQX1GGDN6XZZ","body":"We are currently investigating the issue","created_at":"2025-05-30T05:25:03Z","display_at":"2021-12-08T07:19:07Z","incident_id":"01JWFTD8EVCENVK0M5K60VYM91","status":"investigating","updated_at":"2025-06-19T10:06:08Z"}]},{"id":"01JWFTD8P9FBCE3TGAQBB8B1W4","name":"CAST AI service is down due to GCP cloud loadbalancer outage","status":"resolved","created_at":"2025-05-30T05:25:03Z","updated_at":"2026-03-11T15:56:01Z","resolved_at":"2021-11-17T08:43:28Z","impact":"critical","page_id":"01JWFTD1WG9PADY289K45S1WHV","incident_updates":[{"id":"01JWFTD8P9HVT05J53B9G0MMFD","body":"Monitoring completed. Google Cloud has solved the issue, no further downstream impact have been noticed.","created_at":"2025-05-30T05:25:04Z","display_at":"2021-11-17T08:43:28Z","incident_id":"01JWFTD8P9FBCE3TGAQBB8B1W4","status":"resolved","updated_at":"2025-06-19T10:06:08Z"},{"id":"01JWFTD8P96YM8FCPFRJ21NF31","body":"A fix has been implemented and we are monitoring the results.","created_at":"2025-05-30T05:25:03Z","display_at":"2021-11-16T18:15:56Z","incident_id":"01JWFTD8P9FBCE3TGAQBB8B1W4","status":"monitoring","updated_at":"2025-06-19T10:06:08Z"},{"id":"01JWFTD8P9ARZEV5ZXSAMZC7RH","body":"Root cause: Google Cloud issue https://status.cloud.google.com/incidents/6PM5mNd43NbMqjCZ5REh","created_at":"2025-05-30T05:25:03Z","display_at":"2021-11-16T18:15:51Z","incident_id":"01JWFTD8P9FBCE3TGAQBB8B1W4","status":"identified","updated_at":"2025-06-19T10:06:08Z"},{"id":"01JWFTD8P9GZTEWQ2WA8YF61X9","body":"We are continuing to investigate this issue.","created_at":"2025-05-30T05:25:03Z","display_at":"2021-11-16T18:10:27Z","incident_id":"01JWFTD8P9FBCE3TGAQBB8B1W4","status":"investigating","updated_at":"2025-06-19T10:06:08Z"},{"id":"01JWFTD8P9MDFMYERA44RHMM24","body":"We are currently investigating this issue.","created_at":"2025-05-30T05:25:03Z","display_at":"2021-11-16T18:03:22Z","incident_id":"01JWFTD8P9FBCE3TGAQBB8B1W4","status":"investigating","updated_at":"2025-06-19T10:06:08Z"}]},{"id":"01JWFTD8Z0H18E17F3T4H24F9Y","name":"Users can't login","status":"resolved","created_at":"2025-05-30T05:25:04Z","updated_at":"2026-03-11T15:56:01Z","resolved_at":"2021-10-07T07:24:11Z","impact":"critical","page_id":"01JWFTD1WG9PADY289K45S1WHV","incident_updates":[{"id":"01JWFTD8Z06JTZ9ERKDPN2KRGC","body":"This incident has been resolved.","created_at":"2025-05-30T05:25:04Z","display_at":"2021-10-07T07:24:11Z","incident_id":"01JWFTD8Z0H18E17F3T4H24F9Y","status":"resolved","updated_at":"2025-06-19T10:06:08Z"},{"id":"01JWFTD8Z0XWSA25EQKNJ47KHJ","body":"Investigation is in progress","created_at":"2025-05-30T05:25:04Z","display_at":"2021-10-07T07:19:56Z","incident_id":"01JWFTD8Z0H18E17F3T4H24F9Y","status":"investigating","updated_at":"2025-06-19T10:06:08Z"}]},{"id":"01JWFTD9FRZSD276D5BEVS3X38","name":"Autoscaler out of memory issue","status":"resolved","created_at":"2025-05-30T05:25:04Z","updated_at":"2026-03-11T15:56:01Z","resolved_at":"2021-06-09T19:46:54Z","impact":"critical","page_id":"01JWFTD1WG9PADY289K45S1WHV","incident_updates":[{"id":"01JWFTD9FRMBVBSGSZ68GGRA96","body":"Issue is fixed, we are back to normal operations","created_at":"2025-05-30T05:25:04Z","display_at":"2021-06-09T19:46:54Z","incident_id":"01JWFTD9FRZSD276D5BEVS3X38","status":"resolved","updated_at":"2025-06-19T10:06:08Z"},{"id":"01JWFTD9FRPYSATX465QWQBK2B","body":"We have applied the fix, and currently monitoring the situation.","created_at":"2025-05-30T05:25:04Z","display_at":"2021-06-09T19:41:32Z","incident_id":"01JWFTD9FRZSD276D5BEVS3X38","status":"monitoring","updated_at":"2025-06-19T10:06:08Z"},{"id":"01JWFTD9FR5JPEF32E1KXR0X4Y","body":"We are continuing to investigate this issue.","created_at":"2025-05-30T05:25:04Z","display_at":"2021-06-09T19:19:20Z","incident_id":"01JWFTD9FRZSD276D5BEVS3X38","status":"investigating","updated_at":"2025-06-19T10:06:08Z"},{"id":"01JWFTD9FRYVJ08KQRVC893JM4","body":"We are currently investigating issue that affects our autoscaler capability","created_at":"2025-05-30T05:25:04Z","display_at":"2021-06-09T18:44:01Z","incident_id":"01JWFTD9FRZSD276D5BEVS3X38","status":"investigating","updated_at":"2025-06-19T10:06:08Z"}]},{"id":"01JWFTD9RX5YEYWWX4BE61SHJ1","name":"Github \u0026 GitLab experiencing issues with CDN","status":"resolved","created_at":"2025-05-30T05:25:04Z","updated_at":"2026-03-11T15:56:01Z","resolved_at":"2021-06-08T11:41:07Z","impact":"minor","page_id":"01JWFTD1WG9PADY289K45S1WHV","incident_updates":[{"id":"01JWFTD9RX6DET8QR2SNK7F38M","body":"Github services are operational again. Addons functionality on CAST clusters is working as expected.","created_at":"2025-05-30T05:25:04Z","display_at":"2021-06-08T11:41:07Z","incident_id":"01JWFTD9RX5YEYWWX4BE61SHJ1","status":"resolved","updated_at":"2025-06-19T10:06:08Z"},{"id":"01JWFTD9RXMBZH7N3D37WS14QH","body":"Github \u0026 GitLab experience issues with a CDN provider, as a consequence CAST users can't install addons into clusters. We are monitoring the situation and will provide an update.","created_at":"2025-05-30T05:25:04Z","display_at":"2021-06-08T10:40:08Z","incident_id":"01JWFTD9RX5YEYWWX4BE61SHJ1","status":"monitoring","updated_at":"2025-06-19T10:06:08Z"}]},{"id":"01JWFTDA6DB7NNSBVT7BEWV85B","name":"Unable to create or connect clusters via console","status":"resolved","created_at":"2025-05-30T05:25:05Z","updated_at":"2026-03-11T15:56:01Z","resolved_at":"2021-05-26T11:28:49Z","impact":"major","page_id":"01JWFTD1WG9PADY289K45S1WHV","incident_updates":[{"id":"01JWFTDA6D61CDKBB9WEPTNGXJ","body":"Incident was resolved","created_at":"2025-05-30T05:25:05Z","display_at":"2021-05-26T11:28:49Z","incident_id":"01JWFTDA6DB7NNSBVT7BEWV85B","status":"resolved","updated_at":"2025-06-19T10:06:08Z"},{"id":"01JWFTDA6D1QJZY9EH9Y20RSV3","body":"We are currently investigating an issue affecting our console as users reported not being able to create or connect clusters.","created_at":"2025-05-30T05:25:05Z","display_at":"2021-05-26T11:10:50Z","incident_id":"01JWFTDA6DB7NNSBVT7BEWV85B","status":"investigating","updated_at":"2025-06-19T10:06:08Z"}]},{"id":"01JWFTDAB9D0B2YA9VE4NH0NM6","name":"Login issues","status":"resolved","created_at":"2025-05-30T05:25:05Z","updated_at":"2026-03-11T15:56:01Z","resolved_at":"2021-04-20T20:54:33Z","impact":"major","page_id":"01JWFTD1WG9PADY289K45S1WHV","incident_updates":[{"id":"01JWFTDAB938RM7RYBP88MT64A","body":"Auth0 continues to see performance improvements and rolls back to normal operations.","created_at":"2025-05-30T05:25:05Z","display_at":"2021-04-20T20:54:33Z","incident_id":"01JWFTDAB9D0B2YA9VE4NH0NM6","status":"resolved","updated_at":"2025-06-19T10:06:08Z"},{"id":"01JWFTDAB9QF32P09EEQ02RKC2","body":"Logins and signups are now working although with some possible slow-downs. We are still monitoring Auth0 for further issue clearance.","created_at":"2025-05-30T05:25:05Z","display_at":"2021-04-20T20:05:51Z","incident_id":"01JWFTDAB9D0B2YA9VE4NH0NM6","status":"monitoring","updated_at":"2025-06-19T10:06:08Z"},{"id":"01JWFTDAB9QB1PY37333E8G9XE","body":"We've confirmed that new login sessions can't be initiated. Our team is continuously monitoring Auth0 status. We are also looking into ways to reduce the impact this is generating.","created_at":"2025-05-30T05:25:05Z","display_at":"2021-04-20T16:10:30Z","incident_id":"01JWFTDAB9D0B2YA9VE4NH0NM6","status":"identified","updated_at":"2025-06-19T10:06:08Z"},{"id":"01JWFTDAB97D96TF68ZDWTZ26Q","body":"We’re currently experiencing issues with our authentication provider (Auth0), while that is being resolved, making new sessions to the console.cast.ai might not work.","created_at":"2025-05-30T05:25:05Z","display_at":"2021-04-20T16:09:26Z","incident_id":"01JWFTDAB9D0B2YA9VE4NH0NM6","status":"identified","updated_at":"2025-06-19T10:06:08Z"}]}]}