(original) (raw)

[{"id":"JTgxXCtXTaw3aEVzUUfa","number":"3709370212894413327","begin":"2026-04-10T02:08:00+00:00","created":"2026-04-10T10:07:12+00:00","modified":"2026-04-10T11:27:56+00:00","external_desc":"**Summary**\nGoogle Workspace users using Google IdP are unable to sign into their accounts using security keys and hardware based passkeys.\n**Description**\nWe are experiencing an issue with Workspace users beginning on Thursday, 2026-04-09 19:08 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Friday, 2026-04-10 04:00 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nGoogle Workspace users using Google IdP are unable to sign into their accounts using security keys and hardware based passkeys.\n**Workaround**\nBelow steps should be performed by Admin as the workaround:\n- Admin should revoke the key.\n- Admin to create the verification code to sign-in.\n- The user should sign-in using Backup verification code and then re-enrolls the security key.","updates":[{"created":"2026-04-10T11:27:56+00:00","modified":"2026-04-10T11:27:56+00:00","when":"2026-04-10T11:27:56+00:00","text":"**Summary**\nGoogle Workspace users using Google IdP are unable to sign into their accounts using security keys and hardware based passkeys.\n**Description**\nMitigation work is currently underway by our engineering team.\nWe do not have an ETA for mitigation at this point.\nWe will provide more information by Friday, 2026-04-10 09:00 PDT.\n**Customer Symptoms**\nGoogle Workspace users using Google IdP are unable to sign into their accounts using security keys and hardware based passkeys.\n**Workaround**\nBelow steps should be performed by Admin as the workaround:\n- Admin should revoke the key.\n- Admin to create the verification code to sign-in.\n- The user should sign-in using Backup verification code and then re-enrolls the security key.","status":"SERVICE_DISRUPTION"},{"created":"2026-04-10T10:55:50+00:00","modified":"2026-04-10T11:27:56+00:00","when":"2026-04-10T10:55:50+00:00","text":"**Summary**\nGoogle Workspace users using Google IdP are unable to sign into their accounts using security keys and hardware based passkeys.\n**Description**\nWe are experiencing an issue with Workspace users beginning on Thursday, 2026-04-09 19:08 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Friday, 2026-04-10 05:00 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nGoogle Workspace users using Google IdP are unable to sign into their accounts using security keys and hardware based passkeys.\n**Workaround**\nBelow steps should be performed by Admin as the workaround:\n- Admin should revoke the key.\n- Admin to create the verification code to sign-in.\n- The user should sign-in using Backup verification code and then re-enrolls the security key.","status":"SERVICE_DISRUPTION"},{"created":"2026-04-10T10:07:12+00:00","modified":"2026-04-10T10:56:57+00:00","when":"2026-04-10T10:07:12+00:00","text":"**Summary**\nGoogle Workspace users using Google IdP are unable to sign into their accounts using security keys and hardware based passkeys.\n**Description**\nWe are experiencing an issue with Workspace users beginning on Thursday, 2026-04-09 19:08 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Friday, 2026-04-10 04:00 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nGoogle Workspace users using Google IdP are unable to sign into their accounts using security keys and hardware based passkeys.\n**Workaround**\nBelow steps should be performed by Admin as the workaround:\n- Admin should revoke the key.\n- Admin to create the verification code to sign-in.\n- The user should sign-in using Backup verification code and then re-enrolls the security key.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2026-04-10T11:27:56+00:00","modified":"2026-04-10T11:27:56+00:00","when":"2026-04-10T11:27:56+00:00","text":"**Summary**\nGoogle Workspace users using Google IdP are unable to sign into their accounts using security keys and hardware based passkeys.\n**Description**\nMitigation work is currently underway by our engineering team.\nWe do not have an ETA for mitigation at this point.\nWe will provide more information by Friday, 2026-04-10 09:00 PDT.\n**Customer Symptoms**\nGoogle Workspace users using Google IdP are unable to sign into their accounts using security keys and hardware based passkeys.\n**Workaround**\nBelow steps should be performed by Admin as the workaround:\n- Admin should revoke the key.\n- Admin to create the verification code to sign-in.\n- The user should sign-in using Backup verification code and then re-enrolls the security key.","status":"SERVICE_DISRUPTION"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"xrkLpNMHAvPQczcU2uSb","service_name":"Admin Console","affected_products":[{"title":"Admin Console","id":"xrkLpNMHAvPQczcU2uSb"}],"uri":"incidents/JTgxXCtXTaw3aEVzUUfa"},{"id":"g3Jkha2rqwEMDsK11ThU","number":"13384860394023923849","begin":"2026-04-08T21:05:00+00:00","created":"2026-04-08T22:22:04+00:00","end":"2026-04-08T22:40:00+00:00","modified":"2026-04-08T23:29:45+00:00","external_desc":"**Summary**\nAppSheet customers are experiencing redirection to asia-southeast.appsheet.com instead of their intended application URLs.\n**Description**\nBeginning on Wednesday, 2026-04-08 14:05 PDT, we are experiencing an issue with AppSheet that caused users to be erroneously redirected to the asia-southeast region.\nOur engineers have identified the cause was due to a recent change and have reverted. Our internal telemetry shows the signs of recovery and are currently monitoring the recovery status. We expect full service recovery within an hour.\nWe will provide an update by Wednesday, 2026-04-08 16:30 PDT with current details\n**Customer Symptoms**\nUsers are automatically redirected to asia-southeast.appsheet.com, which may cause \"Forbidden\" or 403 errors when attempting to access their apps.\n**Workaround**\nNone at this time.","updates":[{"created":"2026-04-08T23:27:19+00:00","modified":"2026-04-08T23:27:19+00:00","when":"2026-04-08T23:27:19+00:00","text":"**Summary**\nAppSheet customers experienced redirection to asia-southeast.appsheet.com instead of their intended application URLs.\n**Description**\nThe issue with AppSheet has been resolved for all the affected customers as of Wednesday, 2026-04-08 15:40 US/Pacific.\nAs per the preliminary analysis, the issue was triggered by a recent change that caused the users to be erroneously redirected to the asia-southeast region.\nOur engineering team rolled back the change to mitigate the impact and verified the service stability.\nWe thank you for your patience while we worked on resolving the issue.\n**Customer Symptoms**\nUsers may have experienced automatic redirection to asia-southeast.appsheet.com, resulting in errors when attempting to access their apps.\n**Workaround**\nNone at this time.","status":"AVAILABLE"},{"created":"2026-04-08T22:22:04+00:00","modified":"2026-04-08T23:29:45+00:00","when":"2026-04-08T22:22:04+00:00","text":"**Summary**\nAppSheet customers are experiencing redirection to asia-southeast.appsheet.com instead of their intended application URLs.\n**Description**\nBeginning on Wednesday, 2026-04-08 14:05 PDT, we are experiencing an issue with AppSheet that caused users to be erroneously redirected to the asia-southeast region.\nOur engineers have identified the cause was due to a recent change and have reverted. Our internal telemetry shows the signs of recovery and are currently monitoring the recovery status. We expect full service recovery within an hour.\nWe will provide an update by Wednesday, 2026-04-08 16:30 PDT with current details\n**Customer Symptoms**\nUsers are automatically redirected to asia-southeast.appsheet.com, which may cause \"Forbidden\" or 403 errors when attempting to access their apps.\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2026-04-08T23:27:19+00:00","modified":"2026-04-08T23:27:19+00:00","when":"2026-04-08T23:27:19+00:00","text":"**Summary**\nAppSheet customers experienced redirection to asia-southeast.appsheet.com instead of their intended application URLs.\n**Description**\nThe issue with AppSheet has been resolved for all the affected customers as of Wednesday, 2026-04-08 15:40 US/Pacific.\nAs per the preliminary analysis, the issue was triggered by a recent change that caused the users to be erroneously redirected to the asia-southeast region.\nOur engineering team rolled back the change to mitigate the impact and verified the service stability.\nWe thank you for your patience while we worked on resolving the issue.\n**Customer Symptoms**\nUsers may have experienced automatic redirection to asia-southeast.appsheet.com, resulting in errors when attempting to access their apps.\n**Workaround**\nNone at this time.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"FWjKi5U7KX4FUUPThHAJ","service_name":"AppSheet","affected_products":[{"title":"AppSheet","id":"FWjKi5U7KX4FUUPThHAJ"}],"uri":"incidents/g3Jkha2rqwEMDsK11ThU"},{"id":"224ozRqzW4sFBDK8hLnT","number":"15711552392358063047","begin":"2026-04-08T13:30:00+00:00","created":"2026-04-08T18:22:42+00:00","end":"2026-04-08T21:40:00+00:00","modified":"2026-04-10T02:32:37+00:00","external_desc":"**Title:**\nCustomers may experience delays in receiving and sending emails.\n**Description:**\nWe are aware of an issue with Gmail users experiencing delays in receiving and sending emails beginning at Wednesday, 2026-04-08 06:30 PDT.\nMitigation work is currently underway by our engineering team.\nWe do not have an ETA for mitigation at this point.\nWe will provide an update by Wednesday, 2026-04-08 12:00 PDT with current details.\n**Symptoms:**\nCustomers may experience delays in receiving and sending emails.\n**Workaround:**\nNone at this time.","updates":[{"created":"2026-04-10T02:32:18+00:00","modified":"2026-04-10T02:32:37+00:00","when":"2026-04-10T02:32:18+00:00","text":"# Preliminary Incident Report\nWe apologize for the inconvenience this Gmail service disruption may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. A final Incident Report with preventative actions will be posted once our investigation is complete. If you have experienced an impact outside of what is listed below, please reach out to Google Workspace Support using the help article https://support.google.com/a/answer/1047213.\\n## Date/Time of the Issue (All time US/Pacific)\n- Incident Start: 08 April 2026 06:30\n- Incident End: 08 April 2026 14:40\n- Duration: 8 hours, 10 minutes\n## Summary\nOn Wednesday, 08 April 2026, Gmail customers may have experienced delays/failures when sending or receiving emails. The impact lasted for a duration of 8 hours, 10 minutes.\nThe issue primarily affected the delivery of messages to external recipient domains, where impacted users observed delays or received failure messages when sending emails.\nTo our customers whose communications were impacted during this disruption, we sincerely apologize. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.\n## Preliminary Root Cause\nThe preliminary root cause was identified as increased connection errors to some external mail services.\nGoogle engineers have begun a full root cause analysis and will provide additional information once it is available.\n## Remediation\nThe issue was detected by our engineering team through internal alerts. Engineering teams immediately started an investigation and took the following steps to contain the impact:\n**Account Suspension:** One of the domains responsible for spamming was identified, resulting in the suspension of the abusive account after review.\n**Traffic Blocking:** A block rule was implemented to specifically address and stop traffic originating from an abusive sending domain.\nTeams monitored rejection rates until they returned to normal levels, confirming that the system was healthy for over an hour to confirm resolution.\nGoogle is committed to improving its technology and operations to prevent similar service disruptions. A comprehensive Incident Report that will be published to provide further details and outline preventative measures.\n## Description of Impact\nOn Wednesday, 8 April 2026, starting at approximately 06:30, Gmail customers may have experienced delays/failures when sending or receiving emails. The issue primarily affected the delivery of messages to external recipient domains, with impacted users receiving bounce messages. Specifically, some users observed error messages stating, \"The user you are trying to contact is receiving email at a rate that prevents additional messages from being delivered\".","status":"AVAILABLE"},{"created":"2026-04-08T22:54:46+00:00","modified":"2026-04-10T02:32:18+00:00","when":"2026-04-08T22:54:46+00:00","text":"**Title**\nCustomers may have experienced delays in receiving and sending emails.\n**Description**\nThe issue with Gmail has been resolved for all affected users as of Wednesday, 2026-04-08 14:49 PDT.\nOur engineering team identified a potential issue with a noisy neighbour and has successfully mitigated impact.\nA full root cause analysis is in progress and further details will be shared through a subsequent incident report.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms**\nCustomers may have experienced delays in receiving and sending emails.\n**Workaround**\nNone at this time.","status":"AVAILABLE"},{"created":"2026-04-08T21:49:48+00:00","modified":"2026-04-08T22:54:46+00:00","when":"2026-04-08T21:49:48+00:00","text":"**Title**\nCustomers may have experienced delays in receiving and sending emails.\n**Description**\nWe are aware of an issue with Gmail users experiencing delays in receiving and sending emails beginning on Wednesday, 2026-04-08 06:30 PDT.\nOur engineering team has applied the mitigation, and our internal monitoring indicates that delays and errors have subsided.\nWe will continue to monitor for service stability to ensure full recovery.\nWe will provide an update by Wednesday, 2026-04-08 16:00 PDT with current details.\n**Symptoms**\nCustomers may have experienced delays in receiving and sending emails.\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2026-04-08T20:03:15+00:00","modified":"2026-04-08T21:49:48+00:00","when":"2026-04-08T20:03:15+00:00","text":"**Title:**\nCustomers may experience delays in receiving and sending emails.\n**Description:**\nWe are aware of an issue with Gmail users experiencing delays in receiving and sending emails beginning at Wednesday, 2026-04-08 06:30 PDT.\nMitigation work is currently underway by our engineering team.\nWe do not have an ETA for mitigation at this point.\nWe will provide an update by Wednesday, 2026-04-08 15:00 PDT with current details.\n**Symptoms:**\nCustomers may experience delays in receiving and sending emails.\n**Workaround:**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2026-04-08T19:02:21+00:00","modified":"2026-04-08T20:03:15+00:00","when":"2026-04-08T19:02:21+00:00","text":"**Title:**\nCustomers may experience delays in receiving and sending emails.\n**Description:**\nWe are aware of an issue with Gmail users experiencing delays in receiving and sending emails beginning at Wednesday, 2026-04-08 06:30 PDT.\nMitigation work is currently underway by our engineering team.\nWe do not have an ETA for mitigation at this point.\nWe will provide an update by Wednesday, 2026-04-08 13:00 PDT with current details.\n**Symptoms:**\nCustomers may experience delays in receiving and sending emails.\n**Workaround:**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2026-04-08T18:22:42+00:00","modified":"2026-04-08T19:02:21+00:00","when":"2026-04-08T18:22:42+00:00","text":"**Title:**\nCustomers may experience delays in receiving and sending emails.\n**Description:**\nWe are aware of an issue with Gmail users experiencing delays in receiving and sending emails beginning at Wednesday, 2026-04-08 06:30 PDT.\nMitigation work is currently underway by our engineering team.\nWe do not have an ETA for mitigation at this point.\nWe will provide an update by Wednesday, 2026-04-08 12:00 PDT with current details.\n**Symptoms:**\nCustomers may experience delays in receiving and sending emails.\n**Workaround:**\nNone at this time.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2026-04-10T02:32:18+00:00","modified":"2026-04-10T02:32:37+00:00","when":"2026-04-10T02:32:18+00:00","text":"# Preliminary Incident Report\nWe apologize for the inconvenience this Gmail service disruption may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. A final Incident Report with preventative actions will be posted once our investigation is complete. If you have experienced an impact outside of what is listed below, please reach out to Google Workspace Support using the help article https://support.google.com/a/answer/1047213.\\n## Date/Time of the Issue (All time US/Pacific)\n- Incident Start: 08 April 2026 06:30\n- Incident End: 08 April 2026 14:40\n- Duration: 8 hours, 10 minutes\n## Summary\nOn Wednesday, 08 April 2026, Gmail customers may have experienced delays/failures when sending or receiving emails. The impact lasted for a duration of 8 hours, 10 minutes.\nThe issue primarily affected the delivery of messages to external recipient domains, where impacted users observed delays or received failure messages when sending emails.\nTo our customers whose communications were impacted during this disruption, we sincerely apologize. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.\n## Preliminary Root Cause\nThe preliminary root cause was identified as increased connection errors to some external mail services.\nGoogle engineers have begun a full root cause analysis and will provide additional information once it is available.\n## Remediation\nThe issue was detected by our engineering team through internal alerts. Engineering teams immediately started an investigation and took the following steps to contain the impact:\n**Account Suspension:** One of the domains responsible for spamming was identified, resulting in the suspension of the abusive account after review.\n**Traffic Blocking:** A block rule was implemented to specifically address and stop traffic originating from an abusive sending domain.\nTeams monitored rejection rates until they returned to normal levels, confirming that the system was healthy for over an hour to confirm resolution.\nGoogle is committed to improving its technology and operations to prevent similar service disruptions. A comprehensive Incident Report that will be published to provide further details and outline preventative measures.\n## Description of Impact\nOn Wednesday, 8 April 2026, starting at approximately 06:30, Gmail customers may have experienced delays/failures when sending or receiving emails. The issue primarily affected the delivery of messages to external recipient domains, with impacted users receiving bounce messages. Specifically, some users observed error messages stating, \"The user you are trying to contact is receiving email at a rate that prevents additional messages from being delivered\".","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"prPt2Yra2CbGsbEm9cpC","service_name":"Gmail","affected_products":[{"title":"Gmail","id":"prPt2Yra2CbGsbEm9cpC"}],"uri":"incidents/224ozRqzW4sFBDK8hLnT"},{"id":"T5eKSXRWb6cNKTj4625p","number":"18104305937708421215","begin":"2026-03-20T22:30:00+00:00","created":"2026-03-21T03:19:14+00:00","end":"2026-03-21T05:03:00+00:00","modified":"2026-03-21T07:17:03+00:00","external_desc":"**Summary**\nWe are experiencing an issue with Google Keep where users may be unable to access keep.google.com via browser.\n**Description**\nThe issue began on Friday, 2026-03-20 15:30 US/Pacific.\nOur engineering team is aware of the situation and is actively implementing mitigations. We do not have any ETA at the moment.\nWe will provide the next update by Friday, 2026-03-20 21:00 US/Pacific.\n**Customer Symptoms**\nImpacted users may encounter connection timeouts or error pages (such as 502 Bad Gateway) when attempting to reach keep.google.com via a web browser. The Google Keep mobile applications are currently unaffected and remain operational.\n**Workaround**\nUsers are able to use Google Keep via the Android or iOS mobile app.","updates":[{"created":"2026-03-21T07:17:03+00:00","modified":"2026-03-21T07:17:03+00:00","when":"2026-03-21T07:17:03+00:00","text":"**Description**\nThe issue has been mitigated as of Friday, 2026-03-20 22:03 US/Pacific.\nThe service issue which resulted in connection timeouts and 502 error pages, was caused by excess traffic impacting the keep.google.com. Our engineering team has implemented a mitigation and is continuing to work to ensure long-term service stability and prevent recurrence.\nWe thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},{"created":"2026-03-21T05:33:44+00:00","modified":"2026-03-21T07:17:03+00:00","when":"2026-03-21T05:33:44+00:00","text":"**Summary**\nWe are experiencing an issue with Google Keep where users may be unable to access keep.google.com via browser.\n**Description**\nThe issue began on Friday, 2026-03-20 15:30 US/Pacific.\nOur mitigation efforts are underway and we are seeing the signs of recovery. We do not have an ETA for full mitigation.\nWe will provide the next update by Saturday, 2026-03-21 00:30 US/Pacific.\n**Customer Symptoms**\nImpacted users may encounter connection timeouts or error pages (such as 502 Bad Gateway) when attempting to reach keep.google.com via a web browser. The Google Keep mobile applications are currently unaffected and remain operational.\n**Workaround**\nUsers are able to use Google Keep via the Android or iOS mobile app.","status":"SERVICE_OUTAGE"},{"created":"2026-03-21T03:46:17+00:00","modified":"2026-03-21T05:33:44+00:00","when":"2026-03-21T03:46:17+00:00","text":"**Summary**\nWe are experiencing an issue with Google Keep where users may be unable to access keep.google.com via browser.\n**Description**\nThe issue began on Friday, 2026-03-20 15:30 US/Pacific.\nOur engineering team is aware of the situation and is actively implementing mitigations. We do not have any ETA at the moment.\nWe will provide the next update by Friday, 2026-03-20 23:00 US/Pacific.\n**Customer Symptoms**\nImpacted users may encounter connection timeouts or error pages (such as 502 Bad Gateway) when attempting to reach keep.google.com via a web browser. The Google Keep mobile applications are currently unaffected and remain operational.\n**Workaround**\nUsers are able to use Google Keep via the Android or iOS mobile app.","status":"SERVICE_OUTAGE"},{"created":"2026-03-21T03:19:14+00:00","modified":"2026-03-21T03:46:17+00:00","when":"2026-03-21T03:19:14+00:00","text":"**Summary**\nWe are experiencing an issue with Google Keep where users may be unable to access keep.google.com via browser.\n**Description**\nThe issue began on Friday, 2026-03-20 15:30 US/Pacific.\nOur engineering team is aware of the situation and is actively implementing mitigations. We do not have any ETA at the moment.\nWe will provide the next update by Friday, 2026-03-20 21:00 US/Pacific.\n**Customer Symptoms**\nImpacted users may encounter connection timeouts or error pages (such as 502 Bad Gateway) when attempting to reach keep.google.com via a web browser. The Google Keep mobile applications are currently unaffected and remain operational.\n**Workaround**\nUsers are able to use Google Keep via the Android or iOS mobile app.","status":"SERVICE_OUTAGE"}],"most_recent_update":{"created":"2026-03-21T07:17:03+00:00","modified":"2026-03-21T07:17:03+00:00","when":"2026-03-21T07:17:03+00:00","text":"**Description**\nThe issue has been mitigated as of Friday, 2026-03-20 22:03 US/Pacific.\nThe service issue which resulted in connection timeouts and 502 error pages, was caused by excess traffic impacting the keep.google.com. Our engineering team has implemented a mitigation and is continuing to work to ensure long-term service stability and prevent recurrence.\nWe thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},"status_impact":"SERVICE_OUTAGE","severity":"high","service_key":"aqVkUR9CFSoDHFUMD25G","service_name":"Google Keep","affected_products":[{"title":"Google Keep","id":"aqVkUR9CFSoDHFUMD25G"}],"uri":"incidents/T5eKSXRWb6cNKTj4625p"},{"id":"KamjizntSiNtRETcBL2Q","number":"3525893700290602912","begin":"2026-03-13T12:00:00+00:00","created":"2026-03-13T13:58:39+00:00","end":"2026-03-13T14:30:00+00:00","modified":"2026-03-13T14:56:52+00:00","external_desc":"**Description**\nWe are experiencing intermittent errors with Google Docs, Apps Script, Google Slides, Google Drive, Google Calendar beginning on Friday, 2026-03-13 05:00 PDT.\nOur engineering teams are actively working to mitigate the impact.\nWe will provide an update by Friday, 2026-03-13 08:00 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nA subset of customers may experience elevated error rates when accessing the impacted products in the US region.\n**Workaround**\nImpacted users may re-attempt the unsuccessful action.","updates":[{"created":"2026-03-13T14:56:52+00:00","modified":"2026-03-13T14:56:52+00:00","when":"2026-03-13T14:56:52+00:00","text":"**Description**\nThe issue with Google Workspace has been resolved for all affected users as of Friday, 2026-03-13 07:30 PDT.\nFrom preliminary analysis, the incident was triggered by resource contention in the impacted location, following a recent update. This led to elevated error rates and latency for several Workspace products.\nOur engineering teams mitigated the impact by increasing infrastructure capacity,performing traffic restarts, and rolling back the identified change\nWe thank you for your patience while we worked to resolve this issue.\n**Customer Symptoms**\nA subset of customers may have experienced elevated error rates when accessing the impacted products in the US region.\n**Workaround**\nNA. Issue is now mitigated","status":"AVAILABLE"},{"created":"2026-03-13T14:34:36+00:00","modified":"2026-03-13T14:56:52+00:00","when":"2026-03-13T14:34:36+00:00","text":"**Description**\nWe are experiencing intermittent errors with Google Docs, Apps Script, Google Slides, Google Drive, Google Calendar beginning on Friday, 2026-03-13 05:00 PDT.\nWe have confirmed that the issue affecting Google Drive is now mitigated. Users should no longer experience unavailability when accessing or interacting with Drive files.\nOur engineering teams continue to actively work on mitigating the remaining impacted products.\nWe will provide an update by Friday, 2026-03-13 09:00 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nA subset of customers may experience elevated error rates when accessing the impacted products in the US region.\n**Workaround**\nImpacted users may re-attempt the unsuccessful action.","status":"SERVICE_INFORMATION"},{"created":"2026-03-13T13:58:39+00:00","modified":"2026-03-13T14:34:36+00:00","when":"2026-03-13T13:58:39+00:00","text":"**Description**\nWe are experiencing intermittent errors with Google Docs, Apps Script, Google Slides, Google Drive, Google Calendar beginning on Friday, 2026-03-13 05:00 PDT.\nOur engineering teams are actively working to mitigate the impact.\nWe will provide an update by Friday, 2026-03-13 08:00 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nA subset of customers may experience elevated error rates when accessing the impacted products in the US region.\n**Workaround**\nImpacted users may re-attempt the unsuccessful action.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2026-03-13T14:56:52+00:00","modified":"2026-03-13T14:56:52+00:00","when":"2026-03-13T14:56:52+00:00","text":"**Description**\nThe issue with Google Workspace has been resolved for all affected users as of Friday, 2026-03-13 07:30 PDT.\nFrom preliminary analysis, the incident was triggered by resource contention in the impacted location, following a recent update. This led to elevated error rates and latency for several Workspace products.\nOur engineering teams mitigated the impact by increasing infrastructure capacity,performing traffic restarts, and rolling back the identified change\nWe thank you for your patience while we worked to resolve this issue.\n**Customer Symptoms**\nA subset of customers may have experienced elevated error rates when accessing the impacted products in the US region.\n**Workaround**\nNA. Issue is now mitigated","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"zall","service_name":"Multiple Products","affected_products":[{"title":"Apps Script","id":"tjQKAokSTcX1h4huHNF2"},{"title":"Google Calendar","id":"n5hVexuq1PM9onY9qrmo"},{"title":"Google Docs","id":"wNHuVFtZEWU5Mmj2eRCK"},{"title":"Google Drive","id":"VHNA7p3Z5p3iakj5sA8V"},{"title":"Google Slides","id":"DvWBgkXVhodA3WYwgZoB"}],"uri":"incidents/KamjizntSiNtRETcBL2Q"},{"id":"Cs3FwweNNVYwQzLnoy9D","number":"555014016045677938","begin":"2026-02-23T06:32:00+00:00","created":"2026-02-23T08:17:51+00:00","end":"2026-02-23T08:35:00+00:00","modified":"2026-02-23T08:54:39+00:00","external_desc":"**Summary**\nGoogle Chat is experiencing Issue with creating a new conversations\n**Description**\nOur engineering team continues to investigate the issue.\nWe will provide an update by Monday, 2026-02-23 00:45 PST with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nUsers may experience errors when attempting to create new conversations or spaces in Google Chat. Existing conversations remain functional.\n**Workaround**\nNone at the moment","updates":[{"created":"2026-02-23T08:54:39+00:00","modified":"2026-02-23T08:54:39+00:00","when":"2026-02-23T08:54:39+00:00","text":"The issue with Google Chat has been resolved for all affected users as of Monday, 2026-02-23 00:35 PST.\nAs per our preliminary analysis, the incident was triggered by a recent configuration update. Our engineering team performed a rollback to mitigate the impact and verified the stability.\nWe thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},{"created":"2026-02-23T08:34:22+00:00","modified":"2026-02-23T08:54:39+00:00","when":"2026-02-23T08:34:22+00:00","text":"**Summary**\nGoogle Chat is experiencing issues with creating new conversations.\n**Description**\nOur engineering teams are actively working on mitigation efforts to restore the ability to create new conversations as quickly as possible. We do not have any ETA at the moment.\nWe will provide an update by Monday, 2026-02-23 01:30 PST with current details.\n**Customer Symptoms**\nUsers may experience errors when attempting to create new Direct Messages (DM) or new chat spaces in Google Chat. This issue specifically affects the creation of new conversations; however, users can continue to send and receive messages in existing conversations and spaces\n**Workaround**\nNone at the moment","status":"SERVICE_DISRUPTION"},{"created":"2026-02-23T08:17:51+00:00","modified":"2026-02-23T08:34:22+00:00","when":"2026-02-23T08:17:51+00:00","text":"**Summary**\nGoogle Chat is experiencing Issue with creating a new conversations\n**Description**\nOur engineering team continues to investigate the issue.\nWe will provide an update by Monday, 2026-02-23 00:45 PST with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nUsers may experience errors when attempting to create new conversations or spaces in Google Chat. Existing conversations remain functional.\n**Workaround**\nNone at the moment","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2026-02-23T08:54:39+00:00","modified":"2026-02-23T08:54:39+00:00","when":"2026-02-23T08:54:39+00:00","text":"The issue with Google Chat has been resolved for all affected users as of Monday, 2026-02-23 00:35 PST.\nAs per our preliminary analysis, the incident was triggered by a recent configuration update. Our engineering team performed a rollback to mitigate the impact and verified the stability.\nWe thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"Ht9Vx5PFzDPHYvrnrvSk","service_name":"Google Chat","affected_products":[{"title":"Google Chat","id":"Ht9Vx5PFzDPHYvrnrvSk"}],"uri":"incidents/Cs3FwweNNVYwQzLnoy9D"},{"id":"nK23ZsSxV33wfkqXVeh3","number":"9941420584529604408","begin":"2026-02-18T16:30:00+00:00","created":"2026-02-19T10:57:52+00:00","end":"2026-02-22T20:08:00+00:00","modified":"2026-02-23T01:21:11+00:00","external_desc":"**Title** : Some customers may have experienced an issue with Gemini where conversation history may not have been visible.\n**Description**\nWe are investigating an issue where chat histories are not visible on Gemini Web (gemini.google.com) and the Gemini mobile apps on Wednesday, 2026-02-18 08:30 PST.\nOur engineering team has identified a background process that was causing the missing user conversation metadata and has successfully stopped the process to prevent further impact.\nMitigation efforts are underway, and we do not have an ETA for completion at this time.\nWe will provide an update by Thursday, 2026-02-19 09:00 PST with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms**\n* Users may find that their sidebar is empty or that previously saved conversations are missing.\n* Users may receive an empty response when attempting to list or search for existing conversations.\n* New conversations can still be created and utilized normally.\n**Workaround**\n* Impacted web users on gemini.google.com can navigate to Menu (top left) \u003e Settings \u0026 Help \u003e Activity.\n* Alternatively, users can navigate to myactivity.google.com/product/gemini to view their conversation history.\n* Mobile users could navigate to their profile (top right) and select Gemini Apps Activity.","updates":[{"created":"2026-02-23T01:20:58+00:00","modified":"2026-02-23T01:21:11+00:00","when":"2026-02-23T01:20:58+00:00","text":"**Title :** Some customers may have experienced an issue with Gemini where conversation history may not have been visible.\n**Description :**\nThe issue with Gemini has been resolved for all affected users as of Sunday, 2026-02-22 12:08 PST.\nBased on our initial analysis, the incident was caused by an internal job that caused some valid conversation metadata to be deleted, which resulted in the conversations from the side navigation to be hidden.\nUsers could still access the hidden conversations requests and responses from the activity section.\nOur engineering team mitigated the impact by restoring the data from the appropriate backups.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms :**\n* Users may have found that their sidebar was empty or that previously saved conversations were missing.\n* Users may have received an empty response when attempting to list or search for existing conversations.\n* New conversations were still created and utilized normally.\n**Workaround :**\n* Impacted web users on gemini.google.com can navigate to Menu (top left) \u003e Settings \u0026 Help \u003e Activity.\n* Alternatively, users can navigate to myactivity.google.com/product/gemini to view their conversation history.\n* Mobile users could navigate to their profile (top right) and select Gemini Apps Activity.","status":"AVAILABLE"},{"created":"2026-02-20T03:03:22+00:00","modified":"2026-02-23T01:20:58+00:00","when":"2026-02-20T03:03:22+00:00","text":"**Title**\nSome customers may have experienced an issue with Gemini where conversation history may not have been visible.\n**Description**\nWe are investigating an issue where chat histories are not visible on Gemini Web (gemini.google.com) and the Gemini mobile apps on Wednesday, 2026-02-18 08:30 PST.\nOur engineering team has identified a background process that was causing the missing user conversation metadata and has successfully stopped the process to prevent further impact.\nMitigation efforts are underway, and our engineers are working to restore the missing chat histories. We do not have an ETA for completion at this time.\nWe will provide an update by Friday, 2026-02-20 20:00 PST with current details.\n**Symptoms**\n** Users may find that their sidebar is empty or that previously saved conversations are missing.\n** Users may receive an empty response when attempting to list or search for existing conversations.\n** New conversations can still be created and utilized normally.\n**Workaround**\n* Impacted web users on gemini.google.com can navigate to Menu (top left) \u003e Settings \u0026 Help \u003e Activity.\n* Alternatively, users can navigate to myactivity.google.com/product/gemini to view their conversation history.\n* Mobile users could navigate to their profile (top right) and select Gemini Apps Activity.","status":"SERVICE_DISRUPTION"},{"created":"2026-02-20T03:02:15+00:00","modified":"2026-02-20T03:03:22+00:00","when":"2026-02-20T03:02:15+00:00","text":"**Title**\nSome customers may have experienced an issue with Gemini where conversation history may not have been visible.\n**Description**\nWe are investigating an issue where chat histories are not visible on Gemini Web (gemini.google.com) and the Gemini mobile apps on Wednesday, 2026-02-18 08:30 PST.\nOur engineering team has identified a background process that was causing the missing user conversation metadata and has successfully stopped the process to prevent further impact.\nMitigation efforts are underway, and our engineers are working to restore the missing chat histories. We do not have an ETA for completion at this time.\nWe will provide an update by Thursday, 2026-02-19 18:00 PST with current details.\n**Symptoms**\n** Users may find that their sidebar is empty or that previously saved conversations are missing.\n** Users may receive an empty response when attempting to list or search for existing conversations.\n** New conversations can still be created and utilized normally.\n**Workaround**\n* Impacted web users on gemini.google.com can navigate to Menu (top left) \u003e Settings \u0026 Help \u003e Activity.\n* Alternatively, users can navigate to myactivity.google.com/product/gemini to view their conversation history.\n* Mobile users could navigate to their profile (top right) and select Gemini Apps Activity.","status":"SERVICE_DISRUPTION"},{"created":"2026-02-19T16:49:12+00:00","modified":"2026-02-20T03:02:15+00:00","when":"2026-02-19T16:49:12+00:00","text":"**Title**\nSome customers may have experienced an issue with Gemini where conversation history may not have been visible.\n**Description**\nWe are investigating an issue where chat histories are not visible on Gemini Web (gemini.google.com) and the Gemini mobile apps on Wednesday, 2026-02-18 08:30 PST.\nOur engineering team has identified a background process that was causing the missing user conversation metadata and has successfully stopped the process to prevent further impact.\nMitigation efforts are underway, and our engineers are working to restore the missing chat histories. We do not have an ETA for completion at this time.\nWe will provide an update by Thursday, 2026-02-19 15:00 PST with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms**\n* Users may find that their sidebar is empty or that previously saved conversations are missing.\n* Users may receive an empty response when attempting to list or search for existing conversations.\n* New conversations can still be created and utilized normally.\n**Workaround**\n* Impacted web users on gemini.google.com can navigate to Menu (top left) \u003e Settings \u0026 Help \u003e Activity.\n* Alternatively, users can navigate to myactivity.google.com/product/gemini to view their conversation history.\n* Mobile users could navigate to their profile (top right) and select Gemini Apps Activity.","status":"SERVICE_DISRUPTION"},{"created":"2026-02-19T10:57:52+00:00","modified":"2026-02-19T16:49:12+00:00","when":"2026-02-19T10:57:52+00:00","text":"**Title** : Some customers may have experienced an issue with Gemini where conversation history may not have been visible.\n**Description**\nWe are investigating an issue where chat histories are not visible on Gemini Web (gemini.google.com) and the Gemini mobile apps on Wednesday, 2026-02-18 08:30 PST.\nOur engineering team has identified a background process that was causing the missing user conversation metadata and has successfully stopped the process to prevent further impact.\nMitigation efforts are underway, and we do not have an ETA for completion at this time.\nWe will provide an update by Thursday, 2026-02-19 09:00 PST with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms**\n* Users may find that their sidebar is empty or that previously saved conversations are missing.\n* Users may receive an empty response when attempting to list or search for existing conversations.\n* New conversations can still be created and utilized normally.\n**Workaround**\n* Impacted web users on gemini.google.com can navigate to Menu (top left) \u003e Settings \u0026 Help \u003e Activity.\n* Alternatively, users can navigate to myactivity.google.com/product/gemini to view their conversation history.\n* Mobile users could navigate to their profile (top right) and select Gemini Apps Activity.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2026-02-23T01:20:58+00:00","modified":"2026-02-23T01:21:11+00:00","when":"2026-02-23T01:20:58+00:00","text":"**Title :** Some customers may have experienced an issue with Gemini where conversation history may not have been visible.\n**Description :**\nThe issue with Gemini has been resolved for all affected users as of Sunday, 2026-02-22 12:08 PST.\nBased on our initial analysis, the incident was caused by an internal job that caused some valid conversation metadata to be deleted, which resulted in the conversations from the side navigation to be hidden.\nUsers could still access the hidden conversations requests and responses from the activity section.\nOur engineering team mitigated the impact by restoring the data from the appropriate backups.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms :**\n* Users may have found that their sidebar was empty or that previously saved conversations were missing.\n* Users may have received an empty response when attempting to list or search for existing conversations.\n* New conversations were still created and utilized normally.\n**Workaround :**\n* Impacted web users on gemini.google.com can navigate to Menu (top left) \u003e Settings \u0026 Help \u003e Activity.\n* Alternatively, users can navigate to myactivity.google.com/product/gemini to view their conversation history.\n* Mobile users could navigate to their profile (top right) and select Gemini Apps Activity.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"npdyhgECDJ6tB66MxXyo","service_name":"Gemini","affected_products":[{"title":"Gemini","id":"npdyhgECDJ6tB66MxXyo"}],"uri":"incidents/nK23ZsSxV33wfkqXVeh3"},{"id":"NNnDkY9CJ36annsfytjQ","number":"17176513564181414062","begin":"2026-01-24T13:02:00+00:00","created":"2026-01-24T16:34:41+00:00","end":"2026-01-24T17:55:00+00:00","modified":"2026-02-09T05:04:23+00:00","external_desc":"Gmail users might see banners indicating missing spam checks.\nWe are experiencing an issue with Gmail beginning on Saturday, 2026-01-24 05:02 US/Pacific.\nUsers might experience the following banner: “Be careful with this message. Gmail hasn't scanned this message for spam, unverified senders, or harmful software.”\nWe advise the users to be extra diligent in lieu of missing spam checks.\nOur engineering team is investigating the issue.\nWe will provide an update by Saturday, 2026-01-24 09:30 with current details.","updates":[{"created":"2026-02-06T04:32:31+00:00","modified":"2026-02-09T05:03:28+00:00","when":"2026-02-06T04:32:31+00:00","text":"# Incident Report\n## Summary\nOn Saturday, 24 January 2026, beginning at 05:02 US/Pacific, Gmail experienced high failure rates in spam checking for a duration of 4 hours and 53 minutes. Affected users saw a banner indicating their messages were not scanned for spam and saw some messages not labelled accordingly as promotional and social.\nFollowing recovery of the above-mentioned issues, some Gmail users experienced delivery delays up to 10 minutes or less. This issue was resolved for most users by 09:55 US/Pacific on Saturday, 24 January 2026.\nWe recognize the critical nature of Gmail's reliability to your operations and extend our sincere apologies for any disruption this incident may have caused to your organization. We are implementing immediate measures to enhance system performance and stability to prevent such an occurrence from happening again.\n## Root Cause\nThis incident was caused by an overload of Gmail's spam checking systems, which in turn was triggered by a temporary failure in a backend service. This led to a cascading failure due to excessive retries.\nThe sequence of events leading to customer impact was as follows:\n* **Trigger:** The incident began with a temporary error in a backend component within Gmail's mail processing pipeline, which is responsible for analyzing incoming messages.\n* **Cause:** While the initial error was temporary, the system's response to it caused the impact. A retry mechanism, designed to handle such transient failures, generated a massive and sustained volume of retry traffic. This cascading retry storm overwhelmed the capacity of multiple backend systems for several hours.\n* **Effect:** The overload on the spam checking services and related systems meant that there were mail delivery delays and not all incoming mail was fully classified. To protect users from potentially malicious content, Gmail displayed a warning banner on the affected emails. This issue resulted in some Gmail users experiencing message delivery delays, which were generally 10 minutes or less.\n## Remediation and Prevention\nGoogle engineers were alerted to the issue via automated monitoring at 05:12 US/Pacific on Saturday, 24 January 2026, and immediately began an investigation. The investigation identified high error rates and latency across the classification pipeline.\nTo mitigate the impact of the incident, Google engineers implemented several measures focused on increasing system capacity and temporarily reducing load.\nThe key mitigation actions taken were:\n* **Increasing Capacity:** Engineers increased resources for the affected services. This included general capacity increases for the spam classification services and related backend components. One specific action was a significant, emergency increase of resources for a backend service responsible for abuse detection.\n* **Reducing System Load:** To alleviate pressure on the systems, engineers temporarily disabled certain functionalities to free up headroom for the spam classification services to recover.\n* **Tuning Retry Logic:** To address the cascading issues caused by excessive retries, the team rolled out changes to the retry logic in the mail delivery system.\nGoogle is committed to preventing recurrence by implementing the following action items:\n* **Enhanced Emergency Controls :** We are enhancing our emergency control mechanisms and accelerating mitigation protocols to facilitate rapid recovery through temporary manual traffic tuning overrides during similar service disruptions.\n* **Improved Load shedding:** Implement criticality-aware load shedding across the spam system to protect services and subsystems during overload.\n* **Tune Message Delivery Retries:** Optimize the retry logic in the mail delivery system to avoid overwhelming downstream services.\n* **Improved Deadline Propagation:** Ensure that service deadlines are correctly propagated and honored across the entire spam classification stack.\n* **Reshape Processing Tasks:** Increase the memory and compute resources allocated to individual anti-abuse systems to handle spikes more gracefully.\n## Detailed Description of Impact\nOn Saturday, 24 January 2026 from 05:02 to 09:55 US/Pacific, many Gmail messages experienced elevated failure rates in spam checking. Affected users saw a banner indicating their messages were not scanned for spam and saw some messages not labelled accordingly as promotional and social, and experienced delivery delays, generally up to 10 minutes.\n**User Experience:**\n* **Warning Banners:** Users received a banner on incoming messages stating: _\"Be careful with this message. Gmail hasn't scanned this message for spam, unverified senders, or harmful software.\"_\n* **Delivery Delays:** Impacted messages experienced delivery delays, up to 10 minutes. However, there were no instances of lost emails or erroneously delivered during this period.\n* **Message Labeling:** Users may have noticed that promotional or social labels were not correctly applied during the disruption.","status":"AVAILABLE"},{"created":"2026-01-30T19:26:57+00:00","modified":"2026-02-09T05:04:23+00:00","when":"2026-01-30T19:26:57+00:00","text":"# Preliminary Incident Report\nWe apologize for the inconvenience this service disruption may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. A final Incident Report with preventative actions will be posted once our investigation is complete. If you have experienced an impact outside of what is listed below, please reach out to Google Workspace Support using the help article https://support.google.com/a/answer/1047213.\\n## Date/Time of the Issue (All time US/Pacific)\nIncident Start: 24 January 2026 05:02\nIncident End: 24 January 2026 09:55\nDuration: 4 hours and 53 minutes\n## Summary\nOn Saturday, 24 January 2026, Gmail experienced elevated email delivery time and reduction in the number of emails that were checked for spam for a duration of 4 hours and 53 minutes. Affected users saw email delivery delays and a banner indicating their messages were not scanned for spam on some email. We understand how important Gmail's reliability is for our users and sincerely apologize for the impact this disruption may have caused to your organization. We are taking immediate steps to improve our systems' performance and availability to prevent recurrence.\n## Preliminary Root Cause\nThis incident was caused by an overload of Gmail's spam checking systems, which in turn was triggered by a temporary failure in a backend service. This led to a cascading failure due to excessive retries.\nThe sequence of events leading to customer impact was as follows:\n- **Trigger:** The incident began when a backend component within Gmail's mail processing pipeline, which is responsible for analyzing incoming messages, demonstrated increased latency.\n- **Cause:** While the initial error was temporary, the system's response to it caused the impact. A retry mechanism, designed to handle such transient failures, generated a massive and sustained volume of retry traffic. This cascading retry storm overwhelmed the capacity of multiple backend systems for several hours.\n- **Effect:** The overload on the spam checking services and related systems meant that there were mail delivery delays and not all incoming mail was fully classified. To protect users from potentially malicious content, Gmail displayed a warning banner on the affected emails. This issue resulted in a subset of Gmail users experiencing message delivery delays, which were generally 10 minutes or less.\nGoogle engineers are performing a full root cause analysis and will provide additional information once it is available.\n## Remediation\nGoogle engineers were alerted to the issue via automated monitoring at 05:12 US/Pacific on Saturday, 24 January 2026, and immediately began an investigation. The investigation identified high error rates and latency across the classification pipeline.\nTo mitigate the impact of the incident, Google engineers implemented several measures focused on increasing system capacity and temporarily reducing load.\nThe key mitigation actions taken were:\n- **Increasing Capacity:** Google engineers increased resources for the affected services. This included general capacity increases for the spam classification services and related backend components. One specific action was a significant, emergency increase of resources for a backend service responsible for abuse detection.\n- **Reducing System Load:** To alleviate pressure on the systems, Google engineers temporarily disabled certain non essential functionality to increase headroom for the spam classification services. Google engineers also deployed an emergency configuration to the spam classification services that disabled a specific message analysis feature, which was expected to lower the error rate.\n- **Tuning Retry Logic:** To address the cascading issues caused by excessive retries, the team rolled out changes to the retry logic in the mail delivery system. This tuning involved reducing the number of retries and implementing emergency rate limiting for retries, applied at the top of the stack. This approach helped to quickly shed load, allowing backend services to recover.\n## Description of Impact\nOn Saturday, 24 January 2026 from 05:02 to 09:55 US/Pacific, many Gmail messages experienced elevated failure rates in spam checking. Affected users saw a banner indicating their messages were not scanned for spam and experienced delivery delays, generally 10 minutes or less.\n**User Experience:**\n- **Warning Banners:** Users received a banner on incoming messages stating: \"Be careful with this message. Gmail hasn't scanned this message for spam, unverified senders, or harmful software.\"\n- **Delivery Delays:** Impacted messages experienced delivery delays, majorly between 5 to 10 minutes. However, there were no instances of lost emails or erroneously delivered during this period.\n- Some users may have noticed that promotional or social labels were not correctly applied during the disruption.","status":"AVAILABLE"},{"created":"2026-01-25T01:28:46+00:00","modified":"2026-01-30T19:26:57+00:00","when":"2026-01-25T01:28:46+00:00","text":"**Summary :** Some Gmail users experienced a misclassification of emails in their inbox, additional spam warnings, and delays in receiving email.\n**Description :** The issue with Gmail email classification is resolved as of Saturday, 2026-01-24 09:55 US/Pacific.\nWe will publish an analysis of this incident once we have completed our internal investigation.\nWe thank you for your patience while we worked on resolving the issue.\n**Diagnosis / Customer Symptoms :** Some Gmail users experienced a misclassification of emails in their inbox, delays in receiving email. Additionally, misclassified spam warnings from the incident may persist for existing messages received before the issue resolution.","status":"SERVICE_INFORMATION"},{"created":"2026-01-25T00:07:37+00:00","modified":"2026-01-25T01:28:46+00:00","when":"2026-01-25T00:07:37+00:00","text":"**Description** We are experiencing an issue with Gmail beginning on Saturday, 2026-01-24 05:02 US/Pacific.\nWe are aware that some Gmail users are experiencing misclassification of emails in their inbox and additional spam warnings. We are actively working to resolve the issue. As always, we encourage users to follow standard best practices when engaging with messages from unknown senders.\nWe will provide an update by Saturday, 2026-01-24 17:00 US/Pacific with current details.","status":"SERVICE_INFORMATION"},{"created":"2026-01-24T23:00:07+00:00","modified":"2026-01-25T00:07:37+00:00","when":"2026-01-24T23:00:07+00:00","text":"**Description** We are experiencing an issue with Gmail beginning on Saturday, 2026-01-24 05:02 US/Pacific.\nWe are aware that some Gmail users are experiencing misclassification of emails in their inbox and additional spam warnings. We are actively working to resolve the issue. As always, we encourage users to follow standard best practices when engaging with messages from unknown senders.\nWe will provide an update by Saturday, 2026-01-24 14:00 US/Pacific with current details.","status":"SERVICE_INFORMATION"},{"created":"2026-01-24T20:15:55+00:00","modified":"2026-01-24T23:00:07+00:00","when":"2026-01-24T20:15:55+00:00","text":"**Description** We are experiencing an issue with Gmail beginning on Saturday, 2026-01-24 05:02 US/Pacific.\nWe are aware that some Gmail users are experiencing misclassification of emails in their inbox and additional spam warnings. We are actively working to resolve the issue. As always, we encourage users to follow standard best practices when engaging with messages from unknown senders.\nWe will provide an update by Saturday, 2026-01-24 13:30 US/Pacific with current details.","status":"SERVICE_INFORMATION"},{"created":"2026-01-24T18:54:58+00:00","modified":"2026-01-24T20:15:55+00:00","when":"2026-01-24T18:54:58+00:00","text":"**Description** We are experiencing an issue with Gmail beginning on Saturday, 2026-01-24 05:02 US/Pacific.\nWe are aware that some Gmail users are experiencing misclassification of emails in their inbox and additional spam warnings. We are actively working to resolve the issue. As always, we encourage users to follow standard best practices when engaging with messages from unknown senders.\nWe will provide an update by Saturday, 2026-01-24 12:00 US/Pacific with current details.","status":"SERVICE_INFORMATION"},{"created":"2026-01-24T17:51:26+00:00","modified":"2026-01-24T18:54:58+00:00","when":"2026-01-24T17:51:26+00:00","text":"**Description** We are experiencing an issue with Gmail beginning on Saturday, 2026-01-24 05:02 US/Pacific.\nWe are aware that some Gmail users are experiencing misclassification of emails in their inbox and additional spam warnings. We are actively working to resolve the issue. As always, we encourage users to follow standard best practices when engaging with messages from unknown senders.\nWe will provide an update by Saturday, 2026-01-24 11:00 US/Pacific with current details","status":"SERVICE_INFORMATION"},{"created":"2026-01-24T16:34:41+00:00","modified":"2026-01-24T17:51:26+00:00","when":"2026-01-24T16:34:41+00:00","text":"Gmail users might see banners indicating missing spam checks.\nWe are experiencing an issue with Gmail beginning on Saturday, 2026-01-24 05:02 US/Pacific.\nUsers might experience the following banner: “Be careful with this message. Gmail hasn't scanned this message for spam, unverified senders, or harmful software.”\nWe advise the users to be extra diligent in lieu of missing spam checks.\nOur engineering team is investigating the issue.\nWe will provide an update by Saturday, 2026-01-24 09:30 with current details.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2026-02-06T04:32:31+00:00","modified":"2026-02-09T05:03:28+00:00","when":"2026-02-06T04:32:31+00:00","text":"# Incident Report\n## Summary\nOn Saturday, 24 January 2026, beginning at 05:02 US/Pacific, Gmail experienced high failure rates in spam checking for a duration of 4 hours and 53 minutes. Affected users saw a banner indicating their messages were not scanned for spam and saw some messages not labelled accordingly as promotional and social.\nFollowing recovery of the above-mentioned issues, some Gmail users experienced delivery delays up to 10 minutes or less. This issue was resolved for most users by 09:55 US/Pacific on Saturday, 24 January 2026.\nWe recognize the critical nature of Gmail's reliability to your operations and extend our sincere apologies for any disruption this incident may have caused to your organization. We are implementing immediate measures to enhance system performance and stability to prevent such an occurrence from happening again.\n## Root Cause\nThis incident was caused by an overload of Gmail's spam checking systems, which in turn was triggered by a temporary failure in a backend service. This led to a cascading failure due to excessive retries.\nThe sequence of events leading to customer impact was as follows:\n* **Trigger:** The incident began with a temporary error in a backend component within Gmail's mail processing pipeline, which is responsible for analyzing incoming messages.\n* **Cause:** While the initial error was temporary, the system's response to it caused the impact. A retry mechanism, designed to handle such transient failures, generated a massive and sustained volume of retry traffic. This cascading retry storm overwhelmed the capacity of multiple backend systems for several hours.\n* **Effect:** The overload on the spam checking services and related systems meant that there were mail delivery delays and not all incoming mail was fully classified. To protect users from potentially malicious content, Gmail displayed a warning banner on the affected emails. This issue resulted in some Gmail users experiencing message delivery delays, which were generally 10 minutes or less.\n## Remediation and Prevention\nGoogle engineers were alerted to the issue via automated monitoring at 05:12 US/Pacific on Saturday, 24 January 2026, and immediately began an investigation. The investigation identified high error rates and latency across the classification pipeline.\nTo mitigate the impact of the incident, Google engineers implemented several measures focused on increasing system capacity and temporarily reducing load.\nThe key mitigation actions taken were:\n* **Increasing Capacity:** Engineers increased resources for the affected services. This included general capacity increases for the spam classification services and related backend components. One specific action was a significant, emergency increase of resources for a backend service responsible for abuse detection.\n* **Reducing System Load:** To alleviate pressure on the systems, engineers temporarily disabled certain functionalities to free up headroom for the spam classification services to recover.\n* **Tuning Retry Logic:** To address the cascading issues caused by excessive retries, the team rolled out changes to the retry logic in the mail delivery system.\nGoogle is committed to preventing recurrence by implementing the following action items:\n* **Enhanced Emergency Controls :** We are enhancing our emergency control mechanisms and accelerating mitigation protocols to facilitate rapid recovery through temporary manual traffic tuning overrides during similar service disruptions.\n* **Improved Load shedding:** Implement criticality-aware load shedding across the spam system to protect services and subsystems during overload.\n* **Tune Message Delivery Retries:** Optimize the retry logic in the mail delivery system to avoid overwhelming downstream services.\n* **Improved Deadline Propagation:** Ensure that service deadlines are correctly propagated and honored across the entire spam classification stack.\n* **Reshape Processing Tasks:** Increase the memory and compute resources allocated to individual anti-abuse systems to handle spikes more gracefully.\n## Detailed Description of Impact\nOn Saturday, 24 January 2026 from 05:02 to 09:55 US/Pacific, many Gmail messages experienced elevated failure rates in spam checking. Affected users saw a banner indicating their messages were not scanned for spam and saw some messages not labelled accordingly as promotional and social, and experienced delivery delays, generally up to 10 minutes.\n**User Experience:**\n* **Warning Banners:** Users received a banner on incoming messages stating: _\"Be careful with this message. Gmail hasn't scanned this message for spam, unverified senders, or harmful software.\"_\n* **Delivery Delays:** Impacted messages experienced delivery delays, up to 10 minutes. However, there were no instances of lost emails or erroneously delivered during this period.\n* **Message Labeling:** Users may have noticed that promotional or social labels were not correctly applied during the disruption.","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"prPt2Yra2CbGsbEm9cpC","service_name":"Gmail","affected_products":[{"title":"Gmail","id":"prPt2Yra2CbGsbEm9cpC"}],"uri":"incidents/NNnDkY9CJ36annsfytjQ"},{"id":"9q5n4sfDnxYzHj8FxEua","number":"17034918359433891825","begin":"2026-01-19T18:10:00+00:00","created":"2026-01-20T15:32:19+00:00","end":"2026-01-20T19:49:00+00:00","modified":"2026-01-23T02:07:21+00:00","external_desc":"**Title:** Google Workspace customers may be experiencing missing account metadata, including primary email address and account creation time in the Admin Console\n**Description:** We are experiencing an issue with Admin Console beginning at Monday, 2026-01-19 10:10 PST.\nMitigation work is currently underway by our engineering team.\nThe mitigation is expected to complete by Tuesday, 2026-01-20 07:45 PST.\nWe will provide more information by Tuesday, 2026-01-20 08:00 PST.\nWe apologize to all who are affected by the disruption.\n**Symptoms:**\n- **Admin Console**: Administrators searching for or viewing users may see empty fields for primary email addresses and account creation dates. The issue may also affect display of user information where a user's email address would normally be present (such as Reporting or Activity events).\n- **Directory API**: Applications accessing the Directory API, such as Google Cloud Directory Sync (GCDS), may receive responses with missing email and account creation metadata.\n**Workaround:** None at the time.","updates":[{"created":"2026-01-23T02:07:21+00:00","modified":"2026-01-23T02:07:21+00:00","when":"2026-01-23T02:07:21+00:00","text":"# Incident Report\n## Summary\nStarting 19 January at 10:10, Google Workspace customers may have experienced missing account metadata, including primary email address and account creation time in the Admin Console. Additionally, some customers experienced authentication failures for external services via Single Sign-On (SSO). The issue lasted for 1 day, 1 hour and 39 minutes. To our Workspace customers whose services were impacted during this disruption, we sincerely apologize. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nThis incident was caused by corrupted user account metadata in a secondary data store, triggered by a faulty code change.\nIn Google's infrastructure, user account information is maintained in a primary backend system of record. This data is periodically synchronized to secondary data stores, which applications such as the Admin Console use to retrieve and display user profile information.\nThe trigger for this incident was a faulty code change deployed to a synchronization job. This change caused the job to malfunction and write invalid or empty information into the secondary data store for a subset of user accounts. Although the correct user data remained intact in the primary backend system of record, the corrupted copy in the secondary data store was served to dependent services.\n## Remediation and Prevention\nGoogle engineers were first alerted to the potential issue via a support case submitted on 20 January at 03:32. Upon receiving the alert, engineering teams initiated an investigation and quickly identified that a faulty code change, which had been recently deployed to a synchronization job, was the primary source of the data discrepancies. This specific job was responsible for updating secondary data stores from the primary backend account system.\nTo resolve the incident, a multi-pronged remediation strategy was executed. A rollback of the application service was performed in parallel with a manual synchronization process to restore data for all affected users. This restoration relied on the primary backend account system, which remained accurate throughout the event. A specialized script was developed and run to correct the corrupted metadata for the impacted accounts. To ensure system stability during the recovery phase, the script was deployed in controlled batches. This systematic approach successfully mitigated the issue for all affected accounts and fully restored services by 20 January at 11:49.\nGoogle is committed to preventing a recurrence of this issue and is implementing the following action items:\n- Binary Release Controls: Automated release gates were established for the affected synchronization service to ensure stability and prevent a recurrence.\n- Enhanced Release Validation: We are implementing stricter validation checks within the synchronization job to detect and block invalid metadata writes before they reach secondary stores.\n- Monitoring and Alerting: We are expanding automated monitoring to detect data integrity discrepancies between our primary system of record and secondary data stores, allowing for faster detection.\n## Detailed Description of Impact\nThe service disruption occurred between 19 January at 10:10 and 20 January at 11:49, lasting a total of 1 day, 1 hour and 39 minutes. During this period, Google Workspace customers experienced significant issues with missing account metadata, specifically impacting the visibility and accuracy of user profiles.\n- Admin Console Impact: Administrators searching for or viewing user details found that primary email addresses appeared as empty fields. Additionally, the account creation date for affected users was incorrectly displayed as \"January 1, 1970\". This data corruption also cascaded into other areas where user email addresses are typically populated, such as Reporting logs and Activity events, hindering administrative oversight and auditing.\n- Directory API and Tooling: The issue extended beyond the web interface to programmatic access. Applications utilizing the Directory API, including critical tools like Google Cloud Directory Sync (GCDS), received incomplete responses. These API responses were missing vital fields, including primary emails and account creation metadata, potentially disrupting automated synchronization and provisioning workflows.\n- Single Sign-On (SSO) Failures: A subset of customers experienced authentication failures for external services. These failures occurred when the external services relied on Google Workspace as the identity provider via SSO, as the required user identifiers were unavailable during the disruption.","status":"AVAILABLE"},{"created":"2026-01-21T12:39:29+00:00","modified":"2026-01-23T02:07:21+00:00","when":"2026-01-21T12:39:29+00:00","text":"**Preliminary Incident Report**\nWe apologize for the inconvenience this service disruption may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. A final Incident Report with preventative actions will be posted once our investigation is complete. If you have experienced an impact outside of what is listed below, please reach out to Google Workspace Support using the help article https://support.google.com/a/answer/1047213.\\n\*\*Date/Time of the Issue (All time US/Pacific)**\nIncident Start: 19 January 2026, 10:10\nIncident End: 20 January 2026 11:49\nDuration: 1 day, 1 hour, 39 minutes\n**Summary**\nStarting 19 January at 10:10, Google Workspace customers may have experienced missing account metadata, including primary email address and account creation time in the Admin Console for a duration of 1 day, 1 hour and 39 minutes. To our Workspace customers whose services were impacted during this disruption, we sincerely apologize. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.\n**Preliminary Root Cause**\nThis incident was identified to be caused by corrupted user account metadata in a secondary data store, triggered by a faulty code change.\nIn Google's infrastructure, user account information is maintained in a primary backend system of record. This data is periodically synchronized to secondary data stores, which applications such as the Admin Console use to retrieve and display user profile information.\nThe trigger for this incident was a faulty code change deployed to a synchronization job. This change caused the job to malfunction and write invalid or empty information into the secondary data store for a subset of user accounts. Although the correct user data remained intact in the primary backend system of record, the corrupted copy in the secondary data store was served to dependent services.\nGoogle will complete a full Incident report in the following days that will provide a full root cause.\n**Remediation**\nGoogle engineers were alerted to the issue via support case on 20 January at 03:32. Engineering quickly identified a faulty code change that was deployed to a synchronization job as the source of the issue. A rollback of the application service was performed in parallel with synchronizing the data for affected users from the primary backend account system. A script was run to accomplish this, correcting the data for affected accounts. The script was deployed in batches, which led to the mitigation of the affected accounts and restored services on 20 January at 11:49.\n**Description of Impact**\nBetween 19 January, 10:10 and 20 January, 11:49,\nAdmin Console: Administrators searching for or viewing users may have seen empty fields for primary email addresses and an incorrect account creation date of \"January 1, 1970\". The issue may have also affected the display of user information where a user's email address would normally be present (such as in Reporting or Activity events).\nDirectory API: Applications accessing the Directory API, such as Google Cloud Directory Sync (GCDS), may have received responses with missing fields including email and account creation metadata.\nSingle Sign-On (SSO): Some customers experienced SSO failures for external services that rely on Google Workspace as an identity provider.","status":"AVAILABLE"},{"created":"2026-01-20T17:09:23+00:00","modified":"2026-01-21T12:39:29+00:00","when":"2026-01-20T17:09:23+00:00","text":"**Summary**\nGoogle Workspace customers may have experienced missing account metadata, including primary email address and account creation time in the Admin Console.\n**Description**\nThe issue with Admin Console has been resolved for all affected users as of Tuesday, 2026-01-20 11:49 PST.\nFrom preliminary analysis, the issue was due to a recent change to a job writing to the metadata store. Our engineers mitigated the issue by restoring the metadata from an unaffected source of truth.\nWe thank you for your patience while we worked on resolving the issue.\n**Diagnosis / Customer Symptoms**\nAdmin Console: Administrators searching for or viewing users may have seen empty fields for primary email addresses and account creation dates. The issue may have also affected display of user information where a user's email address would normally be present (such as Reporting or Activity events).\nDirectory API: Applications accessing the Directory API, such as Google Cloud Directory Sync (GCDS), may have received responses with missing email and account creation metadata.\nSome customers saw Single Sign-On (SSO) failures due to this issue.\n**Workaround**\nThe issue is resolved, no workaround necessary.","status":"SERVICE_DISRUPTION"},{"created":"2026-01-20T16:59:53+00:00","modified":"2026-01-20T17:09:23+00:00","when":"2026-01-20T16:59:53+00:00","text":"**Summary**\nGoogle Workspace customers may be experiencing missing account metadata, including primary email address and account creation time in the Admin Console\n**Description**\nWe believe the issue with Admin Console is mitigated for the vast majority of the users.\nEngineering continues to work on full resolution and is expected to complete by Tuesday, 2026-01-20 09:15 PST.\nWe will provide an update by Tuesday, 2026-01-20 09:30 PST with current details.\n**Diagnosis / Customer Symptoms**\nAdmin Console: Administrators searching for or viewing users may see empty fields for primary email addresses and account creation dates. The issue may also affect display of user information where a user's email address would normally be present (such as Reporting or Activity events).\nDirectory API: Applications accessing the Directory API, such as Google Cloud Directory Sync (GCDS), may receive responses with missing email and account creation metadata.\nSome customers may see Single Sign-On (SSO) failures due to this issue.\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2026-01-20T15:59:33+00:00","modified":"2026-01-20T16:59:53+00:00","when":"2026-01-20T15:59:33+00:00","text":"**Title:** Google Workspace customers may be experiencing missing account metadata, including primary email address and account creation time in the Admin Console\n**Description:** We believe the issue with Admin Console is partially resolved.\nFull resolution is expected to complete by Tuesday, 2026-01-20 08:30 PST.\nWe will provide an update by Tuesday, 2026-01-20 08:45 PST with current details.\n**Symptoms:**\n- **Admin Console**: Administrators searching for or viewing users may see empty fields for primary email addresses and account creation dates. The issue may also affect display of user information where a user's email address would normally be present (such as Reporting or Activity events).\n- **Directory API**: Applications accessing the Directory API, such as Google Cloud Directory Sync (GCDS), may receive responses with missing email and account creation metadata.\n**Workaround:** None at the time.","status":"SERVICE_DISRUPTION"},{"created":"2026-01-20T15:32:19+00:00","modified":"2026-01-21T12:32:47+00:00","when":"2026-01-20T15:32:19+00:00","text":"**Title:** Google Workspace customers may be experiencing missing account metadata, including primary email address and account creation time in the Admin Console\n**Description:** We are experiencing an issue with Admin Console beginning at Monday, 2026-01-19 10:10 PST.\nMitigation work is currently underway by our engineering team.\nThe mitigation is expected to complete by Tuesday, 2026-01-20 07:45 PST.\nWe will provide more information by Tuesday, 2026-01-20 08:00 PST.\nWe apologize to all who are affected by the disruption.\n**Symptoms:**\n- **Admin Console**: Administrators searching for or viewing users may see empty fields for primary email addresses and account creation dates. The issue may also affect display of user information where a user's email address would normally be present (such as Reporting or Activity events).\n- **Directory API**: Applications accessing the Directory API, such as Google Cloud Directory Sync (GCDS), may receive responses with missing email and account creation metadata.\n**Workaround:** None at the time.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2026-01-23T02:07:21+00:00","modified":"2026-01-23T02:07:21+00:00","when":"2026-01-23T02:07:21+00:00","text":"# Incident Report\n## Summary\nStarting 19 January at 10:10, Google Workspace customers may have experienced missing account metadata, including primary email address and account creation time in the Admin Console. Additionally, some customers experienced authentication failures for external services via Single Sign-On (SSO). The issue lasted for 1 day, 1 hour and 39 minutes. To our Workspace customers whose services were impacted during this disruption, we sincerely apologize. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nThis incident was caused by corrupted user account metadata in a secondary data store, triggered by a faulty code change.\nIn Google's infrastructure, user account information is maintained in a primary backend system of record. This data is periodically synchronized to secondary data stores, which applications such as the Admin Console use to retrieve and display user profile information.\nThe trigger for this incident was a faulty code change deployed to a synchronization job. This change caused the job to malfunction and write invalid or empty information into the secondary data store for a subset of user accounts. Although the correct user data remained intact in the primary backend system of record, the corrupted copy in the secondary data store was served to dependent services.\n## Remediation and Prevention\nGoogle engineers were first alerted to the potential issue via a support case submitted on 20 January at 03:32. Upon receiving the alert, engineering teams initiated an investigation and quickly identified that a faulty code change, which had been recently deployed to a synchronization job, was the primary source of the data discrepancies. This specific job was responsible for updating secondary data stores from the primary backend account system.\nTo resolve the incident, a multi-pronged remediation strategy was executed. A rollback of the application service was performed in parallel with a manual synchronization process to restore data for all affected users. This restoration relied on the primary backend account system, which remained accurate throughout the event. A specialized script was developed and run to correct the corrupted metadata for the impacted accounts. To ensure system stability during the recovery phase, the script was deployed in controlled batches. This systematic approach successfully mitigated the issue for all affected accounts and fully restored services by 20 January at 11:49.\nGoogle is committed to preventing a recurrence of this issue and is implementing the following action items:\n- Binary Release Controls: Automated release gates were established for the affected synchronization service to ensure stability and prevent a recurrence.\n- Enhanced Release Validation: We are implementing stricter validation checks within the synchronization job to detect and block invalid metadata writes before they reach secondary stores.\n- Monitoring and Alerting: We are expanding automated monitoring to detect data integrity discrepancies between our primary system of record and secondary data stores, allowing for faster detection.\n## Detailed Description of Impact\nThe service disruption occurred between 19 January at 10:10 and 20 January at 11:49, lasting a total of 1 day, 1 hour and 39 minutes. During this period, Google Workspace customers experienced significant issues with missing account metadata, specifically impacting the visibility and accuracy of user profiles.\n- Admin Console Impact: Administrators searching for or viewing user details found that primary email addresses appeared as empty fields. Additionally, the account creation date for affected users was incorrectly displayed as \"January 1, 1970\". This data corruption also cascaded into other areas where user email addresses are typically populated, such as Reporting logs and Activity events, hindering administrative oversight and auditing.\n- Directory API and Tooling: The issue extended beyond the web interface to programmatic access. Applications utilizing the Directory API, including critical tools like Google Cloud Directory Sync (GCDS), received incomplete responses. These API responses were missing vital fields, including primary emails and account creation metadata, potentially disrupting automated synchronization and provisioning workflows.\n- Single Sign-On (SSO) Failures: A subset of customers experienced authentication failures for external services. These failures occurred when the external services relied on Google Workspace as the identity provider via SSO, as the required user identifiers were unavailable during the disruption.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"xrkLpNMHAvPQczcU2uSb","service_name":"Admin Console","affected_products":[{"title":"Admin Console","id":"xrkLpNMHAvPQczcU2uSb"}],"uri":"incidents/9q5n4sfDnxYzHj8FxEua"},{"id":"THtvkDYkSBrQpo3vh82V","number":"6883901939019944266","begin":"2026-01-08T13:00:00+00:00","created":"2026-01-13T23:10:54+00:00","end":"2026-01-14T21:00:00+00:00","modified":"2026-01-14T21:30:52+00:00","external_desc":"**Summary**\nGoogle Voice customers may experience MMS messages not being delivered to recipients.\n**Description**\nWe are experiencing an issue with sending MMS messages via Google Voice beginning on Thursday, 2026-01-08 05:00 PST.\nOur engineering team continues to investigate the issue with a 3rd party vendor.\nWe will provide an update by Tuesday, 2026-01-13 20:30 PST with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nGoogle Voice customers may experience issues while sending MMS messages where the messages may appear to be successfully sent but recipients may not receive them.\n**Workaround**\nSMS messages can be sent successfully, so the users can send 1-to-1 messages or messages without attachments instead of MMS messages.","updates":[{"created":"2026-01-14T21:30:26+00:00","modified":"2026-01-14T21:30:52+00:00","when":"2026-01-14T21:30:26+00:00","text":"The issue with Google Voice has been resolved for all affected users as of Wednesday, 2026-01-14 13:00 PST.\nWe thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},{"created":"2026-01-14T18:33:52+00:00","modified":"2026-01-14T21:30:26+00:00","when":"2026-01-14T18:33:52+00:00","text":"**Summary**\nGoogle Voice customers may experience MMS messages not being delivered to recipients.\n**Description**\nWe are experiencing an issue with sending MMS messages via Google Voice beginning on Thursday, 2026-01-08 05:00 PST.\nOur engineering team continues to investigate the issue with a 3rd party vendor to mitigate the issue.\nWe will provide an update by Thursday, 2026-01-15 10:30 PST with current details.\n**Customer Symptoms**\nGoogle Voice customers may experience issues while sending MMS messages where the messages may appear to be successfully sent but recipients may not receive them.\n**Workaround**\nSMS messages can be sent successfully, so the users can send 1-to-1 messages or messages without attachments instead of MMS messages.","status":"SERVICE_INFORMATION"},{"created":"2026-01-14T04:16:42+00:00","modified":"2026-01-14T18:33:52+00:00","when":"2026-01-14T04:16:42+00:00","text":"**Summary**\nGoogle Voice customers may experience MMS messages not being delivered to recipients.\n**Description**\nWe are experiencing an issue with sending MMS messages via Google Voice beginning on Thursday, 2026-01-08 05:00 PST.\nOur engineering team continues to investigate the issue with a 3rd party vendor to mitigate the issue.\nWe will provide an update by Wednesday, 2026-01-14 10:30 PST with current details.\n**Customer Symptoms**\nGoogle Voice customers may experience issues while sending MMS messages where the messages may appear to be successfully sent but recipients may not receive them.\n**Workaround**\nSMS messages can be sent successfully, so the users can send 1-to-1 messages or messages without attachments instead of MMS messages.","status":"SERVICE_INFORMATION"},{"created":"2026-01-13T23:10:54+00:00","modified":"2026-01-14T04:16:42+00:00","when":"2026-01-13T23:10:54+00:00","text":"**Summary**\nGoogle Voice customers may experience MMS messages not being delivered to recipients.\n**Description**\nWe are experiencing an issue with sending MMS messages via Google Voice beginning on Thursday, 2026-01-08 05:00 PST.\nOur engineering team continues to investigate the issue with a 3rd party vendor.\nWe will provide an update by Tuesday, 2026-01-13 20:30 PST with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nGoogle Voice customers may experience issues while sending MMS messages where the messages may appear to be successfully sent but recipients may not receive them.\n**Workaround**\nSMS messages can be sent successfully, so the users can send 1-to-1 messages or messages without attachments instead of MMS messages.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2026-01-14T21:30:26+00:00","modified":"2026-01-14T21:30:52+00:00","when":"2026-01-14T21:30:26+00:00","text":"The issue with Google Voice has been resolved for all affected users as of Wednesday, 2026-01-14 13:00 PST.\nWe thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"gh79SrQk6wqonWoxPdCm","service_name":"Google Voice","affected_products":[{"title":"Google Voice","id":"gh79SrQk6wqonWoxPdCm"}],"uri":"incidents/THtvkDYkSBrQpo3vh82V"},{"id":"Xck6rcpnzZ4pSTAY2MVa","number":"3255459562132126418","begin":"2025-12-01T08:00:00+00:00","created":"2025-12-02T04:20:49+00:00","end":"2025-12-04T21:23:00+00:00","modified":"2025-12-04T21:28:52+00:00","external_desc":"**Summary**\nSome Google Meet customers may experience an issue with ‘Continuous meeting chat’ feature\n**Description**\nGoogle Workspace released a feature to a subset of Workspace customers [Continue your conversations in Google Chat](https://support.google.com/meet/answer/9308979?visit\_id=639002449653293387-821172296\\u0026p=chat\_in\_meet\\u0026rd=1).\\nSome Google Workspace users are experiencing issues with Google Meet host controls for this feature. This may include inability to disable the feature or the ‘disable’ option not being visible under the host controls within Google Calendar meeting invites.\nWe are investigating and will provide an update by Friday, 2025-12-05 10:00 PST with current details.\nWe apologize to all who are affected by this issue.\n**Symptoms**\nThe impacted users are unable to observe Chat moderation settings under the Host Controls for ‘Continuous meeting chat’ feature.\n**Workaround**\nNone at this time.","updates":[{"created":"2025-12-04T21:28:49+00:00","modified":"2025-12-04T21:28:52+00:00","when":"2025-12-04T21:28:49+00:00","text":"**Summary**\nSome Google Meet customers may experience an issue with ‘Continuous meeting chat’ feature\n**Description**\nThe rollback of the \"Continue your conversations in Google Chat\" feature in Google Meet is now complete. This resolves the issues some users experienced, including the inability to disable the feature and the missing toggle in Calendar host controls.\nWe will commence the rollout of this feature again starting Monday, 2025-12-08. No further updates will be provided here. For more detailed information regarding the updated rollout schedule, please refer to the [Google Workspace Updates Blog](https://workspaceupdates.googleblog.com/2025/11/continue-conversations-google-chat-google-meet.html).\\nWe apologize for the disruption this may have caused.\n**Symptoms**\nThe impacted users are unable to observe Chat moderation settings under the Host Controls for ‘Continuous meeting chat’ feature.\n**Workaround**\nNone at this time.","status":"AVAILABLE"},{"created":"2025-12-02T22:42:20+00:00","modified":"2025-12-04T21:28:49+00:00","when":"2025-12-02T22:42:20+00:00","text":"**Summary**\nSome Google Meet customers may experience an issue with ‘Continuous meeting chat’ feature\n**Description**\nGoogle Workspace released a feature to a subset of Workspace customers [Continue your conversations in Google Chat](https://support.google.com/meet/answer/9308979?visit\_id=639002449653293387-821172296\\u0026p=chat\_in\_meet\\u0026rd=1).\\nSome Google Workspace users are experiencing issues with Google Meet host controls for this feature. This may include inability to disable the feature or the ‘disable’ option not being visible under the host controls within Google Calendar meeting invites.\nTo resolve the issues impacting host controls for the \"Continue your conversations in Google Chat\" feature, we are in the process of rolling back this feature. We will provide a further update by Monday, 2025-12-08 10:00 PST.\nWe apologize to all who are affected by this issue.\n**Symptoms**\nThe impacted users are unable to observe Chat moderation settings under the Host Controls for ‘Continuous meeting chat’ feature.\n**Workaround**\nNone at this time.","status":"SERVICE_INFORMATION"},{"created":"2025-12-02T04:20:49+00:00","modified":"2025-12-02T22:42:20+00:00","when":"2025-12-02T04:20:49+00:00","text":"**Summary**\nSome Google Meet customers may experience an issue with ‘Continuous meeting chat’ feature\n**Description**\nGoogle Workspace released a feature to a subset of Workspace customers [Continue your conversations in Google Chat](https://support.google.com/meet/answer/9308979?visit\_id=639002449653293387-821172296\\u0026p=chat\_in\_meet\\u0026rd=1).\\nSome Google Workspace users are experiencing issues with Google Meet host controls for this feature. This may include inability to disable the feature or the ‘disable’ option not being visible under the host controls within Google Calendar meeting invites.\nWe are investigating and will provide an update by Friday, 2025-12-05 10:00 PST with current details.\nWe apologize to all who are affected by this issue.\n**Symptoms**\nThe impacted users are unable to observe Chat moderation settings under the Host Controls for ‘Continuous meeting chat’ feature.\n**Workaround**\nNone at this time.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-12-04T21:28:49+00:00","modified":"2025-12-04T21:28:52+00:00","when":"2025-12-04T21:28:49+00:00","text":"**Summary**\nSome Google Meet customers may experience an issue with ‘Continuous meeting chat’ feature\n**Description**\nThe rollback of the \"Continue your conversations in Google Chat\" feature in Google Meet is now complete. This resolves the issues some users experienced, including the inability to disable the feature and the missing toggle in Calendar host controls.\nWe will commence the rollout of this feature again starting Monday, 2025-12-08. No further updates will be provided here. For more detailed information regarding the updated rollout schedule, please refer to the [Google Workspace Updates Blog](https://workspaceupdates.googleblog.com/2025/11/continue-conversations-google-chat-google-meet.html).\\nWe apologize for the disruption this may have caused.\n**Symptoms**\nThe impacted users are unable to observe Chat moderation settings under the Host Controls for ‘Continuous meeting chat’ feature.\n**Workaround**\nNone at this time.","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"sUH4BQXzYXma7NiS94Hi","service_name":"Google Meet","affected_products":[{"title":"Google Meet","id":"sUH4BQXzYXma7NiS94Hi"}],"uri":"incidents/Xck6rcpnzZ4pSTAY2MVa"},{"id":"F9Tft7xb3xqenFoKvY4W","number":"3094282244665890171","begin":"2025-11-26T05:34:00+00:00","created":"2025-11-26T07:38:58+00:00","end":"2025-11-26T10:30:00+00:00","modified":"2025-12-03T08:00:57+00:00","external_desc":"**Title**\nGoogle Meet customers are experiencing issues loading meet.google.com in the Asia region.\n**Description**\nWe are experiencing an issue with Google Meet beginning on Tuesday, 2025-11-25 21:34 PST.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Wednesday, 2025-11-26 02:00 PST with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms**\nCustomers are experiencing issues loading meet.google.com.\n**Workaround**\nCustomers can use the mobile version to load Meet.","updates":[{"created":"2025-12-02T12:52:38+00:00","modified":"2025-12-03T08:00:57+00:00","when":"2025-12-02T12:52:38+00:00","text":"**# Incident Report**\n**## Summary**\nBetween 25 November 2025 21:34 US/Pacific and 26 November 2025 2:30 US/Pacific, Google Meet experienced a service disruption for a total duration of 4 hours and 56 minutes. A subset of Google Meet users in Asia were unable to start or join meetings via the web and on meeting devices.\nGoogle is committed to quickly and continually improving our technology and operations to prevent service disruptions. We appreciate your patience and apologize again for the impact to your organization. We thank you for your business.\n**## Root Cause**\nThe incident was caused by an issue within the Meet web frontend server located in the Singapore (SIN) metropolitan area, which is a critical component and responsible for handling Google Meet connections for users in the Asia region.\nThe root cause of the issue was a software update to the Meet web frontend server. The update introduced a blocking call to a dependency in a critical code path, creating a thread deadlock risk.\nApproximately three hours after the update was complete, tasks started deadlocking. The trigger was a combination of regional traffic patterns and latency accessing the dependency involved. This led to stuck requests, Out-Of-Memory (OOM) errors and subsequent task crashes leading to failures when users attempted to load the meeting UI or join meetings.\n**## Remediation and Prevention**\nThe issue started on 25 November 2025 at 21:34 US/Pacific and the problem was first detected by Google's internal alerting systems at 21:41 US/Pacific. Our engineers promptly began working to restore service by taking the following actions in chronological order:\n**25 November, 22:08 US/Pacific** - Shifted traffic away from the most affected Meet web frontend servers.\nEfforts were made to shift traffic away from the specific Meet web frontend servers experiencing the crashes.\n**25 November, 22:20 US/Pacific** - Increase in capacity\nOur engineers increased the resources available to the affected service.\n**25 November, 23:27 US/Pacific** - Rollback of the webfrontend binary\nThe most effective step taken was reverting the software on the Meet web frontend server back to the previous stable version. This action mitigated the issue and restored service availability.\nService was fully restored on **26 November 2025 at 02:30 US/Pacific** after the rollback was complete.\nGoogle is completing the following actions to prevent a repeat of this issue:\n- We are reviewing critical code paths to identify and address potential blocking operations for the dependency that resulted in the deadlocking.\n- We are adding additional checks to prevent adding blocking operations in critical code paths.\n- Increase the load in our canary environments to make it more likely to detect similar issues.\n**## Detailed Description of Impact**\nA subset of Google Meet users in the Asia Pacific region experienced issues loading the meeting user interface (meet.google.com) or were unable to start or join meetings between 25 November 2025 21:34 US/Pacific and 26 November 2025 2:30 US/Pacific. The issue primarily affected users accessing Google Meet via Web browsers and Meeting Devices. Mobile users were not affected.","status":"AVAILABLE"},{"created":"2025-11-27T13:49:49+00:00","modified":"2025-12-02T12:53:24+00:00","when":"2025-11-27T13:49:49+00:00","text":"**# Preliminary Incident Report**\nWe apologize for the inconvenience this service disruption may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Workspace Support using the help article https://support.google.com/a/answer/1047213.\\n\*\*## Summary**\nBetween 25 November 2025 21:34 US/Pacific and 26 November 2025 2:30 US/Pacific, Google Meet experienced a service disruption for a total duration of 5 hours and 10 minutes. A subset of Google Meet users in the Asia region were unable to start or join meetings via the web and on meeting devices.\nGoogle is committed to quickly and continually improving our technology and operations to prevent service disruptions. We appreciate your patience and apologize again for the impact to your organization. We thank you for your business.\n**## Root Cause**\nThe incident was caused by an issue within the Meet web frontend server located in the Singapore (SIN) metropolitan area, which is a critical component and responsible for handling Google Meet connections for users in the Asia region.\nThe issue began with a software update to the Meet web frontend server. Approximately three hours after this update was complete, a combination of traffic patterns and a code change creating a thread deadlock risk which is currently being investigated further, acted as the trigger for the outage.\nThis trigger led to a sustained period of long-hanging requests in the server's processing queue. This abnormal workload ultimately consumed all available resources, causing the server to experience Out-Of-Memory (OOM) errors and subsequent task crashes. The result (effect) was that the server could not properly process requests, leading to failures when users attempted to load the meeting UI or join meetings.\nFurthermore, the issue detection was slightly delayed because traffic patterns in the immediate interval after the software update did not trigger the issue, delaying our response and mitigation efforts.\n**## Remediation**\nThe issue started on 25 November 2025 at 21:34 US/Pacific and the problem was first detected by Google's internal alerting systems at 21:41 US/Pacific. Our engineers promptly began working to restore service by taking the following actions in chronological order:\n**25 November, 22:08 US/Pacific** - Draining of the most affected cluster\nEfforts were made to shift traffic away from the specific server cluster experiencing the crashes.\n**25 November, 22:20 US/Pacific** - Increase in capacity\nOur engineers increased the resources available to the affected service.\n**25 November, 23:27 US/Pacific** - Rollback of the webfrontend binary\nThe most effective step taken was reverting the software on the Meet web frontend server back to the previous stable version. This action mitigated the issue and restored service availability.\nService was fully restored on **26 November 2025 at 02:30 US/Pacific.**\n**## Detailed Description of Impact**\nA subset of Google Meet users in the Asia Pacific region experienced issues loading the meeting user interface (meet.google.com) or were unable to start or join meetings between 25 November 2025 21:34 US/Pacific and 26 November 2025 2:30 US/Pacific. The issue primarily affected users accessing Google Meet via Web browsers and Meeting Devices. Mobile users were not affected.","status":"AVAILABLE"},{"created":"2025-11-26T10:46:48+00:00","modified":"2025-11-27T13:49:49+00:00","when":"2025-11-26T10:46:48+00:00","text":"**Title**\nGoogle Meet customers experienced issues loading meet.google.com in the Asia region.\n**Description**\nThe issue with Google Meet has been resolved for all affected users as of Wednesday, 2025-11-26 02:30 PST.\nWe will publish an analysis of this incident once we have completed our internal investigation.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms**\nCustomers experienced issues loading meet.google.com. This issue affected Google Meet traffic in Asia. Other regions were not affected.\n**Workaround**\nNone as the issue has been mitigated.","status":"AVAILABLE"},{"created":"2025-11-26T09:33:54+00:00","modified":"2025-12-02T12:53:39+00:00","when":"2025-11-26T09:33:54+00:00","text":"**Title**\nGoogle Meet customers are experiencing issues loading meet.google.com in the Asia region.\n**Description**\nWe are experiencing an issue with Google Meet beginning on Tuesday, 2025-11-25 21:34 PST.\nOur engineering team continues to investigate the issue and work on a mitigation strategy.\nWe will provide an update by Wednesday, 2025-11-26 03:00 PST with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms**\nCustomers are experiencing issues loading meet.google.com.\n**Workaround**\nCustomers can use the mobile version to load Meet.","status":"SERVICE_DISRUPTION"},{"created":"2025-11-26T08:38:45+00:00","modified":"2025-12-02T12:53:46+00:00","when":"2025-11-26T08:38:45+00:00","text":"**Title**\nGoogle Meet customers are experiencing issues loading meet.google.com in the Asia region.\n**Description**\nWe are experiencing an issue with Google Meet beginning on Tuesday, 2025-11-25 21:34 PST.\nOur engineering team continues to investigate the issue and work on a mitigation strategy.\nWe will provide an update by Wednesday, 2025-11-26 02:00 PST with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms**\nCustomers are experiencing issues loading meet.google.com.\n**Workaround**\nCustomers can use the mobile version to load Meet.","status":"SERVICE_DISRUPTION"},{"created":"2025-11-26T07:38:58+00:00","modified":"2025-12-02T12:53:53+00:00","when":"2025-11-26T07:38:58+00:00","text":"**Title**\nGoogle Meet customers are experiencing issues loading meet.google.com in the Asia region.\n**Description**\nWe are experiencing an issue with Google Meet beginning on Tuesday, 2025-11-25 21:34 PST.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Wednesday, 2025-11-26 02:00 PST with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms**\nCustomers are experiencing issues loading meet.google.com.\n**Workaround**\nCustomers can use the mobile version to load Meet.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-12-02T12:52:38+00:00","modified":"2025-12-03T08:00:57+00:00","when":"2025-12-02T12:52:38+00:00","text":"**# Incident Report**\n**## Summary**\nBetween 25 November 2025 21:34 US/Pacific and 26 November 2025 2:30 US/Pacific, Google Meet experienced a service disruption for a total duration of 4 hours and 56 minutes. A subset of Google Meet users in Asia were unable to start or join meetings via the web and on meeting devices.\nGoogle is committed to quickly and continually improving our technology and operations to prevent service disruptions. We appreciate your patience and apologize again for the impact to your organization. We thank you for your business.\n**## Root Cause**\nThe incident was caused by an issue within the Meet web frontend server located in the Singapore (SIN) metropolitan area, which is a critical component and responsible for handling Google Meet connections for users in the Asia region.\nThe root cause of the issue was a software update to the Meet web frontend server. The update introduced a blocking call to a dependency in a critical code path, creating a thread deadlock risk.\nApproximately three hours after the update was complete, tasks started deadlocking. The trigger was a combination of regional traffic patterns and latency accessing the dependency involved. This led to stuck requests, Out-Of-Memory (OOM) errors and subsequent task crashes leading to failures when users attempted to load the meeting UI or join meetings.\n**## Remediation and Prevention**\nThe issue started on 25 November 2025 at 21:34 US/Pacific and the problem was first detected by Google's internal alerting systems at 21:41 US/Pacific. Our engineers promptly began working to restore service by taking the following actions in chronological order:\n**25 November, 22:08 US/Pacific** - Shifted traffic away from the most affected Meet web frontend servers.\nEfforts were made to shift traffic away from the specific Meet web frontend servers experiencing the crashes.\n**25 November, 22:20 US/Pacific** - Increase in capacity\nOur engineers increased the resources available to the affected service.\n**25 November, 23:27 US/Pacific** - Rollback of the webfrontend binary\nThe most effective step taken was reverting the software on the Meet web frontend server back to the previous stable version. This action mitigated the issue and restored service availability.\nService was fully restored on **26 November 2025 at 02:30 US/Pacific** after the rollback was complete.\nGoogle is completing the following actions to prevent a repeat of this issue:\n- We are reviewing critical code paths to identify and address potential blocking operations for the dependency that resulted in the deadlocking.\n- We are adding additional checks to prevent adding blocking operations in critical code paths.\n- Increase the load in our canary environments to make it more likely to detect similar issues.\n**## Detailed Description of Impact**\nA subset of Google Meet users in the Asia Pacific region experienced issues loading the meeting user interface (meet.google.com) or were unable to start or join meetings between 25 November 2025 21:34 US/Pacific and 26 November 2025 2:30 US/Pacific. The issue primarily affected users accessing Google Meet via Web browsers and Meeting Devices. Mobile users were not affected.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"sUH4BQXzYXma7NiS94Hi","service_name":"Google Meet","affected_products":[{"title":"Google Meet","id":"sUH4BQXzYXma7NiS94Hi"}],"uri":"incidents/F9Tft7xb3xqenFoKvY4W"},{"id":"KZuQxLyBKvPD1zS6sxVA","number":"574347433464346684","begin":"2025-11-24T08:02:00+00:00","created":"2025-11-24T09:47:18+00:00","end":"2025-11-24T09:55:00+00:00","modified":"2025-11-27T10:00:50+00:00","external_desc":"**Summary**\nGoogle Chat users might not be able to open chat while connecting from Europe.\n**Description**\nWe are experiencing an issue with Google Chat beginning on Monday, 2025-11-24 00:02 PST.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Monday, 2025-11-24 03:00 PST with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms**\nGoogle Chat users might not be able to open chat while connecting from Europe.\n**Workaround**\nNone at this time.","updates":[{"created":"2025-11-26T23:56:10+00:00","modified":"2025-11-27T10:00:50+00:00","when":"2025-11-26T23:56:10+00:00","text":"# Incident Report\n## Summary\nOn Monday, 24 November 2025, Google Chat experienced an overload which led to service disruptions for users for a duration of 1 hour and 53 minutes. Customers encountered issues including the inability to send messages, load the chat application, and start new conversations. Some notifications were also incorrectly displayed with \"Anonymous\" as the sender.\nWe deeply regret the disruption our Google Workspace customers experienced. We are committed to improving our platform's reliability and performance, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nThe trigger of the outage was transient errors served from an internal microservice in Europe that caused Chat backend processes to execute a fallback path to retrieve this data. The root cause was that the data retrieved from the fallback path subsequently caused an update to Chat metadata which were sent to user devices, causing those devices to reload data from Chat backend servers, thereby causing more updates to be sent to user devices resulting in a cascading effect that overloaded Chat servers in Europe.\nGoogle engineers have implemented a mitigation to limit high rates of requests to prevent this cascading loop from recurring.\n## Remediation and Prevention\nGoogle engineers were alerted to the incident via alerts and immediately started an investigation.\nTo fully mitigate the impact, our engineering teams took the following steps:\n1. **Increased Server Capacity:** To handle the unexpected surge in traffic, we significantly increased the capacity of our servers in the affected region to ensure the system could process all incoming requests smoothly.\n2. **Limited the Rate of Metadata Updates**: To prevent the cascading loop of requests from overloading Chat servers again, Google engineers have limited the rate of metadata updates so that Chat servers will not be overloaded in this way.\n3. **Paused Non-Essential Background Tasks:** To prioritize the restoration of core chat functions, we temporarily halted several internal background processes. This action helped to reduce the overall load on our systems, freeing up resources to serve user traffic more effectively.\n4. **Gradual Traffic Restoration:** Our automated defenses had initially blocked some traffic to prevent a complete system overload. A crucial part of the recovery was to gradually and carefully allow this traffic back. This controlled approach ensured that as users reconnected, the system wouldn't be overwhelmed again, allowing everyone to successfully re-establish their connection to Google Chat without getting stuck in a retry loop.\nGoogle is committed preventing a repeat of this issue in the future and is completing the following actions:\n* **Fix the immediate cause**: Stop errors in calls to the microservices from triggering metadata updates to devices. The fix prevents a cascading loop of client devices making requests back to Chat servers resulting in overload.\n* **Prevent devices from sending excessive load**: Implement changes in Chat web and mobile apps to prevent them from performing full reload of data from Chat servers when it is not needed. This reloading contributed greatly to the overload of Chat servers.\n* **Improve Monitoring**: Enhance monitoring to identify which specific metadata updates to user devices cause reloads in order to more quickly mitigate similar issues in the future.\n## Detailed Description of Impact\nOn Monday, 24 November 2025 at 00:02 US/Pacific, Google Chat users in Europe experienced a service disruption affecting core features of the service for a duration of 1 hour and 53 minutes.\nAt its peak, the incident affected approximately 20.2% of users in Europe. The issue was resolved by 01:55 US/Pacific.\nUsers encountered problems with the following core functionality:\n* **Initial Application Load**: Many users had trouble loading the Google Chat application.\n* **Opening Chat Rooms**: Users experienced errors when trying to open existing chat rooms.\n* **Sending Messages**: Users faced delays sending messages.\n* I**ncreased Latency**: There was an increase in message delivery latency for affected users.\n* **Notification Rendering**: Some notifications were also incorrectly displayed with \"Anonymous\" as the sender.","status":"AVAILABLE"},{"created":"2025-11-25T18:28:13+00:00","modified":"2025-11-26T23:56:10+00:00","when":"2025-11-25T18:28:13+00:00","text":"# Preliminary Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced an impact outside of what is listed below, please reach out to Google Workspace Support using the help article https://support.google.com/a/answer/1047213.\\n## Summary\nOn Monday, 24 November 2025, Google Chat experienced an overload which led to service disruptions for users for a duration of 1 hour and 53 minutes. Customers encountered issues including the inability to send messages, load the chat application, and start new conversations. Some notifications were also incorrectly displayed with \"Anonymous\" as the sender\n## Root Cause\nThe trigger of the outage was transient errors served from an internal microservice in Europe that caused Chat backend processes to execute a fallback path to retrieve this data. The root cause was that the data retrieved from the fallback path subsequently caused an update to Chat metadata which were sent to user devices, causing those devices to reload data from Chat backend servers, thereby causing more updates to be sent to user devices, resulting in a cascading effect that overloaded Chat servers in Europe.\nGoogle engineers have implemented a mitigation to limit high rates of requests to prevent this cascading loop from recurring.\n## Remediation\nGoogle engineers were alerted to the incident via alerts and immediately started an investigation.\nTo fully mitigate the impact, our engineering teams took the following steps:\n1. Increased Server Capacity: To handle the unexpected surge in traffic, we significantly increased the capacity of our servers in the affected region to ensure the system could process all incoming requests smoothly.\n2. Limited the Rate of Metadata Updates: To prevent the cascading loop of requests from overloading Chat servers again, Google engineers have limited the rate of metadata updates so that Chat servers will not be overloaded in this way.\n3. Paused Non-Essential Background Tasks: To prioritize the restoration of core chat functions, we temporarily halted several internal background processes. This action helped to reduce the overall load on our systems, freeing up resources to serve user traffic more effectively.\n4. Gradual Traffic Restoration: Our automated defenses had initially blocked some traffic to prevent a complete system overload. A crucial part of the recovery was to gradually and carefully allow this traffic back. This controlled approach ensured that as users reconnected, the system wouldn't be overwhelmed again, allowing everyone to successfully re-establish their connection to Google Chat without getting stuck in a retry loop.\n## Detailed Description of Impact\nFor a duration of 1 hour and 53 minutes, Google Chat users in Europe experienced a service disruption which manifested as an inability to use core features of the service, such as sending messages and starting conversations.","status":"AVAILABLE"},{"created":"2025-11-24T10:06:57+00:00","modified":"2025-11-25T18:28:13+00:00","when":"2025-11-24T10:06:57+00:00","text":"**Summary**\nGoogle Chat users might not be able to open chat while connecting from Europe.\n**Description**\nThe issue with Google Chat has been resolved for all affected users as of Monday, 2025-11-24 01:55 PST.\nWe will publish an analysis of this incident once we have completed our internal investigation.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms**\nGoogle Chat users might not be able to open chat while connecting from Europe.\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-11-24T09:47:18+00:00","modified":"2025-11-24T10:06:57+00:00","when":"2025-11-24T09:47:18+00:00","text":"**Summary**\nGoogle Chat users might not be able to open chat while connecting from Europe.\n**Description**\nWe are experiencing an issue with Google Chat beginning on Monday, 2025-11-24 00:02 PST.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Monday, 2025-11-24 03:00 PST with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms**\nGoogle Chat users might not be able to open chat while connecting from Europe.\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-11-26T23:56:10+00:00","modified":"2025-11-27T10:00:50+00:00","when":"2025-11-26T23:56:10+00:00","text":"# Incident Report\n## Summary\nOn Monday, 24 November 2025, Google Chat experienced an overload which led to service disruptions for users for a duration of 1 hour and 53 minutes. Customers encountered issues including the inability to send messages, load the chat application, and start new conversations. Some notifications were also incorrectly displayed with \"Anonymous\" as the sender.\nWe deeply regret the disruption our Google Workspace customers experienced. We are committed to improving our platform's reliability and performance, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nThe trigger of the outage was transient errors served from an internal microservice in Europe that caused Chat backend processes to execute a fallback path to retrieve this data. The root cause was that the data retrieved from the fallback path subsequently caused an update to Chat metadata which were sent to user devices, causing those devices to reload data from Chat backend servers, thereby causing more updates to be sent to user devices resulting in a cascading effect that overloaded Chat servers in Europe.\nGoogle engineers have implemented a mitigation to limit high rates of requests to prevent this cascading loop from recurring.\n## Remediation and Prevention\nGoogle engineers were alerted to the incident via alerts and immediately started an investigation.\nTo fully mitigate the impact, our engineering teams took the following steps:\n1. **Increased Server Capacity:** To handle the unexpected surge in traffic, we significantly increased the capacity of our servers in the affected region to ensure the system could process all incoming requests smoothly.\n2. **Limited the Rate of Metadata Updates**: To prevent the cascading loop of requests from overloading Chat servers again, Google engineers have limited the rate of metadata updates so that Chat servers will not be overloaded in this way.\n3. **Paused Non-Essential Background Tasks:** To prioritize the restoration of core chat functions, we temporarily halted several internal background processes. This action helped to reduce the overall load on our systems, freeing up resources to serve user traffic more effectively.\n4. **Gradual Traffic Restoration:** Our automated defenses had initially blocked some traffic to prevent a complete system overload. A crucial part of the recovery was to gradually and carefully allow this traffic back. This controlled approach ensured that as users reconnected, the system wouldn't be overwhelmed again, allowing everyone to successfully re-establish their connection to Google Chat without getting stuck in a retry loop.\nGoogle is committed preventing a repeat of this issue in the future and is completing the following actions:\n* **Fix the immediate cause**: Stop errors in calls to the microservices from triggering metadata updates to devices. The fix prevents a cascading loop of client devices making requests back to Chat servers resulting in overload.\n* **Prevent devices from sending excessive load**: Implement changes in Chat web and mobile apps to prevent them from performing full reload of data from Chat servers when it is not needed. This reloading contributed greatly to the overload of Chat servers.\n* **Improve Monitoring**: Enhance monitoring to identify which specific metadata updates to user devices cause reloads in order to more quickly mitigate similar issues in the future.\n## Detailed Description of Impact\nOn Monday, 24 November 2025 at 00:02 US/Pacific, Google Chat users in Europe experienced a service disruption affecting core features of the service for a duration of 1 hour and 53 minutes.\nAt its peak, the incident affected approximately 20.2% of users in Europe. The issue was resolved by 01:55 US/Pacific.\nUsers encountered problems with the following core functionality:\n* **Initial Application Load**: Many users had trouble loading the Google Chat application.\n* **Opening Chat Rooms**: Users experienced errors when trying to open existing chat rooms.\n* **Sending Messages**: Users faced delays sending messages.\n* I**ncreased Latency**: There was an increase in message delivery latency for affected users.\n* **Notification Rendering**: Some notifications were also incorrectly displayed with \"Anonymous\" as the sender.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"Ht9Vx5PFzDPHYvrnrvSk","service_name":"Google Chat","affected_products":[{"title":"Google Chat","id":"Ht9Vx5PFzDPHYvrnrvSk"}],"uri":"incidents/KZuQxLyBKvPD1zS6sxVA"},{"id":"EzZso1djQQfwqEmiJuDi","number":"5127096292705077","begin":"2025-11-14T18:05:00+00:00","created":"2025-11-17T02:29:25+00:00","end":"2025-11-17T03:10:00+00:00","modified":"2025-11-17T04:12:20+00:00","external_desc":"**Summary:**\nA subset of customers may be experiencing UI issue when files are uploaded via Chat to Google Workspace Support.\n**Description:**\nWe are experiencing an intermittent issue with Google Workspace Chat Support beginning at Friday, 2025-11-14 10:05 PST.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Sunday, 2025-11-16 20:30 PST with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms:**\nA subset Customers using Chat to Google Workspace Support might encounter a UI issue where uploaded files appear as a blank message instead of showing the file directly in the chat interface. Despite this UI issue, files are successfully uploaded and accessible by clicking on the blank message.\n**Workaround:**\nNone at this time.","updates":[{"created":"2025-11-17T04:11:45+00:00","modified":"2025-11-17T04:12:20+00:00","when":"2025-11-17T04:11:45+00:00","text":"**Summary:**\nA subset of customers might have experienced UI issue while uploading the files to Google Support via Chat.\n**Description:**\nThe issue with Google Cloud Support has been resolved for all affected users as of Sunday, 2025-11-16 19:10 PST.\nThe issue was due to a recent code change. Our engineers have mitigated the issue by rolling back the system to a previous version.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms:**\nA subset of customers using Chat to Google Support might have encountered a UI issue where uploaded files appeared as a blank message instead of showing the file directly in the chat interface. Despite this UI issue, files were successfully uploaded and accessible by clicking on the blank message.\n**Workaround:**\nNot applicable as the issue has been resolved.\nNone at this time.","status":"AVAILABLE"},{"created":"2025-11-17T02:29:25+00:00","modified":"2025-11-17T04:11:45+00:00","when":"2025-11-17T02:29:25+00:00","text":"**Summary:**\nA subset of customers may be experiencing UI issue when files are uploaded via Chat to Google Workspace Support.\n**Description:**\nWe are experiencing an intermittent issue with Google Workspace Chat Support beginning at Friday, 2025-11-14 10:05 PST.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Sunday, 2025-11-16 20:30 PST with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms:**\nA subset Customers using Chat to Google Workspace Support might encounter a UI issue where uploaded files appear as a blank message instead of showing the file directly in the chat interface. Despite this UI issue, files are successfully uploaded and accessible by clicking on the blank message.\n**Workaround:**\nNone at this time.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-11-17T04:11:45+00:00","modified":"2025-11-17T04:12:20+00:00","when":"2025-11-17T04:11:45+00:00","text":"**Summary:**\nA subset of customers might have experienced UI issue while uploading the files to Google Support via Chat.\n**Description:**\nThe issue with Google Cloud Support has been resolved for all affected users as of Sunday, 2025-11-16 19:10 PST.\nThe issue was due to a recent code change. Our engineers have mitigated the issue by rolling back the system to a previous version.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms:**\nA subset of customers using Chat to Google Support might have encountered a UI issue where uploaded files appeared as a blank message instead of showing the file directly in the chat interface. Despite this UI issue, files were successfully uploaded and accessible by clicking on the blank message.\n**Workaround:**\nNot applicable as the issue has been resolved.\nNone at this time.","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"dnz8HVUHoMMcW9wV91LY","service_name":"Google Workspace Support","affected_products":[{"title":"Google Workspace Support","id":"dnz8HVUHoMMcW9wV91LY"}],"uri":"incidents/EzZso1djQQfwqEmiJuDi"},{"id":"viWmkGEagnWrqYfb7VpS","number":"2706164284614039912","begin":"2025-11-12T17:00:00+00:00","created":"2025-11-12T18:44:45+00:00","end":"2025-11-12T21:45:00+00:00","modified":"2025-11-13T01:23:33+00:00","external_desc":"**Title:**\nWe are investigating access issues with multiple workspace products.\n**Description:**\nWe are experiencing an issue with Google Drive, Google Sheets, Google Docs beginning at Wednesday, 2025-11-12 09:00 PST\nOur engineering team continues to investigate the issue.\nWe will provide an update by Wednesday, 2025-11-12 11:10 PST with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms:**\nSome customers are encountering SSL errors such as \"This site can't provide a secure connection / ERR_SSL_PROTOCOL_ERROR\" when accessing Google Drive, Google Sheets, and Google Docs.\n**Workaround:**\nNone at this time.","updates":[{"created":"2025-11-13T01:23:33+00:00","modified":"2025-11-13T01:23:33+00:00","when":"2025-11-13T01:23:33+00:00","text":"**Title**\nAccess issues with Workspace products are now mitigated.\n**Description**\nThe issue with Google Sheets, Google Docs, Google Slides, and Google Forms has been resolved for all affected users as of Wednesday, 2025-11-12 13:45 PST.\nThe root cause was associated with a third party and external to Google infrastructure.\nWe thank you for your patience while we worked on resolving the issue.\n**Customer Symptoms**\nA small number of customers may have encountered SSL errors such as \"This site can't provide a secure connection / ERR_SSL_PROTOCOL_ERROR\" when accessing Google Sheets, Google Docs, Google Slides, and Google Forms.\n**Workaround**\nNot applicable.","status":"AVAILABLE"},{"created":"2025-11-12T23:32:28+00:00","modified":"2025-11-13T01:23:33+00:00","when":"2025-11-12T23:32:28+00:00","text":"**Title**\nWe are mitigating the access issues with multiple workspace products.\n**Description**\nOur engineering team has identified the root cause. We are continuing to see a decrease in user impact and are currently working to resolve the issue.\nWe will provide an update by Wednesday, 2025-11-12 16:30 PST with current details.\n**Customer Symptoms**\nSome customers are encountering SSL errors such as \"This site can't provide a secure connection / ERR_SSL_PROTOCOL_ERROR\" when accessing Google Drive, Google Sheets, Google Docs, Google Slides, and Google Forms.\n**Workaround**\nAffected customers may try accessing via a mobile device.","status":"SERVICE_DISRUPTION"},{"created":"2025-11-12T22:24:14+00:00","modified":"2025-11-12T23:33:36+00:00","when":"2025-11-12T22:24:14+00:00","text":"**Title**\nWe are mitigating the access issues with multiple workspace products.\n**Description**\nOur engineering team has identified the root cause. We are seeing a decrease in user impact and are currently working to resolve the issue.\nWe will provide an update by Wednesday, 2025-11-12 15:30 PST with current details.\n**Customer Symptoms**\nSome customers are encountering SSL errors such as \"This site can't provide a secure connection / ERR_SSL_PROTOCOL_ERROR\" when accessing Google Drive, Google Sheets, Google Docs, Google Slides, and Google Forms.\n**Workaround**\nAffected customers may try accessing via a mobile device.","status":"SERVICE_DISRUPTION"},{"created":"2025-11-12T21🔞06+00:00","modified":"2025-11-12T23:33:51+00:00","when":"2025-11-12T21🔞06+00:00","text":"**Title:**\nWe are investigating access issues with multiple workspace products.\n**Description:**\nWe are experiencing an issue with Google Drive, Google Sheets, Google Docs, Google Forms, Google Slides beginning at Wednesday, 2025-11-12 09:00 PST.\nOur engineering team is actively investigating the issue.\nWe will provide an update by Wednesday, 2025-11-12 14:15 PST with current details.\n**Symptoms:**\nSome customers are encountering SSL errors such as \"This site can't provide a secure connection / ERR_SSL_PROTOCOL_ERROR\" when accessing Google Drive, Google Sheets, Google Docs, Google Slides, and Google Forms.\n**Workaround:**\nImpacted customers may try and reboot their routers as a potential workaround.\nCustomers may also try to access via a mobile device..","status":"SERVICE_DISRUPTION"},{"created":"2025-11-12T20:57:01+00:00","modified":"2025-11-12T21🔞06+00:00","when":"2025-11-12T20:57:01+00:00","text":"**Title:**\nWe are investigating access issues with multiple workspace products.\n**Description:**\nWe are experiencing an issue with Google Drive, Google Sheets, Google Docs, Google Slides, Google Forms beginning at Wednesday, 2025-11-12 09:00 PST.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Wednesday, 2025-11-12 13:15 PST with current details.\n**Symptoms:**\nSome customers are encountering SSL errors such as \"This site can't provide a secure connection / ERR_SSL_PROTOCOL_ERROR\" when accessing Google Drive, Google Sheets, Google Docs, Google Slides, and Google Forms.\n**Workaround:**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-11-12T20:08:52+00:00","modified":"2025-11-12T20:57:01+00:00","when":"2025-11-12T20:08:52+00:00","text":"**Title:**\nWe are investigating access issues with multiple workspace products.\n**Description:**\nWe are experiencing an issue with Google Drive, Google Sheets, Google Docs beginning at Wednesday, 2025-11-12 09:00 PST\nOur engineering team continues to investigate the issue.\nWe will provide an update by Wednesday, 2025-11-12 12:40 PST with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms:**\nSome customers are encountering SSL errors such as \"This site can't provide a secure connection / ERR_SSL_PROTOCOL_ERROR\" when accessing Google Drive, Google Sheets, Google Docs, Google Slides, and Google Forms.\n**Workaround:**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-11-12T19:46:18+00:00","modified":"2025-11-12T20:08:52+00:00","when":"2025-11-12T19:46:18+00:00","text":"**Title:**\nWe are investigating access issues with multiple workspace products.\n**Description:**\nWe are experiencing an issue with Google Drive, Google Sheets, Google Docs beginning at Wednesday, 2025-11-12 09:00 PST\nOur engineering team continues to investigate the issue.\nWe will provide an update by Wednesday, 2025-11-12 12:10 PST with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms:**\nSome customers are encountering SSL errors such as \"This site can't provide a secure connection / ERR_SSL_PROTOCOL_ERROR\" when accessing Google Drive, Google Sheets, and Google Docs.\n**Workaround:**\nNone at this time.","status":"SERVICE_INFORMATION"},{"created":"2025-11-12T19:05:14+00:00","modified":"2025-11-12T19:46:18+00:00","when":"2025-11-12T19:05:14+00:00","text":"**Title:**\nWe are investigating access issues with multiple workspace products.\n**Description:**\nWe are experiencing an issue with Google Drive, Google Sheets, Google Docs beginning at Wednesday, 2025-11-12 09:00 PST\nOur engineering team continues to investigate the issue.\nWe will provide an update by Wednesday, 2025-11-12 11:45 PST with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms:**\nSome customers are encountering SSL errors such as \"This site can't provide a secure connection / ERR_SSL_PROTOCOL_ERROR\" when accessing Google Drive, Google Sheets, and Google Docs.\n**Workaround:**\nNone at this time.","status":"SERVICE_INFORMATION"},{"created":"2025-11-12T18:44:45+00:00","modified":"2025-11-12T19:05:14+00:00","when":"2025-11-12T18:44:45+00:00","text":"**Title:**\nWe are investigating access issues with multiple workspace products.\n**Description:**\nWe are experiencing an issue with Google Drive, Google Sheets, Google Docs beginning at Wednesday, 2025-11-12 09:00 PST\nOur engineering team continues to investigate the issue.\nWe will provide an update by Wednesday, 2025-11-12 11:10 PST with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms:**\nSome customers are encountering SSL errors such as \"This site can't provide a secure connection / ERR_SSL_PROTOCOL_ERROR\" when accessing Google Drive, Google Sheets, and Google Docs.\n**Workaround:**\nNone at this time.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-11-13T01:23:33+00:00","modified":"2025-11-13T01:23:33+00:00","when":"2025-11-13T01:23:33+00:00","text":"**Title**\nAccess issues with Workspace products are now mitigated.\n**Description**\nThe issue with Google Sheets, Google Docs, Google Slides, and Google Forms has been resolved for all affected users as of Wednesday, 2025-11-12 13:45 PST.\nThe root cause was associated with a third party and external to Google infrastructure.\nWe thank you for your patience while we worked on resolving the issue.\n**Customer Symptoms**\nA small number of customers may have encountered SSL errors such as \"This site can't provide a secure connection / ERR_SSL_PROTOCOL_ERROR\" when accessing Google Sheets, Google Docs, Google Slides, and Google Forms.\n**Workaround**\nNot applicable.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"zall","service_name":"Multiple Products","affected_products":[{"title":"Google Docs","id":"wNHuVFtZEWU5Mmj2eRCK"},{"title":"Google Forms","id":"3mtqLXm6QxeM6bAVkVez"},{"title":"Google Sheets","id":"GsNjeYHsogeRgTiC5coU"},{"title":"Google Slides","id":"DvWBgkXVhodA3WYwgZoB"}],"uri":"incidents/viWmkGEagnWrqYfb7VpS"},{"id":"tR2hwYEXUcAjWVfy1kp8","number":"16997046130160050708","begin":"2025-11-10T14:00:00+00:00","created":"2025-11-10T15:50:36+00:00","end":"2025-11-10T16:58:00+00:00","modified":"2025-11-10T17:26:28+00:00","external_desc":"**Title**\nGmail is experiencing an issue with images not loading properly.\n**Description**\nWe are experiencing an issue with Gmail beginning at Monday, 2025-11-10 06:19 PST.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Monday, 2025-11-10 09:00 PST with current details.\n**Symptoms**\nCustomers may experience issues with images not loading in Gmail. This might appear as broken images in the emails.\n**Workaround**\nNone at this time.","updates":[{"created":"2025-11-10T17:26:12+00:00","modified":"2025-11-10T17:26:28+00:00","when":"2025-11-10T17:26:12+00:00","text":"**Title**\nGmail is experiencing an issue with images not loading properly.\n**Description**\nThe issue with Gmail has been resolved for all affected users as of Monday, 2025-11-10 08:58 US/Pacific.\nThe issue was due to network latency which eventually translated into errors. Our engineers have mitigated the issue by diverting the traffic to other regions.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms**\nCustomers may experience issues with images not loading in Gmail. This might appear as broken images in the emails.\n**Workaround**\nNone at this time.","status":"AVAILABLE"},{"created":"2025-11-10T16:43:08+00:00","modified":"2025-11-10T17:26:12+00:00","when":"2025-11-10T16:43:08+00:00","text":"**Title**\nGmail is experiencing an issue with images not loading properly.\n**Description**\nMitigation work is currently underway by our engineering team.\nWe do not have an ETA for mitigation at this point.\nWe are seeing significant improvement and the error rates have subsided.\nWe will provide an update by Monday, 2025-11-10 10:00 US/Pacific with current details.\n**Symptoms**\nCustomers may experience issues with images not loading in Gmail. This might appear as broken images in the emails.\n**Workaround**\nNone at this time.","status":"SERVICE_INFORMATION"},{"created":"2025-11-10T15:50:36+00:00","modified":"2025-11-10T16:43:08+00:00","when":"2025-11-10T15:50:36+00:00","text":"**Title**\nGmail is experiencing an issue with images not loading properly.\n**Description**\nWe are experiencing an issue with Gmail beginning at Monday, 2025-11-10 06:19 PST.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Monday, 2025-11-10 09:00 PST with current details.\n**Symptoms**\nCustomers may experience issues with images not loading in Gmail. This might appear as broken images in the emails.\n**Workaround**\nNone at this time.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-11-10T17:26:12+00:00","modified":"2025-11-10T17:26:28+00:00","when":"2025-11-10T17:26:12+00:00","text":"**Title**\nGmail is experiencing an issue with images not loading properly.\n**Description**\nThe issue with Gmail has been resolved for all affected users as of Monday, 2025-11-10 08:58 US/Pacific.\nThe issue was due to network latency which eventually translated into errors. Our engineers have mitigated the issue by diverting the traffic to other regions.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms**\nCustomers may experience issues with images not loading in Gmail. This might appear as broken images in the emails.\n**Workaround**\nNone at this time.","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"prPt2Yra2CbGsbEm9cpC","service_name":"Gmail","affected_products":[{"title":"Gmail","id":"prPt2Yra2CbGsbEm9cpC"}],"uri":"incidents/tR2hwYEXUcAjWVfy1kp8"},{"id":"kAS3CYqGhecvgBGnzecT","number":"18333820535903255874","begin":"2025-11-05T05:05:00+00:00","created":"2025-11-05T06:07:01+00:00","end":"2025-11-05T05:55:00+00:00","modified":"2025-11-10T21:31:15+00:00","external_desc":"**Title**\nGoogle Chat customers are experiencing issues loading Chat, creating DMs (Direct Messages) and Chat Spaces.\n**Description**\nWe are experiencing an issue with Google Chat , beginning on Tuesday, 2025-11-04 21:05 US/Pacific.\nOur engineering team continues to investigate the issue.\nWe will provide more information by Tuesday, 2025-11-04 22:30 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms**\nImpacted customers may experience issues while loading Chat, creating DMs (Direct Messages) and Chat Spaces, and may experience high latency while sending and receiving messages.\n**Workaround**\nNone at this time.","updates":[{"created":"2025-11-10T21:31:15+00:00","modified":"2025-11-10T21:31:15+00:00","when":"2025-11-10T21:31:15+00:00","text":"# Incident Report\n## Summary\nGoogle Chat users experienced errors when creating new direct messages (DMs) and chat spaces for a total duration of 50 minutes between 21:05 US/Pacific to 21:55 US/Pacific on Tuesday November 4, 2025. At its peak, the issue impacted approximately 0.3% of users. Affected users also observed increased latency in message delivery and issues with initial application loading and opening chat rooms.\nTo our Google Chat users who were impacted during this disruption, we sincerely apologize. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nThe incident was triggered by database lock contention and hotspotting caused by a large volume of automated background jobs for database maintenance. These resource-intensive tasks created a bottleneck. This resulted in a cascading failure where other unrelated operations that relied on database resources were delayed, causing yet more operations to be delayed in a similar way. This resulted in the errors that users experienced when trying to perform actions like creating new conversations.\n## Remediation and Prevention\nGoogle engineers were alerted to the incident through a series of monitoring alerts at 21:06 US/Pacific on Tuesday November 4, 2025 for high error rates on our internal chat services, and immediately started to investigate.\nThe primary mitigation action was to temporarily halt the queue that processes these background cleanup jobs at 21:45 US/Pacific. This action immediately reduced the load and relieved hotspotting on the database, allowing normal operations to resume. A more specific control was later introduced to disable only the problematic background task on November 5, 2025 at 17:34 US/Pacific.\nWe apologize for the disruption this incident caused. We are committed to learning from this event and implementing changes to prevent similar issues in the future.\nThe following preventative measures are planned and will be implemented:\n- **Improve Monitoring:** Enhance monitoring to quickly identify the specific background tasks causing database contention.\n- **Improve Resilience:** Improve and document procedures for our engineering teams to mitigate similar incidents more quickly, such as by temporarily pausing or slowing down non-essential background tasks.\n- **Fix Underlying Cause:** Implement a fix to prevent root cause from occurring again by reducing the excessive resource usage during background cleanup jobs and to prevent them from overloading the system and cascading to other operations in the future.\n## Detailed Description of Impact\nThe incident had a wide-ranging impact on Google Chat users and connected services, with the most significant effects detailed below:\nAt its peak, the incident affected approximately 0.3% of users. The primary and most direct impact was on core chat functionalities, where users experienced errors when creating new direct messages (DMs) and chat spaces.\n- In addition to the primary issue, users also encountered problems with:\n- Initial Application Load: Many users had trouble loading the Google Chat application.\n- Opening Chat Rooms: Users experienced errors when trying to open existing chat rooms.\n- Sending Messages: Users faced delays sending messages.\n- Increased Latency: There was an increase in message delivery latency for affected users.","status":"AVAILABLE"},{"created":"2025-11-05T10:14:05+00:00","modified":"2025-11-10T21:31:15+00:00","when":"2025-11-05T10:14:05+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Cloud Support using https://cloud.google.com/support or to Google Workspace Support using help article https://support.google.com/a/answer/1047213.\\n(All Times US/Pacific)\n**Incident Start:** 4 November, 2025 21:05\n**Incident End:** 4 November, 2025 21:55\n**Duration:** 50 minutes\n**Affected Services and Features:**\nGoogle Chat\n**Description:**\nGoogle Chat users experienced issues when loading Chat, creating and opening DMs (Direct Messages) and Chat Spaces for a duration of 50 minutes. From preliminary analysis the root cause was a temporary contention in the underlying storage system.\n**Customer Impact:**\nAffected users faced error when trying to load Google Chat, start new direct messages, or create/open chat spaces.","status":"AVAILABLE"},{"created":"2025-11-05T06:19:40+00:00","modified":"2025-11-05T10:14:05+00:00","when":"2025-11-05T06:19:40+00:00","text":"The issue with Google Chat has been resolved for all affected users as of Tuesday, 2025-11-04 21:45 PST.\nWe will publish an analysis of this incident once we have completed our internal investigation.\nWe thank you for your patience while we worked on resolving the issue.","status":"SERVICE_DISRUPTION"},{"created":"2025-11-05T06:07:01+00:00","modified":"2025-11-05T06:19:40+00:00","when":"2025-11-05T06:07:01+00:00","text":"**Title**\nGoogle Chat customers are experiencing issues loading Chat, creating DMs (Direct Messages) and Chat Spaces.\n**Description**\nWe are experiencing an issue with Google Chat , beginning on Tuesday, 2025-11-04 21:05 US/Pacific.\nOur engineering team continues to investigate the issue.\nWe will provide more information by Tuesday, 2025-11-04 22:30 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms**\nImpacted customers may experience issues while loading Chat, creating DMs (Direct Messages) and Chat Spaces, and may experience high latency while sending and receiving messages.\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-11-10T21:31:15+00:00","modified":"2025-11-10T21:31:15+00:00","when":"2025-11-10T21:31:15+00:00","text":"# Incident Report\n## Summary\nGoogle Chat users experienced errors when creating new direct messages (DMs) and chat spaces for a total duration of 50 minutes between 21:05 US/Pacific to 21:55 US/Pacific on Tuesday November 4, 2025. At its peak, the issue impacted approximately 0.3% of users. Affected users also observed increased latency in message delivery and issues with initial application loading and opening chat rooms.\nTo our Google Chat users who were impacted during this disruption, we sincerely apologize. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nThe incident was triggered by database lock contention and hotspotting caused by a large volume of automated background jobs for database maintenance. These resource-intensive tasks created a bottleneck. This resulted in a cascading failure where other unrelated operations that relied on database resources were delayed, causing yet more operations to be delayed in a similar way. This resulted in the errors that users experienced when trying to perform actions like creating new conversations.\n## Remediation and Prevention\nGoogle engineers were alerted to the incident through a series of monitoring alerts at 21:06 US/Pacific on Tuesday November 4, 2025 for high error rates on our internal chat services, and immediately started to investigate.\nThe primary mitigation action was to temporarily halt the queue that processes these background cleanup jobs at 21:45 US/Pacific. This action immediately reduced the load and relieved hotspotting on the database, allowing normal operations to resume. A more specific control was later introduced to disable only the problematic background task on November 5, 2025 at 17:34 US/Pacific.\nWe apologize for the disruption this incident caused. We are committed to learning from this event and implementing changes to prevent similar issues in the future.\nThe following preventative measures are planned and will be implemented:\n- **Improve Monitoring:** Enhance monitoring to quickly identify the specific background tasks causing database contention.\n- **Improve Resilience:** Improve and document procedures for our engineering teams to mitigate similar incidents more quickly, such as by temporarily pausing or slowing down non-essential background tasks.\n- **Fix Underlying Cause:** Implement a fix to prevent root cause from occurring again by reducing the excessive resource usage during background cleanup jobs and to prevent them from overloading the system and cascading to other operations in the future.\n## Detailed Description of Impact\nThe incident had a wide-ranging impact on Google Chat users and connected services, with the most significant effects detailed below:\nAt its peak, the incident affected approximately 0.3% of users. The primary and most direct impact was on core chat functionalities, where users experienced errors when creating new direct messages (DMs) and chat spaces.\n- In addition to the primary issue, users also encountered problems with:\n- Initial Application Load: Many users had trouble loading the Google Chat application.\n- Opening Chat Rooms: Users experienced errors when trying to open existing chat rooms.\n- Sending Messages: Users faced delays sending messages.\n- Increased Latency: There was an increase in message delivery latency for affected users.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"Ht9Vx5PFzDPHYvrnrvSk","service_name":"Google Chat","affected_products":[{"title":"Google Chat","id":"Ht9Vx5PFzDPHYvrnrvSk"}],"uri":"incidents/kAS3CYqGhecvgBGnzecT"},{"id":"AqnPb3cc3JPV96CpxGgL","number":"12714380912420475217","begin":"2025-10-30T15:05:00+00:00","created":"2025-10-30T16:19:13+00:00","end":"2025-10-31T11:56:00+00:00","modified":"2025-10-31T12:42:24+00:00","external_desc":"**Summary**\nGoogle Meet meetings are seeing elevated error rates in upgrading the call on Android.\n**Description**\nThe Dialer on Android has a feature to upgrade a call to a Google Meet meeting. Users taking this action are seeing a high rate of failures to upgrade the call.\nOur engineering team has identified the root cause and is working on mitigation.\nWe will provide more information by Thursday, 2025-10-30 15:00 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nUsers on Android that upgrade calls to Google Meet meetings are seeing high error rates in upgrading the call\n**Workaround**\nUsers can stay in the call, or set up a meeting using the Google Meet app.","updates":[{"created":"2025-10-31T12:41:33+00:00","modified":"2025-10-31T12:42:24+00:00","when":"2025-10-31T12:41:33+00:00","text":"**Summary**\nElevated error rates observed for Android users when upgrading calls to Google Meet meetings.\n**Description**\nThe issue with Google Meet has been resolved for all affected customers as of Friday, 2025-10-31 04:56 US/Pacific.\nFrom preliminary analysis, the issue was triggered by a recent UI change that made a feature for upgrading calls to Google Meet more prominent for Android users. This led to a surge in usage and revealed an underlying issue with the feature, causing a high failure rate when upgrading calls.\nOur engineers mitigated the issue by rolling back the UI change. The team will continue to investigate and address the underlying deficiencies in the feature.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms**\nUsers on Android that upgrade calls to Google Meet meetings were observing high error rates when upgrading the call.\n**Workaround**\nUsers can stay in the call, or set up a meeting using the Google Meet app.","status":"AVAILABLE"},{"created":"2025-10-30T22:50:52+00:00","modified":"2025-10-31T12:41:33+00:00","when":"2025-10-30T22:50:52+00:00","text":"**Summary**\nElevated error rates observed for Android users when upgrading calls to Google Meet meetings.\n**Description**\nThe Dialer on Android has a feature to upgrade a call to a Google Meet meeting. Users taking this action are seeing a high rate of failures when upgrading the call.\nOur engineering team has identified the underlying cause of the issue and a rollback was initiated to mitigate impact. Based on the progress of the rollback, we are observing positive results pertaining to recovery.\nWe expect full mitigation to be completed by Friday, 2025-10-31 12:00 US/Pacific.\nWe will provide more information by Friday, 2025-10-31 12:30 US/Pacific with current details.\n**Symptoms**\nUsers on Android that upgrade calls to Google Meet meetings are observing high error rates when upgrading the call.\n**Workaround**\nUsers can stay in the call, or set up a meeting using the Google Meet app.","status":"SERVICE_INFORMATION"},{"created":"2025-10-30T22:00:54+00:00","modified":"2025-10-30T22:50:52+00:00","when":"2025-10-30T22:00:54+00:00","text":"**Summary**\nGoogle Meet meetings are seeing elevated error rates in upgrading the call on Android.\n**Description**\nThe Dialer on Android has a feature to upgrade a call to a Google Meet meeting. Users taking this action are seeing a high rate of failures to upgrade the call.\nOur engineering team is actively working on the mitigation efforts.\nWe will provide more information by Thursday, 2025-10-30 16:00 US/Pacific with current details.\n**Symptoms**\nUsers on Android that upgrade calls to Google Meet meetings are seeing high error rates in upgrading the call.\n**Workaround**\nUsers can stay in the call, or set up a meeting using the Google Meet app.","status":"SERVICE_INFORMATION"},{"created":"2025-10-30T16:19:13+00:00","modified":"2025-10-30T22:00:54+00:00","when":"2025-10-30T16:19:13+00:00","text":"**Summary**\nGoogle Meet meetings are seeing elevated error rates in upgrading the call on Android.\n**Description**\nThe Dialer on Android has a feature to upgrade a call to a Google Meet meeting. Users taking this action are seeing a high rate of failures to upgrade the call.\nOur engineering team has identified the root cause and is working on mitigation.\nWe will provide more information by Thursday, 2025-10-30 15:00 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nUsers on Android that upgrade calls to Google Meet meetings are seeing high error rates in upgrading the call\n**Workaround**\nUsers can stay in the call, or set up a meeting using the Google Meet app.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-10-31T12:41:33+00:00","modified":"2025-10-31T12:42:24+00:00","when":"2025-10-31T12:41:33+00:00","text":"**Summary**\nElevated error rates observed for Android users when upgrading calls to Google Meet meetings.\n**Description**\nThe issue with Google Meet has been resolved for all affected customers as of Friday, 2025-10-31 04:56 US/Pacific.\nFrom preliminary analysis, the issue was triggered by a recent UI change that made a feature for upgrading calls to Google Meet more prominent for Android users. This led to a surge in usage and revealed an underlying issue with the feature, causing a high failure rate when upgrading calls.\nOur engineers mitigated the issue by rolling back the UI change. The team will continue to investigate and address the underlying deficiencies in the feature.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms**\nUsers on Android that upgrade calls to Google Meet meetings were observing high error rates when upgrading the call.\n**Workaround**\nUsers can stay in the call, or set up a meeting using the Google Meet app.","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"sUH4BQXzYXma7NiS94Hi","service_name":"Google Meet","affected_products":[{"title":"Google Meet","id":"sUH4BQXzYXma7NiS94Hi"}],"uri":"incidents/AqnPb3cc3JPV96CpxGgL"},{"id":"qgaHmpt74UC6CrdZoGmc","number":"10845080150737297681","begin":"2025-10-25T06:05:00+00:00","created":"2025-10-25T07:19:50+00:00","end":"2025-10-25T08:00:00+00:00","modified":"2025-10-27T04:15:53+00:00","external_desc":"**Summary**\nAppSheet customers may experience latency issues\n**Description**\nWe are experiencing an issue with AppSheet beginning at Friday, 2025-10-24 23:05 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Saturday, 2025-10-25 01:30 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms**\nThe impacted customers may experience failures while loading apps or writing data using AppSheet.\n**Workaround**\nNone at this time.","updates":[{"created":"2025-10-27T04:15:53+00:00","modified":"2025-10-27T04:15:54+00:00","when":"2025-10-27T04:15:53+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Workspace Support using help article https://support.google.com/a/answer/1047213.\\n(All Times US/Pacific)\n**Incident Start:** 24 October, 2025 23:05\n**Incident End:** 25 October, 2025 01:00\n**Duration:** 1 hour 55 minutes\n**Affected Services and Features:**\nAppSheet\n**Regions/Zones:**\nGlobal\n**Description:**\nStarting on 24 October 23:05 US/Pacific, AppSheet experienced failures while loading apps or writing data for a duration of 1 hour and 55 minutes.\nFrom preliminary analysis, the issue was caused due to a maintenance event on a database resulting in lock contention. Our engineering mitigated the impact by manually resolving the contention issue.\n**Customer Impact:**\nThe impacted customers might have experienced failures while loading apps or writing data using AppSheet.","status":"AVAILABLE"},{"created":"2025-10-25T08:24:53+00:00","modified":"2025-10-27T04:15:53+00:00","when":"2025-10-25T08:24:53+00:00","text":"**Summary**\nAppSheet customers might have experienced latency issues\n**Description**\nThe issue with AppSheet has been resolved for all affected users as of Saturday, 2025-10-25 01:00 PDT.\nThe issue was triggered due to a maintenance event on a database resulting in lock contention and our engineering mitigated the impact by manually resolving the contention issue.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms**\nThe impacted customers might have experienced failures while loading apps or writing data using AppSheet.\n**Workaround**\nNone","status":"AVAILABLE"},{"created":"2025-10-25T07:19:50+00:00","modified":"2025-10-25T08:24:53+00:00","when":"2025-10-25T07:19:50+00:00","text":"**Summary**\nAppSheet customers may experience latency issues\n**Description**\nWe are experiencing an issue with AppSheet beginning at Friday, 2025-10-24 23:05 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Saturday, 2025-10-25 01:30 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms**\nThe impacted customers may experience failures while loading apps or writing data using AppSheet.\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-10-27T04:15:53+00:00","modified":"2025-10-27T04:15:54+00:00","when":"2025-10-27T04:15:53+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Workspace Support using help article https://support.google.com/a/answer/1047213.\\n(All Times US/Pacific)\n**Incident Start:** 24 October, 2025 23:05\n**Incident End:** 25 October, 2025 01:00\n**Duration:** 1 hour 55 minutes\n**Affected Services and Features:**\nAppSheet\n**Regions/Zones:**\nGlobal\n**Description:**\nStarting on 24 October 23:05 US/Pacific, AppSheet experienced failures while loading apps or writing data for a duration of 1 hour and 55 minutes.\nFrom preliminary analysis, the issue was caused due to a maintenance event on a database resulting in lock contention. Our engineering mitigated the impact by manually resolving the contention issue.\n**Customer Impact:**\nThe impacted customers might have experienced failures while loading apps or writing data using AppSheet.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"FWjKi5U7KX4FUUPThHAJ","service_name":"AppSheet","affected_products":[{"title":"AppSheet","id":"FWjKi5U7KX4FUUPThHAJ"}],"uri":"incidents/qgaHmpt74UC6CrdZoGmc"},{"id":"Jxvg9KeHUPnwfQ4nPgiT","number":"15026016134995879214","begin":"2025-10-24T17:45:00+00:00","created":"2025-10-25T00:15:53+00:00","end":"2025-10-25T19:05:00+00:00","modified":"2025-10-27T23:44:42+00:00","external_desc":"**Summary**\nA subset of customers in EDU domains are experiencing issues accessing Gmail mobile applications.\n**Description**\nWe are experiencing an issue with Gmail beginning on Friday, 2025-10-24 10:45 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Friday, 2025-10-24 20:30 PDT with current details.\n**Symptoms**\nAffected customers are experiencing issues accessing Gmail mobile applications. Gmail web access is unaffected.\n**Workaround**\nCustomers can access Gmail from the web browser.","updates":[{"created":"2025-10-27T23:44:42+00:00","modified":"2025-10-27T23:44:43+00:00","when":"2025-10-27T23:44:42+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service issue may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced an impact outside of what is listed below, please reach out to Google Workspace Support using the help article https://support.google.com/a/answer/1047213.\\n(All Times US/Pacific)\n**Incident Start:** 24 October 2025, 10:45\n**Incident End:** 25 October 2025, 12:05\n**Duration:** 25 hours, 20 minutes\n**Affected Services and Features:**\nGmail - Mobile application users on both ios and android\n**Description:**\nSome Workspace for Education users reported experiencing issues accessing Gmail on their mobile devices.\nPreliminary analysis indicates that the issue was caused by a recent code change in the UX stack of our first-party mobile applications. Our engineering team mitigated the impact by rolling back the problematic changes.\n**Customer Impact:**\nAffected users were unable to access Gmail through mobile applications on Android and iOS devices. Gmail web access was unaffected during the issue.","status":"AVAILABLE"},{"created":"2025-10-26T02🔞18+00:00","modified":"2025-10-27T23:44:42+00:00","when":"2025-10-26T02🔞18+00:00","text":"**Summary**\nA subset of customers in EDU domains experienced issues accessing Gmail mobile applications.\n**Description**\nThe issue with the GMail is resolved for all the users on Saturday, 2025-10-25 12:05 PDT\nAs per our preliminary analysis, the issue was caused due to a recent code change introduced to OneGoogle for Android and iOS apps. Our engineering team mitigated the issue by rolling back these problematic changes.\nWe apologize to all who are affected by the disruption. We thank you for your patience while we worked on resolving the issue.","status":"SERVICE_INFORMATION"},{"created":"2025-10-25T02:55:32+00:00","modified":"2025-10-26T02🔞18+00:00","when":"2025-10-25T02:55:32+00:00","text":"**Summary**\nA subset of customers in EDU domains are experiencing issues accessing Gmail mobile applications.\n**Description**\nWe are experiencing an issue with Gmail beginning on Friday, 2025-10-24 10:45 PDT.\nOur engineering team has identified the root cause and implemented mitigation measures. And, currently we are monitoring the status of mitigation.\nWe will provide an update by Monday, 2025-10-27 11:00 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms**\nAffected customers may have experienced issues accessing Gmail mobile applications. Gmail web access is unaffected.\n**Workaround**\nCustomers can access Gmail from the web browser.","status":"SERVICE_INFORMATION"},{"created":"2025-10-25T00:15:53+00:00","modified":"2025-10-25T02:55:32+00:00","when":"2025-10-25T00:15:53+00:00","text":"**Summary**\nA subset of customers in EDU domains are experiencing issues accessing Gmail mobile applications.\n**Description**\nWe are experiencing an issue with Gmail beginning on Friday, 2025-10-24 10:45 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Friday, 2025-10-24 20:30 PDT with current details.\n**Symptoms**\nAffected customers are experiencing issues accessing Gmail mobile applications. Gmail web access is unaffected.\n**Workaround**\nCustomers can access Gmail from the web browser.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-10-27T23:44:42+00:00","modified":"2025-10-27T23:44:43+00:00","when":"2025-10-27T23:44:42+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service issue may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced an impact outside of what is listed below, please reach out to Google Workspace Support using the help article https://support.google.com/a/answer/1047213.\\n(All Times US/Pacific)\n**Incident Start:** 24 October 2025, 10:45\n**Incident End:** 25 October 2025, 12:05\n**Duration:** 25 hours, 20 minutes\n**Affected Services and Features:**\nGmail - Mobile application users on both ios and android\n**Description:**\nSome Workspace for Education users reported experiencing issues accessing Gmail on their mobile devices.\nPreliminary analysis indicates that the issue was caused by a recent code change in the UX stack of our first-party mobile applications. Our engineering team mitigated the impact by rolling back the problematic changes.\n**Customer Impact:**\nAffected users were unable to access Gmail through mobile applications on Android and iOS devices. Gmail web access was unaffected during the issue.","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"prPt2Yra2CbGsbEm9cpC","service_name":"Gmail","affected_products":[{"title":"Gmail","id":"prPt2Yra2CbGsbEm9cpC"}],"uri":"incidents/Jxvg9KeHUPnwfQ4nPgiT"},{"id":"UYc9QNUSG1gniUUcDkDy","number":"18211409613549623115","begin":"2025-10-20T19:41:00+00:00","created":"2025-10-20T19:50:06+00:00","end":"2025-10-24T00:19:00+00:00","modified":"2025-10-27T11:10:45+00:00","external_desc":"**Summary**\nWe are investigating an issue impacting Endpoint management for Windows Devices\n**Description**\nWe are experiencing an issue with Endpoint management beginning on Wednesday, 2025-10-01 00:00 PDT. We detected the impact progressively increasing on Tuesday, 2025-10-14 10:23 PDT.\nThe Windows native settings app responsible for communicating with Google Identity and Endpoint management is using Internet Explorer 11 which has been deprecated by Microsoft. Google Identity servers are failing to serve requests for IE11 which is impacting device enrollment.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Monday, 2025-10-20 15:00 PDT with current details.\n**Symptoms**\nCustomers are not able to enroll new Windows devices to Endpoint management using Windows Settings app or deeplink. (https://support.google.com/a/answer/9539504?hl=en\\u0026sjid=13524530415918932254-NC\\u0026authuser=0)\\nCustomers using GCPW (Google Credential Provider for Windows) are not impacted. (https://support.google.com/a/answer/9250996?hl=en\\u0026src=supportwidget0\\u0026authuser=0)\\n\*\*Workaround\*\*\\nNone at this time","updates":[{"created":"2025-10-27T11:10:45+00:00","modified":"2025-10-27T11:10:46+00:00","when":"2025-10-27T11:10:45+00:00","text":"# Incident Report\n## Summary\nBeginning on 1 October 2025, Google Endpoint Management customers experienced issues enrolling new Windows devices using the native Windows Settings application or direct enrollment deeplinks. This incident started on 1 October 2025 at 00:00 US/Pacific and lasted for 22 days, 17 hours and 19 minutes, with resolution on 23 October 2025 at 17:19 US/Pacific. We sincerely apologize for any impact this disruption had on your device management workflows. We are taking measures to prevent a recurrence.\n## Root Cause\nThe root cause of this incident was a deprecated[1] browser compatibility issue. The device enrollment process initiated from the Windows Settings application relies on an embedded browser component. This component's behavior is based on the now deprecated Internet Explorer 11. Recent updates to Google's Identity servers, intended to enhance security and modernize the platform, inadvertently removed compatibility with the outdated protocols and behaviors exhibited by the Internet Explorer 11 engine. As a result, authentication requests from the Windows Settings enrollment flow could no longer be processed successfully by the updated Identity servers, causing the enrollment to fail.\n[1] Deprecated browser compatibility issue: https://learn.microsoft.com/en-us/lifecycle/products/internet-explorer-11\\n## Remediation and Prevention\nThe issue was detected on Tuesday, 14 October 2025, at 10:23 US/Pacific, as the impact progressively increased, prompting a detailed investigation. Google engineers traced the failures back to the incompatibility between the Google Identity servers and the enrollment process embedded within the Windows Settings application, which uses a deprecated Internet Explorer 11 component.\nTo remediate the issue, Google engineers developed and deployed a targeted fix to the Google Identity servers. This update restored the servers' ability to handle authentication requests originating from the enrollment flow dependent on the Internet Explorer 11 technology. The rollout of this fix was completed on 23 October 2025 at 17:19 US/Pacific, fully restoring the ability to enroll new Windows devices via the Windows Settings application and deeplinks.\nGoogle is committed to preventing similar incidents in the future and is completing the following actions:\nWe are enhancing our testing procedures for Windows device enrollment flows.\nWe are exploring improvements in monitoring to detect issues on lower-traffic enrollment pathways more rapidly.\n## Detailed Description of Impact\nGoogle Endpoint Management - From 1 October 2025, 00:00 US/Pacific to 23 October 2025, 17:19 US/Pacific, customers were unable to enroll new Windows devices into Google Endpoint Management when using the built-in Windows Settings application (as described in “Enroll a device” in Windows device management) or an enrollment deeplink. The enrollment process would fail during the authentication phase due to the incompatibility.\nCustomers using the Google Credential Provider for Windows (GCPW) for device setup and login (as described in “Install Google Credential Provider for Windows”) were not affected by this incident and could continue to enroll and manage devices without issue. The impact was limited to the specific enrollment method relying on the Windows Settings interface.","status":"AVAILABLE"},{"created":"2025-10-24T06:50:16+00:00","modified":"2025-10-27T11:10:45+00:00","when":"2025-10-24T06:50:16+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Cloud Support using https://cloud.google.com/support or to Google Workspace Support using help article https://support.google.com/a/answer/1047213.\\n(All Times US/Pacific)\n**Incident Start:** 01 October, 2025 00:00\n**Incident End:** 23 October, 2025 17:19\n**Duration:** 22 days, 17 hours, 19 minutes\n**Affected Services and Features:**\nEndpoint Management\n**Description:**\nStarting on 01 October 2025, Endpoint Management customers experienced the inability to enroll new Windows devices using Windows Settings app or deeplink. We detected the impact when it progressively increased on Tuesday, 2025-10-14 10:23 PDT.\nFrom our analysis, the issue originated from a browser compatibility concern. Specifically, the Microsoft-provided Windows Settings application utilized the deprecated Internet Explorer 11, rendering it incompatible with recent modifications to the identity servers. Our engineers resolved this by deploying a fix to the Google Identity servers to restore support for these requests.\n**Customer Impact:**\nThe Windows native settings application, which facilitates communication with Google Identity and Endpoint management, relies on Internet Explorer 11, a browser no longer supported by Microsoft. Device enrollment was affected due to Google Identity servers unable to fulfill requests from Internet Explorer 11.\nCustomers encountered issues enrolling new Windows devices in Google Endpoint management through the Windows Settings app or deeplink1. However, users leveraging Google Credential Provider for Windows (GCPW) were unaffected by this problem2.\n**Reference(s):**\n1 [Enroll a device in Windows device management](https://support.google.com/a/answer/9539504?hl=en\\u0026sjid=13524530415918932254-NC\\u0026authuser=0%29\*%29\[https://support.google.com/a/answer/9539504?hl=en\\u0026sjid=13524530415918932254-NC\\u0026authuser=0)\\n2 [Install Google Credential Provider for Windows](https://support.google.com/a/answer/9250996?hl=en\\u0026src=supportwidget0\\u0026authuser=0)","status":"AVAILABLE"},{"created":"2025-10-24T01:04:22+00:00","modified":"2025-10-24T06:50:16+00:00","when":"2025-10-24T01:04:22+00:00","text":"\*\*Summary\*\*\\nCustomers were unable to enroll new Windows devices to Endpoint management using Windows Settings app or deeplink.\n**Description**\nThe issue with Endpoint management for Windows enrollment has been resolved for all affected customers as of Thursday, October 23, 2025, 17:19 PDT.\nFrom preliminary analysis, the issue was caused by a browser compatibility problem with the deprecated IE11 used by Microsoft provided Windows Settings app. Our engineers resolved this by deploying a fix to the Google Identity servers to restore support for these requests.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms**\nCustomers were not able to enroll new Windows devices to Endpoint management using Windows Settings app or deeplink. (https://support.google.com/a/answer/9539504?hl=en\\u0026sjid=13524530415918932254-NC\\u0026authuser=0)\\nCustomers using GCPW (Google Credential Provider for Windows) were not impacted. (https://support.google.com/a/answer/9250996?hl=en\\u0026src=supportwidget0\\u0026authuser=0)\\n\*\*Workaround\*\*\\nNone.","status":"AVAILABLE"},{"created":"2025-10-23T23:05:38+00:00","modified":"2025-10-24T05:40:57+00:00","when":"2025-10-23T23:05:38+00:00","text":"\*\*Summary\*\*\\nCustomers are unable to enroll new Windows devices to Endpoint management using Windows Settings app or deeplink.\n**Description**\nMitigation work is currently underway by our engineering team.\nWe do not have an ETA for mitigation at this point.\nWe will provide an update by Friday, 2025-10-24 12:00 PDT with current details.\n**Symptoms**\nCustomers are not able to enroll new Windows devices to Endpoint management using Windows Settings app or deeplink. (https://support.google.com/a/answer/9539504?hl=en\\u0026sjid=13524530415918932254-NC\\u0026authuser=0)\\nCustomers using GCPW (Google Credential Provider for Windows) are not impacted. (https://support.google.com/a/answer/9250996?hl=en\\u0026src=supportwidget0\\u0026authuser=0)\\n\*\*Workaround\*\*\\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-10-21T19:17:52+00:00","modified":"2025-10-24T05:41:12+00:00","when":"2025-10-21T19:17:52+00:00","text":"**Summary**\nCustomers are unable to enroll new Windows devices to Endpoint management using Windows Settings app or deeplink.\n**Description**\nMitigation work is currently underway by our engineering team.\nWe do not have an ETA for mitigation at this point.\nWe will provide an update by Thursday, 2025-10-23 17:00 PDT with current details.\n**Symptoms**\nCustomers are not able to enroll new Windows devices to Endpoint management using Windows Settings app or deeplink. (https://support.google.com/a/answer/9539504?hl=en\\u0026sjid=13524530415918932254-NC\\u0026authuser=0)\\nCustomers using GCPW (Google Credential Provider for Windows) are not impacted. (https://support.google.com/a/answer/9250996?hl=en\\u0026src=supportwidget0\\u0026authuser=0)\\n\*\*Workaround\*\*\\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-10-21T03:58:16+00:00","modified":"2025-10-24T05:41:29+00:00","when":"2025-10-21T03:58:16+00:00","text":"**Summary**\nCustomers are unable to enroll new Windows devices to Endpoint management using Windows Settings app or deeplink.\n**Description**\nMitigation work is currently underway by our engineering team.\nWe do not have an ETA for mitigation at this point.\nWe will provide an update by Tuesday, 2025-10-21 11:00 PDT with current details.\n**Symptoms**\nCustomers are not able to enroll new Windows devices to Endpoint management using Windows Settings app or deeplink. (https://support.google.com/a/answer/9539504?hl=en\\u0026sjid=13524530415918932254-NC\\u0026authuser=0)\\nCustomers using GCPW (Google Credential Provider for Windows) are not impacted. (https://support.google.com/a/answer/9250996?hl=en\\u0026src=supportwidget0\\u0026authuser=0)\\n\*\*Workaround\*\*\\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-10-21T02:59:05+00:00","modified":"2025-10-24T05:41:45+00:00","when":"2025-10-21T02:59:05+00:00","text":"**Summary**\nCustomers are unable to enroll new Windows devices to Endpoint management using Windows Settings app or deeplink.\n**Description**\nMitigation work is currently underway by our engineering team.\nWe do not have an ETA for mitigation at this point.\nWe will provide an update by Tuesday, 2025-10-21 01:00 PDT with current details.\n**Symptoms**\nCustomers are not able to enroll new Windows devices to Endpoint management using Windows Settings app or deeplink. (https://support.google.com/a/answer/9539504?hl=en\\u0026sjid=13524530415918932254-NC\\u0026authuser=0)\\nCustomers using GCPW (Google Credential Provider for Windows) are not impacted. (https://support.google.com/a/answer/9250996?hl=en\\u0026src=supportwidget0\\u0026authuser=0)\\n\*\*Workaround\*\*\\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-10-21T01:39:50+00:00","modified":"2025-10-24T05:42:00+00:00","when":"2025-10-21T01:39:50+00:00","text":"**Summary**\nCustomers are unable to enroll new Windows devices to Endpoint management using Windows Settings app or deeplink.\n**Description**\nMitigation work is currently underway by our engineering team.\nWe do not have an ETA for mitigation at this point.\nWe will provide an update by Monday, 2025-10-20 20:00 PDT with current details.\n**Symptoms**\nCustomers are not able to enroll new Windows devices to Endpoint management using Windows Settings app or deeplink. (https://support.google.com/a/answer/9539504?hl=en\\u0026sjid=13524530415918932254-NC\\u0026authuser=0)\\nCustomers using GCPW (Google Credential Provider for Windows) are not impacted. (https://support.google.com/a/answer/9250996?hl=en\\u0026src=supportwidget0\\u0026authuser=0)\\n\*\*Workaround\*\*\\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-10-20T21:41:22+00:00","modified":"2025-10-24T05:42:15+00:00","when":"2025-10-20T21:41:22+00:00","text":"**Summary**\nCustomers are unable to enroll new Windows devices to Endpoint management using Windows Settings app or deeplink.\n**Description**\nMitigation work is currently underway by our engineering team.\nWe do not have an ETA for mitigation at this point.\nWe will provide an update by Monday, 2025-10-20 18:30 PDT with current details.\n**Symptoms**\nCustomers are not able to enroll new Windows devices to Endpoint management using Windows Settings app or deeplink. (https://support.google.com/a/answer/9539504?hl=en\\u0026sjid=13524530415918932254-NC\\u0026authuser=0)\\nCustomers using GCPW (Google Credential Provider for Windows) are not impacted. (https://support.google.com/a/answer/9250996?hl=en\\u0026src=supportwidget0\\u0026authuser=0)\\n\*\*Workaround\*\*\\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-10-20T19:50:06+00:00","modified":"2025-10-24T05:42:29+00:00","when":"2025-10-20T19:50:06+00:00","text":"**Summary**\nWe are investigating an issue impacting Endpoint management for Windows Devices\n**Description**\nWe are experiencing an issue with Endpoint management beginning on Wednesday, 2025-10-01 00:00 PDT. We detected the impact progressively increasing on Tuesday, 2025-10-14 10:23 PDT.\nThe Windows native settings app responsible for communicating with Google Identity and Endpoint management is using Internet Explorer 11 which has been deprecated by Microsoft. Google Identity servers are failing to serve requests for IE11 which is impacting device enrollment.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Monday, 2025-10-20 15:00 PDT with current details.\n**Symptoms**\nCustomers are not able to enroll new Windows devices to Endpoint management using Windows Settings app or deeplink. (https://support.google.com/a/answer/9539504?hl=en\\u0026sjid=13524530415918932254-NC\\u0026authuser=0)\\nCustomers using GCPW (Google Credential Provider for Windows) are not impacted. (https://support.google.com/a/answer/9250996?hl=en\\u0026src=supportwidget0\\u0026authuser=0)\\n\*\*Workaround\*\*\\nNone at this time","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-10-27T11:10:45+00:00","modified":"2025-10-27T11:10:46+00:00","when":"2025-10-27T11:10:45+00:00","text":"# Incident Report\n## Summary\nBeginning on 1 October 2025, Google Endpoint Management customers experienced issues enrolling new Windows devices using the native Windows Settings application or direct enrollment deeplinks. This incident started on 1 October 2025 at 00:00 US/Pacific and lasted for 22 days, 17 hours and 19 minutes, with resolution on 23 October 2025 at 17:19 US/Pacific. We sincerely apologize for any impact this disruption had on your device management workflows. We are taking measures to prevent a recurrence.\n## Root Cause\nThe root cause of this incident was a deprecated[1] browser compatibility issue. The device enrollment process initiated from the Windows Settings application relies on an embedded browser component. This component's behavior is based on the now deprecated Internet Explorer 11. Recent updates to Google's Identity servers, intended to enhance security and modernize the platform, inadvertently removed compatibility with the outdated protocols and behaviors exhibited by the Internet Explorer 11 engine. As a result, authentication requests from the Windows Settings enrollment flow could no longer be processed successfully by the updated Identity servers, causing the enrollment to fail.\n[1] Deprecated browser compatibility issue: https://learn.microsoft.com/en-us/lifecycle/products/internet-explorer-11\\n## Remediation and Prevention\nThe issue was detected on Tuesday, 14 October 2025, at 10:23 US/Pacific, as the impact progressively increased, prompting a detailed investigation. Google engineers traced the failures back to the incompatibility between the Google Identity servers and the enrollment process embedded within the Windows Settings application, which uses a deprecated Internet Explorer 11 component.\nTo remediate the issue, Google engineers developed and deployed a targeted fix to the Google Identity servers. This update restored the servers' ability to handle authentication requests originating from the enrollment flow dependent on the Internet Explorer 11 technology. The rollout of this fix was completed on 23 October 2025 at 17:19 US/Pacific, fully restoring the ability to enroll new Windows devices via the Windows Settings application and deeplinks.\nGoogle is committed to preventing similar incidents in the future and is completing the following actions:\nWe are enhancing our testing procedures for Windows device enrollment flows.\nWe are exploring improvements in monitoring to detect issues on lower-traffic enrollment pathways more rapidly.\n## Detailed Description of Impact\nGoogle Endpoint Management - From 1 October 2025, 00:00 US/Pacific to 23 October 2025, 17:19 US/Pacific, customers were unable to enroll new Windows devices into Google Endpoint Management when using the built-in Windows Settings application (as described in “Enroll a device” in Windows device management) or an enrollment deeplink. The enrollment process would fail during the authentication phase due to the incompatibility.\nCustomers using the Google Credential Provider for Windows (GCPW) for device setup and login (as described in “Install Google Credential Provider for Windows”) were not affected by this incident and could continue to enroll and manage devices without issue. The impact was limited to the specific enrollment method relying on the Windows Settings interface.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"oDCiEGFp9igW5FDawdXx","service_name":"Endpoint Management","affected_products":[{"title":"Endpoint Management","id":"oDCiEGFp9igW5FDawdXx"}],"uri":"incidents/UYc9QNUSG1gniUUcDkDy"},{"id":"JiwWc1AHjeGuPxh6RUwJ","number":"518392520791595651","begin":"2025-10-09T12:25:00+00:00","created":"2025-10-09T15:09:28+00:00","end":"2025-10-09T13:20:00+00:00","modified":"2025-10-10T01:28:44+00:00","external_desc":"Some AppSheet users in Europe and America experienced increased latency, resulting in app synchronization failures and editor loading timeouts from Thursday, 2025-10-09 5:25 AM PDT to 2025-10-09 6:20 PDT\nWe will provide an update by Thursday, 2025-10-09 08:30 PDT with current details.","updates":[{"created":"2025-10-10T01:28:44+00:00","modified":"2025-10-10T01:28:45+00:00","when":"2025-10-10T01:28:44+00:00","text":"# Incident Report\n## Summary\nAppSheet experienced elevated latency and intermittent synchronization failures, resulting in application loading timeouts across two distinct windows for users in the EMEA and AMER regions.\n## Root Cause\nHorizontal Pod Autoscaling (HPA) was deleted as we moved from one actuation method to another. The HPA removal resulted in some regions not scaling with customer traffic, which caused resource contention based on single server load. Additionally, cross region failover in Anthos Service Mesh did not work, resulting in a whole region outage rather than requests getting rerouted to a serving region.\n## Remediation and Prevention\nGoogle was alerted via internal monitoring for elevated latency and service instability starting with the initial impact window on 9 October 2025 at 00:10 US/Pacific. Our engineers were immediately engaged and commenced investigation and mitigation efforts.\nThe problem was resolved by applying manual mitigations in two distinct phases corresponding to the two impact windows:\n* **Impact Window 1:** (9 October 2025, 00:10 - 02:05 US/Pacific): Our engineers identified the initial server overloads caused by the autoscaling failure in the EMEA region. They immediately began manually rerouting customer traffic away from the impacted servers, resolving the first service disruption by 02:05 US/Pacific.\n* **Impact Window 2:** (9 October 2025, 05:25 - 06:20 US/Pacific): Upon recurrence of the issue, which affected the EMEA and partial AMER regions, our engineers manually rerouted all affected traffic away from the two regions where the autoscaling failure originated. This action successfully stabilized the environment and ended the incident at 06:20 US/Pacific.\n**Prevention AIs:**\n* We added an alert to detect missing HPA, ensuring that our engineers can remediate the issue manually if needed.\n* We updated the migration process for our Kubernetes infrastructure to prevent the continuous removal of HPA after migration.\n* We are fixing cross region failover routing in Anthos Service Mesh.\nGoogle is committed to quickly and continually improving our technology and operations to prevent service disruptions. We appreciate your patience and apologize again for the impact to your organization. We thank you for your business.\n## Detailed Description of Impact\nThe overarching issue was that AppSheet servers became overloaded in two regions due to a failure in the automatic scaling mechanism. This resulted in high latency and elevated error rates, causing cascading issues with app synchronization and the ability to load the editor.\nThe impact occurred across two distinct windows, initially affecting users in the EMEA region, and later expanding to include a partial section of the AMER region.","status":"AVAILABLE"},{"created":"2025-10-09T17:52:40+00:00","modified":"2025-10-10T01:28:44+00:00","when":"2025-10-09T17:52:40+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced an impact outside of what is listed below, please reach out to Google Workspace Support using the help article: https://support.google.com/a/answer/1047213.\\n(All Times US/Pacific)\n**Incident Start:** 9 October 2025 at 00:10\n**Incident End:** 9 October 2025 at 06:20\n**Total Duration:** The incident occurred across two distinct impact windows:\n* Impact Window #1 (EMEA): 1 hour, 55 minutes\n* Impact Window #2 (EMEA \u0026 partial AMER): 55 minutes\n**Affected Services and Features:**\nAppSheet - App synchronization and Editor loading\n**Regions/Zones:**\n* EMEA\n* AMER (partial impact)\n**Description:**\nOn Thursday, 9 October 2025, some AppSheet users in the EMEA and AMER regions experienced elevated latency, resulting in app synchronization failures and editor loading timeouts. The cumulative duration of impact across both windows was 2 hours, 50 minutes.\nPreliminary investigation identified the issue was due to failures in autoscaling in 2 regions which caused servers to overload and cross-region failover did not occur. The issue was mitigated by rerouting traffic away from the impacted regions.\nGoogle will complete a full Incident Report in the following days that will provide a detailed root cause.\n**Customer Impact:**\nCustomers in the affected regions may have experienced high latency, resulting in errors when syncing data and loading AppSheet applications. The issue was initially observed in the EMEA region and was later observed in both EMEA and parts of the AMER region.\nThe timeline of the two distinct impact windows is as follows:\n* EMEA only: 9 October 2025, from 00:10 to 02:05 US/Pacific.\n* EMEA and partial AMER: 9 October 2025, from 05:25 to 06:20 US/Pacific.\n---","status":"AVAILABLE"},{"created":"2025-10-09T15:14:14+00:00","modified":"2025-10-09T17:52:40+00:00","when":"2025-10-09T15:14:14+00:00","text":"The issue with AppSheet in Europe and America is now resolved.\nFrom preliminary investigation the issue was due to resource contention that resulted in our servers not being able to startup during the duration of the issue. The issue was mitigated by rerouting traffic away from the impacted servers.","status":"AVAILABLE"},{"created":"2025-10-09T15:09:28+00:00","modified":"2025-10-09T15:14:14+00:00","when":"2025-10-09T15:09:28+00:00","text":"Some AppSheet users in Europe and America experienced increased latency, resulting in app synchronization failures and editor loading timeouts from Thursday, 2025-10-09 5:25 AM PDT to 2025-10-09 6:20 PDT\nWe will provide an update by Thursday, 2025-10-09 08:30 PDT with current details.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-10-10T01:28:44+00:00","modified":"2025-10-10T01:28:45+00:00","when":"2025-10-10T01:28:44+00:00","text":"# Incident Report\n## Summary\nAppSheet experienced elevated latency and intermittent synchronization failures, resulting in application loading timeouts across two distinct windows for users in the EMEA and AMER regions.\n## Root Cause\nHorizontal Pod Autoscaling (HPA) was deleted as we moved from one actuation method to another. The HPA removal resulted in some regions not scaling with customer traffic, which caused resource contention based on single server load. Additionally, cross region failover in Anthos Service Mesh did not work, resulting in a whole region outage rather than requests getting rerouted to a serving region.\n## Remediation and Prevention\nGoogle was alerted via internal monitoring for elevated latency and service instability starting with the initial impact window on 9 October 2025 at 00:10 US/Pacific. Our engineers were immediately engaged and commenced investigation and mitigation efforts.\nThe problem was resolved by applying manual mitigations in two distinct phases corresponding to the two impact windows:\n* **Impact Window 1:** (9 October 2025, 00:10 - 02:05 US/Pacific): Our engineers identified the initial server overloads caused by the autoscaling failure in the EMEA region. They immediately began manually rerouting customer traffic away from the impacted servers, resolving the first service disruption by 02:05 US/Pacific.\n* **Impact Window 2:** (9 October 2025, 05:25 - 06:20 US/Pacific): Upon recurrence of the issue, which affected the EMEA and partial AMER regions, our engineers manually rerouted all affected traffic away from the two regions where the autoscaling failure originated. This action successfully stabilized the environment and ended the incident at 06:20 US/Pacific.\n**Prevention AIs:**\n* We added an alert to detect missing HPA, ensuring that our engineers can remediate the issue manually if needed.\n* We updated the migration process for our Kubernetes infrastructure to prevent the continuous removal of HPA after migration.\n* We are fixing cross region failover routing in Anthos Service Mesh.\nGoogle is committed to quickly and continually improving our technology and operations to prevent service disruptions. We appreciate your patience and apologize again for the impact to your organization. We thank you for your business.\n## Detailed Description of Impact\nThe overarching issue was that AppSheet servers became overloaded in two regions due to a failure in the automatic scaling mechanism. This resulted in high latency and elevated error rates, causing cascading issues with app synchronization and the ability to load the editor.\nThe impact occurred across two distinct windows, initially affecting users in the EMEA region, and later expanding to include a partial section of the AMER region.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"FWjKi5U7KX4FUUPThHAJ","service_name":"AppSheet","affected_products":[{"title":"AppSheet","id":"FWjKi5U7KX4FUUPThHAJ"}],"uri":"incidents/JiwWc1AHjeGuPxh6RUwJ"},{"id":"CBwZ6fszY7KFmncBCg2R","number":"4711405262404186636","begin":"2025-10-09T07:10:00+00:00","created":"2025-10-09T08:29:15+00:00","end":"2025-10-09T09:05:00+00:00","modified":"2025-10-10T01:30:46+00:00","external_desc":"**Summary**\nAppSheet users in Europe may encounter increased latency, resulting in app synchronization failures and editor loading timeouts.\n**Description**\nWe are experiencing an issue with AppSheet beginning on Thursday, 2025-10-09 00:10 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Thursday, 2025-10-09 02:30 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nCustomers in the Europe may experience high latency, resulting in errors when syncing data and loading AppSheet applications\n**Workaround**\nNone at this time.","updates":[{"created":"2025-10-10T01:30:46+00:00","modified":"2025-10-10T01:30:46+00:00","when":"2025-10-10T01:30:46+00:00","text":"# Incident Report\n## Summary\nAppSheet experienced elevated latency and intermittent synchronization failures, resulting in application loading timeouts across two distinct windows for users in the EMEA and AMER regions.\n## Root Cause\nHorizontal Pod Autoscaling (HPA) was deleted as we moved from one actuation method to another. The HPA removal resulted in some regions not scaling with customer traffic, which caused resource contention based on single server load. Additionally, cross region failover in Anthos Service Mesh did not work, resulting in a whole region outage rather than requests getting rerouted to a serving region.\n## Remediation and Prevention\nGoogle was alerted via internal monitoring for elevated latency and service instability starting with the initial impact window on 9 October 2025 at 00:10 US/Pacific. Our engineers were immediately engaged and commenced investigation and mitigation efforts.\nThe problem was resolved by applying manual mitigations in two distinct phases corresponding to the two impact windows:\n* **Impact Window 1:** (9 October 2025, 00:10 - 02:05 US/Pacific): Our engineers identified the initial server overloads caused by the autoscaling failure in the EMEA region. They immediately began manually rerouting customer traffic away from the impacted servers, resolving the first service disruption by 02:05 US/Pacific.\n* **Impact Window 2:** (9 October 2025, 05:25 - 06:20 US/Pacific): Upon recurrence of the issue, which affected the EMEA and partial AMER regions, our engineers manually rerouted all affected traffic away from the two regions where the autoscaling failure originated. This action successfully stabilized the environment and ended the incident at 06:20 US/Pacific.\n**Prevention AIs:**\n* We added an alert to detect missing HPA, ensuring that our engineers can remediate the issue manually if needed.\n* We updated the migration process for our Kubernetes infrastructure to prevent the continuous removal of HPA after migration.\n* We are fixing cross region failover routing in Anthos Service Mesh.\nGoogle is committed to quickly and continually improving our technology and operations to prevent service disruptions. We appreciate your patience and apologize again for the impact to your organization. We thank you for your business.\n## Detailed Description of Impact\nThe overarching issue was that AppSheet servers became overloaded in two regions due to a failure in the automatic scaling mechanism. This resulted in high latency and elevated error rates, causing cascading issues with app synchronization and the ability to load the editor.\nThe impact occurred across two distinct windows, initially affecting users in the EMEA region, and later expanding to include a partial section of the AMER region.","status":"AVAILABLE"},{"created":"2025-10-09T17:53:30+00:00","modified":"2025-10-10T01:30:46+00:00","when":"2025-10-09T17:53:30+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced an impact outside of what is listed below, please reach out to Google Workspace Support using the help article: https://support.google.com/a/answer/1047213.\\n(All Times US/Pacific)\n**Incident Start:** 9 October 2025 at 00:10\n**Incident End:** 9 October 2025 at 06:20\n**Total Duration:** The incident occurred across two distinct impact windows:\n* Impact Window #1 (EMEA): 1 hour, 55 minutes\n* Impact Window #2 (EMEA \u0026 partial AMER): 55 minutes\n**Affected Services and Features:**\nAppSheet - App synchronization and Editor loading\n**Regions/Zones:**\n* EMEA\n* AMER (partial impact)\n**Description:**\nOn Thursday, 9 October 2025, some AppSheet users in the EMEA and AMER regions experienced elevated latency, resulting in app synchronization failures and editor loading timeouts. The cumulative duration of impact across both windows was 2 hours, 50 minutes.\nPreliminary investigation identified the issue was due to failures in autoscaling in 2 regions which caused servers to overload and cross-region failover did not occur. The issue was mitigated by rerouting traffic away from the impacted regions.\nGoogle will complete a full Incident Report in the following days that will provide a detailed root cause.\n**Customer Impact:**\nCustomers in the affected regions may have experienced high latency, resulting in errors when syncing data and loading AppSheet applications. The issue was initially observed in the EMEA region and was later observed in both EMEA and parts of the AMER region.\nThe timeline of the two distinct impact windows is as follows:\n* EMEA only: 9 October 2025, from 00:10 to 02:05 US/Pacific.\n* EMEA and partial AMER: 9 October 2025, from 05:25 to 06:20 US/Pacific.\n---","status":"AVAILABLE"},{"created":"2025-10-09T09:50:43+00:00","modified":"2025-10-09T17:53:30+00:00","when":"2025-10-09T09:50:43+00:00","text":"The problem with AppSheet has been resolved. We apologize for the inconvenience and thank you for your patience and continued support.","status":"AVAILABLE"},{"created":"2025-10-09T08:37:28+00:00","modified":"2025-10-09T09:50:43+00:00","when":"2025-10-09T08:37:28+00:00","text":"**Summary**\nAppSheet users in Europe may encounter increased latency, resulting in app synchronization failures and editor loading timeouts.\n**Description**\nMitigation work is currently underway by our engineering team.\nWe do not have an ETA for mitigation at this point.\nWe will provide more information by Thursday, 2025-10-09 03:00 PDT.\n**Customer Symptoms**\nCustomers in the Europe may experience high latency, resulting in errors when syncing data and loading AppSheet applications\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-10-09T08:29:15+00:00","modified":"2025-10-09T08:37:28+00:00","when":"2025-10-09T08:29:15+00:00","text":"**Summary**\nAppSheet users in Europe may encounter increased latency, resulting in app synchronization failures and editor loading timeouts.\n**Description**\nWe are experiencing an issue with AppSheet beginning on Thursday, 2025-10-09 00:10 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Thursday, 2025-10-09 02:30 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nCustomers in the Europe may experience high latency, resulting in errors when syncing data and loading AppSheet applications\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-10-10T01:30:46+00:00","modified":"2025-10-10T01:30:46+00:00","when":"2025-10-10T01:30:46+00:00","text":"# Incident Report\n## Summary\nAppSheet experienced elevated latency and intermittent synchronization failures, resulting in application loading timeouts across two distinct windows for users in the EMEA and AMER regions.\n## Root Cause\nHorizontal Pod Autoscaling (HPA) was deleted as we moved from one actuation method to another. The HPA removal resulted in some regions not scaling with customer traffic, which caused resource contention based on single server load. Additionally, cross region failover in Anthos Service Mesh did not work, resulting in a whole region outage rather than requests getting rerouted to a serving region.\n## Remediation and Prevention\nGoogle was alerted via internal monitoring for elevated latency and service instability starting with the initial impact window on 9 October 2025 at 00:10 US/Pacific. Our engineers were immediately engaged and commenced investigation and mitigation efforts.\nThe problem was resolved by applying manual mitigations in two distinct phases corresponding to the two impact windows:\n* **Impact Window 1:** (9 October 2025, 00:10 - 02:05 US/Pacific): Our engineers identified the initial server overloads caused by the autoscaling failure in the EMEA region. They immediately began manually rerouting customer traffic away from the impacted servers, resolving the first service disruption by 02:05 US/Pacific.\n* **Impact Window 2:** (9 October 2025, 05:25 - 06:20 US/Pacific): Upon recurrence of the issue, which affected the EMEA and partial AMER regions, our engineers manually rerouted all affected traffic away from the two regions where the autoscaling failure originated. This action successfully stabilized the environment and ended the incident at 06:20 US/Pacific.\n**Prevention AIs:**\n* We added an alert to detect missing HPA, ensuring that our engineers can remediate the issue manually if needed.\n* We updated the migration process for our Kubernetes infrastructure to prevent the continuous removal of HPA after migration.\n* We are fixing cross region failover routing in Anthos Service Mesh.\nGoogle is committed to quickly and continually improving our technology and operations to prevent service disruptions. We appreciate your patience and apologize again for the impact to your organization. We thank you for your business.\n## Detailed Description of Impact\nThe overarching issue was that AppSheet servers became overloaded in two regions due to a failure in the automatic scaling mechanism. This resulted in high latency and elevated error rates, causing cascading issues with app synchronization and the ability to load the editor.\nThe impact occurred across two distinct windows, initially affecting users in the EMEA region, and later expanding to include a partial section of the AMER region.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"FWjKi5U7KX4FUUPThHAJ","service_name":"AppSheet","affected_products":[{"title":"AppSheet","id":"FWjKi5U7KX4FUUPThHAJ"}],"uri":"incidents/CBwZ6fszY7KFmncBCg2R"},{"id":"xZEAwY6ckkm8kkjRa2Kp","number":"4868498765151813095","begin":"2025-10-03T17:16:00+00:00","created":"2025-10-03T20:07:52+00:00","end":"2025-10-03T22:00:00+00:00","modified":"2025-10-03T22:57:02+00:00","external_desc":"**Summary:**\nUsers using non-Gmail accounts in the Gmail app may encounter an error message requiring additional steps.\n**Description:**\nOur engineering team has identified the root cause and mitigation work is currently underway.\nThe mitigation is expected to complete by Friday, 2025-10-03 15:00 PDT\nWe will provide an update by Friday, 2025-10-03 15:30 PDT with details.\n**Diagnosis / Customer Symptoms:**\nImpacted users may see an error message when opening the Gmail app with a non-Gmail account (e.g. Yahoo, Hotmail, etc) saying they need to complete additional steps before using the Gmail app. After that, their inbox will have a banner saying their account is offline.\nThe app can be used normally even while the \"offline\" banner is visible.\nThis is only impacting mobile Android users at the moment, both consumers and enterprise users.\nDesktop users are not impacted.\n**Workaround:**\nUsers can keep using the app as usual, or they can use alternative clients (e.g. desktop).","updates":[{"created":"2025-10-03T22:57:02+00:00","modified":"2025-10-03T22:57:02+00:00","when":"2025-10-03T22:57:02+00:00","text":"The issue with Gmail has been resolved for all affected users as of Friday, 2025-10-03 15:00 PDT.\nFrom preliminary analysis, the issue was triggered by a recent change that was deployed as part of some improvements to Android authentication. Once the underlying cause was identified, our engineers mitigated the issue by rolling back the change.\nWe thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},{"created":"2025-10-03T20:07:52+00:00","modified":"2025-10-03T22:57:02+00:00","when":"2025-10-03T20:07:52+00:00","text":"**Summary:**\nUsers using non-Gmail accounts in the Gmail app may encounter an error message requiring additional steps.\n**Description:**\nOur engineering team has identified the root cause and mitigation work is currently underway.\nThe mitigation is expected to complete by Friday, 2025-10-03 15:00 PDT\nWe will provide an update by Friday, 2025-10-03 15:30 PDT with details.\n**Diagnosis / Customer Symptoms:**\nImpacted users may see an error message when opening the Gmail app with a non-Gmail account (e.g. Yahoo, Hotmail, etc) saying they need to complete additional steps before using the Gmail app. After that, their inbox will have a banner saying their account is offline.\nThe app can be used normally even while the \"offline\" banner is visible.\nThis is only impacting mobile Android users at the moment, both consumers and enterprise users.\nDesktop users are not impacted.\n**Workaround:**\nUsers can keep using the app as usual, or they can use alternative clients (e.g. desktop).","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-10-03T22:57:02+00:00","modified":"2025-10-03T22:57:02+00:00","when":"2025-10-03T22:57:02+00:00","text":"The issue with Gmail has been resolved for all affected users as of Friday, 2025-10-03 15:00 PDT.\nFrom preliminary analysis, the issue was triggered by a recent change that was deployed as part of some improvements to Android authentication. Once the underlying cause was identified, our engineers mitigated the issue by rolling back the change.\nWe thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"prPt2Yra2CbGsbEm9cpC","service_name":"Gmail","affected_products":[{"title":"Gmail","id":"prPt2Yra2CbGsbEm9cpC"}],"uri":"incidents/xZEAwY6ckkm8kkjRa2Kp"},{"id":"JWwiKhWRMBJB6Pi9gdws","number":"5905233132377111505","begin":"2025-09-29T17:53:00+00:00","created":"2025-09-29T18:57:13+00:00","end":"2025-09-29T18:42:00+00:00","modified":"2025-10-02T20:17:40+00:00","external_desc":"We're investigating reports of an issue with NotebookLM. We will provide more information shortly.","updates":[{"created":"2025-10-02T20:17:40+00:00","modified":"2025-10-02T20:17:40+00:00","when":"2025-10-02T20:17:40+00:00","text":"# Incident Report\n## Summary\nOn Monday, 29 September 2025, several AI products experienced elevated 500 (internal) errors globally for 49 minutes, between 10:53 and 11:42 US/Pacific.\nTo our customers whose operations were impacted during this disruption, we sincerely apologize. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nThe incident occurred on a core machine learning (ML) serving platform at Google, responsible for hosting and delivering the Large Language Models that power many of our AI products and services.\nA software update intended to improve the platform's performance and scalability introduced a bug in a central control system. This system directs traffic to the correct machine learning models, and the bug caused it to incorrectly determine that some models were unavailable, leading to errors for our customers.\nThe impact of this issue was unintentionally magnified because the faulty component was responding more quickly, which caused our traffic management system to direct more requests to that component.\n## Remediation and Prevention\nGoogle engineers were alerted to the outage via manual escalation from multiple products on Monday, 29 September 2025 at 11:09 US/Pacific and immediately started an investigation. Once the underlying cause was identified, the problematic change was rolled back. All the errors returned by ML serving were then reduced to pre-fault levels, mitigating customer impact at 11:42 US/Pacific.\nGoogle is committed preventing a repeat of this issue in the future and is completing the following actions:\n* Update error handling for the ML serving platform to not incorrectly return an unavailable status for deployed models.\n* Shard the deployed models such that the critical ones always get new updates in the last waves of rollouts.\n* Add missing checks to automatically rollback this class of changes.\n* Review and update internal processes to ensure timely communications with external customers.\nWe apologize for the length and severity of this incident. We are taking immediate steps to prevent a recurrence and improve reliability in the future.\n## Detailed Description of Impact\nOn Monday, 29 September 2025, from 10:53 to 11:42 US/Pacific, Google Workspace products with a dependency on the LM Serving infrastructure experienced routing issues globally. The incident resulted in 21 percent of the request failing due to [model] NOT_FOUND error, which cascaded to customers as an internal error.\n---","status":"AVAILABLE"},{"created":"2025-09-30T05:15:40+00:00","modified":"2025-10-02T20:17:40+00:00","when":"2025-09-30T05:15:40+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Cloud Support using https://cloud.google.com/support or to Google Workspace Support using help article https://support.google.com/a/answer/1047213.\\n(All Times US/Pacific)\n**Incident Start:** 29 September, 2025 10:53\n**Incident End:** 29 September, 2025 11:42\n**Duration:** 49 minutes\n**GWS Affected Services and Features:**\nWorkspace Gemini and NotebookLM\n**Regions/Zones:** Global\n**Description:**\nVarious AI products experienced elevated 500 (internal) errors due to the rollout of a faulty change to production. Google will complete a full IR in the following days that will provide a full root cause.\n**Customer Impact:**\nCustomers affected by this issue may have observed up to a 100% of requests failing with '500' errors during the impacted time, between 10:53 and 11:42 US/Pacific on 29 September 2025.\n**Additional details:**\nA faulty change was rolled out to the control plane of the canonical Machine Learning (ML) serving stack in one cluster. Traffic routers that received their configuration from the faulty cluster assumed that some models were deleted and returned errors to the clients. Due to the error, the faulty control plane cluster had a significantly lower load, which resulted in more routers getting their configuration from the faulty cluster. This exacerbated the impact of the change.\nThe bad change was rolled back and all the errors returned by ML serving were reduced to pre-fault levels, mitigating customer impact.","status":"AVAILABLE"},{"created":"2025-09-29T19:56:21+00:00","modified":"2025-09-30T05:15:40+00:00","when":"2025-09-29T19:56:21+00:00","text":"**Summary:**\nWorkSpace NotebookLM customers may experience elevated errors for online users\n**Description:**\nThe issue with Workspace NotebookLM has been mitigated as of Monday, 2025-09-29 11:49 PDT.\nBased on preliminary investigation, the root cause of the issue was due to a recent change, which has been rolled back.\n**Customer Symptoms:**\nCustomers are experiencing elevated \"INTERNAL\\_ERROR\" with error code 500s","status":"AVAILABLE"},{"created":"2025-09-29T19:39:58+00:00","modified":"2025-09-29T19:56:21+00:00","when":"2025-09-29T19:39:58+00:00","text":"**Summary:**\nNotebookLM customers may experience elevated errors for online users\n**Description:**\nOur engineering team has identified the cause of the issue and a mitigation has been put in place. We are currently monitoring and believe we are seeing full recovery.\n**Customer Symptoms:**\nCustomers are experiencing elevated \"INTERNAL\\_ERROR\" with error code 500s","status":"SERVICE_INFORMATION"},{"created":"2025-09-29T18:57:13+00:00","modified":"2025-09-29T19:39:58+00:00","when":"2025-09-29T18:57:13+00:00","text":"We're investigating reports of an issue with NotebookLM. We will provide more information shortly. Some users would observe elevated errors from Gemini for Workspace.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-10-02T20:17:40+00:00","modified":"2025-10-02T20:17:40+00:00","when":"2025-10-02T20:17:40+00:00","text":"# Incident Report\n## Summary\nOn Monday, 29 September 2025, several AI products experienced elevated 500 (internal) errors globally for 49 minutes, between 10:53 and 11:42 US/Pacific.\nTo our customers whose operations were impacted during this disruption, we sincerely apologize. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nThe incident occurred on a core machine learning (ML) serving platform at Google, responsible for hosting and delivering the Large Language Models that power many of our AI products and services.\nA software update intended to improve the platform's performance and scalability introduced a bug in a central control system. This system directs traffic to the correct machine learning models, and the bug caused it to incorrectly determine that some models were unavailable, leading to errors for our customers.\nThe impact of this issue was unintentionally magnified because the faulty component was responding more quickly, which caused our traffic management system to direct more requests to that component.\n## Remediation and Prevention\nGoogle engineers were alerted to the outage via manual escalation from multiple products on Monday, 29 September 2025 at 11:09 US/Pacific and immediately started an investigation. Once the underlying cause was identified, the problematic change was rolled back. All the errors returned by ML serving were then reduced to pre-fault levels, mitigating customer impact at 11:42 US/Pacific.\nGoogle is committed preventing a repeat of this issue in the future and is completing the following actions:\n* Update error handling for the ML serving platform to not incorrectly return an unavailable status for deployed models.\n* Shard the deployed models such that the critical ones always get new updates in the last waves of rollouts.\n* Add missing checks to automatically rollback this class of changes.\n* Review and update internal processes to ensure timely communications with external customers.\nWe apologize for the length and severity of this incident. We are taking immediate steps to prevent a recurrence and improve reliability in the future.\n## Detailed Description of Impact\nOn Monday, 29 September 2025, from 10:53 to 11:42 US/Pacific, Google Workspace products with a dependency on the LM Serving infrastructure experienced routing issues globally. The incident resulted in 21 percent of the request failing due to [model] NOT_FOUND error, which cascaded to customers as an internal error.\n---","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"sqTm5ZmzCmb66kvyzcNS","service_name":"NotebookLM","affected_products":[{"title":"NotebookLM","id":"sqTm5ZmzCmb66kvyzcNS"}],"uri":"incidents/JWwiKhWRMBJB6Pi9gdws"},{"id":"6GdahVtGTryxTdh56iLP","number":"3413083329319526622","begin":"2025-09-29T17:53:00+00:00","created":"2025-09-29T18:58:21+00:00","end":"2025-09-29T18:42:00+00:00","modified":"2025-10-02T20:17:06+00:00","external_desc":"We're investigating reports of an issue with Gemini. We will provide more information shortly.","updates":[{"created":"2025-10-02T20:17:06+00:00","modified":"2025-10-02T20:17:06+00:00","when":"2025-10-02T20:17:06+00:00","text":"# Incident Report\n## Summary\nOn Monday, 29 September 2025, several AI products experienced elevated 500 (internal) errors globally for 49 minutes, between 10:53 and 11:42 US/Pacific.\nTo our customers whose operations were impacted during this disruption, we sincerely apologize. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nThe incident occurred on a core machine learning (ML) serving platform at Google, responsible for hosting and delivering the Large Language Models that power many of our AI products and services.\nA software update intended to improve the platform's performance and scalability introduced a bug in a central control system. This system directs traffic to the correct machine learning models, and the bug caused it to incorrectly determine that some models were unavailable, leading to errors for our customers.\nThe impact of this issue was unintentionally magnified because the faulty component was responding more quickly, which caused our traffic management system to direct more requests to that component.\n## Remediation and Prevention\nGoogle engineers were alerted to the outage via manual escalation from multiple products on Monday, 29 September 2025 at 11:09 US/Pacific and immediately started an investigation. Once the underlying cause was identified, the problematic change was rolled back. All the errors returned by ML serving were then reduced to pre-fault levels, mitigating customer impact at 11:42 US/Pacific.\nGoogle is committed preventing a repeat of this issue in the future and is completing the following actions:\n* Update error handling for the ML serving platform to not incorrectly return an unavailable status for deployed models.\n* Shard the deployed models such that the critical ones always get new updates in the last waves of rollouts.\n* Add missing checks to automatically rollback this class of changes.\n* Review and update internal processes to ensure timely communications with external customers.\nWe apologize for the length and severity of this incident. We are taking immediate steps to prevent a recurrence and improve reliability in the future.\n## Detailed Description of Impact\nOn Monday, 29 September 2025, from 10:53 to 11:42 US/Pacific, Google Workspace products with a dependency on the LM Serving infrastructure experienced routing issues globally. The incident resulted in 21 percent of the request failing due to [model] NOT_FOUND error, which cascaded to customers as an internal error.\n---","status":"AVAILABLE"},{"created":"2025-09-30T05:12:51+00:00","modified":"2025-10-02T20:17:06+00:00","when":"2025-09-30T05:12:51+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Cloud Support using https://cloud.google.com/support or to Google Workspace Support using help article https://support.google.com/a/answer/1047213.\\n(All Times US/Pacific)\n**Incident Start:** 29 September, 2025 10:53\n**Incident End:** 29 September, 2025 11:42\n**Duration:** 49 minutes\n**GWS Affected Services and Features:**\nWorkspace Gemini and NotebookLM\n**Regions/Zones:** Global\n**Description:**\nVarious AI products experienced elevated 500 (internal) errors due to the rollout of a faulty change to production. Google will complete a full IR in the following days that will provide a full root cause.\n**Customer Impact:**\nCustomers affected by this issue may have observed up to a 100% of requests failing with '500' errors during the impacted time, between 10:53 and 11:42 US/Pacific on 29 September 2025.\n**Additional details:**\nA faulty change was rolled out to the control plane of the canonical Machine Learning (ML) serving stack in one cluster. Traffic routers that received their configuration from the faulty cluster assumed that some models were deleted and returned errors to the clients. Due to the error, the faulty control plane cluster had a significantly lower load, which resulted in more routers getting their configuration from the faulty cluster. This exacerbated the impact of the change.\nThe bad change was rolled back and all the errors returned by ML serving were reduced to pre-fault levels, mitigating customer impact.","status":"AVAILABLE"},{"created":"2025-09-29T19:48:03+00:00","modified":"2025-09-30T05:12:51+00:00","when":"2025-09-29T19:48:03+00:00","text":"**Summary:**\nWorkSpace Gemini customers may experience elevated errors for online users\n**Description:**\nThe issue with Workspace Gemini has been mitigated as of Monday, 2025-09-29 11:42 PDT.\nBased on preliminary investigation, the root cause of the issue was due to a recent change, which has been rolled back.\n**Customer Symptoms:**\nCustomers are experiencing elevated \"INTERNAL\\_ERROR\" with error code 500s","status":"SERVICE_INFORMATION"},{"created":"2025-09-29T19:37:23+00:00","modified":"2025-09-29T19:48:03+00:00","when":"2025-09-29T19:37:23+00:00","text":"**Summary:**\nWorkSpace Gemini customers may experience elevated errors for online users\n**Description:**\nOur engineering team has identified the cause of the issue and a mitigation has been put in place. We are currently monitoring and believe we are seeing full recovery.\n**Customer Symptoms:**\nCustomers are experiencing elevated \"INTERNAL\\_ERROR\" with error code 500s","status":"SERVICE_INFORMATION"},{"created":"2025-09-29T18:58:21+00:00","modified":"2025-09-29T19:37:23+00:00","when":"2025-09-29T18:58:21+00:00","text":"We're investigating reports of an issue with Gemini. We will provide more information shortly. Some users would observe elevated errors from Gemini for Workspace.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-10-02T20:17:06+00:00","modified":"2025-10-02T20:17:06+00:00","when":"2025-10-02T20:17:06+00:00","text":"# Incident Report\n## Summary\nOn Monday, 29 September 2025, several AI products experienced elevated 500 (internal) errors globally for 49 minutes, between 10:53 and 11:42 US/Pacific.\nTo our customers whose operations were impacted during this disruption, we sincerely apologize. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nThe incident occurred on a core machine learning (ML) serving platform at Google, responsible for hosting and delivering the Large Language Models that power many of our AI products and services.\nA software update intended to improve the platform's performance and scalability introduced a bug in a central control system. This system directs traffic to the correct machine learning models, and the bug caused it to incorrectly determine that some models were unavailable, leading to errors for our customers.\nThe impact of this issue was unintentionally magnified because the faulty component was responding more quickly, which caused our traffic management system to direct more requests to that component.\n## Remediation and Prevention\nGoogle engineers were alerted to the outage via manual escalation from multiple products on Monday, 29 September 2025 at 11:09 US/Pacific and immediately started an investigation. Once the underlying cause was identified, the problematic change was rolled back. All the errors returned by ML serving were then reduced to pre-fault levels, mitigating customer impact at 11:42 US/Pacific.\nGoogle is committed preventing a repeat of this issue in the future and is completing the following actions:\n* Update error handling for the ML serving platform to not incorrectly return an unavailable status for deployed models.\n* Shard the deployed models such that the critical ones always get new updates in the last waves of rollouts.\n* Add missing checks to automatically rollback this class of changes.\n* Review and update internal processes to ensure timely communications with external customers.\nWe apologize for the length and severity of this incident. We are taking immediate steps to prevent a recurrence and improve reliability in the future.\n## Detailed Description of Impact\nOn Monday, 29 September 2025, from 10:53 to 11:42 US/Pacific, Google Workspace products with a dependency on the LM Serving infrastructure experienced routing issues globally. The incident resulted in 21 percent of the request failing due to [model] NOT_FOUND error, which cascaded to customers as an internal error.\n---","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"npdyhgECDJ6tB66MxXyo","service_name":"Gemini","affected_products":[{"title":"Gemini","id":"npdyhgECDJ6tB66MxXyo"}],"uri":"incidents/6GdahVtGTryxTdh56iLP"},{"id":"Cy8B1om17oecUkKm3G9Q","number":"7174538375727097808","begin":"2025-09-26T15:49:00+00:00","created":"2025-09-26T16:52:53+00:00","end":"2025-09-26T16:47:00+00:00","modified":"2025-10-02T02:01:02+00:00","external_desc":"**Summary:**\nGoogle Meet may be unavailable for users globally.\n**Description:**\nWe are experiencing an issue with Google Meet beginning on Friday, 2025-09-26 08:49 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Friday, 2025-09-26 11:00 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms:**\nUsers are unable to join calls on Google Meet.\n**Workaround:**\nNone at this time.","updates":[{"created":"2025-10-02T00:08:13+00:00","modified":"2025-10-02T02:01:02+00:00","when":"2025-10-02T00:08:13+00:00","text":"# Incident Report\n## Summary\nOn Friday, 26 September 2025 at 08:40 US/Pacific, Google Meet users experienced elevated error rates globally for 1 hour, 1 minute. To our Google Meet customers who were impacted during this disruption, we sincerely apologize. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nThe root cause of the incident was a flaw in the error-handling logic within the experimentation module's configuration evaluation. This flaw created the conditions for resource contention, where too many processes attempted to access the same system resource at once.\nThe trigger was a specific change to the Meet experimentation configuration file, which caused an elevated error level during evaluation and led to the contention in the Meet backend. Our monitoring systems detected the issue right away, and engineers quickly identified the change as the source. They reverted the update, which resolved the issue.\n## Remediation and Prevention\nGoogle engineers were alerted to the issue through the internal alerts at 08:49 US/Pacific on 26 September, 2025 and immediately started an investigation. They identified several recent changes as potential causes, and began systematically rolling back those changes.\nAt 09:25 US/Pacific, engineers reverted the Meet API Server binary to a previous version.\nAt 09:41 US/Pacific, engineers reverted and paused subsequent Meet backend API server configuration rollouts across all Google Meet production clusters thus mitigating the issue.\nAt 11:52 US/Pacific, engineers reverted the change that caused lock contention.\nWe sincerely apologize to our customers who were impacted by this disruption. This is not the level of quality and reliability we strive to offer, and we are taking the following steps to ensure this does not happen again. The Google Meet Engineering team is accelerating the deprecation of legacy code that caused lock contention.\n## Detailed Description of Impact\nOn Friday, 26 September 2025, from 08:40 to 09:41 US/Pacific, users were either unable to connect to Google Meet or experienced problems with establishing meetings. Users already in Google Meet sessions were likely not impacted. The issue affected up to 20 percent of Google Meet users globally.","status":"AVAILABLE"},{"created":"2025-09-26T19:08:58+00:00","modified":"2025-10-02T00:08:13+00:00","when":"2025-09-26T19:08:58+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Workspace Support using the help article https://support.google.com/a/answer/1047213.\\n(All Times US/Pacific)\n**Incident Start:** 26 September, 2025 08:40\n**Incident End:** 26 September, 2025 09:47\n**Duration:** 1 hours, 7 minutes\n**Affected Services and Features:**\nGoogle Meet\n**Regions/Zones:** Global\n**Description:**\nGoogle Meet users experienced elevated error rates globally for a duration of 1 hour, 7 minutes. The preliminary root cause was a recent rollout to a backend API server, which was mitigated after the rollback completed.\nGoogle will complete a full Incident Report in the following days that will provide a full root cause.\n**Customer Impact:**\nAffected users experienced 504 errors when attempting to access Google Meet.\n---","status":"AVAILABLE"},{"created":"2025-09-26T17:17:51+00:00","modified":"2025-09-26T19:08:58+00:00","when":"2025-09-26T17:17:51+00:00","text":"**Summary:**\nSome users experienced errors when trying to access Google Meet.\n**Description:**\nThe issue with Google Meet has been resolved for all affected users as of Friday, 2025-09-26 09:47 PDT.\nA root cause analysis is in progress. Google will be publishing a preliminary Incident Report shortly and a full Incident Report in the coming days.\nWe thank you for your patience while we worked on resolving the issue.\n**Customer Symptoms:**\nImpacted users experienced errors when attempting to access Google Meet.","status":"AVAILABLE"},{"created":"2025-09-26T16:52:53+00:00","modified":"2025-09-26T17:17:51+00:00","when":"2025-09-26T16:52:53+00:00","text":"**Summary:**\nGoogle Meet may be unavailable for users globally.\n**Description:**\nWe are experiencing an issue with Google Meet beginning on Friday, 2025-09-26 08:49 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Friday, 2025-09-26 11:00 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms:**\nUsers are unable to join calls on Google Meet.\n**Workaround:**\nNone at this time.","status":"SERVICE_OUTAGE"}],"most_recent_update":{"created":"2025-10-02T00:08:13+00:00","modified":"2025-10-02T02:01:02+00:00","when":"2025-10-02T00:08:13+00:00","text":"# Incident Report\n## Summary\nOn Friday, 26 September 2025 at 08:40 US/Pacific, Google Meet users experienced elevated error rates globally for 1 hour, 1 minute. To our Google Meet customers who were impacted during this disruption, we sincerely apologize. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nThe root cause of the incident was a flaw in the error-handling logic within the experimentation module's configuration evaluation. This flaw created the conditions for resource contention, where too many processes attempted to access the same system resource at once.\nThe trigger was a specific change to the Meet experimentation configuration file, which caused an elevated error level during evaluation and led to the contention in the Meet backend. Our monitoring systems detected the issue right away, and engineers quickly identified the change as the source. They reverted the update, which resolved the issue.\n## Remediation and Prevention\nGoogle engineers were alerted to the issue through the internal alerts at 08:49 US/Pacific on 26 September, 2025 and immediately started an investigation. They identified several recent changes as potential causes, and began systematically rolling back those changes.\nAt 09:25 US/Pacific, engineers reverted the Meet API Server binary to a previous version.\nAt 09:41 US/Pacific, engineers reverted and paused subsequent Meet backend API server configuration rollouts across all Google Meet production clusters thus mitigating the issue.\nAt 11:52 US/Pacific, engineers reverted the change that caused lock contention.\nWe sincerely apologize to our customers who were impacted by this disruption. This is not the level of quality and reliability we strive to offer, and we are taking the following steps to ensure this does not happen again. The Google Meet Engineering team is accelerating the deprecation of legacy code that caused lock contention.\n## Detailed Description of Impact\nOn Friday, 26 September 2025, from 08:40 to 09:41 US/Pacific, users were either unable to connect to Google Meet or experienced problems with establishing meetings. Users already in Google Meet sessions were likely not impacted. The issue affected up to 20 percent of Google Meet users globally.","status":"AVAILABLE"},"status_impact":"SERVICE_OUTAGE","severity":"high","service_key":"sUH4BQXzYXma7NiS94Hi","service_name":"Google Meet","affected_products":[{"title":"Google Meet","id":"sUH4BQXzYXma7NiS94Hi"}],"uri":"incidents/Cy8B1om17oecUkKm3G9Q"},{"id":"rQD4joDQXiqfecRaVPRR","number":"9848531984137254167","begin":"2025-09-26T09:02:00+00:00","created":"2025-09-26T12:02:29+00:00","end":"2025-09-26T15:02:00+00:00","modified":"2025-10-06T15:01:54+00:00","external_desc":"We're investigating reports of an issue with Admin Console. We will provide more information shortly.","updates":[{"created":"2025-10-04T00:56:40+00:00","modified":"2025-10-06T11:25:00+00:00","when":"2025-10-04T00:56:40+00:00","text":"# Incident Report\n**Incident Start:** 26 September 2025 02:02\n**Incident End:** 26 September 2025 08:02\n**Duration:** 6 hours\n## Summary\nStarting on Friday, 26 September 2025 02:02 US/Pacific, a power equipment failure in zone us-central1-a caused service disruptions. Google Workspace Admin Console experienced elevated latency and error rates. Service was fully restored by 08:02 US/Pacific.\n## Root Cause\nThe incident was triggered by a power equipment failure at one of the data centers in us-central1-a at 02:02 US/Pacific. This failure resulted in a power disruption to a subset of storage servers within the zone, leading to the observed impact across the listed services.\n## Detailed Description of Impact\n#### Admin Console (Google Workspace)\nA majority of Admin-exposed services, such as the Security Investigation Tool (SIT), Email Log Search (ELS), and Admin Quarantine, experienced issues where reporting pages appeared blank, charts failed to load, or a buffering circle was continuously displayed. Some services also returned ‘502’ (Bad Gateway) errors. The services were fully mitigated by 08:02 US/Pacific. The delayed recovery was caused by a dependency on Google BigQuery, which was also impacted by the incident.\n## Remediation and Prevention\nGoogle engineers were alerted to the incident from an internal monitoring alert at 02:13 US/Pacific on 26 September 2025 and addressed the incident by transferring the affected servers to alternate power equipment, which was completed by 02:35 US/Pacific.\nA majority of the impacted services recovered quickly upon the resumption of power. However, several products had extended time to recover due to their dependency on the impacted storage system.\nTo protect the data during a power event, a safety feature was auto activated. This feature temporarily placed affected storage servers into a protective, read-only mode to prevent any data from being written that might not be saved correctly in case of another power loss.\nWhile this safety feature worked as intended to ensure data durability, it had an unintended side effect. Our underlying file system, which orchestrates data storage, needs to perform very small, internal \"write\" operations to coordinate its tasks, even when you are only reading data. Because a large number of storage servers were in the read-only state, the file system was unable to perform these essential internal writes. This caused parts of the file system to become temporarily unavailable, which in turn impacted services like Persistent Disk that rely on it.\nWe sincerely apologize to our customers whose services were impacted by this disruption. This is not the level of quality and reliability we strive to offer, and we are taking steps to ensure this does not happen again.\n* **Speeding storage system recovery:** We are reverting the recent change, to the previous behavior. We will improve testing of this feature to ensure it does not introduce availability issues after power loss events before reintroducing it at a later date.\n* **Enhancing our monitoring:** We are building a new dashboard to give us a real-time, comprehensive view of the power and battery status of all our storage servers. This will allow us to detect and respond to potential power-related issues much more quickly.\n* **Strengthening data-protection protocols:** We are creating clearer guidelines for our engineers on when it is appropriate to override automated data-protection features. This will help ensure that we always prioritize the safety and durability of your data.\n* **Rigorously testing our safety features:** We are implementing more thorough fault testing to confirm that our data-protection mechanisms work exactly as expected, especially after they have been activated and then reset.\n* **Improving dependent system recovery:** We are implementing changes in BigQuery to improve the time to recovery of the BigQuery processing for recently ingested data after a partial or full cluster level storage system outage.","status":"AVAILABLE"},{"created":"2025-09-27T01:12:11+00:00","modified":"2025-10-06T15:01:54+00:00","when":"2025-09-27T01:12:11+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Workspace Support using help article https://support.google.com/a/answer/1047213.\\n(All Times US/Pacific)\n**Incident Start:** 26 September 2025 02:02\n**Incident End:** 26 September 2025 08:00\n**Duration:** 5 hours, 58 minutes\n**Affected Services and Features:**\nAdmin Console\n**Description:**\nStarting at 02:02 on Friday, 26 September 2025, multiple Google services in a specific zone began experiencing elevated latency and error rates. A power equipment failure at one of the datacenters in the zone resulted in power disruption for a subset of storage servers.\nTo restore services, the underlying power issue was immediately addressed by transferring the affected servers to an alternate power equipment by 02:35.\nA majority of the impacted services were recovered quickly upon resumption of power. Due to manual intervention required to restore a small subset of the storage servers, a few services continued to experience partial unavailability and were fully recovered by 08:00.\nGoogle will complete a full Incident Report in the following days that will provide a full root cause.\n**Customer Impact:**\nWorkspace Admin Console users may have observed ‘502’ errors (Bad Gateway) or continuous loading.","status":"AVAILABLE"},{"created":"2025-09-26T15:49:11+00:00","modified":"2025-09-27T01:12:11+00:00","when":"2025-09-26T15:49:11+00:00","text":"**Description :**\nThe issue with Admin Console has been resolved for all affected users as of Friday, 2025-09-26 07:56 PT.\nThe issue was triggered by a backend storage service issue. We are investigating the full root cause, and we will publish an analysis of this incident once we have completed our internal investigation.\nWe thank you for your patience while we worked on resolving the issue.","status":"SERVICE_DISRUPTION"},{"created":"2025-09-26T15:01:44+00:00","modified":"2025-09-26T15:49:11+00:00","when":"2025-09-26T15:01:44+00:00","text":"**Summary**\nWorkspace Admin Console users might observe 502 errors (Bad Gateway) or continuous loading.\n**Description**\nWe are experiencing an issue with Admin Console (Security Investigation Tool, Email Log Search, Admin Quarantine) and AdminSDK: Reports API beginning on Friday, 2025-09-26 02:13 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Friday, 2025-09-26 09:00 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nWorkspace Admin Console users might observe 502 errors (Bad Gateway) or continuous loading.\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-09-26T13:53:11+00:00","modified":"2025-09-26T15:01:44+00:00","when":"2025-09-26T13:53:11+00:00","text":"**Summary**\nWorkspace Admin Console users might observe 502 errors (Bad Gateway) or continuous loading.\n**Description**\nWe are experiencing an issue with Admin Console (Security Investigation Tool, Email Log Search, Admin Quarantine) and AdminSDK: Reports API beginning on Friday, 2025-09-26 02:13 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Friday, 2025-09-26 08:00 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nWorkspace Admin Console users might observe 502 errors (Bad Gateway) or continuous loading.\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-09-26T13:25:10+00:00","modified":"2025-09-26T13:53:11+00:00","when":"2025-09-26T13:25:10+00:00","text":"**Summary**\nWorkspace Admin Console users might observe 502 errors (Bad Gateway) or continuous loading.\n**Description**\nWe are experiencing an issue with Admin Console (Security Investigation Tool, Email Log Search, Admin Quarantine) and AdminSDK: Reports API beginning on Friday, 2025-09-26 02:13 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Friday, 2025-09-26 07:00 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nWorkspace Admin Console users might observe 502 errors (Bad Gateway) or continuous loading.\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-09-26T12:51:28+00:00","modified":"2025-09-26T13:25:10+00:00","when":"2025-09-26T12:51:28+00:00","text":"**Summary**\nWorkspace Admin Console users might observe 502 errors (Bad Gateway) or continuous loading.\n**Description**\nWe are experiencing an issue with Admin Console (Security Investigation Tool, Email Log Search, Admin Quarantine) and AdminSDK: Reports API beginning on Friday, 2025-09-26 02:13 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Friday, 2025-09-26 06:30 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nWorkspace Admin Console users might observe 502 errors (Bad Gateway) or continuous loading.\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-09-26T12:24:50+00:00","modified":"2025-09-26T12:51:28+00:00","when":"2025-09-26T12:24:50+00:00","text":"**Summary**\nWorkspace Admin Console users might observe 502 errors (Bad Gateway) or continuous loading.\n**Description**\nWe are experiencing an issue with Admin Console (Security Investigation Tool, Email Log Search, Admin Quarantine) and AdminSDK: Reports API beginning on Friday, 2025-09-26 02:13 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Friday, 2025-09-26 06:00 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nWorkspace Admin Console users might observe 502 errors (Bad Gateway) or continuous loading.\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-09-26T12:07:15+00:00","modified":"2025-09-26T12:24:50+00:00","when":"2025-09-26T12:07:15+00:00","text":"**Summary**\nWorkspace Admin Console users might observe 502 errors (Bad Gateway) or continuous loading.\n**Description**\nWe are experiencing an issue with Admin Console (Security Investigation Tool, Email Log Search, Admin Quarantine) and AdminSDK: Reports API beginning on Friday, 2025-09-26 02:13 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Friday, 2025-09-26 05:30 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nWorkspace Admin Console users might observe 502 errors (Bad Gateway) or continuous loading.\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-09-26T12:02:29+00:00","modified":"2025-09-26T12:07:15+00:00","when":"2025-09-26T12:02:29+00:00","text":"We're investigating reports of an issue with Admin Console. We will provide more information shortly.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-10-04T00:56:40+00:00","modified":"2025-10-06T11:25:00+00:00","when":"2025-10-04T00:56:40+00:00","text":"# Incident Report\n**Incident Start:** 26 September 2025 02:02\n**Incident End:** 26 September 2025 08:02\n**Duration:** 6 hours\n## Summary\nStarting on Friday, 26 September 2025 02:02 US/Pacific, a power equipment failure in zone us-central1-a caused service disruptions. Google Workspace Admin Console experienced elevated latency and error rates. Service was fully restored by 08:02 US/Pacific.\n## Root Cause\nThe incident was triggered by a power equipment failure at one of the data centers in us-central1-a at 02:02 US/Pacific. This failure resulted in a power disruption to a subset of storage servers within the zone, leading to the observed impact across the listed services.\n## Detailed Description of Impact\n#### Admin Console (Google Workspace)\nA majority of Admin-exposed services, such as the Security Investigation Tool (SIT), Email Log Search (ELS), and Admin Quarantine, experienced issues where reporting pages appeared blank, charts failed to load, or a buffering circle was continuously displayed. Some services also returned ‘502’ (Bad Gateway) errors. The services were fully mitigated by 08:02 US/Pacific. The delayed recovery was caused by a dependency on Google BigQuery, which was also impacted by the incident.\n## Remediation and Prevention\nGoogle engineers were alerted to the incident from an internal monitoring alert at 02:13 US/Pacific on 26 September 2025 and addressed the incident by transferring the affected servers to alternate power equipment, which was completed by 02:35 US/Pacific.\nA majority of the impacted services recovered quickly upon the resumption of power. However, several products had extended time to recover due to their dependency on the impacted storage system.\nTo protect the data during a power event, a safety feature was auto activated. This feature temporarily placed affected storage servers into a protective, read-only mode to prevent any data from being written that might not be saved correctly in case of another power loss.\nWhile this safety feature worked as intended to ensure data durability, it had an unintended side effect. Our underlying file system, which orchestrates data storage, needs to perform very small, internal \"write\" operations to coordinate its tasks, even when you are only reading data. Because a large number of storage servers were in the read-only state, the file system was unable to perform these essential internal writes. This caused parts of the file system to become temporarily unavailable, which in turn impacted services like Persistent Disk that rely on it.\nWe sincerely apologize to our customers whose services were impacted by this disruption. This is not the level of quality and reliability we strive to offer, and we are taking steps to ensure this does not happen again.\n* **Speeding storage system recovery:** We are reverting the recent change, to the previous behavior. We will improve testing of this feature to ensure it does not introduce availability issues after power loss events before reintroducing it at a later date.\n* **Enhancing our monitoring:** We are building a new dashboard to give us a real-time, comprehensive view of the power and battery status of all our storage servers. This will allow us to detect and respond to potential power-related issues much more quickly.\n* **Strengthening data-protection protocols:** We are creating clearer guidelines for our engineers on when it is appropriate to override automated data-protection features. This will help ensure that we always prioritize the safety and durability of your data.\n* **Rigorously testing our safety features:** We are implementing more thorough fault testing to confirm that our data-protection mechanisms work exactly as expected, especially after they have been activated and then reset.\n* **Improving dependent system recovery:** We are implementing changes in BigQuery to improve the time to recovery of the BigQuery processing for recently ingested data after a partial or full cluster level storage system outage.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"xrkLpNMHAvPQczcU2uSb","service_name":"Admin Console","affected_products":[{"title":"Admin Console","id":"xrkLpNMHAvPQczcU2uSb"}],"uri":"incidents/rQD4joDQXiqfecRaVPRR"},{"id":"mLrAjo9VbGGzQJRt7TQM","number":"6479781004903047668","begin":"2025-09-24T18:51:00+00:00","created":"2025-09-25T00:43:43+00:00","end":"2025-09-25T08:00:00+00:00","modified":"2025-09-25T09:26:40+00:00","external_desc":"**Summary**\nUsers may experience elevated latency and timeout errors with Admin SDK APIs.\n**Description**\nWe are experiencing an issue with Admin SDK APIs beginning on Wednesday, 2025-09-24 11:51 PDT.\nMitigation work is currently underway by our engineering team. We do not have an ETA for mitigation at this point.\nWe will provide more information by Wednesday, 2025-09-24 19:30 PDT.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nUsers may experience elevated latency and timeout errors with Admin SDK APIs.\n**Workaround**\nImpacted customers may retry Admin SDK API calls.","updates":[{"created":"2025-09-25T09:26:40+00:00","modified":"2025-09-25T09:26:40+00:00","when":"2025-09-25T09:26:40+00:00","text":"The problem with Admin Console has been resolved. We apologize for the inconvenience and thank you for your patience and continued support.","status":"AVAILABLE"},{"created":"2025-09-25T07:43:35+00:00","modified":"2025-09-25T09:26:40+00:00","when":"2025-09-25T07:43:35+00:00","text":"**Summary**\nUsers may experience elevated latency and timeout errors with Admin SDK APIs.\n**Description**\nWe are experiencing an issue with Admin SDK APIs beginning on Wednesday, 2025-09-24 11:51 PDT.\nMitigation work is currently underway by our engineering team. We do not have an ETA for mitigation at this point.\nWe will provide more information by Thursday, 2025-09-25 05:00 PDT.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nUsers may experience elevated latency and timeout errors with Admin SDK APIs.\n**Workaround**\nImpacted customers may retry Admin SDK API calls.","status":"SERVICE_DISRUPTION"},{"created":"2025-09-25T03:05:27+00:00","modified":"2025-09-25T07:43:35+00:00","when":"2025-09-25T03:05:27+00:00","text":"**Summary**\nUsers may experience elevated latency and timeout errors with Admin SDK APIs.\n**Description**\nWe are experiencing an issue with Admin SDK APIs beginning on Wednesday, 2025-09-24 11:51 PDT.\nMitigation work is currently underway by our engineering team. We do not have an ETA for mitigation at this point.\nWe will provide more information by Thursday, 2025-09-25 01:00 PDT.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nUsers may experience elevated latency and timeout errors with Admin SDK APIs.\n**Workaround**\nImpacted customers may retry Admin SDK API calls.","status":"SERVICE_DISRUPTION"},{"created":"2025-09-25T02🔞46+00:00","modified":"2025-09-25T03:05:27+00:00","when":"2025-09-25T02🔞46+00:00","text":"**Summary**\nUsers may experience elevated latency and timeout errors with Admin SDK APIs.\n**Description**\nWe are experiencing an issue with Admin SDK APIs beginning on Wednesday, 2025-09-24 11:51 PDT.\nMitigation work is currently underway by our engineering team. We do not have an ETA for mitigation at this point.\nWe will provide more information by Wednesday, 2025-09-24 21:00 PDT.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nUsers may experience elevated latency and timeout errors with Admin SDK APIs.\n**Workaround**\nImpacted customers may retry Admin SDK API calls.","status":"SERVICE_DISRUPTION"},{"created":"2025-09-25T00:43:43+00:00","modified":"2025-09-25T02🔞46+00:00","when":"2025-09-25T00:43:43+00:00","text":"**Summary**\nUsers may experience elevated latency and timeout errors with Admin SDK APIs.\n**Description**\nWe are experiencing an issue with Admin SDK APIs beginning on Wednesday, 2025-09-24 11:51 PDT.\nMitigation work is currently underway by our engineering team. We do not have an ETA for mitigation at this point.\nWe will provide more information by Wednesday, 2025-09-24 19:30 PDT.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nUsers may experience elevated latency and timeout errors with Admin SDK APIs.\n**Workaround**\nImpacted customers may retry Admin SDK API calls.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-09-25T09:26:40+00:00","modified":"2025-09-25T09:26:40+00:00","when":"2025-09-25T09:26:40+00:00","text":"The problem with Admin Console has been resolved. We apologize for the inconvenience and thank you for your patience and continued support.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"xrkLpNMHAvPQczcU2uSb","service_name":"Admin Console","affected_products":[{"title":"Admin Console","id":"xrkLpNMHAvPQczcU2uSb"}],"uri":"incidents/mLrAjo9VbGGzQJRt7TQM"},{"id":"5V5yK8N8heBKnmdqS1eW","number":"1782113629924000856","begin":"2025-09-18T14:22:00+00:00","created":"2025-09-18T16:03:13+00:00","end":"2025-09-18T15:35:00+00:00","modified":"2025-09-24T17:30:38+00:00","external_desc":"**Summary:**\nWe have mitigated an issue where some customers experienced login issues to Google services.\n**Description:**\nBetween 07:22 and 08:30 PDT, customers experienced login issues to Google services.\nOur engineering team has identified the cause and applied appropriate mitigations by 08:30 PDT. Customers should now be able to login to the Google services.\nWe are closely monitoring the environment for stability.\nWe will provide an update by Thursday, 2025-09-18 09:20 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms:**\nSome customers experienced login issues to Google services.\n**Workaround:**\nNone at this time.","updates":[{"created":"2025-09-24T17:27:52+00:00","modified":"2025-09-24T17:30:38+00:00","when":"2025-09-24T17:27:52+00:00","text":"# Incident Report\n## Summary\nOn Thursday, 18 September 2025, between 07:22 and 08:35 US/Pacific, Google Workspace products experienced elevated login failures globally for a duration of 1 hour 13 minutes. Some users may have been unable to login to various Google Workspace services during that time.\nThis is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nA shift in traffic led to a resource contention issue within Google's authentication system. The system's capacity for handling concurrent requests was exhausted in several east coast regions, creating a bottleneck that blocked new login requests. This resulted in elevated login failures and errors for users across various Google services.\n## Remediation and Prevention\nGoogle engineers were alerted at 07:31 US/Pacific via internal monitoring. After identifying the bottleneck, engineers made configuration changes to increase the replica count of the affected frontend service component by 50%. This additional capacity relieved the resource contention, and normal service was fully restored by 08:35 US/Pacific.\nGoogle is committed preventing a repeat of this issue in the future and is completing the following actions:\n- We will tune the current authentication frontend's deployment parameters to ensure ongoing system stability\n- Longer term, we plan to eliminate this class of resource contention by migrating to a new architecture.","status":"AVAILABLE"},{"created":"2025-09-18T23:44:08+00:00","modified":"2025-09-24T17:27:52+00:00","when":"2025-09-18T23:44:08+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Workspace Support using the help article https://support.google.com/a/answer/1047213.\\n(All Times US/Pacific)\n**Incident Start:** 18 September 2025, 07:20\n**Incident End:** 18 September 2025, 08:30\n**Duration:** 1 hour, 10 minutes\n**Affected Services and Features:**\nAll Google Workspace products\n**Description:**\nMultiple customers experienced elevated login failures within various Workspace products. From preliminary analysis, the incident was suspected to be caused by a resource contention issue in Google’s internal authentication system.\nGoogle will complete a full incident report in the following days that will provide a full root cause.\n**Customer Impact:**\nMultiple users may have been unable to login to various Google services during the impacted time.","status":"AVAILABLE"},{"created":"2025-09-18T16:53:36+00:00","modified":"2025-09-18T23:44:08+00:00","when":"2025-09-18T16:53:36+00:00","text":"**Summary:**\nWe have mitigated an issue where some customers experienced login issues to Google services.\n**Description:**\nThe issue with Google services has been resolved for all affected customers as of Thursday, 2025-09-18 08:30 US/Pacific.\nFrom preliminary analysis, we have identified that the issue is caused by a resource contention issue. Our engineers have mitigated the issue by adding in additional resources.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms:**\nSome customers experienced login issues to Google services.\n**Workaround:**\nNone at this time.","status":"SERVICE_INFORMATION"},{"created":"2025-09-18T16:03:13+00:00","modified":"2025-09-18T16:53:36+00:00","when":"2025-09-18T16:03:13+00:00","text":"**Summary:**\nWe have mitigated an issue where some customers experienced login issues to Google services.\n**Description:**\nBetween 07:22 and 08:30 PDT, customers experienced login issues to Google services.\nOur engineering team has identified the cause and applied appropriate mitigations by 08:30 PDT. Customers should now be able to login to the Google services.\nWe are closely monitoring the environment for stability.\nWe will provide an update by Thursday, 2025-09-18 09:20 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms:**\nSome customers experienced login issues to Google services.\n**Workaround:**\nNone at this time.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-09-24T17:27:52+00:00","modified":"2025-09-24T17:30:38+00:00","when":"2025-09-24T17:27:52+00:00","text":"# Incident Report\n## Summary\nOn Thursday, 18 September 2025, between 07:22 and 08:35 US/Pacific, Google Workspace products experienced elevated login failures globally for a duration of 1 hour 13 minutes. Some users may have been unable to login to various Google Workspace services during that time.\nThis is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nA shift in traffic led to a resource contention issue within Google's authentication system. The system's capacity for handling concurrent requests was exhausted in several east coast regions, creating a bottleneck that blocked new login requests. This resulted in elevated login failures and errors for users across various Google services.\n## Remediation and Prevention\nGoogle engineers were alerted at 07:31 US/Pacific via internal monitoring. After identifying the bottleneck, engineers made configuration changes to increase the replica count of the affected frontend service component by 50%. This additional capacity relieved the resource contention, and normal service was fully restored by 08:35 US/Pacific.\nGoogle is committed preventing a repeat of this issue in the future and is completing the following actions:\n- We will tune the current authentication frontend's deployment parameters to ensure ongoing system stability\n- Longer term, we plan to eliminate this class of resource contention by migrating to a new architecture.","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"zall","service_name":"Multiple Products","affected_products":[{"title":"Admin Console","id":"xrkLpNMHAvPQczcU2uSb"},{"title":"Chrome Browser","id":"sfPUGuj9d6b63iYijuYV"},{"title":"ChromeOS","id":"RjKbsvuGeE7M1pvBHmvX"},{"title":"Gmail","id":"prPt2Yra2CbGsbEm9cpC"},{"title":"Google Chat","id":"Ht9Vx5PFzDPHYvrnrvSk"}],"uri":"incidents/5V5yK8N8heBKnmdqS1eW"},{"id":"mYvDLhU69KiPAdE967TZ","number":"4739091364163717407","begin":"2025-09-11T17:04:00+00:00","created":"2025-09-11T20:46:35+00:00","end":"2025-09-11T21:27:00+00:00","modified":"2025-09-12T02:58:16+00:00","external_desc":"**Summary:**\nWe are experiencing an issue with NotebookLM\n**Description:**\nWe are experiencing an issue with AudioOverview, VideoOverview, NAPPS (Notebook Apps) for NotebookLM users, beginning on Thursday, 2025-09-11 10:04 PDT.\nMitigation work is currently underway by our engineering team.\nWe do not have a full mitigation ETA at this time.\nWe will provide an update by Thursday, 2025-09-11 15:00 PDT with current details.\n**Symptoms:**\nNotebookLM users may see failures to generate:\n- Audio Overviews\n- Video Overviews\n- Reports\n- Flashcards\n- Quizzes\nFeatures launched earlier this week have been rolled back for now.\n(Quizzes, Flashcards, Reports) (https://blog.google/technology/google-labs/notebooklm-student-features/)\\nAny previously generated artifacts are accessible. All other features and CUJs are believed to be working. E.g. text Q\u0026A chat, adding sources should work\n**Workaround:**\nNone at this time.","updates":[{"created":"2025-09-11T22:49:59+00:00","modified":"2025-09-12T02:58:16+00:00","when":"2025-09-11T22:49:59+00:00","text":"**Summary:**\nNotebookLM users may have encountered failures when generating some features.\n**Description:**\nThe issue with Audio Overviews and Video Overviews has been resolved for all affected users as of Thursday, 2025-09-11 14:27 PDT.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms:**\nNotebookLM users may have encountered failures to generate:\n* Audio Overviews\n* Video Overviews\n* Reports\n* Flashcards\n* Quizzes\nFeatures launched earlier this week (Quizzes, Flashcards, Reports) have been temporarily rolled back as part of the mitigation. (https://blog.google/technology/google-labs/notebooklm-student-features/)\\nAny previously generated artifacts were accessible. All other features and CUJs were unaffected by this issue; e.g. Text Q\u0026A chat, adding sources.","status":"AVAILABLE"},{"created":"2025-09-11T20:46:35+00:00","modified":"2025-09-11T22:49:59+00:00","when":"2025-09-11T20:46:35+00:00","text":"**Summary:**\nWe are experiencing an issue with NotebookLM\n**Description:**\nWe are experiencing an issue with AudioOverview, VideoOverview, NAPPS (Notebook Apps) for NotebookLM users, beginning on Thursday, 2025-09-11 10:04 PDT.\nMitigation work is currently underway by our engineering team.\nWe do not have a full mitigation ETA at this time.\nWe will provide an update by Thursday, 2025-09-11 15:00 PDT with current details.\n**Symptoms:**\nNotebookLM users may see failures to generate:\n- Audio Overviews\n- Video Overviews\n- Reports\n- Flashcards\n- Quizzes\nFeatures launched earlier this week have been rolled back for now.\n(Quizzes, Flashcards, Reports) (https://blog.google/technology/google-labs/notebooklm-student-features/)\\nAny previously generated artifacts are accessible. All other features and CUJs are believed to be working. E.g. text Q\u0026A chat, adding sources should work\n**Workaround:**\nNone at this time.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-09-11T22:49:59+00:00","modified":"2025-09-12T02:58:16+00:00","when":"2025-09-11T22:49:59+00:00","text":"**Summary:**\nNotebookLM users may have encountered failures when generating some features.\n**Description:**\nThe issue with Audio Overviews and Video Overviews has been resolved for all affected users as of Thursday, 2025-09-11 14:27 PDT.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms:**\nNotebookLM users may have encountered failures to generate:\n* Audio Overviews\n* Video Overviews\n* Reports\n* Flashcards\n* Quizzes\nFeatures launched earlier this week (Quizzes, Flashcards, Reports) have been temporarily rolled back as part of the mitigation. (https://blog.google/technology/google-labs/notebooklm-student-features/)\\nAny previously generated artifacts were accessible. All other features and CUJs were unaffected by this issue; e.g. Text Q\u0026A chat, adding sources.","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"sqTm5ZmzCmb66kvyzcNS","service_name":"NotebookLM","affected_products":[{"title":"NotebookLM","id":"sqTm5ZmzCmb66kvyzcNS"}],"uri":"incidents/mYvDLhU69KiPAdE967TZ"},{"id":"muX1XjAhRG1krLmwbQsW","number":"5192794371337674379","begin":"2025-09-08T17:25:00+00:00","created":"2025-09-08T17:49:31+00:00","end":"2025-09-08T18:17:00+00:00","modified":"2025-09-22T13:48:55+00:00","external_desc":"**Summary:**\nWe are investigating reports of Google Meet service issues.\n**Description:**\nWe are investigating reports of Google Meet degradation\nWe will provide more information by Monday, 2025-09-08 11:15 PDT.\n**Symptoms:**\nCustomers may be experiencing elevated errors when joining meet sessions.\n**Workaround:**\nNone at this time.","updates":[{"created":"2025-09-18T17:00:33+00:00","modified":"2025-09-22T13:48:55+00:00","when":"2025-09-18T17:00:33+00:00","text":"# Incident Report\n## Summary\nOn 8 September 2025, beginning at 00:00 US/Pacific, Google Meet experienced intermittent issues loading meet.google.com for a total duration of 7 hours 57 minutes.\n- Limited impact in Central Europe occurred between 00:00 - 7:05 US/Pacific.\n- Widespread global impact, with many customers unable to load meet.google.com occurred between 10:25 -11:17 US/Pacific.\nTo our Google Meet customers whose business was impacted during this disruption, we sincerely apologize. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nThe initial incident was caused by an overloaded cache service fleet that is part of Google’s Static Content Service (SCS). SCS is used to serve aspects of the Google Meet UI that are not tied to specific to individual users or meetings. If it fails to respond, users will typically be unable to load the Google Meet landing page.\nThe exact reason for the system overload is still being investigated. However, evidence points to a specific set of Meet hot keys bypassed internal caching layers starting on Sep 8, 2025 12:00 AM US/Pacific (9:00 AM Central European Time). This issue affected a limited number of cache servers, leading to their overload, and consequently, an increase in error rates for Google Meet users.\nThe global impact was caused by a configuration change that was attempting to permanently remedy the first issue by configuring a dedicated cache for Google Meet. This dedicated cache was inadequately sized and not pre-warmed. Subsequent traffic spikes resulted in cache misses, which overloaded the cache servers and triggered the global outage.\n## Remediation and Prevention\nGoogle Meet engineers were alerted to the initial outage on 8 September 2025 at 03:12 US/Pacific via an alert indicating increased search activity about Google Meet outages. They immediately began an investigation which identified the overloaded cache service. By 07:05 US/Pacific, engineers had redirected traffic, mitigating the issue for users in Central Europe.\nAt 10:25 US/Pacific, Google engineers rolled out the configuration change to configure a dedicated cache for Google Meet as a permanent fix for the cache overload issue.\nAt 10:38 US/Pacific, Google engineers were alerted to increased error rates for the cache service fleet, and reopened the investigation. Once the global nature of the impact was identified, engineers began rolling back the configuration change at 10:56 US/Pacific.\nThe rollback of the configuration change completed at 11:09 US/Pacific and impact was confirmed to be mitigated by 11:17 US/Pacific.\nGoogle is committed preventing a repeat of this issue in the future and is completing the following actions:\n- Implement early detection monitoring and alerting mechanism for proactive future identification.\nTarget: Q3 2025\n- Implement configuration rollout safety improvements to proactively identify load nuances.\nTarget: Q4 2025\n- Validate Static Content Service (SCS) resilience to overload triggers to avoid a recurrence.\nTarget: Q4 2025\n## Detailed Description of Impact\nOn Monday 8 September 2025, Meet users may have experienced issues when attempting to launch or join meetings for a total duration of 7 hours and 57 minutes.\n- Impact Timeframe 1: 00:00 - 07:05 US/Pacific (Impact limited to Central Europe)\n- Impact Timeframe 2: 10:25 - 11:17 US/Pacific (Global Impact)\nWhen a user navigates to meet.google.com and attempts to join a meeting, multiple requests are being made from the user’s browser to SCS. Depending on which request failed, the user will experience different symptoms. Typical symptoms include blank pages and missing controls. Some customers were likely able to successfully reload a page that was broken on the initial load.\nUsers from a total of 1.8 million individual IP addresses were served at least one error when making an SCS request.","status":"AVAILABLE"},{"created":"2025-09-08T20:42:38+00:00","modified":"2025-09-18T17:00:33+00:00","when":"2025-09-08T20:42:38+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Cloud Support using https://cloud.google.com/support or to Google Workspace Support using help article https://support.google.com/a/answer/1047213.\\n(All Times US/Pacific)\n**Incident Start:** 8 September 2025 03:12\n**Incident End:** 8 September 2025 11:18\n**Duration:** 8 hours, 6 minutes\n**Affected Services and Features:** Google Meet\n**Regions/Zones:** Global\n**Description:**\nImpacted users may have experienced intermittent issues loading meet.google.com for a duration of 8 hours 6 minutes.\nThis incident was caused by errors in the system responsible for signing static content, such as Javascript and CSS files. The issue initially impacted users in Germany and possibly in the United Kingdom and Ireland, before spreading globally to all regions.\nThe trigger was identified as a recent code change. Google engineers redirected traffic away from the affected systems to mitigate the European impact. The subsequent global impact was addressed by rolling back the change, which fully mitigated the issue.\nGoogle will complete and share a full Incident Report in the following days that will provide a full root cause analysis.\n**Customer Impact:**\nUsers may have experienced intermittent issues loading meet.google.com.","status":"AVAILABLE"},{"created":"2025-09-08T18:37:45+00:00","modified":"2025-09-08T20:42:38+00:00","when":"2025-09-08T18:37:45+00:00","text":"The issue with Google Meet began at 10:25 PDT on 2025-09-08 and has been resolved for all affected users as of 2025-09-08 11:18 PDT.\nFrom preliminary analysis, the root cause of the issue is a recent change in content edge cache. Our engineers have resolved the issue by rolling back the change.\nWe thank you for your patience while we worked on resolving the issue.","status":"SERVICE_DISRUPTION"},{"created":"2025-09-08T18:30:57+00:00","modified":"2025-09-08T18:37:45+00:00","when":"2025-09-08T18:30:57+00:00","text":"**Summary:**\nResolved issues with Google Meet customers experiencing issues with loading the meet UI\n**Description:**\nOur engineers have identified the root cause and rolled back a recent change.\nCurrent internal telemetry data indicates that the Google Meet service is operating normally as of 11:18 PDT. Our engineers are continuing to monitor the service stability closely.\nWe will provide more information by Monday, 2025-09-08 11:55 PDT.\n**Symptoms:**\nAffected customers experienced slow UI loads and as a result are unable to join meet sessions.\n**Workaround:**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-09-08T18:07:46+00:00","modified":"2025-09-08T18:30:57+00:00","when":"2025-09-08T18:07:46+00:00","text":"**Summary:**\nSome Google Meet customers are experiencing issues with loading the meet UI\n**Description:**\nWe are experiencing an issue with Google Meet beginning at Monday, 2025-09-08 10:25 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Monday, 2025-09-08 11:30 PDT with current details.\n**Symptoms:**\nAffected customers are experiencing slow UI loads and as a result are unable to join meet sessions\n**Workaround:**\nNone at this time.","status":"SERVICE_INFORMATION"},{"created":"2025-09-08T17:49:31+00:00","modified":"2025-09-08T18:07:46+00:00","when":"2025-09-08T17:49:31+00:00","text":"**Summary:**\nWe are investigating reports of Google Meet service issues.\n**Description:**\nWe are investigating reports of Google Meet degradation\nWe will provide more information by Monday, 2025-09-08 11:15 PDT.\n**Symptoms:**\nCustomers may be experiencing elevated errors when joining meet sessions.\n**Workaround:**\nNone at this time.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-09-18T17:00:33+00:00","modified":"2025-09-22T13:48:55+00:00","when":"2025-09-18T17:00:33+00:00","text":"# Incident Report\n## Summary\nOn 8 September 2025, beginning at 00:00 US/Pacific, Google Meet experienced intermittent issues loading meet.google.com for a total duration of 7 hours 57 minutes.\n- Limited impact in Central Europe occurred between 00:00 - 7:05 US/Pacific.\n- Widespread global impact, with many customers unable to load meet.google.com occurred between 10:25 -11:17 US/Pacific.\nTo our Google Meet customers whose business was impacted during this disruption, we sincerely apologize. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nThe initial incident was caused by an overloaded cache service fleet that is part of Google’s Static Content Service (SCS). SCS is used to serve aspects of the Google Meet UI that are not tied to specific to individual users or meetings. If it fails to respond, users will typically be unable to load the Google Meet landing page.\nThe exact reason for the system overload is still being investigated. However, evidence points to a specific set of Meet hot keys bypassed internal caching layers starting on Sep 8, 2025 12:00 AM US/Pacific (9:00 AM Central European Time). This issue affected a limited number of cache servers, leading to their overload, and consequently, an increase in error rates for Google Meet users.\nThe global impact was caused by a configuration change that was attempting to permanently remedy the first issue by configuring a dedicated cache for Google Meet. This dedicated cache was inadequately sized and not pre-warmed. Subsequent traffic spikes resulted in cache misses, which overloaded the cache servers and triggered the global outage.\n## Remediation and Prevention\nGoogle Meet engineers were alerted to the initial outage on 8 September 2025 at 03:12 US/Pacific via an alert indicating increased search activity about Google Meet outages. They immediately began an investigation which identified the overloaded cache service. By 07:05 US/Pacific, engineers had redirected traffic, mitigating the issue for users in Central Europe.\nAt 10:25 US/Pacific, Google engineers rolled out the configuration change to configure a dedicated cache for Google Meet as a permanent fix for the cache overload issue.\nAt 10:38 US/Pacific, Google engineers were alerted to increased error rates for the cache service fleet, and reopened the investigation. Once the global nature of the impact was identified, engineers began rolling back the configuration change at 10:56 US/Pacific.\nThe rollback of the configuration change completed at 11:09 US/Pacific and impact was confirmed to be mitigated by 11:17 US/Pacific.\nGoogle is committed preventing a repeat of this issue in the future and is completing the following actions:\n- Implement early detection monitoring and alerting mechanism for proactive future identification.\nTarget: Q3 2025\n- Implement configuration rollout safety improvements to proactively identify load nuances.\nTarget: Q4 2025\n- Validate Static Content Service (SCS) resilience to overload triggers to avoid a recurrence.\nTarget: Q4 2025\n## Detailed Description of Impact\nOn Monday 8 September 2025, Meet users may have experienced issues when attempting to launch or join meetings for a total duration of 7 hours and 57 minutes.\n- Impact Timeframe 1: 00:00 - 07:05 US/Pacific (Impact limited to Central Europe)\n- Impact Timeframe 2: 10:25 - 11:17 US/Pacific (Global Impact)\nWhen a user navigates to meet.google.com and attempts to join a meeting, multiple requests are being made from the user’s browser to SCS. Depending on which request failed, the user will experience different symptoms. Typical symptoms include blank pages and missing controls. Some customers were likely able to successfully reload a page that was broken on the initial load.\nUsers from a total of 1.8 million individual IP addresses were served at least one error when making an SCS request.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"sUH4BQXzYXma7NiS94Hi","service_name":"Google Meet","affected_products":[{"title":"Google Meet","id":"sUH4BQXzYXma7NiS94Hi"}],"uri":"incidents/muX1XjAhRG1krLmwbQsW"},{"id":"H4H2Z9QdKvqdT92ke1Pb","number":"17539000878079652329","begin":"2025-09-04T17:33:00+00:00","created":"2025-09-04T18:14:04+00:00","end":"2025-09-04T19:45:00+00:00","modified":"2025-09-04T19:59:00+00:00","external_desc":"We're investigating reports of an issue with Google Drive. We will provide more information shortly.","updates":[{"created":"2025-09-04T19:51:10+00:00","modified":"2025-09-04T19:59:00+00:00","when":"2025-09-04T19:51:10+00:00","text":"**Title**\nGoogle Drive customers may experience elevated errors when downloading content\n**Description**\nThe issue with Google Drive has been resolved for all affected users as of Thursday, 2025-09-04 12:45 US/Pacific.\nThe preliminary root cause of the issue was due a recent change rollout and was mitigated by rolling back the change.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms**\nCustomer may experience errors when downloading content from Google Drive.\n**Workaround**\nNone at this time.","status":"AVAILABLE"},{"created":"2025-09-04T18:21:21+00:00","modified":"2025-09-04T19:51:10+00:00","when":"2025-09-04T18:21:21+00:00","text":"**Title**\nGoogle Drive customers may experience elevated errors when downloading content.\n**Description**\nMitigation work is currently underway by our engineering team.\nThe mitigation is expected to complete by Thursday, 2025-09-04 13:00 PDT.\nWe will provide more information by Thursday, 2025-09-04 13:00 PDT.\n**Symptoms**\nCustomer may experience errors when downloading content from Google Drive.\n**Workaround**\nNone at this time","status":"SERVICE_DISRUPTION"},{"created":"2025-09-04T18:17:33+00:00","modified":"2025-09-04T18:21:21+00:00","when":"2025-09-04T18:17:33+00:00","text":"**Title**\nGoogle Drive customers may experience elevated errors when downloading content.\n**Description**\nWe are experiencing an issue with Google Drive beginning at Thursday, 2025-09-04 10:33 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Thursday, 2025-09-04 12:30 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms**\nCustomer may experience errors when downloading content from Google Drive.\n**Workaround**\nNone at this time","status":"SERVICE_DISRUPTION"},{"created":"2025-09-04T18:14:04+00:00","modified":"2025-09-04T18:17:33+00:00","when":"2025-09-04T18:14:04+00:00","text":"We're investigating reports of an issue with Google Drive. We will provide more information shortly.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-09-04T19:51:10+00:00","modified":"2025-09-04T19:59:00+00:00","when":"2025-09-04T19:51:10+00:00","text":"**Title**\nGoogle Drive customers may experience elevated errors when downloading content\n**Description**\nThe issue with Google Drive has been resolved for all affected users as of Thursday, 2025-09-04 12:45 US/Pacific.\nThe preliminary root cause of the issue was due a recent change rollout and was mitigated by rolling back the change.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms**\nCustomer may experience errors when downloading content from Google Drive.\n**Workaround**\nNone at this time.","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"VHNA7p3Z5p3iakj5sA8V","service_name":"Google Drive","affected_products":[{"title":"Google Drive","id":"VHNA7p3Z5p3iakj5sA8V"}],"uri":"incidents/H4H2Z9QdKvqdT92ke1Pb"},{"id":"UX3VQZM4R3vLkbSq4Mrf","number":"8467497216017881467","begin":"2025-09-04T07:08:00+00:00","created":"2025-09-04T09:21:05+00:00","end":"2025-09-04T08:12:00+00:00","modified":"2025-09-11T00:29:28+00:00","external_desc":"We are experiencing an issue with multiple Google Workspace services beginning on Thursday, 2025-09-04 00:43 PDT. Google Workspace services may not load for impacted users.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Thursday, 2025-09-04 04:00 PDT with current details.\nWe apologize to all who are affected by the disruption.","updates":[{"created":"2025-09-11T00:29:28+00:00","modified":"2025-09-11T00:29:28+00:00","when":"2025-09-11T00:29:28+00:00","text":"# Incident Report\n## Summary\nOn Thursday, 4 September, 2025, customer traffic from any source region to end-users served by the Sofia, Bulgaria POP locations may have experienced network disruptions for a total of 12 hours and 5 minutes. A period of severe impact, lasting 1 hour and 4 minutes, resulted in network congestion, elevated latency, and timeouts. Google Workspace services were impacted during this 1 hour and 4 minute period of severe impact.\nWe sincerely apologize for this incident, which does not reflect the level of quality and reliability we strive to offer. We are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nGoogle’s production backbone is a global network that enables connectivity for all user-facing traffic via Points of Presence (POPs) or internet exchanges.\nThe incident was caused by a disruption of multiple fiber paths in the production backbone within Bulgaria which reduced the available network bandwidth for the POP location in Sofia, Bulgaria. Although capacity for POP locations is planned with multiple redundant fiber paths, the remaining capacity for the Sofia, Bulgaria sites to the rest of the backbone could not support the traffic load after the series of fiber failures.\nOur network is designed to be resilient and can automatically handle multiple, isolated fiber cuts by rerouting traffic. In this instance, the network successfully handled the initial fiber failures without significant customer impact. However, a subsequent fiber failure overwhelmed the remaining network capacity, which led to the service disruption.\n## Remediation and Prevention\nGoogle engineers were alerted to the outage via our internal monitoring on Thursday, 4 September, 2025 00:19 US/Pacific and immediately started an investigation. Once the nature and scope of the issue became clear, Google engineers first quickly rerouted serving traffic to adjacent locations, restoring connectivity for most services by approximately Thursday, 4 September, 2025 01:12 US/Pacific.\nSufficient fiber connectivity was restored on Thursday, 4 September, 2025 07:52 US/Pacific and Google engineers began the process of reactivating the capacity, and after validation, allowing traffic serving from the Sofia, Bulgaria POP location.\nAll traffic migrations back to the pre-incident state were completed by Thursday, 4 September, 2025 12:13 US/Pacific.\nGoogle is committed to preventing a repeat of this issue in the future and is completing the following actions:\n* To enhance service resilience, we have implemented additional redundancy on the Milan- Sofia fiber path.\n* While adding diversity will improve the overall availability of this site, we are also investigating ways to automatically signal fail-over to our external connection partners when internal capacity reduces beyond a critical threshold to satisfy demand. This will more quickly benefit those customers who connect in multiple regions.\nGoogle is committed to quickly and continually improving our technology and operations to prevent service disruptions. We appreciate your patience and apologize again for the impact to your organization. We thank you for your business.\n## Detailed Description of Impact\nThe issue primarily resulted in inaccessibility to Google Workspace services for customers in various locations served by the POP in Sofia, Bulgaria. The impact was first detected at approximately 00:19 US/Pacific on Thursday, 4 September, 2025. Google Workspace users that were routed to the affected POPs would have observed errors and high latency trying to connect to Google services. Additionally, some users encountered DNS resolution issues, including an inability to resolve some publicly accessible Google services.\nGoogle Workspace services were restored by Thursday, 4 September, 2025 01:12 US/Pacific after Google engineers rerouted serving traffic to adjacent locations, restoring connectivity for services. All fiber repairs were completed on Thursday, 4 September, 2025 12:13 US/Pacific.","status":"AVAILABLE"},{"created":"2025-09-05T20:33:50+00:00","modified":"2025-09-11T00:29:28+00:00","when":"2025-09-05T20:33:50+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Workspace Support using the help article https://support.google.com/a/answer/1047213.\\n(All Times US/Pacific)\n**Incident Start:** 4 September 2025 00:08\n**Incident End:** 4 September 2025 01:12\n**Duration:** 1 hour, 4 minutes\n**GWS- Affected Services and Features:**\nPotentially all Google Workspace services may have been impacted for users depending on the point of ingress.\n**Locations:**\nTurkey, Greece, and Bulgaria among other regions\n**Description:**\nGoogle’s production backbone is a global network that enables connectivity for user-facing traffic via Points of Presence (POPs) or internet exchanges.\nA disruption of fiber paths in the production backbone within Bulgaria reduced the available network bandwidth for the POP location in Sofia, Bulgaria. Although capacity for POP locations is planned with multiple redundant fiber paths, the remaining capacity for the Sofia, Bulgaria sites to the rest of the backbone could not support the traffic load after the series of fiber disruptions.\nGoogle engineers mitigated the impact by rerouting serving traffic to adjacent locations, restoring connectivity by approximately Thursday, 4 September, 2025 01:12 US/Pacific.\n**Customer Impact:**\nImpacted users would have encountered network errors and been unable to connect to Google Workspace services.","status":"AVAILABLE"},{"created":"2025-09-04T09:33:40+00:00","modified":"2025-09-05T20:33:50+00:00","when":"2025-09-04T09:33:40+00:00","text":"The issue with Google Workspace services has been resolved for all affected users as of Thursday, 2025-09-04 02:18 PDT.\nWe thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},{"created":"2025-09-04T09:21:05+00:00","modified":"2025-09-04T09:33:40+00:00","when":"2025-09-04T09:21:05+00:00","text":"We are experiencing an issue with multiple Google Workspace services beginning on Thursday, 2025-09-04 00:43 PDT. Google Workspace services may not load for impacted users.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Thursday, 2025-09-04 04:00 PDT with current details.\nWe apologize to all who are affected by the disruption.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-09-11T00:29:28+00:00","modified":"2025-09-11T00:29:28+00:00","when":"2025-09-11T00:29:28+00:00","text":"# Incident Report\n## Summary\nOn Thursday, 4 September, 2025, customer traffic from any source region to end-users served by the Sofia, Bulgaria POP locations may have experienced network disruptions for a total of 12 hours and 5 minutes. A period of severe impact, lasting 1 hour and 4 minutes, resulted in network congestion, elevated latency, and timeouts. Google Workspace services were impacted during this 1 hour and 4 minute period of severe impact.\nWe sincerely apologize for this incident, which does not reflect the level of quality and reliability we strive to offer. We are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nGoogle’s production backbone is a global network that enables connectivity for all user-facing traffic via Points of Presence (POPs) or internet exchanges.\nThe incident was caused by a disruption of multiple fiber paths in the production backbone within Bulgaria which reduced the available network bandwidth for the POP location in Sofia, Bulgaria. Although capacity for POP locations is planned with multiple redundant fiber paths, the remaining capacity for the Sofia, Bulgaria sites to the rest of the backbone could not support the traffic load after the series of fiber failures.\nOur network is designed to be resilient and can automatically handle multiple, isolated fiber cuts by rerouting traffic. In this instance, the network successfully handled the initial fiber failures without significant customer impact. However, a subsequent fiber failure overwhelmed the remaining network capacity, which led to the service disruption.\n## Remediation and Prevention\nGoogle engineers were alerted to the outage via our internal monitoring on Thursday, 4 September, 2025 00:19 US/Pacific and immediately started an investigation. Once the nature and scope of the issue became clear, Google engineers first quickly rerouted serving traffic to adjacent locations, restoring connectivity for most services by approximately Thursday, 4 September, 2025 01:12 US/Pacific.\nSufficient fiber connectivity was restored on Thursday, 4 September, 2025 07:52 US/Pacific and Google engineers began the process of reactivating the capacity, and after validation, allowing traffic serving from the Sofia, Bulgaria POP location.\nAll traffic migrations back to the pre-incident state were completed by Thursday, 4 September, 2025 12:13 US/Pacific.\nGoogle is committed to preventing a repeat of this issue in the future and is completing the following actions:\n* To enhance service resilience, we have implemented additional redundancy on the Milan- Sofia fiber path.\n* While adding diversity will improve the overall availability of this site, we are also investigating ways to automatically signal fail-over to our external connection partners when internal capacity reduces beyond a critical threshold to satisfy demand. This will more quickly benefit those customers who connect in multiple regions.\nGoogle is committed to quickly and continually improving our technology and operations to prevent service disruptions. We appreciate your patience and apologize again for the impact to your organization. We thank you for your business.\n## Detailed Description of Impact\nThe issue primarily resulted in inaccessibility to Google Workspace services for customers in various locations served by the POP in Sofia, Bulgaria. The impact was first detected at approximately 00:19 US/Pacific on Thursday, 4 September, 2025. Google Workspace users that were routed to the affected POPs would have observed errors and high latency trying to connect to Google services. Additionally, some users encountered DNS resolution issues, including an inability to resolve some publicly accessible Google services.\nGoogle Workspace services were restored by Thursday, 4 September, 2025 01:12 US/Pacific after Google engineers rerouted serving traffic to adjacent locations, restoring connectivity for services. All fiber repairs were completed on Thursday, 4 September, 2025 12:13 US/Pacific.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"VHNA7p3Z5p3iakj5sA8V","service_name":"Google Drive","affected_products":[{"title":"Google Drive","id":"VHNA7p3Z5p3iakj5sA8V"}],"uri":"incidents/UX3VQZM4R3vLkbSq4Mrf"},{"id":"VGfCcPeEwQMdXwDjFYEX","number":"10927389432659257465","begin":"2025-09-04T07:08:00+00:00","created":"2025-09-04T09:16:07+00:00","end":"2025-09-04T08:12:00+00:00","modified":"2025-09-11T00:30:58+00:00","external_desc":"We are experiencing an issue with multiple Google Workspace services beginning on Thursday, 2025-09-04 00:43 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Thursday, 2025-09-04 04:00 PDT with current details.\nWe apologize to all who are affected by the disruption.","updates":[{"created":"2025-09-11T00:30:58+00:00","modified":"2025-09-11T00:30:58+00:00","when":"2025-09-11T00:30:58+00:00","text":"A full incident report has been posted at https://www.google.com/appsstatus/dashboard/incidents/UX3VQZM4R3vLkbSq4Mrf","status":"AVAILABLE"},{"created":"2025-09-05T20:58:13+00:00","modified":"2025-09-11T00:30:58+00:00","when":"2025-09-05T20:58:13+00:00","text":"A mini incident report has been posted at https://www.google.com/appsstatus/dashboard/incidents/UX3VQZM4R3vLkbSq4Mrf","status":"AVAILABLE"},{"created":"2025-09-04T09:32:34+00:00","modified":"2025-09-05T20:58:13+00:00","when":"2025-09-04T09:32:34+00:00","text":"The issue with Google Workspace services has been resolved for all affected users as of Thursday, 2025-09-04 02:18 PDT.\nWe thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},{"created":"2025-09-04T09:16:07+00:00","modified":"2025-09-04T09:32:34+00:00","when":"2025-09-04T09:16:07+00:00","text":"We are experiencing an issue with multiple Google Workspace services beginning on Thursday, 2025-09-04 00:43 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Thursday, 2025-09-04 04:00 PDT with current details.\nWe apologize to all who are affected by the disruption.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-09-11T00:30:58+00:00","modified":"2025-09-11T00:30:58+00:00","when":"2025-09-11T00:30:58+00:00","text":"A full incident report has been posted at https://www.google.com/appsstatus/dashboard/incidents/UX3VQZM4R3vLkbSq4Mrf","status":"AVAILABLE"},"status\_impact":"SERVICE\_DISRUPTION","severity":"medium","service\_key":"n5hVexuq1PM9onY9qrmo","service\_name":"Google Calendar","affected_products":[{"title":"Google Calendar","id":"n5hVexuq1PM9onY9qrmo"}],"uri":"incidents/VGfCcPeEwQMdXwDjFYEX"},{"id":"3cc92cDxkMbGrvFaiySR","number":"16411630852503207546","begin":"2025-09-04T07:08:00+00:00","created":"2025-09-04T09:13:39+00:00","end":"2025-09-04T08:12:00+00:00","modified":"2025-09-11T00:31:29+00:00","external_desc":"**Summary**\nMultiple Google Workspace services may be inaccessible to some customers in the EMEA region.\n**Description**\nWe are experiencing an issue with multiple Google Workspace services beginning on Thursday, 2025-09-04 00:43 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Thursday, 2025-09-04 04:00 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nGoogle Workspace services may not load for impacted users.\n**Workaround**\nNone at this time.","updates":[{"created":"2025-09-11T00:31:29+00:00","modified":"2025-09-11T00:31:29+00:00","when":"2025-09-11T00:31:29+00:00","text":"A full incident report has been posted at https://www.google.com/appsstatus/dashboard/incidents/UX3VQZM4R3vLkbSq4Mrf","status":"AVAILABLE"},{"created":"2025-09-05T20:56:35+00:00","modified":"2025-09-11T00:31:29+00:00","when":"2025-09-05T20:56:35+00:00","text":"A mini incident report has been posted at https://www.google.com/appsstatus/dashboard/incidents/UX3VQZM4R3vLkbSq4Mrf","status":"AVAILABLE"},{"created":"2025-09-04T09:34:34+00:00","modified":"2025-09-05T20:56:35+00:00","when":"2025-09-04T09:34:34+00:00","text":"The issue with Google Workspace services has been resolved for all affected users as of Thursday, 2025-09-04 02:18 PDT.\nWe thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},{"created":"2025-09-04T09:13:39+00:00","modified":"2025-09-04T09:34:34+00:00","when":"2025-09-04T09:13:39+00:00","text":"**Summary**\nMultiple Google Workspace services may be inaccessible to some customers in the EMEA region.\n**Description**\nWe are experiencing an issue with multiple Google Workspace services beginning on Thursday, 2025-09-04 00:43 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Thursday, 2025-09-04 04:00 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nGoogle Workspace services may not load for impacted users.\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-09-11T00:31:29+00:00","modified":"2025-09-11T00:31:29+00:00","when":"2025-09-11T00:31:29+00:00","text":"A full incident report has been posted at https://www.google.com/appsstatus/dashboard/incidents/UX3VQZM4R3vLkbSq4Mrf","status":"AVAILABLE"},"status\_impact":"SERVICE\_DISRUPTION","severity":"medium","service\_key":"prPt2Yra2CbGsbEm9cpC","service\_name":"Gmail","affected\_products":\[{"title":"Gmail","id":"prPt2Yra2CbGsbEm9cpC"}\],"uri":"incidents/3cc92cDxkMbGrvFaiySR"},{"id":"R6N8442fvbXvxRFsbLd4","number":"11980598393880330639","begin":"2025-08-25T21:27:00+00:00","created":"2025-08-25T22:23:08+00:00","end":"2025-08-25T21:52:00+00:00","modified":"2025-08-25T22:48:49+00:00","external\_desc":"\*\*Summary\*\*\\nGoogle Docs customers may have observed '500' errors when accessing editors.\n**Description**\nWe were experiencing an issue with Google Docs beginning at Monday, 2025-08-25 14:27 PDT.\nOur engineering team has successfully mitigated the issue and we are currently monitoring our system for stability.\nWe will provide an update by Monday, 2025-08-25 15:30 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms**\nImpacted customers may have observed '500' errors when accessing Docs editors, including docs, spreadsheets, presentations, and drawings\n**Workaround**\nNone at this time.","updates":[{"created":"2025-08-25T22:48:12+00:00","modified":"2025-08-25T22:48:49+00:00","when":"2025-08-25T22:48:12+00:00","text":"**Summary**\nGoogle Docs customers may have observed '500' errors when accessing editors\n**Description**\nThe issue with Google Docs has been resolved for all affected users as of Monday, 2025-08-25 14:52 PDT.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms**\nImpacted customers may have observed '500' errors when accessing Docs editors, including docs, spreadsheets, presentations, and drawings.","status":"AVAILABLE"},{"created":"2025-08-25T22:23:08+00:00","modified":"2025-08-25T22:48:12+00:00","when":"2025-08-25T22:23:08+00:00","text":"**Summary**\nGoogle Docs customers may have observed '500' errors when accessing editors.\n**Description**\nWe were experiencing an issue with Google Docs beginning at Monday, 2025-08-25 14:27 PDT.\nOur engineering team has successfully mitigated the issue and we are currently monitoring our system for stability.\nWe will provide an update by Monday, 2025-08-25 15:30 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms**\nImpacted customers may have observed '500' errors when accessing Docs editors, including docs, spreadsheets, presentations, and drawings\n**Workaround**\nNone at this time.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-08-25T22:48:12+00:00","modified":"2025-08-25T22:48:49+00:00","when":"2025-08-25T22:48:12+00:00","text":"**Summary**\nGoogle Docs customers may have observed '500' errors when accessing editors\n**Description**\nThe issue with Google Docs has been resolved for all affected users as of Monday, 2025-08-25 14:52 PDT.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms**\nImpacted customers may have observed '500' errors when accessing Docs editors, including docs, spreadsheets, presentations, and drawings.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"wNHuVFtZEWU5Mmj2eRCK","service_name":"Google Docs","affected_products":[{"title":"Google Docs","id":"wNHuVFtZEWU5Mmj2eRCK"}],"uri":"incidents/R6N8442fvbXvxRFsbLd4"},{"id":"jkx7uigB29SrPe8zwvfZ","number":"16032085283414294175","begin":"2025-08-25T16:25:00+00:00","created":"2025-08-25T20:01:27+00:00","end":"2025-08-25T20:25:00+00:00","modified":"2025-08-29T00:55:22+00:00","external_desc":"**Summary:**\nGoogle Voice customers may have been unable to place or receive calls on their Google Voice clients and desk phones\n**Description:**\nMitigation work is currently underway by our engineering team.\nWe do not have an ETA for mitigation at this point.\nWe will provide more information by Monday, 2025-08-25 15:00 PDT.\n**Diagnosis / Customer Symptoms:**\nAffected customers may have experienced dropped calls and connection issues intermittently.\n**Workaround:**\nNone at this time.","updates":[{"created":"2025-08-29T00:55:22+00:00","modified":"2025-08-29T00:55:23+00:00","when":"2025-08-29T00:55:22+00:00","text":"# Incident Report\n## Summary\n**Incident Start:** 25 August 2025 09:25 US/Pacific\n**Incident End:** 25 August 2025 13:25 US/Pacific\n**Duration:** 4 hours\nOn Monday, 25 August 2025, Google Voice customers experienced intermittent dropped calls and connection issues globally for a duration of 4 hours.\nTo our affected customers whose business was impacted during this disruption, we sincerely apologize. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nA software update inadvertently caused a massive number of devices to simultaneously connect to a critical component of our voice infrastructure. This surge of connection attempts overwhelmed the component, resulting in server tasks dropping traffic and repeatedly crashing for approximately 4 hours.\nThe issue was compounded because the devices were configured to immediately and repeatedly retry failed connections, creating a feedback loop that intensified the overload.\nAs a result, this core component was unable to process traffic from other clients, including Google Voice. This resulted in intermittent dropped calls and a significant percentage of call failures during the incident’s peak.\n## Remediation and Prevention\nGoogle engineers were alerted to the issue via an internal monitoring at 07:20 US/Pacific on 25 August and immediately started an investigation. Once the nature and scope of the issue was identified, Google engineers rolled back the software update and increased capacity to the affected servers.These actions successfully mitigated the issue for all affected users by 13:25 US/Pacific.\nGoogle is committed preventing a repeat of this issue in the future and is completing the following actions:\n- To prevent service degradation, client applications implement a longer retry backoff upon receiving an HTTP 503 error for connection failures.\n- Improve and update software rollout procedures and tooling to protect against unintentional changes and resulting traffic spikes.\n- Improve capacity planning and server utilization monitoring for the affected servers.\nWe apologize for the length and severity of this incident. We are taking immediate steps to prevent a recurrence and improve reliability in the future.\n## Detailed Description of Impact\nFrom 09:25 to 13:25 US/Pacific on 25 August 2025, Google Voice customers experienced intermittent dropped calls and connection issues globally for a duration of 4 hours.\nBoth enterprise and consumer users experienced intermittent dropped calls and connection issues across all devices such as mobile, web, and desk phones. At the peak of the incident, call failure rates reached up to 25%, with approximately one call dropped per second due to peer timeouts.","status":"AVAILABLE"},{"created":"2025-08-26T07:46:45+00:00","modified":"2025-08-29T00:55:22+00:00","when":"2025-08-26T07:46:45+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Workspace Support using help article https://support.google.com/a/answer/1047213.\\n(All Times US/Pacific)\n**Incident Start:** 25 August 2025 09:25\n**Incident End:** 25 August 2025 13:25\n**Duration:** 4 hours\n**Affected Services and Features:**\nGoogle Voice\n**Regions/Zones:** Global\n**Description:**\nGoogle Voice customers experienced intermittent dropped calls and connection issues globally for a duration of 4 hours. Based on preliminary analysis, the root cause is a software rollout which resulted in increased traffic and overloaded key infrastructure Google uses for voice calls.\n**Customer Impact:**\nAffected customers intermittently experienced dropped calls and connection issues. This issue may have affected our enterprise and consumer account customers across all devices (mobile, web, deskphones or dedicated hardware).","status":"AVAILABLE"},{"created":"2025-08-25T21:12:05+00:00","modified":"2025-08-26T07:46:45+00:00","when":"2025-08-25T21:12:05+00:00","text":"**Summary:**\nGoogle Voice customers may have been unable to place or receive calls on their Google Voice clients and deskphones\n**Description:**\nThe issue with Google Voice has been resolved for all affected users as of Monday, 2025-08-25 12:57 PDT\nThe issue was due to a hardware device firmware update causing higher than expected traffic to an overloaded server. Our engineers have mitigated the issue by disabling the software change and adding extra capacity to the affected servers.\nWe thank you for your patience while we worked on resolving the issue.\n**Diagnosis / Customer Symptoms:**\nAffected customers may have experienced dropped calls and connection issues intermittently.\n**Workaround:**\nNone at this time.","status":"AVAILABLE"},{"created":"2025-08-25T20:01:27+00:00","modified":"2025-08-26T14:32:50+00:00","when":"2025-08-25T20:01:27+00:00","text":"**Summary:**\nGoogle Voice customers may have been unable to place or receive calls on their Google Voice clients and desk phones\n**Description:**\nMitigation work is currently underway by our engineering team.\nWe do not have an ETA for mitigation at this point.\nWe will provide more information by Monday, 2025-08-25 15:00 PDT.\n**Diagnosis / Customer Symptoms:**\nAffected customers may have experienced dropped calls and connection issues intermittently.\n**Workaround:**\nNone at this time.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-08-29T00:55:22+00:00","modified":"2025-08-29T00:55:23+00:00","when":"2025-08-29T00:55:22+00:00","text":"# Incident Report\n## Summary\n**Incident Start:** 25 August 2025 09:25 US/Pacific\n**Incident End:** 25 August 2025 13:25 US/Pacific\n**Duration:** 4 hours\nOn Monday, 25 August 2025, Google Voice customers experienced intermittent dropped calls and connection issues globally for a duration of 4 hours.\nTo our affected customers whose business was impacted during this disruption, we sincerely apologize. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform’s performance and availability.\n## Root Cause\nA software update inadvertently caused a massive number of devices to simultaneously connect to a critical component of our voice infrastructure. This surge of connection attempts overwhelmed the component, resulting in server tasks dropping traffic and repeatedly crashing for approximately 4 hours.\nThe issue was compounded because the devices were configured to immediately and repeatedly retry failed connections, creating a feedback loop that intensified the overload.\nAs a result, this core component was unable to process traffic from other clients, including Google Voice. This resulted in intermittent dropped calls and a significant percentage of call failures during the incident’s peak.\n## Remediation and Prevention\nGoogle engineers were alerted to the issue via an internal monitoring at 07:20 US/Pacific on 25 August and immediately started an investigation. Once the nature and scope of the issue was identified, Google engineers rolled back the software update and increased capacity to the affected servers.These actions successfully mitigated the issue for all affected users by 13:25 US/Pacific.\nGoogle is committed preventing a repeat of this issue in the future and is completing the following actions:\n- To prevent service degradation, client applications implement a longer retry backoff upon receiving an HTTP 503 error for connection failures.\n- Improve and update software rollout procedures and tooling to protect against unintentional changes and resulting traffic spikes.\n- Improve capacity planning and server utilization monitoring for the affected servers.\nWe apologize for the length and severity of this incident. We are taking immediate steps to prevent a recurrence and improve reliability in the future.\n## Detailed Description of Impact\nFrom 09:25 to 13:25 US/Pacific on 25 August 2025, Google Voice customers experienced intermittent dropped calls and connection issues globally for a duration of 4 hours.\nBoth enterprise and consumer users experienced intermittent dropped calls and connection issues across all devices such as mobile, web, and desk phones. At the peak of the incident, call failure rates reached up to 25%, with approximately one call dropped per second due to peer timeouts.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"gh79SrQk6wqonWoxPdCm","service_name":"Google Voice","affected_products":[{"title":"Google Voice","id":"gh79SrQk6wqonWoxPdCm"}],"uri":"incidents/jkx7uigB29SrPe8zwvfZ"},{"id":"B6TDyZ3a2dhAg59QcpZE","number":"18008682940474766646","begin":"2025-08-19T07:00:00+00:00","created":"2025-08-27T19:14:43+00:00","end":"2025-08-28T20:20:00+00:00","modified":"2025-08-28T21:32:44+00:00","external_desc":"Some ChromeOS customers experiencing issues signing on.\n**Description:**\nWe are experiencing an issue with ChromeOS M-139.\nOur engineering team continues to investigate the issue.\nWe will provide more information by Wednesday, 2025-08-27 13:00 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms:**\nAuthentication failures are occurring on ChromeOS M-139, affecting Single Sign-On (SSO) with educational partners Clever and Classlink. The issue is also causing 2-Step Verification (2SV) to fail for some users.\n**Workarounds:**\nThere are 2 workarounds and you can follow one of them.\n1. Rollback to the previous version M138. Detailed instructions https://support.google.com/chrome/a/answer/12569990.\\nOR\\n2\. Change LoginAuthenticationBehavior (https://chromeenterprise.google/policies/#LoginAuthenticationBehavior) to \"Authentication via the default GAIA flow\"","updates":[{"created":"2025-08-28T21:32:03+00:00","modified":"2025-08-28T21:32:44+00:00","when":"2025-08-28T21:32:03+00:00","text":"Some ChromeOS customers may have experienced issues signing on.\n**Description**\nThe issue with ChromeOS has been resolved for all affected users as of Thursday, 2025-08-28 13:20 PDT.\nFrom preliminary analysis, the issue was triggered by a change in the backend authentication method, which resulted in Single Sign-On (SSO) failures or 2-Step Verification (2SV) to fail for some users.\nAfter identifying the root cause, our engineering team immediately halted the release of the affected Chrome OS/Browser version (16328.55.0/139.0.7258.137) to additional devices. The fix was identified and tested, after which it has been successfully deployed in version 16328.65.0/ 139.0.7258.164.\nCustomers previously affected by this issue may update their Chrome OS/Browser versions to 16328.65.0/139.0.7258.164.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms**\nAuthentication failures were occurring on ChromeOS devices with OS/Browser versions 16328.55.0/139.0.7258.137, affecting Single Sign-On (SSO) with educational partners Clever and Classlink. The issue also caused 2-Step Verification (2SV) to fail for some users.","status":"AVAILABLE"},{"created":"2025-08-28T17:21:04+00:00","modified":"2025-08-28T21:32:03+00:00","when":"2025-08-28T17:21:04+00:00","text":"Some ChromeOS customers experiencing issues signing on.\n**Description:**\nThis affects only ChromeOS devices on the following OS/Browser Version:\n16328.55.0/139.0.7258.137\nMitigation work is taking longer than anticipated.\nFix is undergoing automated testing. Once completed, engineering will validate the results and we'll make the fix available to users.\nWe currently don't have ETA for the version with the fix to be available.\nWe will provide an update by Thursday, 2025-08-28 17:30 US/Pacific with current details.\n**Symptoms:**\nAuthentication failures are occurring on ChromeOS devices with OS/Browser versions 16328.55.0/139.0.7258.137, affecting Single Sign-On (SSO) with educational partners Clever and Classlink. The issue is also causing 2-Step Verification (2SV) to fail for some users.\n**Workaround:**\nThere are 2 workarounds and you can follow one of them.\n1. Rollback to the previous version M138. Detailed instructions https://support.google.com/chrome/a/answer/12569990.\\nOR\\n2\. Change LoginAuthenticationBehavior (https://chromeenterprise.google/policies/#LoginAuthenticationBehavior) to \"Authentication via the default GAIA flow\"","status":"SERVICE_INFORMATION"},{"created":"2025-08-27T20:22:22+00:00","modified":"2025-08-28T17:21:04+00:00","when":"2025-08-27T20:22:22+00:00","text":"Some ChromeOS customers experiencing issues signing on.\n**Description:**\nThis affects only ChromeOS devices on the following OS/Browser Version:\n16328.55.0/139.0.7258.137\nMitigation work is currently underway by our engineering team.\nAfter identifying the root cause, our engineering team immediately halted the release of the affected version to additional devices. The fix has been identified and tested, and the build for the fix is currently underway.\nThe version with the fix is expected to be available by tomorrow.\nWe will provide an update by Thursday, 2025-08-28 13:30 US/Pacific with current details.\n**Symptoms:**\nAuthentication failures are occurring on ChromeOS devices with OS/Browser versions 16328.55.0/139.0.7258.137, affecting Single Sign-On (SSO) with educational partners Clever and Classlink. The issue is also causing 2-Step Verification (2SV) to fail for some users.\n**Workaround:**\nThere are 2 workarounds and you can follow one of them.\n1. Rollback to the previous version M138. Detailed instructions https://support.google.com/chrome/a/answer/12569990.\\nOR\\n2\. Change LoginAuthenticationBehavior (https://chromeenterprise.google/policies/#LoginAuthenticationBehavior) to \"Authentication via the default GAIA flow\"","status":"SERVICE_INFORMATION"},{"created":"2025-08-27T19:14:43+00:00","modified":"2025-08-27T20:22:22+00:00","when":"2025-08-27T19:14:43+00:00","text":"Some ChromeOS customers experiencing issues signing on.\n**Description:**\nWe are experiencing an issue with ChromeOS M-139.\nOur engineering team continues to investigate the issue.\nWe will provide more information by Wednesday, 2025-08-27 13:00 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\n**Symptoms:**\nAuthentication failures are occurring on ChromeOS M-139, affecting Single Sign-On (SSO) with educational partners Clever and Classlink. The issue is also causing 2-Step Verification (2SV) to fail for some users.\n**Workarounds:**\nThere are 2 workarounds and you can follow one of them.\n1. Rollback to the previous version M138. Detailed instructions https://support.google.com/chrome/a/answer/12569990.\\nOR\\n2\. Change LoginAuthenticationBehavior (https://chromeenterprise.google/policies/#LoginAuthenticationBehavior) to \"Authentication via the default GAIA flow\"","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-08-28T21:32:03+00:00","modified":"2025-08-28T21:32:44+00:00","when":"2025-08-28T21:32:03+00:00","text":"Some ChromeOS customers may have experienced issues signing on.\n**Description**\nThe issue with ChromeOS has been resolved for all affected users as of Thursday, 2025-08-28 13:20 PDT.\nFrom preliminary analysis, the issue was triggered by a change in the backend authentication method, which resulted in Single Sign-On (SSO) failures or 2-Step Verification (2SV) to fail for some users.\nAfter identifying the root cause, our engineering team immediately halted the release of the affected Chrome OS/Browser version (16328.55.0/139.0.7258.137) to additional devices. The fix was identified and tested, after which it has been successfully deployed in version 16328.65.0/ 139.0.7258.164.\nCustomers previously affected by this issue may update their Chrome OS/Browser versions to 16328.65.0/139.0.7258.164.\nWe thank you for your patience while we worked on resolving the issue.\n**Symptoms**\nAuthentication failures were occurring on ChromeOS devices with OS/Browser versions 16328.55.0/139.0.7258.137, affecting Single Sign-On (SSO) with educational partners Clever and Classlink. The issue also caused 2-Step Verification (2SV) to fail for some users.","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"RjKbsvuGeE7M1pvBHmvX","service_name":"ChromeOS","affected_products":[{"title":"ChromeOS","id":"RjKbsvuGeE7M1pvBHmvX"}],"uri":"incidents/B6TDyZ3a2dhAg59QcpZE"},{"id":"oFcAZTr4EVieF5Fr6Ee9","number":"6540446163670387082","begin":"2025-07-18T15:10:00+00:00","created":"2025-07-18T15:35:43+00:00","end":"2025-07-18T15:56:00+00:00","modified":"2025-07-22T13:52:47+00:00","external_desc":"Various Workspace services are experiencing elevated latency and error rates","updates":[{"created":"2025-07-22T13:52:47+00:00","modified":"2025-07-22T13:52:47+00:00","when":"2025-07-22T13:52:47+00:00","text":"The Incident Report is now available on the https://status.cloud.google.com/incidents/8cY8jdUpEGGbsSMSQk7J\\n\[Google Cloud Service Health Dashboard] while we prepare a full Issue Summary Report.","status":"AVAILABLE"},{"created":"2025-07-18T22:21:25+00:00","modified":"2025-07-22T13:52:47+00:00","when":"2025-07-18T22:21:25+00:00","text":"The Mini Incident Report is available on the https://status.cloud.google.com/incidents/8cY8jdUpEGGbsSMSQk7J (Google Cloud Service Health Dashboard) while we prepare a full Incident Summary Report.","status":"AVAILABLE"},{"created":"2025-07-18T16:05:39+00:00","modified":"2025-07-18T22:21:25+00:00","when":"2025-07-18T16:05:39+00:00","text":"The issue has been mitigated for all affected users as of 2025-07-18 08:56 US/Pacific.\nA subset of workspace customers experienced elevated latency and error rates using workspace services.\nOur initial investigation points to a hardware infrastructure failure as the likely cause. Engineers have mitigated the workspace impact by routing the traffic away from the affected hardware infrastructure.\nWe thank you for your patience while we're working on resolving the issue.","status":"AVAILABLE"},{"created":"2025-07-18T15:35:43+00:00","modified":"2025-07-18T16:05:39+00:00","when":"2025-07-18T15:35:43+00:00","text":"Various Workspace services are experiencing elevated latency and error rates","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-07-22T13:52:47+00:00","modified":"2025-07-22T13:52:47+00:00","when":"2025-07-22T13:52:47+00:00","text":"The Incident Report is now available on the https://status.cloud.google.com/incidents/8cY8jdUpEGGbsSMSQk7J\\n\[Google Cloud Service Health Dashboard] while we prepare a full Issue Summary Report.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"zall","service_name":"Multiple Products","affected_products":[{"title":"Gmail","id":"prPt2Yra2CbGsbEm9cpC"},{"title":"Google Chat","id":"Ht9Vx5PFzDPHYvrnrvSk"},{"title":"Google Docs","id":"wNHuVFtZEWU5Mmj2eRCK"},{"title":"Google Drive","id":"VHNA7p3Z5p3iakj5sA8V"},{"title":"Google Meet","id":"sUH4BQXzYXma7NiS94Hi"},{"title":"Google Workspace Support","id":"dnz8HVUHoMMcW9wV91LY"}],"uri":"incidents/oFcAZTr4EVieF5Fr6Ee9"},{"id":"1hSNYy2ihYPewRrRji6R","number":"14610264225459492371","begin":"2025-07-16T01:27:00+00:00","created":"2025-07-16T18:48:30+00:00","end":"2025-07-17T00:16:00+00:00","modified":"2025-07-18T15:47:54+00:00","external_desc":"**Summary:**\nGoogle Drive users may be unexpectedly logged out of Drive for Desktop.\n**Description:**\nWe are experiencing an issue with Google Drive for desktop, beginning on Wednesday, 2025-07-16 10:24:07 PDT.\nOur engineering team has identified the source of the issue and is implementing a fix.\nWe will provide more information by Wednesday, 2025-07-16 12:30:00 PDT with additional details.\nWe apologize to all who are affected by the disruption.\n**Diagnosis / Customer Symptoms:**\nGoogle Drive users may experience being logged out of Drive for Desktop and see error “Your administrator has restricted Google Drive access to authorized devices. You have been logged out, and all Google Drive content related to your account has been wiped from this computer. Please contact your administrator to learn more.” Only enterprise customers are impacted.\n**Workaround:**\nAdministrators may go to the Drive for Desktop settings in Admin Console under Apps \u003e Drive and Docs \u003e Drive for desktop \u003e Enable Drive for desktop and uncheck “Only allow Google Drive for desktop on authorized devices” For details see https://support.google.com/a/answer/7491144.","updates":\[{"created":"2025-07-18T08:22:38+00:00","modified":"2025-07-18T15:47:54+00:00","when":"2025-07-18T08:22:38+00:00","text":"# Incident Report\n## Summary\nOn Tuesday, 15 July 2025, for a duration of up to 22 hours and 49 minutes, Google Drive users were denied access to the Google Drive for desktop application. Affected users received an error message ‘Your administrator has restricted Google Drive access to authorized devices. You have been logged out, and all Google Drive content related to your account has been wiped from this computer. Please contact your administrator to learn more’. This message was erroneously triggered and no data was programmatically wiped.\nWe deeply regret the disruption our Google Drive for Desktop users experienced. We recognize the impact this had on your businesses, and we are truly sorry for falling short of your expectations. We have investigated the root cause of the issue and are implementing measures to improve the resilience of our services.\n## Root Cause\nThe root cause of the issue was an update to Drive's policy configuration. As part of this update, the admin setting that restricts Google Drive for desktop to authorized devices had its default unintentionally changed from FALSE to TRUE. This change took effect for domains which hadn't previously overridden the setting.\nAs a result, a number of users were unexpectedly prevented from launching the Google Drive for desktop application, requiring them to restart the application. The impact was limited to Windows and Mac devices used by Enterprise customers. The incident began at 18:27 US/Pacific on 15 July 2025 and deteriorated further on 16 July 2025.\n## Remediation and Prevention\nAt 18:27 US/Pacific on Tuesday, 15 July 2025, a 1% rollout release began causing minor impact. As the release progressed, the situation escalated significantly and became evident on Wednesday, 16 July 2025 from an increase in support cases. Our engineering team was alerted at 09:25 US/Pacific and immediately began the investigation.\nAs the first step to mitigate impact, Google engineers deployed a client-side mitigation to all Google Drive for desktop clients by 11:47 US/Pacific on Wednesday, 16 July 2025. This deployment disabled Enterprise Mobility Management support within Google Drive for Desktop. While this aimed to prevent further application launch blocks, it wasn't guaranteed to reach all devices before the block signal.\nConcurrently, a policy fix was deployed via a rollback. This rollback targeted the incorrect policy configuration that triggered the blocks, which completed at 17:16 US/Pacific on Wednesday, 16 July 2025.\nWe are deeply sorry for the length and severity of this incident. Google is committed to preventing a repeat of this incident in the future and we are completing the following actions :\n* Improve the error message displayed to our users when devices that are not company owned are blocked from using Google Drive for desktop.\n* Improve monitoring on the number and rate of policy related actions taken by Google Drive for desktop.\n## Detailed Description of Impact\nOn Tuesday, 15 July 2025 from 18:27 US/Pacific to Wednesday, 16 July 2025, 17:16 US/Pacific:\n#### Google Drive\nApproximately 1% users may have been prevented from launching Google Drive for desktop and encountered the error ‘Your administrator has restricted Google Drive access to authorized devices. You have been logged out, and all Google Drive content related to your account has been wiped from this computer. Please contact your administrator to learn more.’\nAs a temporary workaround, administrators were advised to manually disable the ‘Only allow Google Drive for desktop on authorized devices’ setting to prevent blocks on their managed devices.\n## Additional Information for Customers\nOnce the incident was mitigated, users who exited the application without taking any further action were able to access the application after they relaunched it using existing account configuration. Some users who may have disconnected their account using the button that was presented as an option, were able to regain access after signing into the application and performing additional actions to reconfigure any mirror roots and re-pin any files that required offline availability. These users who disconnected their account may also have had to download the files again that were previously cached.","status":"AVAILABLE"},{"created":"2025-07-17T17:41:09+00:00","modified":"2025-07-18T08:22:38+00:00","when":"2025-07-17T17:41:09+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Workspace Support using help article https://support.google.com/a/answer/1047213.\\n(All Times US/Pacific)\n**Incident Start:** 15 July 2025 18:27\n**Incident End:** 16 July 2025 18:19\n**Duration:** 23 hours, 52 minutes\n**Affected Services and Features:**\nGoogle Drive for desktop\n**Regions/Zones:** Global\n**Description:**\nA subset of Google Drive users who had their account in Drive for desktop disabled, observed the error: “Your administrator has restricted Google Drive access to authorized devices. You have been logged out, and all Google Drive content related to your account has been wiped from this computer. Please contact your administrator to learn more”. Despite the message, no Google Drive content was actually wiped from the users’ computers. The message was a result of a configuration change to the default admin policies. Google will complete a full Incident Report in the following days.\n**Customer Impact:**\nA subset of Google Drive users had their account in Drive for desktop disabled and observed the error: “Your administrator has restricted Google Drive access to authorized devices. You have been logged out, and all Google Drive content related to your account has been wiped from this computer. Please contact your administrator to learn more”. Despite the message, no Google Drive content was actually wiped from the users’ computers.","status":"AVAILABLE"},{"created":"2025-07-17T02:05:13+00:00","modified":"2025-07-17T17:41:09+00:00","when":"2025-07-17T02:05:13+00:00","text":"The issue with Google Drive for Desktop has been resolved for all affected users as of Wednesday, 2025-07-16 17:16 PDT.\nThe issue was due to a configuration change to the default admin policies. Our engineers mitigated the issue by rolling back the configuration.\n**Action required:** If any users are still experiencing issues, they should quit the Drive for desktop app and relaunch it.\nWe thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},{"created":"2025-07-17T01:09:00+00:00","modified":"2025-07-17T02:05:13+00:00","when":"2025-07-17T01:09:00+00:00","text":"**Summary**:\nGoogle Drive users may be unexpectedly logged out of Drive for Desktop.\n**Description**:\nThe engineering team believes that the incident mitigation is complete and is confirming its effectiveness.\nAn estimated time of resolution provided is Wednesday, 2025-07-16 at 19:00:00 PDT.\nWe will provide more information by Wednesday, 2025-07-16 19:30:00 PDT with additional details.\n**Diagnosis / Customer Symptoms**:\nGoogle Drive users may experience being logged out of Drive for Desktop and see error “Your administrator has restricted Google Drive access to authorized devices. You have been logged out, and all Google Drive content related to your account has been wiped from this computer. Please contact your administrator to learn more.” Only enterprise customers are impacted.\n**Workaround**:\nAdministrators may go to the Drive for Desktop settings in Admin Console under Apps \u003e Drive and Docs \u003e Drive for desktop \u003e Enable Drive for desktop and uncheck “Only allow Google Drive for desktop on authorized devices” For details see https://support.google.com/a/answer/7491144.","status":"SERVICE\_DISRUPTION"},{"created":"2025-07-16T19:50:49+00:00","modified":"2025-07-17T01:09:00+00:00","when":"2025-07-16T19:50:49+00:00","text":"\*\*Summary:\*\*\\nGoogle Drive users may be unexpectedly logged out of Drive for Desktop.\n**Description:**\nWe are experiencing an issue with Google Drive for Desktop, beginning on Wednesday, 2025-07-16 10:24:07 PDT.\nOur engineers have implemented a mitigation strategy and have partially mitigated the issue.\nThe engineering team continues to work towards full mitigation.\nAn estimated time of resolution provided is Wednesday, 2025-07-16 at 17:00:00 PDT.\nWe will provide more information by Wednesday, 2025-07-16 18:00:00 PDT with additional details.\nWe apologize to all who are affected by the disruption.\n**Diagnosis / Customer Symptoms:**\nGoogle Drive users may experience being logged out of Drive for Desktop and see error “Your administrator has restricted Google Drive access to authorized devices. You have been logged out, and all Google Drive content related to your account has been wiped from this computer. Please contact your administrator to learn more.” Only enterprise customers are impacted.\n**Workaround:**\nAdministrators may go to the Drive for Desktop settings in Admin Console under Apps \u003e Drive and Docs \u003e Drive for desktop \u003e Enable Drive for desktop and uncheck “Only allow Google Drive for desktop on authorized devices” For details see https://support.google.com/a/answer/7491144.","status":"SERVICE\_DISRUPTION"},{"created":"2025-07-16T19:25:42+00:00","modified":"2025-07-16T19:50:49+00:00","when":"2025-07-16T19:25:42+00:00","text":"\*\*Summary:\*\*\\nGoogle Drive users may be unexpectedly logged out of Drive for Desktop.\n**Description:**\nWe are experiencing an issue with Google Drive for desktop, beginning on Wednesday, 2025-07-16 10:24:07 PDT.\nOur engineering team has identified the source of the issue and is working on implementing a fix.\nWe will provide more information by Wednesday, 2025-07-16 13:30:00 PDT with additional details.\nWe apologize to all who are affected by the disruption.\n**Diagnosis / Customer Symptoms:**\nGoogle Drive users may experience being logged out of Drive for Desktop and see error “Your administrator has restricted Google Drive access to authorized devices. You have been logged out, and all Google Drive content related to your account has been wiped from this computer. Please contact your administrator to learn more.” Only enterprise customers are impacted.\n**Workaround:**\nAdministrators may go to the Drive for Desktop settings in Admin Console under Apps \u003e Drive and Docs \u003e Drive for desktop \u003e Enable Drive for desktop and uncheck “Only allow Google Drive for desktop on authorized devices” For details see https://support.google.com/a/answer/7491144.","status":"SERVICE\_DISRUPTION"},{"created":"2025-07-16T18:48:30+00:00","modified":"2025-07-16T19:27:53+00:00","when":"2025-07-16T18:48:30+00:00","text":"\*\*Summary:\*\*\\nGoogle Drive users may be unexpectedly logged out of Drive for Desktop.\n**Description:**\nWe are experiencing an issue with Google Drive for desktop, beginning on Wednesday, 2025-07-16 10:24:07 PDT.\nOur engineering team has identified the source of the issue and is implementing a fix.\nWe will provide more information by Wednesday, 2025-07-16 12:30:00 PDT with additional details.\nWe apologize to all who are affected by the disruption.\n**Diagnosis / Customer Symptoms:**\nGoogle Drive users may experience being logged out of Drive for Desktop and see error “Your administrator has restricted Google Drive access to authorized devices. You have been logged out, and all Google Drive content related to your account has been wiped from this computer. Please contact your administrator to learn more.” Only enterprise customers are impacted.\n**Workaround:**\nAdministrators may go to the Drive for Desktop settings in Admin Console under Apps \u003e Drive and Docs \u003e Drive for desktop \u003e Enable Drive for desktop and uncheck “Only allow Google Drive for desktop on authorized devices” For details see https://support.google.com/a/answer/7491144.","status":"SERVICE\_DISRUPTION"}\],"most\_recent\_update":{"created":"2025-07-18T08:22:38+00:00","modified":"2025-07-18T15:47:54+00:00","when":"2025-07-18T08:22:38+00:00","text":"# Incident Report\n## Summary\nOn Tuesday, 15 July 2025, for a duration of up to 22 hours and 49 minutes, Google Drive users were denied access to the Google Drive for desktop application. Affected users received an error message ‘Your administrator has restricted Google Drive access to authorized devices. You have been logged out, and all Google Drive content related to your account has been wiped from this computer. Please contact your administrator to learn more’. This message was erroneously triggered and no data was programmatically wiped.\nWe deeply regret the disruption our Google Drive for Desktop users experienced. We recognize the impact this had on your businesses, and we are truly sorry for falling short of your expectations. We have investigated the root cause of the issue and are implementing measures to improve the resilience of our services.\n## Root Cause\nThe root cause of the issue was an update to Drive's policy configuration. As part of this update, the admin setting that restricts Google Drive for desktop to authorized devices had its default unintentionally changed from FALSE to TRUE. This change took effect for domains which hadn't previously overridden the setting.\nAs a result, a number of users were unexpectedly prevented from launching the Google Drive for desktop application, requiring them to restart the application. The impact was limited to Windows and Mac devices used by Enterprise customers. The incident began at 18:27 US/Pacific on 15 July 2025 and deteriorated further on 16 July 2025.\n## Remediation and Prevention\nAt 18:27 US/Pacific on Tuesday, 15 July 2025, a 1% rollout release began causing minor impact. As the release progressed, the situation escalated significantly and became evident on Wednesday, 16 July 2025 from an increase in support cases. Our engineering team was alerted at 09:25 US/Pacific and immediately began the investigation.\nAs the first step to mitigate impact, Google engineers deployed a client-side mitigation to all Google Drive for desktop clients by 11:47 US/Pacific on Wednesday, 16 July 2025. This deployment disabled Enterprise Mobility Management support within Google Drive for Desktop. While this aimed to prevent further application launch blocks, it wasn't guaranteed to reach all devices before the block signal.\nConcurrently, a policy fix was deployed via a rollback. This rollback targeted the incorrect policy configuration that triggered the blocks, which completed at 17:16 US/Pacific on Wednesday, 16 July 2025.\nWe are deeply sorry for the length and severity of this incident. Google is committed to preventing a repeat of this incident in the future and we are completing the following actions :\n* Improve the error message displayed to our users when devices that are not company owned are blocked from using Google Drive for desktop.\n* Improve monitoring on the number and rate of policy related actions taken by Google Drive for desktop.\n## Detailed Description of Impact\nOn Tuesday, 15 July 2025 from 18:27 US/Pacific to Wednesday, 16 July 2025, 17:16 US/Pacific:\n#### Google Drive\nApproximately 1% users may have been prevented from launching Google Drive for desktop and encountered the error ‘Your administrator has restricted Google Drive access to authorized devices. You have been logged out, and all Google Drive content related to your account has been wiped from this computer. Please contact your administrator to learn more.’\nAs a temporary workaround, administrators were advised to manually disable the ‘Only allow Google Drive for desktop on authorized devices’ setting to prevent blocks on their managed devices.\n## Additional Information for Customers\nOnce the incident was mitigated, users who exited the application without taking any further action were able to access the application after they relaunched it using existing account configuration. Some users who may have disconnected their account using the button that was presented as an option, were able to regain access after signing into the application and performing additional actions to reconfigure any mirror roots and re-pin any files that required offline availability. These users who disconnected their account may also have had to download the files again that were previously cached.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"VHNA7p3Z5p3iakj5sA8V","service_name":"Google Drive","affected_products":[{"title":"Google Drive","id":"VHNA7p3Z5p3iakj5sA8V"}],"uri":"incidents/1hSNYy2ihYPewRrRji6R"},{"id":"owtYrdrh7EwXtfAmyu3U","number":"3633811992985588295","begin":"2025-07-02T18:40:00+00:00","created":"2025-07-02T20:17:49+00:00","end":"2025-07-02T20:02:00+00:00","modified":"2025-07-02T21:10:25+00:00","external_desc":"We have received reports of an intermittent issue with Google Meet beginning at Wednesday, 2025-07-02 11:49 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Wednesday, 2025-07-02 13:45 PDT with current details.\nWe apologize to all who are affected by the disruption.","updates":[{"created":"2025-07-02T21:10:22+00:00","modified":"2025-07-02T21:10:25+00:00","when":"2025-07-02T21:10:22+00:00","text":"A small number of users may have experienced intermittent issues connecting to or creating new Google Meet sessions.","status":"AVAILABLE"},{"created":"2025-07-02T20:49:20+00:00","modified":"2025-07-02T21:10:22+00:00","when":"2025-07-02T20:49:20+00:00","text":"Our engineering team has applied the mitigation and our internal monitoring indicates recovery. Engineering team continues to closely monitor the environment.\nWe will provide an update by Wednesday, 2025-07-02 14:45 PDT with current details.","status":"SERVICE_INFORMATION"},{"created":"2025-07-02T20:17:49+00:00","modified":"2025-07-02T20:49:20+00:00","when":"2025-07-02T20:17:49+00:00","text":"We have received reports of an intermittent issue with Google Meet beginning at Wednesday, 2025-07-02 11:49 PDT.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Wednesday, 2025-07-02 13:45 PDT with current details.\nWe apologize to all who are affected by the disruption.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-07-02T21:10:22+00:00","modified":"2025-07-02T21:10:25+00:00","when":"2025-07-02T21:10:22+00:00","text":"A small number of users may have experienced intermittent issues connecting to or creating new Google Meet sessions.","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"sUH4BQXzYXma7NiS94Hi","service_name":"Google Meet","affected_products":[{"title":"Google Meet","id":"sUH4BQXzYXma7NiS94Hi"}],"uri":"incidents/owtYrdrh7EwXtfAmyu3U"},{"id":"Eab7zGLWSaUJyh9c9R9L","number":"1558320242640410399","begin":"2025-06-12T17:58:00+00:00","created":"2025-06-12T19:01:47+00:00","end":"2025-06-12T23:00:00+00:00","modified":"2025-06-14T00:07:48+00:00","external_desc":"We're investigating reports of an issue with Gmail, Google Calendar, Google Chat, Google Cloud Search, Google Docs, Google Drive, Google Meet, Google Tasks, and Google Voice. We will provide more information shortly.","updates":[{"created":"2025-06-13T23:59:37+00:00","modified":"2025-06-14T00:07:48+00:00","when":"2025-06-13T23:59:37+00:00","text":"The detailed Incident Report is now available on the Google Cloud Service Health Dashboard at https://status.cloud.google.com/incidents/ow5i3PPK96RduMcb1SsW","status":"AVAILABLE"},{"created":"2025-06-13T07:00:35+00:00","modified":"2025-06-13T23:59:37+00:00","when":"2025-06-13T07:00:35+00:00","text":"# Mini Incident Report\nWe are deeply sorry for the impact to all of our users and their customers that this service disruption/outage caused. Businesses large and small trust Google Cloud with your workloads and we will do better. In the coming days, we will publish a full incident report of the root cause, detailed timeline and robust remediation steps we will be taking. Given the size and impact of this incident, we would like to provide some information below.\nPlease note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Cloud Support using https://cloud.google.com/support or to Google Workspace Support using help article https://support.google.com/a/answer/1047213.\\n\*\*(All Times US/Pacific)**\n**Incident Start:** 12 June, 2025 10:49\n**All regions except us-central1 mitigated:** 12 June, 2025 12:48\n**Incident End:** 12 June, 2025 13:49\n**Duration:** 3 hours\n**Regions/Zones:** Global\n**Description:**\nMultiple Google Cloud and Google Workspace products experienced increased 503 errors in external API requests, impacting customers.\nFrom our initial analysis, the issue occurred due to an invalid automated quota update to our API management system which was distributed globally, causing external API requests to be rejected. To recover we bypassed the offending quota check, which allowed recovery in most regions within 2 hours. However, the quota policy database in us-central1 became overloaded, resulting in much longer recovery in that region. Several products had moderate residual impact (e.g. backlogs) for up to an hour after the primary issue was mitigated and a small number recovering after that.\nGoogle will complete a full Incident Report in the following days that will provide a detailed root cause.\n**Customer Impact:**\nCustomers had intermittent API and user-interface access issues to the impacted services. Existing streaming and IaaS resources were not impacted.\n**Additional details:**\nThis incident should not have happened, and we will take the following measures to prevent future recurrence:\nPrevent our API management platform from failing due to invalid or corrupt data.\nPrevent metadata from propagating globally without appropriate protection, testing and monitoring in place.\nImprove system error handling and comprehensive testing for handling of invalid data.\n**Affected Services and Features:**\n**Google Cloud Products:**\n* Identity and Access Management\n* Cloud Build\n* Cloud Key Management Service\n* Google Cloud Storage\n* Cloud Monitoring\n* Google Cloud Dataproc\n* Cloud Security Command Center\n* Artifact Registry\n* Cloud Workflows\n* Cloud Healthcare\n* Resource Manager API\n* Dataproc Metastore\n* Cloud Run\n* VMWare engine\n* Dataplex\n* Migrate to Virtual Machines\n* Google BigQuery\n* Contact Center AI Platform\n* Google Cloud Deploy\n* Media CDN\n* Colab Enterprise\n* Vertex Gemini API\n* Cloud Data Fusion\n* Cloud Asset Inventory\n* Datastream\n* Integration Connectors\n* Apigee\n* Google Cloud NetApp Volumes\n* Google Cloud Bigtable\n* Looker (Google Cloud core)\n* Looker Studio\n* Google Cloud Functions\n* Cloud Load Balancing\n* Traffic Director\n* Document AI\n* AutoML Translation\n* Pub/Sub Lite\n* API Gateway\n* Agent Assist\n* AlloyDB for PostgreSQL\n* Cloud Firestore\n* Cloud Logging\n* Cloud Shell\n* Cloud Memorystore\n* Cloud Spanner\n* Contact Center Insights\n* Database Migration Service\n* Dialogflow CX\n* Dialogflow ES\n* Google App Engine\n* Google Cloud Composer\n* Google Cloud Console\n* Google Cloud DNS\n* Google Cloud Pub/Sub\n* Google Cloud SQL\n* Google Compute Engine\n* Identity Platform\n* Managed Service for Apache Kafka\n* Memorystore for Memcached\n* Memorystore for Redis\n* Memorystore for Redis Cluster\n* Persistent Disk\n* Personalized Service Health\n* Speech-to-Text\n* Text-to-Speech\n* Vertex AI Search\n* Retail API\n* Vertex AI Feature Store\n* BigQuery Data Transfer Service\n* Google Cloud Marketplace\n* Cloud NAT\n* Hybrid Connectivity\n* Cloud Vision\n* Network Connectivity Center\n* Cloud Workstations\n**Google Workspace Products:**\n* AppSheet\n* Gmail\n* Google Calendar\n* Google Drive\n* Google Chat\n* Google Voice\n* Google Docs\n* Google Meet\n* Google Cloud Search\n* Google Tasks","status":"AVAILABLE"},{"created":"2025-06-12T19:53:15+00:00","modified":"2025-06-13T07:00:35+00:00","when":"2025-06-12T19:53:15+00:00","text":"The issue with Google Chat, Google Meet, Gmail, Google Calendar, Google Drive, Google Cloud Search, Google Tasks, Google Voice has been resolved for all affected users as of Thursday, 2025-06-12 12:48 PDT.\nThe issue with AppSheet has been resolved for all affected users as of Thursday, 2025-06-12 16:00 PDT.\nWe will publish an analysis of this incident once we have completed our internal investigation.\nWe thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},{"created":"2025-06-12T19:30:51+00:00","modified":"2025-06-12T19:53:15+00:00","when":"2025-06-12T19:30:51+00:00","text":"All product impacts except Google Meet have recovered.\nGoogle engineers continue to work on full mitigation.","status":"SERVICE_DISRUPTION"},{"created":"2025-06-12T19:01:47+00:00","modified":"2025-06-12T19:30:51+00:00","when":"2025-06-12T19:01:47+00:00","text":"We're investigating reports of an issue with Gmail, Google Calendar, Google Chat, Google Cloud Search, Google Docs, Google Drive, Google Meet, Google Tasks, and Google Voice. We will provide more information shortly.\nMultiple Workspace products beginning on Thursday, at 2025-06-12 10:58 PDT may be experiencing service issues.\nOur engineers are currently investigating the issue.\nWe apologize to all who are affected by the disruption.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-06-13T23:59:37+00:00","modified":"2025-06-14T00:07:48+00:00","when":"2025-06-13T23:59:37+00:00","text":"The detailed Incident Report is now available on the Google Cloud Service Health Dashboard at https://status.cloud.google.com/incidents/ow5i3PPK96RduMcb1SsW","status":"AVAILABLE"},"status\_impact":"SERVICE\_INFORMATION","severity":"low","service\_key":"zall","service\_name":"Multiple Products","affected_products":[{"title":"AppSheet","id":"FWjKi5U7KX4FUUPThHAJ"},{"title":"Gmail","id":"prPt2Yra2CbGsbEm9cpC"},{"title":"Google Calendar","id":"n5hVexuq1PM9onY9qrmo"},{"title":"Google Chat","id":"Ht9Vx5PFzDPHYvrnrvSk"},{"title":"Google Cloud Search","id":"iJPnujhwzow3TaPE1dM8"},{"title":"Google Docs","id":"wNHuVFtZEWU5Mmj2eRCK"},{"title":"Google Drive","id":"VHNA7p3Z5p3iakj5sA8V"},{"title":"Google Meet","id":"sUH4BQXzYXma7NiS94Hi"},{"title":"Google Tasks","id":"Vups7kniH52V5x4UX1gM"},{"title":"Google Voice","id":"gh79SrQk6wqonWoxPdCm"}],"uri":"incidents/Eab7zGLWSaUJyh9c9R9L"},{"id":"KjFLjecwg9xeewnUYjh1","number":"15423978574401052094","begin":"2025-06-09T22:07:00+00:00","created":"2025-06-09T23:28:15+00:00","end":"2025-06-10T00:00:00+00:00","modified":"2025-06-12T12:39:18+00:00","external_desc":"**Summary:**\nGoogle Classroom experiencing elevated latency issue\n**Description:**\nWe are experiencing an Classroom beginning at Monday, 2025-06-09 15:00 PDT\nOur engineering team continues to investigate the issue.\nWe will provide an update by Monday, 2025-06-09 17:00 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Diagnosis:**\nThe impacted users would experience increased latency while using Classroom.\n**Workaround:**\nNone at this time.","updates":[{"created":"2025-06-12T12:39:18+00:00","modified":"2025-06-12T12:39:18+00:00","when":"2025-06-12T12:39:18+00:00","text":"# Incident Report\n## Summary\nOn Monday, 9 June 2025, at 15:07 US/Pacific, Google Classroom experienced increased latency for a duration of 1 hour 53 minutes. We deeply regret the disruption our Google Classroom users experienced. We know you rely on our services, and we fell short of meeting your expectations. We are taking immediate action to improve our platform's reliability and prevent future disruptions.\n## Root Cause\nThe incident was triggered by a schema change applied to Google's globally distributed database, which stores and serves a majority of Classroom data. This schema change caused a surge in CPU utilization across multiple clusters, reaching the maximum utilization at 100 percent. As a result, a backend service relying on the database experienced a significant drop in availability and performance, impacting Google Classroom users.\n## Remediation and Prevention\nGoogle engineers were alerted to the outage via a monitoring alert on June 9 at 15:07 US/Pacific and immediately started an investigation. The issue was mitigated on 9 June at 17:00 US/Pacific once the schema change was fully rolled back.\nGoogle is committed to preventing a repeat of this issue in the future and is completing the following actions:\nUpdate Google Classrooms database release process efficiency to use incremental rollout, to aid in identifying potential impact and prevent a recurrence.\nImplement proactive identification for schema changes with potential wide-reaching impact prior to rollout.\nGoogle is committed to quickly and continually improving our technology and operations to prevent service disruptions. We appreciate your patience and apologize again for the impact to your organization. We thank you for your business.\n## Detailed Description of Impact\n### Google Classroom\nOn 9 June 2025, from 15:07 to 17:00 US/Pacific Google Classroom users experienced increased latency wherein the application may have failed to load for an extended period.\nGoogle Classroom experienced a significant drop in availability and performance for approximately ~0.1% of users for a duration of 1 hour and 53 minutes. Most requests made to the database experienced latency or unavailability during the incident time.","status":"AVAILABLE"},{"created":"2025-06-10T04:02:24+00:00","modified":"2025-06-12T12:39:18+00:00","when":"2025-06-10T04:02:24+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Cloud Support using https://cloud.google.com/support or to Google Workspace Support using help article https://support.google.com/a/answer/1047213.\\n(All Times US/Pacific)\n**Incident Start:** 9 June, 2025 15:07\n**Incident End:** 9 June, 2025 17:00\n**Duration:** 1 hour, 53 minutes\n**Affected Services and Features:**\nGoogle Classroom\n**Regions/Zones:** Global\n**Description:**\nGoogle Classroom users experienced elevated latency for a duration of 1 hour and 53 minutes. The incident was triggered by a schema change applied to Google's globally distributed database. This schema change caused a surge in utilization across multiple zones, reaching the maximum utilization at 100%. As a result, a backend service relying on the database experienced a significant drop in availability and performance, impacting Classroom users. The incident was mitigated by rolling back the schema change.\nGoogle will complete a detailed Incident report in the following days that will provide a full root cause.\n**Customer Impact:**\nImpacted users experienced increased latency while using Google Classroom.","status":"AVAILABLE"},{"created":"2025-06-10T00:12:23+00:00","modified":"2025-06-10T04:02:24+00:00","when":"2025-06-10T00:12:23+00:00","text":"**Summary:**\nGoogle Classroom experienced elevated latency issue\n**Description:**\nThe issue with Classroom has been resolved for all affected users as of Monday, 2025-06-09 17:00 PDT.\nThe issue was due to a recent change in the Spanner Database which prevented many users from being able to access Classroom. Our engineers have mitigated the issue by rolling back the recent release to the previous version.\nWe will publish an analysis of this incident once we have completed our internal investigation.\nWe thank you for your patience while we worked on resolving the issue.\n**Diagnosis:**\nThe impacted users experienced increased latency while using Classroom.\n**Workaround:**\nNone at this time.","status":"AVAILABLE"},{"created":"2025-06-09T23:51:12+00:00","modified":"2025-06-10T00:12:23+00:00","when":"2025-06-09T23:51:12+00:00","text":"**Summary:**\nGoogle Classroom experiencing elevated latency issue\n**Description:**\nMitigation work is currently underway by our engineering team.\nWe do not have an ETA for mitigation at this point.\nWe will provide more information by Monday, 2025-06-09 17:50 PDT.\n**Diagnosis:**\nThe impacted users would experience increased latency while using Classroom.\n**Workaround:**\nNone at this time.","status":"SERVICE_DISRUPTION"},{"created":"2025-06-09T23:28:15+00:00","modified":"2025-06-09T23:51:12+00:00","when":"2025-06-09T23:28:15+00:00","text":"**Summary:**\nGoogle Classroom experiencing elevated latency issue\n**Description:**\nWe are experiencing an Classroom beginning at Monday, 2025-06-09 15:00 PDT\nOur engineering team continues to investigate the issue.\nWe will provide an update by Monday, 2025-06-09 17:00 PDT with current details.\nWe apologize to all who are affected by the disruption.\n**Diagnosis:**\nThe impacted users would experience increased latency while using Classroom.\n**Workaround:**\nNone at this time.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-06-12T12:39:18+00:00","modified":"2025-06-12T12:39:18+00:00","when":"2025-06-12T12:39:18+00:00","text":"# Incident Report\n## Summary\nOn Monday, 9 June 2025, at 15:07 US/Pacific, Google Classroom experienced increased latency for a duration of 1 hour 53 minutes. We deeply regret the disruption our Google Classroom users experienced. We know you rely on our services, and we fell short of meeting your expectations. We are taking immediate action to improve our platform's reliability and prevent future disruptions.\n## Root Cause\nThe incident was triggered by a schema change applied to Google's globally distributed database, which stores and serves a majority of Classroom data. This schema change caused a surge in CPU utilization across multiple clusters, reaching the maximum utilization at 100 percent. As a result, a backend service relying on the database experienced a significant drop in availability and performance, impacting Google Classroom users.\n## Remediation and Prevention\nGoogle engineers were alerted to the outage via a monitoring alert on June 9 at 15:07 US/Pacific and immediately started an investigation. The issue was mitigated on 9 June at 17:00 US/Pacific once the schema change was fully rolled back.\nGoogle is committed to preventing a repeat of this issue in the future and is completing the following actions:\nUpdate Google Classrooms database release process efficiency to use incremental rollout, to aid in identifying potential impact and prevent a recurrence.\nImplement proactive identification for schema changes with potential wide-reaching impact prior to rollout.\nGoogle is committed to quickly and continually improving our technology and operations to prevent service disruptions. We appreciate your patience and apologize again for the impact to your organization. We thank you for your business.\n## Detailed Description of Impact\n### Google Classroom\nOn 9 June 2025, from 15:07 to 17:00 US/Pacific Google Classroom users experienced increased latency wherein the application may have failed to load for an extended period.\nGoogle Classroom experienced a significant drop in availability and performance for approximately ~0.1% of users for a duration of 1 hour and 53 minutes. Most requests made to the database experienced latency or unavailability during the incident time.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"3GoNVrVz48jeNZSyLJMV","service_name":"Classroom","affected_products":[{"title":"Classroom","id":"3GoNVrVz48jeNZSyLJMV"}],"uri":"incidents/KjFLjecwg9xeewnUYjh1"},{"id":"gwMkZKeZCE1eAC6z5EvH","number":"7701776215533702872","begin":"2025-05-22T17:00:00+00:00","created":"2025-05-23T02:02:34+00:00","end":"2025-05-23T01:45:00+00:00","modified":"2025-05-23T04:41:12+00:00","external_desc":"**Summary**\nAppSheet application and Dashboards are not showing within Looker Studio.\n**Description**\nWe are experiencing an issue with AppSheet beginning on Thursday, 2025-05-22 10:00 US/Pacific.\nOur engineering team identified the cause was due to a recent change and are actively working on the rollback.\nThe mitigation is expected to complete by Thursday, 2025-05-22 19:45 US/Pacific.\nWe will provide more information by Thursday, 2025-05-22 20:15 US/Pacific.\n**Customer Symptoms**\nAppSheet apps will not show within Looker Studio and Dashboards will return Error.\n**Workaround**\nNone at this moment","updates":[{"created":"2025-05-23T04:40:46+00:00","modified":"2025-05-23T04:41:12+00:00","when":"2025-05-23T04:40:46+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Cloud Support using https://cloud.google.com/support or to Google Workspace Support using help article https://support.google.com/a/answer/1047213.\\n(All Times US/Pacific)\n**Incident Start:** 22 May, 2025 10:00\n**Incident End:** 22 May, 2025 18:45\n**Duration:** 8 hours, 45 minutes\n**Affected Services and Features:**\nAppSheet\n**Regions/Zones:** Global\n**Description:**\nAppSheet in Looker Studio experienced issues where applications and dashboards were not visible. Based on our analysis, this was caused by a recent change that AppSheet introduced to enhance its authentication for the Looker Studio connector. Once the impact was identified by our Engineering team, we promptly rolled back the change, restoring the connector's stability.\n**Customer Impact:**\nImpacted customers may have experienced AppSheet apps not showing within Looker Studio, and dependent dashboards may have returned an exception: ‘Community Connector Error’.","status":"AVAILABLE"},{"created":"2025-05-23T02:15:46+00:00","modified":"2025-05-23T04:40:46+00:00","when":"2025-05-23T02:15:46+00:00","text":"**Description**\nThe issue with AppSheet has been resolved for all affected users as of Thursday, 2025-05-22 19:05 US/Pacific.\nA recent change introduced by our engineering team caused the issue. To mitigate the impact, the change was rolled back, and the stability of the service has been confirmed.\nWe thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},{"created":"2025-05-23T02:02:34+00:00","modified":"2025-05-23T02:15:46+00:00","when":"2025-05-23T02:02:34+00:00","text":"**Summary**\nAppSheet application and Dashboards are not showing within Looker Studio.\n**Description**\nWe are experiencing an issue with AppSheet beginning on Thursday, 2025-05-22 10:00 US/Pacific.\nOur engineering team identified the cause was due to a recent change and are actively working on the rollback.\nThe mitigation is expected to complete by Thursday, 2025-05-22 19:45 US/Pacific.\nWe will provide more information by Thursday, 2025-05-22 20:15 US/Pacific.\n**Customer Symptoms**\nAppSheet apps will not show within Looker Studio and Dashboards will return Error.\n**Workaround**\nNone at this moment","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-05-23T04:40:46+00:00","modified":"2025-05-23T04:41:12+00:00","when":"2025-05-23T04:40:46+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Cloud Support using https://cloud.google.com/support or to Google Workspace Support using help article https://support.google.com/a/answer/1047213.\\n(All Times US/Pacific)\n**Incident Start:** 22 May, 2025 10:00\n**Incident End:** 22 May, 2025 18:45\n**Duration:** 8 hours, 45 minutes\n**Affected Services and Features:**\nAppSheet\n**Regions/Zones:** Global\n**Description:**\nAppSheet in Looker Studio experienced issues where applications and dashboards were not visible. Based on our analysis, this was caused by a recent change that AppSheet introduced to enhance its authentication for the Looker Studio connector. Once the impact was identified by our Engineering team, we promptly rolled back the change, restoring the connector's stability.\n**Customer Impact:**\nImpacted customers may have experienced AppSheet apps not showing within Looker Studio, and dependent dashboards may have returned an exception: ‘Community Connector Error’.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"FWjKi5U7KX4FUUPThHAJ","service_name":"AppSheet","affected_products":[{"title":"AppSheet","id":"FWjKi5U7KX4FUUPThHAJ"}],"uri":"incidents/gwMkZKeZCE1eAC6z5EvH"},{"id":"SNDHnqEixWu9UKSJNjrR","number":"11844245915375502324","begin":"2025-05-13T15:08:00+00:00","created":"2025-05-13T15:47:25+00:00","end":"2025-05-13T18:36:00+00:00","modified":"2025-05-13T19:06:00+00:00","external_desc":"**Summary:**\nMultiple Workspace services are experiencing elevated error rates\n**Description:**\nBeginning at 08:08 US/Pacific time on Tuesday, May 13, 2025, there is an ongoing issue affecting Google Docs, Google Sheets, Google Slides, Gmail, Google Chat, and Google Meet.\nInternal monitoring indicates that Google Docs was recovered as of 08:15 US/Pacific time, and the engineering team is still investigating the issue for the remaining impact.\nWe will provide an update by Tuesday, 2025-05-13 09:30 US/Pacific with current details.\n**Symptoms:**\nAffected customers are experiencing elevated error rates for the workspace services\n**Workaround:**\nCustomers can retry the errored operation.","updates":[{"created":"2025-05-13T19:05:47+00:00","modified":"2025-05-13T19:06:00+00:00","when":"2025-05-13T19:05:47+00:00","text":"Google Docs, Google Sheets, Google Slides recovered as of 08:15 US/Pacific time.\nGmail User Interface, Google Meet recovered as of 09:10 US/Pacific time.\nGmail SMTP issues were resolved as of 10:00 US/Pacific time.\nGoogle Chat issues were resolved as of 11:36 US/Pacific time.\nPreliminary analysis indicates a datacenter power issue overloaded the backend storage system, causing intermittent customer traffic disruptions. Automated systems recovered some services, while engineers mitigated the remaining impact by rerouting traffic from the affected datacenter and also provisioned additional resources.\nWe thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},{"created":"2025-05-13T18:21:37+00:00","modified":"2025-05-13T19:05:47+00:00","when":"2025-05-13T18:21:37+00:00","text":"**Summary:**\nSome customers are experiencing elevated error rates with multiple Workspace services.\n**Description:**\nGoogle Docs, Google Sheets, Google Slides recovered as of 08:15 US/Pacific time.\nGmail User Interface, Google Meet recovered as of 09:10 US/Pacific time.\nGmail SMTP issues were resolved as of 10:00 US/Pacific time.\nGoogle Chat mitigation efforts are ongoing. Engineers are implementing various actions and observing a gradual decrease in impact.\nWe do not have an ETA for mitigation at this point.\nWe will provide an update by Tuesday, 2025-05-13 12:45 US/Pacific with current details.\n**Symptoms:**\nAffected customers are experiencing elevated error rates for the workspace services.\n**Workaround:**\nCustomers can retry the errored operation.","status":"SERVICE_INFORMATION"},{"created":"2025-05-13T17:39:35+00:00","modified":"2025-05-13T18:21:37+00:00","when":"2025-05-13T17:39:35+00:00","text":"**Summary:**\nSome customers are experiencing elevated error rates with multiple Workspace services.\n**Description:**\nGoogle Docs, Google Sheets, Google Slides recovered as of 08:15 US/Pacific time.\nGmail User Interface, Google Meet recovered as of 09:10 US/Pacific time.\nGmail SMTP issues were resolved as of 10:00 US/Pacific time.\nMitigation work is currently underway by our engineering team for Google Chat.\nThe mitigation is expected to complete by Tuesday, 2025-05-13 11:15 US/Pacific.\nWe will provide an update by Tuesday, 2025-05-13 11:30 US/Pacific with current details.\n**Symptoms:**\nAffected customers are experiencing elevated error rates for the workspace services.\n**Workaround:**\nCustomers can retry the errored operation.","status":"SERVICE_INFORMATION"},{"created":"2025-05-13T16:22:00+00:00","modified":"2025-05-13T17:39:35+00:00","when":"2025-05-13T16:22:00+00:00","text":"**Summary:**\nSome customers are experiencing elevated error rates with multiple Workspace services.\n**Description:**\nInternal monitoring indicates that various workspace services are now recovered.\nGoogle Docs, Google Sheets, Google Slides recovered as of 08:15 US/Pacific time.\nGmail User Interface, Google Meet recovered as of 09:10 US/Pacific time.\nMitigation work is currently underway by our engineering team for Google Chat.\nThe mitigation is expected to complete by Tuesday, 2025-05-13 09:45 US/Pacific.\nOur engineers are continuing to investigate Gmail SMTP related issues.\nWe will provide an update by Tuesday, 2025-05-13 10:30 US/Pacific with current details.\n**Symptoms:**\nAffected customers are experiencing elevated error rates for the workspace services.\n**Workaround:**\nCustomers can retry the errored operation.","status":"SERVICE_INFORMATION"},{"created":"2025-05-13T15:58:11+00:00","modified":"2025-05-13T16:22:00+00:00","when":"2025-05-13T15:58:11+00:00","text":"**Summary:**\nSome customers are experiencing elevated error rates with multiple Workspace services.\n**Description:**\nInternal monitoring indicates that Google Docs, Google Sheets, Google Slides recovered as of 08:15 US/Pacific time, and the engineering team is still investigating the issue for the remaining impact for Google Chat, Gmail, and Google Meet.\nWe will provide an update by Tuesday, 2025-05-13 10:00 US/Pacific with current details.\n**Symptoms:**\nAffected customers are experiencing elevated error rates for the workspace services.\n**Workaround:**\nCustomers can retry the errored operation.","status":"SERVICE_INFORMATION"},{"created":"2025-05-13T15:47:25+00:00","modified":"2025-05-13T15:58:11+00:00","when":"2025-05-13T15:47:25+00:00","text":"**Summary:**\nMultiple Workspace services are experiencing elevated error rates\n**Description:**\nBeginning at 08:08 US/Pacific time on Tuesday, May 13, 2025, there is an ongoing issue affecting Google Docs, Google Sheets, Google Slides, Gmail, Google Chat, and Google Meet.\nInternal monitoring indicates that Google Docs was recovered as of 08:15 US/Pacific time, and the engineering team is still investigating the issue for the remaining impact.\nWe will provide an update by Tuesday, 2025-05-13 09:30 US/Pacific with current details.\n**Symptoms:**\nAffected customers are experiencing elevated error rates for the workspace services\n**Workaround:**\nCustomers can retry the errored operation.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-05-13T19:05:47+00:00","modified":"2025-05-13T19:06:00+00:00","when":"2025-05-13T19:05:47+00:00","text":"Google Docs, Google Sheets, Google Slides recovered as of 08:15 US/Pacific time.\nGmail User Interface, Google Meet recovered as of 09:10 US/Pacific time.\nGmail SMTP issues were resolved as of 10:00 US/Pacific time.\nGoogle Chat issues were resolved as of 11:36 US/Pacific time.\nPreliminary analysis indicates a datacenter power issue overloaded the backend storage system, causing intermittent customer traffic disruptions. Automated systems recovered some services, while engineers mitigated the remaining impact by rerouting traffic from the affected datacenter and also provisioned additional resources.\nWe thank you for your patience while we worked on resolving the issue.","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"zall","service_name":"Multiple Products","affected_products":[{"title":"Gmail","id":"prPt2Yra2CbGsbEm9cpC"},{"title":"Google Chat","id":"Ht9Vx5PFzDPHYvrnrvSk"},{"title":"Google Docs","id":"wNHuVFtZEWU5Mmj2eRCK"},{"title":"Google Meet","id":"sUH4BQXzYXma7NiS94Hi"},{"title":"Google Sheets","id":"GsNjeYHsogeRgTiC5coU"},{"title":"Google Slides","id":"DvWBgkXVhodA3WYwgZoB"}],"uri":"incidents/SNDHnqEixWu9UKSJNjrR"},{"id":"q2AU1kpKWNdBCDtAyjLi","number":"15997718865756572109","begin":"2025-05-13T00:45:00+00:00","created":"2025-05-13T03:42:11+00:00","end":"2025-05-13T04:13:00+00:00","modified":"2025-05-13T06:46:36+00:00","external_desc":"**Summary**\nGoogle Cloud Workspace customers are experiencing an error while using https://gemini.google.com\\n\*\*Description\*\*\\nWe are experiencing an issue with Gemini beginning at Monday, 2025-05-12 20:04 US/Pacific.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Monday, 2025-05-12 22:00 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nWhen accessing https://gemini.google.com, the users may observe an error, \"Gemini is currently on a break. Give it a moment and try again later.\"\n**Workaround**\nNone at this time.","updates":[{"created":"2025-05-13T06:38:47+00:00","modified":"2025-05-13T06:39:10+00:00","when":"2025-05-13T06:38:47+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Cloud Support using https://cloud.google.com/support or to Google Workspace Support using help article https://support.google.com/a/answer/1047213.\\n(All Times US/Pacific)\n**Incident Start:** 12 May, 2025 17:45\n**Incident End:** 12 May, 2025 21:13\n**Duration:** 3 hour 28 minutes\n**Affected Services and Features:**\nGemini\n**Description:**\nGoogle Cloud Workspace customers experienced an error while using https://gemini.google.com. From preliminary analysis, the root cause of the issue was a recent code change and Google engineering quickly mitigated the issue by rolling back the code change.\n**Customer Impact:**\nWhen accessing https://gemini.google.com, the users observed an error, \"Gemini is currently on a break. Give it a moment and try again later.\"","status":"AVAILABLE"},{"created":"2025-05-13T04:27:12+00:00","modified":"2025-05-13T06:46:20+00:00","when":"2025-05-13T04:27:12+00:00","text":"**Summary**\nGoogle Cloud Workspace customers are experiencing an error while using https://gemini.google.com\\n\*\*Closing Message**\nThe issue with Gemini has been resolved for all affected users as of Monday, 2025-05-12 21:13 US/Pacific.\nGoogle engineers identified the recent code change causing this issue and mitigated the issue by rolling back the changes.\nWe thank you for your patience while we worked on resolving the issue.","status":"SERVICE_OUTAGE"},{"created":"2025-05-13T03:42:11+00:00","modified":"2025-05-13T06:46:36+00:00","when":"2025-05-13T03:42:11+00:00","text":"**Summary**\nGoogle Cloud Workspace customers are experiencing an error while using https://gemini.google.com\\n\*\*Description\*\*\\nWe are experiencing an issue with Gemini beginning at Monday, 2025-05-12 20:04 US/Pacific.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Monday, 2025-05-12 22:00 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms**\nWhen accessing https://gemini.google.com, the users may observe an error, \"Gemini is currently on a break. Give it a moment and try again later.\"\n**Workaround**\nNone at this time.","status":"SERVICE_OUTAGE"}],"most_recent_update":{"created":"2025-05-13T06:38:47+00:00","modified":"2025-05-13T06:39:10+00:00","when":"2025-05-13T06:38:47+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced impact outside of what is listed below, please reach out to Google Cloud Support using https://cloud.google.com/support or to Google Workspace Support using help article https://support.google.com/a/answer/1047213.\\n(All Times US/Pacific)\n**Incident Start:** 12 May, 2025 17:45\n**Incident End:** 12 May, 2025 21:13\n**Duration:** 3 hour 28 minutes\n**Affected Services and Features:**\nGemini\n**Description:**\nGoogle Cloud Workspace customers experienced an error while using https://gemini.google.com. From preliminary analysis, the root cause of the issue was a recent code change and Google engineering quickly mitigated the issue by rolling back the code change.\n**Customer Impact:**\nWhen accessing https://gemini.google.com, the users observed an error, \"Gemini is currently on a break. Give it a moment and try again later.\"","status":"AVAILABLE"},"status_impact":"SERVICE_OUTAGE","severity":"high","service_key":"npdyhgECDJ6tB66MxXyo","service_name":"Gemini","affected_products":[{"title":"Gemini","id":"npdyhgECDJ6tB66MxXyo"}],"uri":"incidents/q2AU1kpKWNdBCDtAyjLi"},{"id":"SwA8chvNSrGA9WPLtyYu","number":"4385767702209736005","begin":"2025-05-07T14:47:00+00:00","created":"2025-05-07T15:33:33+00:00","end":"2025-05-07T22:12:00+00:00","modified":"2025-05-14T00:30:35+00:00","external_desc":"**Summary:**\nGoogle Drive and Gmail customers are experiencing rendering issues with some PDF files\n**Description:**\nWe are experiencing an issue with Google Drive, Gmail beginning at Wednesday, 2025-05-07 07:47 US/Pacific.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Wednesday, 2025-05-07 09:15 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms:**\nSome PDF files in Google Drive and Gmail are not being displayed properly\n**Workaround:**\nNone at this time.","updates":[{"created":"2025-05-12T21:46:07+00:00","modified":"2025-05-14T00:30:35+00:00","when":"2025-05-12T21:46:07+00:00","text":"# Incident Report\n## Summary\nOn Friday, 2 May 2025, Gmail, Google Chat, and Google Drive experienced an issue where some customers using Android devices and web browsers were unable to properly display Portable Document Format (PDF) files globally. Generative AI features associated with the impacted PDF files were also affected, for a duration of 5 days, 12 hours, and 4 minutes.\nTo our customers whose workflows were impacted during this disruption, we sincerely apologize. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform's performance and availability.\n## Root Cause\nThe root cause of the issue was an update to an underlying software library that affected how PDF files were interpreted during the rendering process.\nA recent software update to an underlying, third-party library inadvertently introduced a change in how PDF files were interpreted during the rendering process. This change led to some Gmail, Google Chat, and Google Drive users on Android devices and web browsers seeing incomplete or inaccurate content within certain PDFs. This issue also impacted the generative AI features for the affected PDF files. iOS users were unaffected.\n## Remediation and Prevention\nGoogle engineers were alerted to the outage on Wednesday, 7 May at 05:52 US/Pacific via a customer support case and immediately started an investigation.\nOnce the root cause was identified, the issue was mitigated by reverting to the previous software version, which restored the correct rendering of PDFs. The incident was fully mitigated on Wednesday, 7 May 16:21 US/Pacific.\nGoogle is committed to preventing a repeat of this issue in the future and is completing the following actions:\n- Add comprehensive test coverage for all new features and critical existing functionality.\n- Improve release-time validation of code changes.\n## Detailed Description of Impact\n- Google Drive: PDF files in drive were not displayed correctly due to rendering issues. Furthermore, Gemini was also affected by the same rendering issue as the Drive viewer for some PDFs.\n- Gmail: PDF files in email were not displayed correctly due to rendering issues.\n- Google Chat: PDF files in chat were not displayed correctly due to rendering issues.","status":"AVAILABLE"},{"created":"2025-05-08T11:19:13+00:00","modified":"2025-05-12T21:46:07+00:00","when":"2025-05-08T11:19:13+00:00","text":"# Mini Incident Report\nWe apologize for the inconvenience this service disruption/outage may have caused. We would like to provide some information about this incident below. Please note, this information is based on our best knowledge at the time of posting and is subject to change as our investigation continues. If you have experienced an impact outside of what is listed below, please reach out to Google Workspace Support using the help article https://support.google.com/a/answer/1047213.\\n(All Times US/Pacific)\n**Incident Start:** 30 Apr 2025, 09:00\n**Incident End:** 7 May 2025, 15:12\n**Duration:** 7 days, 6 hours, 12 minutes\n**Affected Services and Features:**\nGoogle Drive, Gmail, Google Chat\n**Regions/Zones:** Global\n**Description:**\nSome PDF files in Google Drive, Gmail and Google Chat were not displayed properly for customers using Android devices and web browsers. The issue also affected the generative AI features for the affected PDF files. This occurred for a duration of 7 days, 6 hours, 12 minutes. The root cause of the incident was a code change in an internal service. This update triggered the incorrect rendering of information in PDFs, impacting users of services dependent on this rendering service. The impact was mitigated by rolling back the internal services to a previous version.\nGoogle will complete a full Incident Report in the following days that will provide a full root cause analysis.\n**Customer Impact:**\nSome PDF files in Google Drive, Gmail, and Google Chat were not displayed properly for customers using android devices and web browsers.\nThe issue also affected the generative AI features for the affected PDF files.\nThe issue did not affect customers using iOS devices.","status":"AVAILABLE"},{"created":"2025-05-08T08:52:01+00:00","modified":"2025-05-08T11:19:13+00:00","when":"2025-05-08T08:52:01+00:00","text":"The problem with Gmail, Google Chat, and Google Drive has been resolved. We apologize for the inconvenience and thank you for your patience and continued support.","status":"SERVICE_INFORMATION"},{"created":"2025-05-08T01:05:17+00:00","modified":"2025-05-08T08:52:01+00:00","when":"2025-05-08T01:05:17+00:00","text":"**Summary:**\nGoogle Drive, Gmail, and Google Chat customers using Android devices, and Web browser are experiencing rendering issues with some PDF files\n**Description:**\nThe recent issue with PDF rendering affecting Gmail, Google Chat, Google Drive (Web and Android) have been fully resolved.\nGemini responses for some pdfs which were uploaded to Google Drive between April 23rd and May 7th would have degraded responses in the Gemini side panel on Gmail, Google Chat and Google Drive web apps.\nOur engineering team continues to work on mitigating the Gen AI functionality and is expected to complete by Thursday, 2025-05-08 18:00 US/Pacific.\nWe will provide an update by Thursday, 2025-05-08 19:00 US/Pacific with current details.\n**Symptoms:**\nSome PDF files in Google Drive, Gmail, and Google Chat are not being displayed properly for customers using Android devices and Web browsers.\nThe issue may also affect the Gen AI features for the affected PDF Files.\nThe issue does not affect customers using iOS devices.\n**Workarounds:**\nCustomers can download the affected PDF files and view them without any issues. If Gemini response is degraded users can use a copy of original pdf and that would work.","status":"SERVICE_INFORMATION"},{"created":"2025-05-07T22:34:17+00:00","modified":"2025-05-08T01:05:17+00:00","when":"2025-05-07T22:34:17+00:00","text":"**Summary:**\nGoogle Drive, Gmail, and Google Chat customers using Android devices, and Web browser are experiencing rendering issues with some PDF files\n**Description:**\nWe believe the issue with Gmail Web, Google Chat Web, Google Drive Web is fully resolved.\nOur engineers are working on mitigating the issue for Gmail, Google Chat and Google Drive Android customers.\nFull resolution is expected to complete complete by Wednesday, 2025-05-07 18:30 US/Pacific.\nWe will provide an update by Wednesday, 2025-05-07 19:00 US/Pacific with current details.\n**Symptoms:**\nSome PDF files in Google Drive, Gmail, and Google Chat are not being displayed properly for customers using Android devices and Web browsers.\nThe issue may also affect the Gen AI features for the affected PDF Files.\nThe issue does not affect customers using iOS devices.\n**Workarounds:**\nCustomers can download the affected PDF files and view them without any issues.","status":"SERVICE_INFORMATION"},{"created":"2025-05-07T20:01:50+00:00","modified":"2025-05-07T22:34:17+00:00","when":"2025-05-07T20:01:50+00:00","text":"**Summary:**\nGoogle Drive, Gmail, and Google Chat customers using Android devices, and Web browser are experiencing rendering issues with some PDF files\n**Description:**\nWe believe the issue with Gmail Web , Google Chat Web is fully resolved.\nOur engineers are working on mitigating the issue for Gmail and Google Chat Android customers, and Google Drive Android and Web customers.\nFull resolution is expected to complete complete by Wednesday, 2025-05-07 15:30 US/Pacific.\nWe will provide an update by Wednesday, 2025-05-07 15:45 US/Pacific with current details.\n**Symptoms:**\nSome PDF files in Google Drive, Gmail, and Google Chat are not being displayed properly for customers using Android devices and Web browsers.\nThe issue may also affect the Gen AI features for the affected PDF Files.\nThe issue does not affect customers using iOS devices.\n**Workarounds:**\nCustomers can download the affected PDF files and view them without any issues.","status":"SERVICE_INFORMATION"},{"created":"2025-05-07T17:58:24+00:00","modified":"2025-05-07T20:01:50+00:00","when":"2025-05-07T17:58:24+00:00","text":"**Summary:**\nGoogle Drive, Gmail, and Google Chat customers using Android devices, and Web browser are experiencing rendering issues with some PDF files\n**Description:**\nWe believe the issue with Gmail , Google Chat Web is fully resolved.\nOur engineers are working on mitigating the issue for Gmail and Google Chat Android customers, and Google Drive Android and Web customers.\nFull resolution is expected to complete by Wednesday, 2025-05-07 12:30 US/Pacific.\nWe will provide an update by Wednesday, 2025-05-07 12:45 US/Pacific with current details.\n**Symptoms:**\nSome PDF files in Google Drive, Gmail, and Google Chat are not being displayed properly for customers using Android devices and Web browsers.\nThe issue may also affect the Gen AI features for the affected PDF Files.\nThe issue does not affect customers using iOS devices.\n**Workarounds:**\nCustomers can download the affected PDF files and view them without any issues.","status":"SERVICE_INFORMATION"},{"created":"2025-05-07T16:13:25+00:00","modified":"2025-05-07T17:58:24+00:00","when":"2025-05-07T16:13:25+00:00","text":"**Summary:**\nGoogle Drive, Gmail, and Google Chat customers are experiencing rendering issues with some PDF files\n**Description:**\nWe are experiencing an issue with Google Drive, Gmail, and Google Chat beginning at Wednesday, 2025-05-07 07:47 US/Pacific.\nMitigation work is currently underway by our engineering team.\nThe mitigation is expected to complete by Wednesday, 2025-05-07 11:00 US/Pacific.\nWe will provide more information by Wednesday, 2025-05-07 11:15 US/Pacific.\n**Customer Symptoms:**\nSome PDF files in Google Drive , Gmail, and Google Chat are not being displayed properly\n**Workaround:**\nCustomers can download the affected PDF files and view them without any issues.","status":"SERVICE_INFORMATION"},{"created":"2025-05-07T15:48:14+00:00","modified":"2025-05-07T16:13:25+00:00","when":"2025-05-07T15:48:14+00:00","text":"**Summary:**\nGoogle Drive, Gmail, and Google Chat customers are experiencing rendering issues with some PDF files\n**Description:**\nWe are experiencing an issue with Google Drive, Gmail, and Google Chat beginning at Wednesday, 2025-05-07 07:47 US/Pacific.\nMitigation work is currently underway by our engineering team.\nThe mitigation is expected to complete by Wednesday, 2025-05-07 11:00 US/Pacific.\nWe will provide more information by Wednesday, 2025-05-07 11:15 US/Pacific.\n**Customer Symptoms:**\nSome PDF files in Google Drive , Gmail, and Google Chat are not being displayed properly\n**Workaround:**\nNone at this time.","status":"SERVICE_INFORMATION"},{"created":"2025-05-07T15:33:33+00:00","modified":"2025-05-07T15:48:14+00:00","when":"2025-05-07T15:33:33+00:00","text":"**Summary:**\nGoogle Drive and Gmail customers are experiencing rendering issues with some PDF files\n**Description:**\nWe are experiencing an issue with Google Drive, Gmail beginning at Wednesday, 2025-05-07 07:47 US/Pacific.\nOur engineering team continues to investigate the issue.\nWe will provide an update by Wednesday, 2025-05-07 09:15 US/Pacific with current details.\nWe apologize to all who are affected by the disruption.\n**Customer Symptoms:**\nSome PDF files in Google Drive and Gmail are not being displayed properly\n**Workaround:**\nNone at this time.","status":"SERVICE_INFORMATION"}],"most_recent_update":{"created":"2025-05-12T21:46:07+00:00","modified":"2025-05-14T00:30:35+00:00","when":"2025-05-12T21:46:07+00:00","text":"# Incident Report\n## Summary\nOn Friday, 2 May 2025, Gmail, Google Chat, and Google Drive experienced an issue where some customers using Android devices and web browsers were unable to properly display Portable Document Format (PDF) files globally. Generative AI features associated with the impacted PDF files were also affected, for a duration of 5 days, 12 hours, and 4 minutes.\nTo our customers whose workflows were impacted during this disruption, we sincerely apologize. This is not the level of quality and reliability we strive to offer you, and we are taking immediate steps to improve the platform's performance and availability.\n## Root Cause\nThe root cause of the issue was an update to an underlying software library that affected how PDF files were interpreted during the rendering process.\nA recent software update to an underlying, third-party library inadvertently introduced a change in how PDF files were interpreted during the rendering process. This change led to some Gmail, Google Chat, and Google Drive users on Android devices and web browsers seeing incomplete or inaccurate content within certain PDFs. This issue also impacted the generative AI features for the affected PDF files. iOS users were unaffected.\n## Remediation and Prevention\nGoogle engineers were alerted to the outage on Wednesday, 7 May at 05:52 US/Pacific via a customer support case and immediately started an investigation.\nOnce the root cause was identified, the issue was mitigated by reverting to the previous software version, which restored the correct rendering of PDFs. The incident was fully mitigated on Wednesday, 7 May 16:21 US/Pacific.\nGoogle is committed to preventing a repeat of this issue in the future and is completing the following actions:\n- Add comprehensive test coverage for all new features and critical existing functionality.\n- Improve release-time validation of code changes.\n## Detailed Description of Impact\n- Google Drive: PDF files in drive were not displayed correctly due to rendering issues. Furthermore, Gemini was also affected by the same rendering issue as the Drive viewer for some PDFs.\n- Gmail: PDF files in email were not displayed correctly due to rendering issues.\n- Google Chat: PDF files in chat were not displayed correctly due to rendering issues.","status":"AVAILABLE"},"status_impact":"SERVICE_INFORMATION","severity":"low","service_key":"zall","service_name":"Multiple Products","affected_products":[{"title":"Gmail","id":"prPt2Yra2CbGsbEm9cpC"},{"title":"Google Chat","id":"Ht9Vx5PFzDPHYvrnrvSk"},{"title":"Google Drive","id":"VHNA7p3Z5p3iakj5sA8V"}],"uri":"incidents/SwA8chvNSrGA9WPLtyYu"},{"id":"fV4x9UvX32Uv4oWEzKU8","number":"2601336434679622824","begin":"2025-04-16T12:33:00+00:00","created":"2025-04-16T13:22:54+00:00","end":"2025-04-16T13:16:00+00:00","modified":"2025-04-16T13:56:49+00:00","external_desc":"We're investigating reports of an issue with Google Workspace Support. We will provide more information shortly. The affected users are able to access Google Workspace Support, but are seeing error messages, high latency, and/or other unexpected behavior.","updates":[{"created":"2025-04-16T13:56:49+00:00","modified":"2025-04-16T13:56:50+00:00","when":"2025-04-16T13:56:49+00:00","text":"The problem with Google Workspace Support has been resolved. We apologize for the inconvenience and thank you for your patience and continued support.","status":"AVAILABLE"},{"created":"2025-04-16T13:22:54+00:00","modified":"2025-04-16T13:56:49+00:00","when":"2025-04-16T13:22:54+00:00","text":"We're investigating reports of an issue with Google Workspace Support. We will provide more information shortly. The affected users are able to access Google Workspace Support, but are seeing error messages, high latency, and/or other unexpected behavior. Customer symptoms: Customers may experience an issue with creating cases. Workaround: Customers may attempt to create a case again.","status":"SERVICE_DISRUPTION"}],"most_recent_update":{"created":"2025-04-16T13:56:49+00:00","modified":"2025-04-16T13:56:50+00:00","when":"2025-04-16T13:56:49+00:00","text":"The problem with Google Workspace Support has been resolved. We apologize for the inconvenience and thank you for your patience and continued support.","status":"AVAILABLE"},"status_impact":"SERVICE_DISRUPTION","severity":"medium","service_key":"dnz8HVUHoMMcW9wV91LY","service_name":"Google Workspace Support","affected_products":[{"title":"Google Workspace Support","id":"dnz8HVUHoMMcW9wV91LY"}],"uri":"incidents/fV4x9UvX32Uv4oWEzKU8"}]