From 932f6c8a41a0ee4a86e60d570d987624a7de5523 Mon Sep 17 00:00:00 2001 From: Milan Burda Date: Wed, 12 Sep 2018 16:18:35 +0200 Subject: [PATCH] feat: add screen reader support to Win32 toast notifications (#13834) --- brightray/BUILD.gn | 2 + .../win/win32_desktop_notifications/toast.cc | 43 +++ .../win/win32_desktop_notifications/toast.h | 3 + .../win32_desktop_notifications/toast_uia.cc | 250 ++++++++++++++++++ .../win32_desktop_notifications/toast_uia.h | 80 ++++++ 5 files changed, 378 insertions(+) create mode 100644 brightray/browser/win/win32_desktop_notifications/toast_uia.cc create mode 100644 brightray/browser/win/win32_desktop_notifications/toast_uia.h diff --git a/brightray/BUILD.gn b/brightray/BUILD.gn index a318c479f291d..38081f7beac33 100644 --- a/brightray/BUILD.gn +++ b/brightray/BUILD.gn @@ -103,6 +103,8 @@ static_library("brightray") { "browser/win/win32_desktop_notifications/common.h", "browser/win/win32_desktop_notifications/desktop_notification_controller.cc", "browser/win/win32_desktop_notifications/desktop_notification_controller.h", + "browser/win/win32_desktop_notifications/toast_uia.cc", + "browser/win/win32_desktop_notifications/toast_uia.h", "browser/win/win32_desktop_notifications/toast.cc", "browser/win/win32_desktop_notifications/toast.h", "browser/win/win32_notification.cc", diff --git a/brightray/browser/win/win32_desktop_notifications/toast.cc b/brightray/browser/win/win32_desktop_notifications/toast.cc index 061eb977a8dc5..990ba63bcd6a2 100644 --- a/brightray/browser/win/win32_desktop_notifications/toast.cc +++ b/brightray/browser/win/win32_desktop_notifications/toast.cc @@ -2,11 +2,17 @@ #define NOMINMAX #endif #include "brightray/browser/win/win32_desktop_notifications/toast.h" + +#include + +#include #include #include #include + #include "base/logging.h" #include "brightray/browser/win/win32_desktop_notifications/common.h" +#include "brightray/browser/win/win32_desktop_notifications/toast_uia.h" #pragma comment(lib, "msimg32.lib") #pragma comment(lib, "uxtheme.lib") @@ -196,6 +202,22 @@ DesktopNotificationController::Toast::Toast(HWND hwnd, } DesktopNotificationController::Toast::~Toast() { + if (uia_) { + auto* UiaDisconnectProvider = + reinterpret_cast(GetProcAddress( + GetModuleHandle(L"uiautomationcore.dll"), "UiaDisconnectProvider")); + // first detach from the toast, then call UiaDisconnectProvider; + // UiaDisconnectProvider may call WM_GETOBJECT and we don't want + // it to return the object that we're disconnecting + uia_->DetachToast(); + + if (UiaDisconnectProvider) + UiaDisconnectProvider(uia_); + + uia_->Release(); + uia_ = nullptr; + } + DeleteDC(hdc_); if (bitmap_) DeleteBitmap(bitmap_); @@ -232,6 +254,13 @@ LRESULT DesktopNotificationController::Toast::WndProc(HWND hwnd, SetWindowLongPtr(hwnd, 0, 0); return 0; + case WM_DESTROY: + if (Get(hwnd)->uia_) { + // free UI Automation resources associated with this window + UiaReturnRawElementProvider(hwnd, 0, 0, nullptr); + } + break; + case WM_MOUSEACTIVATE: return MA_NOACTIVATE; @@ -295,6 +324,20 @@ LRESULT DesktopNotificationController::Toast::WndProc(HWND hwnd, Get(hwnd)->is_highlighted_ = false; } } break; + + case WM_GETOBJECT: + if (lparam == UiaRootObjectId) { + auto* inst = Get(hwnd); + if (!inst->uia_) { + inst->uia_ = new UIAutomationInterface(inst); + inst->uia_->AddRef(); + } + // don't return the interface if it's being disconnected + if (!inst->uia_->IsDetached()) { + return UiaReturnRawElementProvider(hwnd, wparam, lparam, inst->uia_); + } + } + break; } return DefWindowProc(hwnd, message, wparam, lparam); diff --git a/brightray/browser/win/win32_desktop_notifications/toast.h b/brightray/browser/win/win32_desktop_notifications/toast.h index 5a0e91b5e5ca0..4e8ba8949e563 100644 --- a/brightray/browser/win/win32_desktop_notifications/toast.h +++ b/brightray/browser/win/win32_desktop_notifications/toast.h @@ -70,6 +70,9 @@ class DesktopNotificationController::Toast { HDC hdc_; HBITMAP bitmap_ = NULL; + class UIAutomationInterface; + UIAutomationInterface* uia_ = nullptr; + const std::shared_ptr data_; // never null SIZE toast_size_ = {}; diff --git a/brightray/browser/win/win32_desktop_notifications/toast_uia.cc b/brightray/browser/win/win32_desktop_notifications/toast_uia.cc new file mode 100644 index 0000000000000..0d51c61967eb7 --- /dev/null +++ b/brightray/browser/win/win32_desktop_notifications/toast_uia.cc @@ -0,0 +1,250 @@ +#include "brightray/browser/win/win32_desktop_notifications/toast_uia.h" +#include +#include "brightray/browser/win/win32_desktop_notifications/common.h" + +#pragma comment(lib, "uiautomationcore.lib") + +namespace brightray { + +DesktopNotificationController::Toast::UIAutomationInterface:: + UIAutomationInterface(Toast* toast) + : hwnd_(toast->hwnd_) { + text_ = toast->data_->caption; + if (!toast->data_->body_text.empty()) { + if (!text_.empty()) + text_.append(L", "); + text_.append(toast->data_->body_text); + } +} + +ULONG DesktopNotificationController::Toast::UIAutomationInterface::AddRef() { + return InterlockedIncrement(&cref_); +} + +ULONG DesktopNotificationController::Toast::UIAutomationInterface::Release() { + LONG ret = InterlockedDecrement(&cref_); + if (ret == 0) { + delete this; + return 0; + } + _ASSERT(ret > 0); + return ret; +} + +STDMETHODIMP +DesktopNotificationController::Toast::UIAutomationInterface::QueryInterface( + REFIID riid, + LPVOID* ppv) { + if (!ppv) + return E_INVALIDARG; + + if (riid == IID_IUnknown) { + *ppv = + static_cast(static_cast(this)); + } else if (riid == __uuidof(IRawElementProviderSimple)) { + *ppv = static_cast(this); + } else if (riid == __uuidof(IWindowProvider)) { + *ppv = static_cast(this); + } else if (riid == __uuidof(IInvokeProvider)) { + *ppv = static_cast(this); + } else if (riid == __uuidof(ITextProvider)) { + *ppv = static_cast(this); + } else { + *ppv = nullptr; + return E_NOINTERFACE; + } + + this->AddRef(); + return S_OK; +} + +HRESULT DesktopNotificationController::Toast::UIAutomationInterface:: + get_ProviderOptions(ProviderOptions* retval) { + *retval = ProviderOptions_ServerSideProvider; + return S_OK; +} + +HRESULT +DesktopNotificationController::Toast::UIAutomationInterface::GetPatternProvider( + PATTERNID pattern_id, + IUnknown** retval) { + switch (pattern_id) { + case UIA_WindowPatternId: + *retval = static_cast(this); + break; + case UIA_InvokePatternId: + *retval = static_cast(this); + break; + case UIA_TextPatternId: + *retval = static_cast(this); + break; + default: + *retval = nullptr; + return S_OK; + } + this->AddRef(); + return S_OK; +} + +HRESULT +DesktopNotificationController::Toast::UIAutomationInterface::GetPropertyValue( + PROPERTYID property_id, + VARIANT* retval) { + // Note: In order to have the toast read by the NVDA screen reader, we + // pretend that we're a Windows 8 native toast notification by reporting + // these property values: + // ClassName: ToastContentHost + // ControlType: UIA_ToolTipControlTypeId + + retval->vt = VT_EMPTY; + switch (property_id) { + case UIA_NamePropertyId: + retval->vt = VT_BSTR; + retval->bstrVal = SysAllocString(text_.c_str()); + break; + + case UIA_ClassNamePropertyId: + retval->vt = VT_BSTR; + retval->bstrVal = SysAllocString(L"ToastContentHost"); + break; + + case UIA_ControlTypePropertyId: + retval->vt = VT_I4; + retval->lVal = UIA_ToolTipControlTypeId; + break; + + case UIA_LiveSettingPropertyId: + retval->vt = VT_I4; + retval->lVal = Assertive; + break; + + case UIA_IsContentElementPropertyId: + case UIA_IsControlElementPropertyId: + case UIA_IsPeripheralPropertyId: + retval->vt = VT_BOOL; + retval->lVal = VARIANT_TRUE; + break; + + case UIA_HasKeyboardFocusPropertyId: + case UIA_IsKeyboardFocusablePropertyId: + case UIA_IsOffscreenPropertyId: + retval->vt = VT_BOOL; + retval->lVal = VARIANT_FALSE; + break; + } + return S_OK; +} + +HRESULT +DesktopNotificationController::Toast::UIAutomationInterface:: + get_HostRawElementProvider(IRawElementProviderSimple** retval) { + if (!hwnd_) + return E_FAIL; + return UiaHostProviderFromHwnd(hwnd_, retval); +} + +HRESULT DesktopNotificationController::Toast::UIAutomationInterface::Invoke() { + return E_NOTIMPL; +} + +HRESULT +DesktopNotificationController::Toast::UIAutomationInterface::SetVisualState( + WindowVisualState state) { + // setting the visual state is not supported + return E_FAIL; +} + +HRESULT DesktopNotificationController::Toast::UIAutomationInterface::Close() { + return E_NOTIMPL; +} + +HRESULT +DesktopNotificationController::Toast::UIAutomationInterface::WaitForInputIdle( + int milliseconds, + BOOL* retval) { + return E_NOTIMPL; +} + +HRESULT +DesktopNotificationController::Toast::UIAutomationInterface::get_CanMaximize( + BOOL* retval) { + *retval = FALSE; + return S_OK; +} + +HRESULT +DesktopNotificationController::Toast::UIAutomationInterface::get_CanMinimize( + BOOL* retval) { + *retval = FALSE; + return S_OK; +} + +HRESULT +DesktopNotificationController::Toast::UIAutomationInterface::get_IsModal( + BOOL* retval) { + *retval = FALSE; + return S_OK; +} + +HRESULT DesktopNotificationController::Toast::UIAutomationInterface:: + get_WindowVisualState(WindowVisualState* retval) { + *retval = WindowVisualState_Normal; + return S_OK; +} + +HRESULT DesktopNotificationController::Toast::UIAutomationInterface:: + get_WindowInteractionState(WindowInteractionState* retval) { + if (!hwnd_) + *retval = WindowInteractionState_Closing; + else + *retval = WindowInteractionState_ReadyForUserInteraction; + + return S_OK; +} + +HRESULT +DesktopNotificationController::Toast::UIAutomationInterface::get_IsTopmost( + BOOL* retval) { + *retval = TRUE; + return S_OK; +} + +HRESULT +DesktopNotificationController::Toast::UIAutomationInterface::GetSelection( + SAFEARRAY** retval) { + return E_NOTIMPL; +} + +HRESULT +DesktopNotificationController::Toast::UIAutomationInterface::GetVisibleRanges( + SAFEARRAY** retval) { + return E_NOTIMPL; +} + +HRESULT +DesktopNotificationController::Toast::UIAutomationInterface::RangeFromChild( + IRawElementProviderSimple* child_element, + ITextRangeProvider** retval) { + return E_NOTIMPL; +} + +HRESULT +DesktopNotificationController::Toast::UIAutomationInterface::RangeFromPoint( + UiaPoint point, + ITextRangeProvider** retval) { + return E_NOTIMPL; +} + +HRESULT +DesktopNotificationController::Toast::UIAutomationInterface::get_DocumentRange( + ITextRangeProvider** retval) { + return E_NOTIMPL; +} + +HRESULT DesktopNotificationController::Toast::UIAutomationInterface:: + get_SupportedTextSelection(SupportedTextSelection* retval) { + *retval = SupportedTextSelection_None; + return S_OK; +} + +} // namespace brightray diff --git a/brightray/browser/win/win32_desktop_notifications/toast_uia.h b/brightray/browser/win/win32_desktop_notifications/toast_uia.h new file mode 100644 index 0000000000000..4f14e480fc9df --- /dev/null +++ b/brightray/browser/win/win32_desktop_notifications/toast_uia.h @@ -0,0 +1,80 @@ +#ifndef BRIGHTRAY_BROWSER_WIN_WIN32_DESKTOP_NOTIFICATIONS_TOAST_UIA_H_ +#define BRIGHTRAY_BROWSER_WIN_WIN32_DESKTOP_NOTIFICATIONS_TOAST_UIA_H_ + +#include "brightray/browser/win/win32_desktop_notifications/toast.h" + +#include + +#include + +namespace brightray { + +class DesktopNotificationController::Toast::UIAutomationInterface + : public IRawElementProviderSimple, + public IWindowProvider, + public IInvokeProvider, + public ITextProvider { + public: + explicit UIAutomationInterface(Toast* toast); + + void DetachToast() { hwnd_ = NULL; } + + bool IsDetached() const { return !hwnd_; } + + private: + virtual ~UIAutomationInterface() = default; + + // IUnknown + public: + ULONG STDMETHODCALLTYPE AddRef() override; + ULONG STDMETHODCALLTYPE Release() override; + STDMETHODIMP QueryInterface(REFIID riid, LPVOID* ppv) override; + + // IRawElementProviderSimple + public: + STDMETHODIMP get_ProviderOptions(ProviderOptions* retval) override; + STDMETHODIMP GetPatternProvider(PATTERNID pattern_id, + IUnknown** retval) override; + STDMETHODIMP GetPropertyValue(PROPERTYID property_id, + VARIANT* retval) override; + STDMETHODIMP get_HostRawElementProvider( + IRawElementProviderSimple** retval) override; + + // IWindowProvider + public: + STDMETHODIMP SetVisualState(WindowVisualState state) override; + STDMETHODIMP Close() override; + STDMETHODIMP WaitForInputIdle(int milliseconds, BOOL* retval) override; + STDMETHODIMP get_CanMaximize(BOOL* retval) override; + STDMETHODIMP get_CanMinimize(BOOL* retval) override; + STDMETHODIMP get_IsModal(BOOL* retval) override; + STDMETHODIMP get_WindowVisualState(WindowVisualState* retval) override; + STDMETHODIMP get_WindowInteractionState( + WindowInteractionState* retval) override; + STDMETHODIMP get_IsTopmost(BOOL* retval) override; + + // IInvokeProvider + public: + STDMETHODIMP Invoke() override; + + // ITextProvider + public: + STDMETHODIMP GetSelection(SAFEARRAY** retval) override; + STDMETHODIMP GetVisibleRanges(SAFEARRAY** retval) override; + STDMETHODIMP RangeFromChild(IRawElementProviderSimple* child_element, + ITextRangeProvider** retval) override; + STDMETHODIMP RangeFromPoint(UiaPoint point, + ITextRangeProvider** retval) override; + STDMETHODIMP get_DocumentRange(ITextRangeProvider** retval) override; + STDMETHODIMP get_SupportedTextSelection( + SupportedTextSelection* retval) override; + + private: + volatile LONG cref_ = 0; + HWND hwnd_; + std::wstring text_; +}; + +} // namespace brightray + +#endif // BRIGHTRAY_BROWSER_WIN_WIN32_DESKTOP_NOTIFICATIONS_TOAST_UIA_H_