# HG changeset patch # User Paper # Date 1718853235 14400 # Node ID 74e2365326c687ebf442ad1d3d2caed2d867d3d1 # Parent eac06513db861348548d3d85f96b9f30ebbc98e4 dep/animone: add experimental accessibility strategy I also moved most of the functions out of util/win32.cc, because that file is meant for things that are shared between the different functions, and currently that is only wide string conversion helpers. diff -r eac06513db86 -r 74e2365326c6 dep/animone/Makefile.am --- a/dep/animone/Makefile.am Wed Jun 19 14:02:11 2024 -0400 +++ b/dep/animone/Makefile.am Wed Jun 19 23:13:55 2024 -0400 @@ -10,6 +10,7 @@ include/animone/types.h noinst_HEADERS = \ + include/animone/a11y/win32.h \ include/animone/fd/freebsd.h \ include/animone/fd/openbsd.h \ include/animone/fd/netbsd.h \ @@ -20,13 +21,14 @@ include/animone/win/quartz.h \ include/animone/win/win32.h \ include/animone/win/x11.h \ + include/animone/a11y.h \ include/animone/fd.h \ include/animone/strategies.h \ include/animone/util.h \ include/animone/win.h if BUILD_WIN -files_win = src/fd/win32.cc src/win/win32.cc src/util/win32.cc +files_win = src/a11y/win32.cc src/fd/win32.cc src/win/win32.cc src/util/win32.cc libs_win = -lole32 -luuid endif @@ -65,6 +67,7 @@ libanimone_la_SOURCES = \ src/animone.cc \ + src/a11y.cc \ src/fd.cc \ src/player.cc \ src/strategist.cc \ diff -r eac06513db86 -r 74e2365326c6 dep/animone/include/animone/a11y.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/dep/animone/include/animone/a11y.h Wed Jun 19 23:13:55 2024 -0400 @@ -0,0 +1,30 @@ +#ifndef ANIMONE_ANIMONE_A11Y_H_ +#define ANIMONE_ANIMONE_A11Y_H_ + +#include +#include + +#include "animone.h" + +namespace animone { +namespace internal { + +enum class WebBrowserInformationType { + Address, + Tab, + Title, +}; + +struct WebBrowserInformation { + WebBrowserInformationType type = WebBrowserInformationType::Title; + std::string value; +}; + +using web_browser_proc_t = std::function; + +bool GetWebBrowserInformation(const Window& window, web_browser_proc_t web_browser_proc); + +} // namespace internal +} // namespace animone + +#endif // ANIMONE_ANIMONE_A11Y_H_ diff -r eac06513db86 -r 74e2365326c6 dep/animone/include/animone/a11y/win32.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/dep/animone/include/animone/a11y/win32.h Wed Jun 19 23:13:55 2024 -0400 @@ -0,0 +1,14 @@ +#ifndef ANIMONE_ANIMONE_A11Y_WIN32_H_ +#define ANIMONE_ANIMONE_A11Y_WIN32_H_ + +#include "animone.h" +#include "animone/a11y.h" +#include "animone/a11y/win32.h" + +namespace animone::internal::win32 { + +bool GetWebBrowserInformation(const Window& window, web_browser_proc_t web_browser_proc); + +} // namespace animone::internal::win32 + +#endif // ANIMONE_ANIMONE_A11Y_WIN32_H_ diff -r eac06513db86 -r 74e2365326c6 dep/animone/include/animone/util/win32.h --- a/dep/animone/include/animone/util/win32.h Wed Jun 19 14:02:11 2024 -0400 +++ b/dep/animone/include/animone/util/win32.h Wed Jun 19 23:13:55 2024 -0400 @@ -23,10 +23,7 @@ std::string ToUtf8String(const UNICODE_STRING& string); std::wstring ToWstring(const std::string& string); -std::wstring GetProcessPath(DWORD process_id); -std::wstring GetFileNameFromPath(const std::wstring& path); -std::wstring GetFileNameWithoutExtension(const std::wstring& filename); - +/* XXX can this stuff be moved to fd/win32.cc? */ bool IsSystemDirectory(const std::string& path); bool IsSystemDirectory(std::wstring path); diff -r eac06513db86 -r 74e2365326c6 dep/animone/src/a11y.cc --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/dep/animone/src/a11y.cc Wed Jun 19 23:13:55 2024 -0400 @@ -0,0 +1,19 @@ +#include "animone/a11y.h" + +#ifdef USE_WIN32 +# include "animone/a11y/win32.h" +#endif + +namespace animone::internal { + +bool GetWebBrowserInformation(const Window& window, web_browser_proc_t web_browser_proc) { + bool success = false; + +#ifdef USE_WIN32 + success ^= win32::GetWebBrowserInformation(window, web_browser_proc); +#endif + + return success; +} + +} // namespace animone::internal diff -r eac06513db86 -r 74e2365326c6 dep/animone/src/a11y/win32.cc --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/dep/animone/src/a11y/win32.cc Wed Jun 19 23:13:55 2024 -0400 @@ -0,0 +1,243 @@ +#include +#include +#include + +#include +#include + +#include "animone/a11y.h" +#include "animone/a11y/win32.h" + +namespace animone::internal::win32 { + +// Windows Accessibility API reference: +// https://msdn.microsoft.com/en-us/library/windows/desktop/ff486375.aspx + +// Commonly used interfaces +using Element = IUIAutomationElement; +using TreeWalker = IUIAutomationTreeWalker; +using ValuePattern = IUIAutomationValuePattern; + +using element_proc_t = std::function; +using properties_t = std::vector>; + +// The main interface that is used throughout this file. Must be initialized +// before it can be used for the first time. +static ComInterface ui_automation; + +//////////////////////////////////////////////////////////////////////////////// + +static bool InitializeUIAutomation() { + if (ui_automation) + return true; + + // COM library must be initialized on the current thread before calling + // CoCreateInstance. + ::CoInitialize(nullptr); + + IUIAutomation* ui_automation_interface = nullptr; + const auto result = ::CoCreateInstance( + CLSID_CUIAutomation, nullptr, CLSCTX_INPROC_SERVER, IID_IUIAutomation, + reinterpret_cast(&ui_automation_interface)); + ui_automation.reset(ui_automation_interface); + + return SUCCEEDED(result); +} + +//////////////////////////////////////////////////////////////////////////////// + +static Element* GetElementFromHandle(HWND hwnd) { + Element* element = nullptr; + ui_automation->ElementFromHandle(static_cast(hwnd), &element); + return element; +} + +static std::wstring GetElementName(Element& element) { + std::wstring element_name; + + BSTR bstr = nullptr; + if (SUCCEEDED(element.get_CurrentName(&bstr)) && bstr) { + element_name = bstr; + ::SysFreeString(bstr); + } + + return element_name; +} + +static std::wstring GetElementValue(Element& element) { + std::wstring element_value; + + ValuePattern* value_pattern_interface = nullptr; + element.GetCurrentPatternAs( + UIA_ValuePatternId, IID_PPV_ARGS(&value_pattern_interface)); + ComInterface value_pattern(value_pattern_interface); + + if (value_pattern) { + BSTR bstr = nullptr; + if (SUCCEEDED(value_pattern->get_CurrentValue(&bstr)) && bstr) { + element_value = bstr; + ::SysFreeString(bstr); + } + } + + return element_value; +} + +//////////////////////////////////////////////////////////////////////////////// + +static bool VerifyElementProperties(Element& element, const properties_t& properties) { + VARIANT v = {}; + for (const auto& pair : properties) { + if (FAILED(element.GetCurrentPropertyValue(pair.first, &v))) + return false; + if (v.boolVal != (pair.second ? VARIANT_TRUE : VARIANT_FALSE)) + return false; + } + + return true; +} + +static bool IsAddressBarElement(Element& element) { + static const properties_t properties = { + {UIA_IsEnabledPropertyId, true}, + {UIA_IsKeyboardFocusablePropertyId, true}, + {UIA_IsValuePatternAvailablePropertyId, true}, + {UIA_ValueIsReadOnlyPropertyId, false}, + }; + + return VerifyElementProperties(element, properties); +} + +static bool IsTabsElement(Element& element) { + static const properties_t properties = { + {UIA_ValueIsReadOnlyPropertyId, true}, + }; + + return VerifyElementProperties(element, properties); +} + +//////////////////////////////////////////////////////////////////////////////// + +static void WalkElements(TreeWalker& tree_walker, Element& parent, TreeScope scope, + size_t depth, element_proc_t element_proc) { + constexpr size_t kMaxTreeDepth = 16; // arbitrary value + if (depth > kMaxTreeDepth) + return; + + if (scope & TreeScope_Element) + scope = element_proc(parent); + + auto descend = [](TreeScope scope) { + return (scope & TreeScope_Children) || (scope & TreeScope_Descendants); + }; + + if (descend(scope)) { + Element* first_element = nullptr; + tree_walker.GetFirstChildElement(&parent, &first_element); + ComInterface element(first_element); + + while (element) { + scope = element_proc(*element); + + if (descend(scope)) + WalkElements(tree_walker, *element, scope, depth + 1, element_proc); + + Element* next_element = nullptr; + tree_walker.GetNextSiblingElement(element.get(), &next_element); + element.reset(next_element); + } + } +} + +static bool FindWebBrowserElements(Element& parent, std::wstring& address, + std::vector& tabs) { + TreeWalker* tree_walker_interface = nullptr; + ui_automation->get_ControlViewWalker(&tree_walker_interface); + ComInterface tree_walker(tree_walker_interface); + + if (!tree_walker) + return false; + + auto element_proc = [&](Element& element) -> TreeScope { + CONTROLTYPEID control_type_id = 0; + element.get_CurrentControlType(&control_type_id); + + switch (control_type_id) { + default: + // Are we done? + if (!address.empty() && !tabs.empty()) + return TreeScope_Element; + // Otherwise continue descending the tree. + return TreeScope_Descendants; + + case UIA_DocumentControlTypeId: + case UIA_MenuBarControlTypeId: + case UIA_TitleBarControlTypeId: + // We do not need to walk through these nodes. In fact, skipping + // documents dramatically improves our performance on worst case + // scenarios. This is the whole reason we are walking the tree rather + // than using FindFirst and FindAll methods. + return TreeScope_Element; + + case UIA_EditControlTypeId: + // Here we assume that the first edit control that fits our properties + // is the address bar (e.g. "Omnibox" on Chrome, "Awesome Bar" on + // Firefox). This element is named differently on each web browser + // (e.g. "Address and search bar" on Chrome, "Search or enter address" + // on Firefox). This name can change depending on the browser + // language. However, we are only interested in the element value, + // which usually gives us the URL of the current page. + if (address.empty() && IsAddressBarElement(element)) { + address = GetElementValue(element); + return TreeScope_Element; + } else { + // Opera has an edit control ("Address field") within another edit + // control ("Address bar"). + return TreeScope_Descendants; + } + + case UIA_TabControlTypeId: + if (tabs.empty() && IsTabsElement(element)) + return TreeScope_Children; + return TreeScope_Element; + + case UIA_TabItemControlTypeId: + tabs.push_back(GetElementName(element)); + return TreeScope_Element; + } + }; + + WalkElements(*tree_walker, parent, TreeScope_Subtree, 0, element_proc); + return true; +} + +/* ------------------------------------------------------------------------------------ */ + +bool GetWebBrowserInformation(const Window& window, web_browser_proc_t web_browser_proc) { + if (!web_browser_proc) + return false; + + if (!InitializeUIAutomation()) + return false; + + ComInterface parent(GetElementFromHandle(hwnd)); + if (!parent) + return false; + + const std::string title = ToUtf8String(GetElementName(*parent)); + web_browser_proc({WebBrowserInformationType::Title, title}); + + std::wstring address; + std::vector tabs; + + if (!FindWebBrowserElements(*parent, address, tabs)) + return false; + + web_browser_proc({WebBrowserInformationType::Address, ToUtf8String(address)}); + for (const auto& tab : tabs) + web_browser_proc({WebBrowserInformationType::Tab, ToUtf8String(tab)}); + + return true; +} + +} // namespace animone::internal::win32 diff -r eac06513db86 -r 74e2365326c6 dep/animone/src/fd/win32.cc --- a/dep/animone/src/fd/win32.cc Wed Jun 19 14:02:11 2024 -0400 +++ b/dep/animone/src/fd/win32.cc Wed Jun 19 23:13:55 2024 -0400 @@ -32,15 +32,10 @@ #include /* SystemExtendedHandleInformation is only available in NT 5.1+ (XP and higher) and provides information for - * 32-bit PIDs, unlike SystemHandleInformation - */ + * 32-bit PIDs, unlike SystemHandleInformation */ static constexpr SYSTEM_INFORMATION_CLASS SystemExtendedHandleInformation = static_cast(0x40); static constexpr NTSTATUS STATUS_INFO_LENGTH_MISMATCH = 0xC0000004UL; -/* this is filled in at runtime because it's not guaranteed to be (and isn't) - * constant between different versions of Windows */ -static unsigned short file_type_index = 0; - struct SYSTEM_HANDLE_TABLE_ENTRY_INFO_EX { PVOID Object; ULONG_PTR UniqueProcessId; @@ -136,22 +131,58 @@ static std::wstring GetFinalPathNameByHandle(HANDLE handle) { std::wstring buffer; - DWORD size = ::GetFinalPathNameByHandleW(handle, NULL, 0, FILE_NAME_NORMALIZED | VOLUME_NAME_DOS); + DWORD size = ::GetFinalPathNameByHandleW(handle, nullptr, 0, FILE_NAME_NORMALIZED | VOLUME_NAME_DOS); buffer.resize(size); ::GetFinalPathNameByHandleW(handle, &buffer.front(), buffer.size(), FILE_NAME_NORMALIZED | VOLUME_NAME_DOS); return buffer; } +/* ------------------------------------------------------------------- */ + +static bool GetSystemDirectory(std::wstring& str) { + PWSTR path_wch; + + if (FAILED(::SHGetFolderPathW(NULL, CSIDL_WINDOWS, NULL, SHGFP_TYPE_CURRENT, &path_wch))) + return false; + + str.assign(path_wch); + + ::CoTaskMemFree(path_wch); + return true; +} + +static bool IsSystemDirectory(const std::string& path) { + return IsSystemDirectory(ToWstring(path)); +} + +static bool IsSystemDirectory(std::wstring path) { + std::wstring windir; + if (!GetSystemDirectory(windir)) + return false; + + ::CharUpperBuffW(&path.front(), path.length()); + ::CharUpperBuffW(&windir.front(), windir.length()); + + // XXX wtf is 4? + return path.find(windir) == 4; +} + static bool IsFileHandle(HANDLE handle, unsigned short object_type_index) { - if (file_type_index) - return object_type_index == file_type_index; - else if (!handle) + /* this is filled in at runtime because it's not guaranteed to be (and isn't) + * constant between different versions of Windows */ + static std::optional file_type_index; + + if (file_type_index.has_value()) { + return object_type_index == file_type_index.value(); + } else if (!handle) { + /* XXX what? */ return true; - else if (GetHandleType(handle) == L"File") { - file_type_index = object_type_index; + } else if (GetHandleType(handle) == L"File") { + file_type_index.reset(object_type_index); return true; } + return false; } @@ -179,13 +210,72 @@ return true; } +/* ------------------------------------------------------------------- */ + +static std::string GetProcessPath(DWORD process_id) { + // If we try to open a SYSTEM process, this function fails and the last error + // code is ERROR_ACCESS_DENIED. + // + // Note that if we requested PROCESS_QUERY_INFORMATION access right instead + // of PROCESS_QUERY_LIMITED_INFORMATION, this function would fail when used + // to open an elevated process. + Handle process_handle(::OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, FALSE, process_id)); + + if (!process_handle) + return std::wstring(); + + std::wstring buffer(MAX_PATH, L'\0'); + DWORD buf_size = buffer.length(); + + // Note that this function requires Windows Vista or above. You may use + // GetProcessImageFileName or GetModuleFileNameEx on earlier versions. + if (!::QueryFullProcessImageNameW(process_handle.get(), 0, &buffer.front(), &buf_size)) + return std::wstring(); + + buffer.resize(buf_size); + return ToUtf8String(buffer); +} + +static std::string GetFilenameFromPath(const std::string& path) { + const auto pos = path.find_last_of(L"/\\"); + return pos != std::wstring::npos ? path.substr(pos + 1) : path; +} + +static bool VerifyProcessPath(const std::string& path) { + return !path.empty() && !IsSystemDirectory(path); +} + +static bool VerifyProcessFilename(const std::string& name) { + static const std::set invalid_names = { + // System files + "explorer.exe", // Windows Explorer + "taskeng.exe", // Task Scheduler Engine + "taskhost.exe", // Host Process for Windows Tasks + "taskhostex.exe", // Host Process for Windows Tasks + "taskmgr.exe", // Task Manager + "services.exe", // Service Control Manager + }; + + if (name.empty()) + return false; + + for (const auto& invalid_name : invalid_names) + if (util::EqualStrings(name, invalid_name)) + return false; + + return true; +} + +/* ------------------------------------------------------------------- */ +/* extern functions */ + bool GetProcessName(pid_t pid, std::string& name) { std::string path = GetProcessPath(pid); if (path.empty() || !VerifyProcessPath(path)) return false; - name = GetFileNameFromPath(path); - if (!VerifyProcessFileName(name)) + name = GetFilenameFromPath(path); + if (!VerifyProcessFilename(name)) return false; return true; @@ -214,7 +304,6 @@ return true; } -/* this could be changed to being a callback, but... I'm too lazy right now :) */ bool EnumerateOpenFiles(const std::set& pids, open_file_proc_t open_file_proc) { if (!open_file_proc) return false; @@ -247,7 +336,7 @@ if (handle.get() == INVALID_HANDLE_VALUE) continue; - if (GetFileType(handle.get()) != FILE_TYPE_DISK) + if (::GetFileType(handle.get()) != FILE_TYPE_DISK) continue; const std::wstring path = GetFinalPathNameByHandle(handle.get()); diff -r eac06513db86 -r 74e2365326c6 dep/animone/src/strategist.cc --- a/dep/animone/src/strategist.cc Wed Jun 19 14:02:11 2024 -0400 +++ b/dep/animone/src/strategist.cc Wed Jun 19 23:13:55 2024 -0400 @@ -2,11 +2,14 @@ #include #include "animone.h" +#include "animone/a11y.h" #include "animone/fd.h" #include "animone/strategies.h" #include "animone/util.h" -/* this was STUPIDLY slow in Anisthesia, oops! */ +/* This file was changed lots from anisthesia. Most notably we don't use classes here + * anymore, and we just pass the result vector to different function that append + * to the result (which is better imo) */ namespace animone::internal { @@ -45,13 +48,17 @@ if (media_information.value.empty()) return false; - Media media; - media.information.push_back(media_information); + Media media = { + .information = {media_information} + }; result.media.push_back(std::move(media)); return true; } +/* ------------------------------------------------------------------------- */ +/* strategies */ + static bool ApplyWindowTitleStrategy(std::vector& results) { bool success = false; @@ -94,15 +101,43 @@ return success; } +static bool ApplyAccessibilityStrategy(std::vector& results) { + bool success = false; + + for (Result& result : results) { + auto web_browser_proc = [&result](const WebBrowserInformation& info) { + auto value = info.value; + + switch (info.type) { + case WebBrowserInformationType::Address: + AddMedia(result, {MediaInfoType::Url, value}); + break; + case WebBrowserInformationType::Title: + ApplyWindowTitleFormat(result.player.window_title_format, value); + AddMedia(result, {MediaInfoType::Title, value}); + break; + case WebBrowserInformationType::Tab: + AddMedia(result, {MediaInfoType::Tab, value}); + break; + } + }; + + success |= GetWebBrowserInformation(result.window, web_browser_proc); + } + + return success; +} + +/* ------------------------------------------------------------------------- */ + bool ApplyStrategies(std::vector& results) { bool success = false; success |= ApplyWindowTitleStrategy(results); success |= ApplyOpenFilesStrategy(results); + success |= ApplyAccessibilityStrategy(results); return success; } -//////////////////////////////////////////////////////////////////////////////// - } // namespace animone::internal diff -r eac06513db86 -r 74e2365326c6 dep/animone/src/util/win32.cc --- a/dep/animone/src/util/win32.cc Wed Jun 19 14:02:11 2024 -0400 +++ b/dep/animone/src/util/win32.cc Wed Jun 19 23:13:55 2024 -0400 @@ -47,80 +47,4 @@ return ret; } -std::string GetProcessPath(DWORD process_id) { - // If we try to open a SYSTEM process, this function fails and the last error - // code is ERROR_ACCESS_DENIED. - // - // Note that if we requested PROCESS_QUERY_INFORMATION access right instead - // of PROCESS_QUERY_LIMITED_INFORMATION, this function would fail when used - // to open an elevated process. - Handle process_handle(::OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, FALSE, process_id)); - - if (!process_handle) - return std::wstring(); - - std::wstring buffer(MAX_PATH, L'\0'); - DWORD buf_size = buffer.length(); - - // Note that this function requires Windows Vista or above. You may use - // GetProcessImageFileName or GetModuleFileNameEx on earlier versions. - if (!::QueryFullProcessImageNameW(process_handle.get(), 0, &buffer.front(), &buf_size)) - return std::wstring(); - - buffer.resize(buf_size); - return ToUtf8String(buffer); -} - -std::string GetFileNameFromPath(const std::string& path) { - const auto pos = path.find_last_of(L"/\\"); - return pos != std::wstring::npos ? path.substr(pos + 1) : path; -} - -static std::wstring GetSystemDirectory() { - PWSTR path_wch; - SHGetFolderPathW(NULL, CSIDL_WINDOWS, NULL, SHGFP_TYPE_CURRENT, &path_wch); - std::wstring path_wstr(path_wch); - CoTaskMemFree(path_wch); - return path_wstr; -} - -bool IsSystemDirectory(const std::string& path) { - return IsSystemDirectory(ToWstring(path)); -} - -bool IsSystemDirectory(std::wstring path) { - ::CharUpperBuffW(&path.front(), path.length()); - - std::wstring windir = GetSystemDirectory(); - ::CharUpperBuffW(&windir.front(), windir.length()); - - // XXX wtf is 4? - return path.find(windir) == 4; -} - -bool VerifyProcessPath(const std::string& path) { - return !path.empty() && !IsSystemDirectory(path); -} - -bool VerifyProcessFileName(const std::string& name) { - static const std::set invalid_names = { - // System files - "explorer.exe", // Windows Explorer - "taskeng.exe", // Task Scheduler Engine - "taskhost.exe", // Host Process for Windows Tasks - "taskhostex.exe", // Host Process for Windows Tasks - "taskmgr.exe", // Task Manager - "services.exe", // Service Control Manager - }; - - if (name.empty()) - return false; - - for (const auto& invalid_name : invalid_names) - if (util::EqualStrings(name, invalid_name)) - return false; - - return true; -} - } // namespace animone::internal::win32