Flutter iOS Embedder
accessibility_bridge.mm
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
6 
7 #include <utility>
8 
9 #include "flutter/fml/logging.h"
14 
15 #include "flutter/common/constants.h"
16 
17 #pragma GCC diagnostic error "-Wundeclared-selector"
18 
20 
21 namespace flutter {
22 namespace {
23 
24 class DefaultIosDelegate : public AccessibilityBridge::IosDelegate {
25  public:
26  bool IsFlutterViewControllerPresentingModalViewController(
27  FlutterViewController* view_controller) override {
28  if (view_controller) {
29  return view_controller.isPresentingViewController;
30  } else {
31  return false;
32  }
33  }
34 
35  void PostAccessibilityNotification(UIAccessibilityNotifications notification,
36  id argument) override {
37  UIAccessibilityPostNotification(notification, argument);
38  }
39 };
40 } // namespace
41 
43  FlutterViewController* view_controller,
44  PlatformViewIOS* platform_view,
45  __weak FlutterPlatformViewsController* platform_views_controller,
46  std::unique_ptr<IosDelegate> ios_delegate)
47  : view_controller_(view_controller),
48  accessibility_element_init_container_([[UIView alloc] initWithFrame:CGRectZero]),
49  platform_view_(platform_view),
50  platform_views_controller_(platform_views_controller),
51  objects_([[NSMutableDictionary alloc] init]),
52  previous_routes_({}),
53  ios_delegate_(ios_delegate ? std::move(ios_delegate)
54  : std::make_unique<DefaultIosDelegate>()),
55  weak_factory_(this) {
56  accessibility_channel_ = [[FlutterBasicMessageChannel alloc]
57  initWithName:@"flutter/accessibility"
58  binaryMessenger:platform_view->GetOwnerViewController().engine.binaryMessenger
59  codec:[FlutterStandardMessageCodec sharedInstance]];
60  fml::WeakPtr<AccessibilityBridge> weak_self = GetWeakPtr();
61  [accessibility_channel_ setMessageHandler:^(id message, FlutterReply reply) {
62  if (weak_self) {
63  weak_self->HandleEvent((NSDictionary*)message);
64  }
65  }];
66 }
67 
68 AccessibilityBridge::~AccessibilityBridge() {
69  [accessibility_channel_ setMessageHandler:nil];
70  clearState();
71 }
72 
73 void AccessibilityBridge::SetViewController(FlutterViewController* viewController,
74  FlutterView* previousView) {
75  if (viewController) {
76  FML_DCHECK(viewController.platformViewsController == platform_views_controller_)
77  << "A reattached FlutterViewController must belong to the same FlutterEngine.";
78  }
79  view_controller_ = viewController;
80  UIView* currentView = ViewIfLoaded();
81  if (previousView != currentView) {
82  ClearAccessibilityElementsIfOwnedByBridge(previousView);
83  }
84  UpdateAccessibilityElementsForCurrentView();
85  NotifySemanticsObjectsViewChanged();
86 }
87 
88 UIView<UITextInput>* AccessibilityBridge::textInputView() {
89  return [[platform_view_->GetOwnerViewController().engine textInputPlugin] textInputView];
90 }
91 
92 bool AccessibilityBridge::HasSemantics() const {
93  return objects_[@(kRootNodeId)] != nil;
94 }
95 
96 void AccessibilityBridge::AccessibilityObjectDidBecomeFocused(int32_t id) {
97  last_focused_semantics_object_id_ = id;
98  [accessibility_channel_ sendMessage:@{@"type" : @"didGainFocus", @"nodeId" : @(id)}];
99 }
100 
101 void AccessibilityBridge::AccessibilityObjectDidLoseFocus(int32_t id) {
102  if (last_focused_semantics_object_id_ == id) {
103  last_focused_semantics_object_id_ = kSemanticObjectIdInvalid;
104  }
105 }
106 
107 void AccessibilityBridge::UpdateSemantics(
108  flutter::SemanticsNodeUpdates nodes,
109  const flutter::CustomAccessibilityActionUpdates& actions) {
110  BOOL layoutChanged = NO;
111  BOOL scrollOccured = NO;
112  BOOL needsAnnouncement = NO;
113  for (const auto& entry : actions) {
114  const flutter::CustomAccessibilityAction& action = entry.second;
115  actions_[action.id] = action;
116  }
117  // The semantics cache is updated even while the owner view is not loaded. UIKit
118  // accessibility notifications are only useful once there is a loaded view with
119  // accessibility elements to inspect.
120  BOOL shouldPostAccessibilityNotifications =
121  view_controller_ && ViewIfLoaded() &&
122  !ios_delegate_->IsFlutterViewControllerPresentingModalViewController(view_controller_);
123  for (const auto& entry : nodes) {
124  const flutter::SemanticsNode& node = entry.second;
125  SemanticsObject* object = GetOrCreateObject(node.id, nodes);
126  layoutChanged = layoutChanged || [object nodeWillCauseLayoutChange:&node];
127  scrollOccured = scrollOccured || [object nodeWillCauseScroll:&node];
128  needsAnnouncement = [object nodeShouldTriggerAnnouncement:&node];
129  [object setSemanticsNode:&node];
130  NSUInteger newChildCountInTraversalOrder = node.childrenInTraversalOrder.size();
131  NSMutableArray* newChildren =
132  [[NSMutableArray alloc] initWithCapacity:newChildCountInTraversalOrder];
133  for (NSUInteger i = 0; i < newChildCountInTraversalOrder; ++i) {
134  SemanticsObject* child = GetOrCreateObject(node.childrenInTraversalOrder[i], nodes);
135  [newChildren addObject:child];
136  }
137  NSUInteger newChildCountInHitTestOrder = node.childrenInHitTestOrder.size();
138  NSMutableArray* newChildrenInHitTestOrder =
139  [[NSMutableArray alloc] initWithCapacity:newChildCountInHitTestOrder];
140  for (NSUInteger i = 0; i < newChildCountInHitTestOrder; ++i) {
141  SemanticsObject* child = GetOrCreateObject(node.childrenInHitTestOrder[i], nodes);
142  [newChildrenInHitTestOrder addObject:child];
143  }
144  object.children = newChildren;
145  object.childrenInHitTestOrder = newChildrenInHitTestOrder;
146  if (!node.customAccessibilityActions.empty()) {
147  NSMutableArray<FlutterCustomAccessibilityAction*>* accessibilityCustomActions =
148  [[NSMutableArray alloc] init];
149  for (int32_t action_id : node.customAccessibilityActions) {
150  flutter::CustomAccessibilityAction& action = actions_[action_id];
151  if (action.overrideId != -1) {
152  // iOS does not support overriding standard actions, so we ignore any
153  // custom actions that have an override id provided.
154  continue;
155  }
156  NSString* label = @(action.label.data());
157  SEL selector = @selector(onCustomAccessibilityAction:);
158  FlutterCustomAccessibilityAction* customAction =
159  [[FlutterCustomAccessibilityAction alloc] initWithName:label
160  target:object
161  selector:selector];
162  customAction.uid = action_id;
163  [accessibilityCustomActions addObject:customAction];
164  }
165  object.accessibilityCustomActions = accessibilityCustomActions;
166  }
167 
168  if (needsAnnouncement && shouldPostAccessibilityNotifications) {
169  // Try to be more polite - iOS 11+ supports
170  // UIAccessibilitySpeechAttributeQueueAnnouncement which should avoid
171  // interrupting system notifications or other elements.
172  // Expectation: roughly match the behavior of polite announcements on
173  // Android.
174  NSString* announcement = [[NSString alloc] initWithUTF8String:object.node.label.c_str()];
175  UIAccessibilityPostNotification(
176  UIAccessibilityAnnouncementNotification,
177  [[NSAttributedString alloc] initWithString:announcement
178  attributes:@{
179  UIAccessibilitySpeechAttributeQueueAnnouncement : @YES
180  }]);
181  }
182  }
183 
184  SemanticsObject* root = objects_[@(kRootNodeId)];
185 
186  bool routeChanged = false;
187  SemanticsObject* lastAdded = nil;
188 
189  if (root) {
190  UpdateAccessibilityElementsForCurrentView();
191  NSMutableArray<SemanticsObject*>* newRoutes = [[NSMutableArray alloc] init];
192  [root collectRoutes:newRoutes];
193  // Finds the last route that is not in the previous routes.
194  for (SemanticsObject* route in newRoutes) {
195  if (std::find(previous_routes_.begin(), previous_routes_.end(), [route uid]) ==
196  previous_routes_.end()) {
197  lastAdded = route;
198  }
199  }
200  // If all the routes are in the previous route, get the last route.
201  if (lastAdded == nil && [newRoutes count] > 0) {
202  int index = [newRoutes count] - 1;
203  lastAdded = [newRoutes objectAtIndex:index];
204  }
205  // There are two cases if lastAdded != nil
206  // 1. lastAdded is not in previous routes. In this case,
207  // [lastAdded uid] != previous_route_id_
208  // 2. All new routes are in previous routes and
209  // lastAdded = newRoutes.last.
210  // In the first case, we need to announce new route. In the second case,
211  // we need to announce if one list is shorter than the other.
212  if (lastAdded != nil &&
213  ([lastAdded uid] != previous_route_id_ || [newRoutes count] != previous_routes_.size())) {
214  previous_route_id_ = [lastAdded uid];
215  routeChanged = true;
216  }
217  previous_routes_.clear();
218  for (SemanticsObject* route in newRoutes) {
219  previous_routes_.push_back([route uid]);
220  }
221  } else {
222  UIView* view = ViewIfLoaded();
223  ClearAccessibilityElementsIfOwnedByBridge(view);
224  }
225 
226  NSMutableArray<NSNumber*>* doomed_uids = [NSMutableArray arrayWithArray:objects_.allKeys];
227  if (root) {
228  VisitObjectsRecursivelyAndRemove(root, doomed_uids);
229  }
230  [objects_ removeObjectsForKeys:doomed_uids];
231 
232  for (SemanticsObject* object in objects_.allValues) {
233  [object accessibilityBridgeDidFinishUpdate];
234  }
235 
236  if (!shouldPostAccessibilityNotifications) {
237  return;
238  }
239 
240  layoutChanged = layoutChanged || [doomed_uids count] > 0;
241 
242  if (routeChanged) {
243  NSString* routeName = [lastAdded routeName];
244  ios_delegate_->PostAccessibilityNotification(UIAccessibilityScreenChangedNotification,
245  routeName);
246  }
247 
248  if (layoutChanged) {
249  SemanticsObject* next = FindNextFocusableIfNecessary();
250  SemanticsObject* lastFocused = [objects_ objectForKey:@(last_focused_semantics_object_id_)];
251  // Only specify the focus item if the new focus is different, avoiding double focuses on the
252  // same item. See: https://github.com/flutter/flutter/issues/104176. If there is a route
253  // change, we always refocus.
254  ios_delegate_->PostAccessibilityNotification(
255  UIAccessibilityLayoutChangedNotification,
256  (routeChanged || next != lastFocused) ? next.nativeAccessibility : NULL);
257  } else if (scrollOccured) {
258  // TODO(chunhtai): figure out what string to use for notification. At this
259  // point, it is guarantee the previous focused object is still in the tree
260  // so that we don't need to worry about focus lost. (e.g. "Screen 0 of 3")
261  ios_delegate_->PostAccessibilityNotification(
262  UIAccessibilityPageScrolledNotification,
263  FindNextFocusableIfNecessary().nativeAccessibility);
264  }
265 }
266 
267 void AccessibilityBridge::DispatchSemanticsAction(int32_t node_uid,
268  flutter::SemanticsAction action) {
269  // TODO(team-ios): Remove implicit view assumption.
270  // https://github.com/flutter/flutter/issues/142845
271  platform_view_->DispatchSemanticsAction(kFlutterImplicitViewId, node_uid, action, {});
272 }
273 
274 void AccessibilityBridge::DispatchSemanticsAction(int32_t node_uid,
275  flutter::SemanticsAction action,
276  fml::MallocMapping args) {
277  // TODO(team-ios): Remove implicit view assumption.
278  // https://github.com/flutter/flutter/issues/142845
279  platform_view_->DispatchSemanticsAction(kFlutterImplicitViewId, node_uid, action,
280  std::move(args));
281 }
282 
283 static void ReplaceSemanticsObject(SemanticsObject* oldObject,
284  SemanticsObject* newObject,
285  NSMutableDictionary<NSNumber*, SemanticsObject*>* objects) {
286  // `newObject` should represent the same id as `oldObject`.
287  FML_DCHECK(oldObject.node.id == newObject.uid);
288  NSNumber* nodeId = @(oldObject.node.id);
289  NSUInteger positionInChildlist = [oldObject.parent.children indexOfObject:oldObject];
290  oldObject.children = @[];
291  [oldObject.parent replaceChildAtIndex:positionInChildlist withChild:newObject];
292  [objects removeObjectForKey:nodeId];
293  objects[nodeId] = newObject;
294 }
295 
296 static SemanticsObject* CreateObject(const flutter::SemanticsNode& node,
297  const fml::WeakPtr<AccessibilityBridge>& weak_ptr) {
298  if (node.flags.isTextField && !node.flags.isReadOnly) {
299  // Text fields are backed by objects that implement UITextInput.
300  return [[TextInputSemanticsObject alloc] initWithBridge:weak_ptr uid:node.id];
301  } else if (!node.flags.isInMutuallyExclusiveGroup &&
302  (node.flags.isToggled != flutter::SemanticsTristate::kNone ||
303  node.flags.isChecked != flutter::SemanticsCheckState::kNone)) {
304  return [[FlutterSwitchSemanticsObject alloc] initWithBridge:weak_ptr uid:node.id];
305  } else if (node.flags.hasImplicitScrolling) {
306  return [[FlutterScrollableSemanticsObject alloc] initWithBridge:weak_ptr uid:node.id];
307  } else if (node.IsPlatformViewNode()) {
308  FlutterPlatformViewsController* platformViewsController =
309  weak_ptr->GetPlatformViewsController();
310  FlutterTouchInterceptingView* touchInterceptingView =
311  [platformViewsController flutterTouchInterceptingViewForId:node.platformViewId];
312  return [[FlutterPlatformViewSemanticsContainer alloc] initWithBridge:weak_ptr
313  uid:node.id
314  platformView:touchInterceptingView];
315  } else {
316  return [[FlutterSemanticsObject alloc] initWithBridge:weak_ptr uid:node.id];
317  }
318 }
319 
320 SemanticsObject* AccessibilityBridge::GetOrCreateObject(int32_t uid,
321  flutter::SemanticsNodeUpdates& updates) {
322  SemanticsObject* object = objects_[@(uid)];
323  if (!object) {
324  object = CreateObject(updates[uid], GetWeakPtr());
325  objects_[@(uid)] = object;
326  } else {
327  // Existing node case
328  auto nodeEntry = updates.find(object.node.id);
329  if (nodeEntry != updates.end()) {
330  // There's an update for this node
331  flutter::SemanticsNode node = nodeEntry->second;
332  if (object.node.flags.isTextField != node.flags.isTextField ||
333  object.node.flags.isReadOnly != node.flags.isReadOnly ||
334  (object.node.flags.isChecked == flutter::SemanticsCheckState::kNone) !=
335  (node.flags.isChecked == flutter::SemanticsCheckState::kNone) ||
336  ((object.node.flags.isToggled == flutter::SemanticsTristate::kNone) !=
337  (node.flags.isToggled == flutter::SemanticsTristate::kNone)) ||
338  object.node.flags.hasImplicitScrolling != node.flags.hasImplicitScrolling
339 
340  ) {
341  // The node changed its type. In this case, we cannot reuse the existing
342  // SemanticsObject implementation. Instead, we replace it with a new
343  // instance.
344  SemanticsObject* newSemanticsObject = CreateObject(node, GetWeakPtr());
345  ReplaceSemanticsObject(object, newSemanticsObject, objects_);
346  object = newSemanticsObject;
347  }
348  }
349  }
350  return object;
351 }
352 
353 void AccessibilityBridge::VisitObjectsRecursivelyAndRemove(SemanticsObject* object,
354  NSMutableArray<NSNumber*>* doomed_uids) {
355  [doomed_uids removeObject:@(object.uid)];
356  for (SemanticsObject* child in [object children])
357  VisitObjectsRecursivelyAndRemove(child, doomed_uids);
358 }
359 
360 SemanticsObject* AccessibilityBridge::FindNextFocusableIfNecessary() {
361  // This property will be -1 if the focus is outside of the flutter
362  // application. In this case, we should not refocus anything.
363  if (last_focused_semantics_object_id_ == kSemanticObjectIdInvalid) {
364  return nil;
365  }
366 
367  // Tries to refocus the previous focused semantics object to avoid random jumps.
368  return FindFirstFocusable(objects_[@(last_focused_semantics_object_id_)]);
369 }
370 
371 SemanticsObject* AccessibilityBridge::FindFirstFocusable(SemanticsObject* parent) {
372  SemanticsObject* currentObject = parent ?: objects_[@(kRootNodeId)];
373  if (!currentObject) {
374  return nil;
375  }
376  if (currentObject.isAccessibilityElement) {
377  return currentObject;
378  }
379 
380  for (SemanticsObject* child in [currentObject children]) {
381  SemanticsObject* candidate = FindFirstFocusable(child);
382  if (candidate) {
383  return candidate;
384  }
385  }
386  return nil;
387 }
388 
389 void AccessibilityBridge::HandleEvent(NSDictionary<NSString*, id>* annotatedEvent) {
390  NSString* type = annotatedEvent[@"type"];
391  if ([type isEqualToString:@"announce"]) {
392  NSString* message = annotatedEvent[@"data"][@"message"];
393  ios_delegate_->PostAccessibilityNotification(UIAccessibilityAnnouncementNotification, message);
394  }
395  if ([type isEqualToString:@"focus"]) {
396  SemanticsObject* node = objects_[annotatedEvent[@"nodeId"]];
397  ios_delegate_->PostAccessibilityNotification(UIAccessibilityLayoutChangedNotification, node);
398  }
399 }
400 
401 fml::WeakPtr<AccessibilityBridge> AccessibilityBridge::GetWeakPtr() {
402  return weak_factory_.GetWeakPtr();
403 }
404 
405 bool AccessibilityBridge::AccessibilityElementsWereInstalledByBridge(NSArray* elements) const {
406  if (elements.count != 1) {
407  return false;
408  }
409  id element = elements.firstObject;
410  if (![element isKindOfClass:[SemanticsObjectContainer class]]) {
411  return false;
412  }
413  SemanticsObject* semanticsObject = ((SemanticsObjectContainer*)element).semanticsObject;
414  return semanticsObject && semanticsObject.bridge == this;
415 }
416 
417 void AccessibilityBridge::ClearAccessibilityElementsIfOwnedByBridge(UIView* view) {
418  if (!view) {
419  return;
420  }
421  if (AccessibilityElementsWereInstalledByBridge(view.accessibilityElements)) {
422  view.accessibilityElements = nil;
423  }
424 }
425 
426 void AccessibilityBridge::UpdateAccessibilityElementsForCurrentView() {
427  UIView* view = ViewIfLoaded();
428  if (!view) {
429  return;
430  }
431  SemanticsObject* root = objects_[@(kRootNodeId)];
432  if (!root) {
433  view.accessibilityElements = nil;
434  return;
435  }
436  id accessibilityContainer = [root accessibilityContainer];
437  if (!accessibilityContainer) {
438  view.accessibilityElements = nil;
439  return;
440  }
441  view.accessibilityElements = @[ accessibilityContainer ];
442 }
443 
444 void AccessibilityBridge::NotifySemanticsObjectsViewChanged() {
445  for (SemanticsObject* object in objects_.allValues) {
446  [object accessibilityBridgeDidChangeView];
447  }
448 }
449 
450 void AccessibilityBridge::clearState() {
451  [objects_ removeAllObjects];
452  previous_route_id_ = 0;
453  previous_routes_.clear();
454  view_controller_.viewIfLoaded.accessibilityElements = nil;
455 }
456 
457 } // namespace flutter
NS_ASSUME_NONNULL_BEGIN typedef void(^ FlutterReply)(id _Nullable reply)
std::unique_ptr< flutter::PlatformViewIOS > platform_view
instancetype initWithFrame
FlutterViewController * viewController
FlutterTextInputPlugin * textInputPlugin
constexpr NS_ASSUME_NONNULL_BEGIN int32_t kRootNodeId
AccessibilityBridge()
Creates a new instance of a accessibility bridge.
NSArray< SemanticsObject * > * children
SemanticsObject * parent
nullable flutter::AccessibilityBridgeIos * bridge()
flutter::SemanticsNode node