Flutter iOS Embedder
All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Properties Friends Macros
accessibility_bridge.mm
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
6 
7 #include <utility>
8 
9 #include "flutter/fml/logging.h"
14 
15 #pragma GCC diagnostic error "-Wundeclared-selector"
16 
18 
19 namespace flutter {
20 namespace {
21 
22 constexpr int32_t kSemanticObjectIdInvalid = -1;
23 
24 class DefaultIosDelegate : public AccessibilityBridge::IosDelegate {
25  public:
26  bool IsFlutterViewControllerPresentingModalViewController(
27  FlutterViewController* view_controller) override {
28  if (view_controller) {
29  return view_controller.isPresentingViewController;
30  } else {
31  return false;
32  }
33  }
34 
35  void PostAccessibilityNotification(UIAccessibilityNotifications notification,
36  id argument) override {
37  UIAccessibilityPostNotification(notification, argument);
38  }
39 };
40 } // namespace
41 
43  FlutterViewController* view_controller,
44  PlatformViewIOS* platform_view,
45  __weak FlutterPlatformViewsController* platform_views_controller,
46  std::unique_ptr<IosDelegate> ios_delegate)
47  : view_controller_(view_controller),
48  platform_view_(platform_view),
49  platform_views_controller_(platform_views_controller),
50  last_focused_semantics_object_id_(kSemanticObjectIdInvalid),
51  objects_([[NSMutableDictionary alloc] init]),
52  previous_routes_({}),
53  ios_delegate_(ios_delegate ? std::move(ios_delegate)
54  : std::make_unique<DefaultIosDelegate>()),
55  weak_factory_(this) {
56  accessibility_channel_ = [[FlutterBasicMessageChannel alloc]
57  initWithName:@"flutter/accessibility"
58  binaryMessenger:platform_view->GetOwnerViewController().engine.binaryMessenger
59  codec:[FlutterStandardMessageCodec sharedInstance]];
60  [accessibility_channel_ setMessageHandler:^(id message, FlutterReply reply) {
61  HandleEvent((NSDictionary*)message);
62  }];
63 }
64 
66  [accessibility_channel_ setMessageHandler:nil];
67  clearState();
68  view_controller_.viewIfLoaded.accessibilityElements = nil;
69 }
70 
71 UIView<UITextInput>* AccessibilityBridge::textInputView() {
72  return [[platform_view_->GetOwnerViewController().engine textInputPlugin] textInputView];
73 }
74 
76  last_focused_semantics_object_id_ = id;
77  [accessibility_channel_ sendMessage:@{@"type" : @"didGainFocus", @"nodeId" : @(id)}];
78 }
79 
81  if (last_focused_semantics_object_id_ == id) {
82  last_focused_semantics_object_id_ = kSemanticObjectIdInvalid;
83  }
84 }
85 
87  flutter::SemanticsNodeUpdates nodes,
88  const flutter::CustomAccessibilityActionUpdates& actions) {
89  BOOL layoutChanged = NO;
90  BOOL scrollOccured = NO;
91  BOOL needsAnnouncement = NO;
92  for (const auto& entry : actions) {
93  const flutter::CustomAccessibilityAction& action = entry.second;
94  actions_[action.id] = action;
95  }
96  for (const auto& entry : nodes) {
97  const flutter::SemanticsNode& node = entry.second;
98  SemanticsObject* object = GetOrCreateObject(node.id, nodes);
99  layoutChanged = layoutChanged || [object nodeWillCauseLayoutChange:&node];
100  scrollOccured = scrollOccured || [object nodeWillCauseScroll:&node];
101  needsAnnouncement = [object nodeShouldTriggerAnnouncement:&node];
102  [object setSemanticsNode:&node];
103  NSUInteger newChildCount = node.childrenInTraversalOrder.size();
104  NSMutableArray* newChildren = [[NSMutableArray alloc] initWithCapacity:newChildCount];
105  for (NSUInteger i = 0; i < newChildCount; ++i) {
106  SemanticsObject* child = GetOrCreateObject(node.childrenInTraversalOrder[i], nodes);
107  [newChildren addObject:child];
108  }
109  NSMutableArray* newChildrenInHitTestOrder =
110  [[NSMutableArray alloc] initWithCapacity:newChildCount];
111  for (NSUInteger i = 0; i < newChildCount; ++i) {
112  SemanticsObject* child = GetOrCreateObject(node.childrenInHitTestOrder[i], nodes);
113  [newChildrenInHitTestOrder addObject:child];
114  }
115  object.children = newChildren;
116  object.childrenInHitTestOrder = newChildrenInHitTestOrder;
117  if (!node.customAccessibilityActions.empty()) {
118  NSMutableArray<FlutterCustomAccessibilityAction*>* accessibilityCustomActions =
119  [[NSMutableArray alloc] init];
120  for (int32_t action_id : node.customAccessibilityActions) {
121  flutter::CustomAccessibilityAction& action = actions_[action_id];
122  if (action.overrideId != -1) {
123  // iOS does not support overriding standard actions, so we ignore any
124  // custom actions that have an override id provided.
125  continue;
126  }
127  NSString* label = @(action.label.data());
128  SEL selector = @selector(onCustomAccessibilityAction:);
129  FlutterCustomAccessibilityAction* customAction =
130  [[FlutterCustomAccessibilityAction alloc] initWithName:label
131  target:object
132  selector:selector];
133  customAction.uid = action_id;
134  [accessibilityCustomActions addObject:customAction];
135  }
136  object.accessibilityCustomActions = accessibilityCustomActions;
137  }
138 
139  if (needsAnnouncement) {
140  // Try to be more polite - iOS 11+ supports
141  // UIAccessibilitySpeechAttributeQueueAnnouncement which should avoid
142  // interrupting system notifications or other elements.
143  // Expectation: roughly match the behavior of polite announcements on
144  // Android.
145  NSString* announcement = [[NSString alloc] initWithUTF8String:object.node.label.c_str()];
146  UIAccessibilityPostNotification(
147  UIAccessibilityAnnouncementNotification,
148  [[NSAttributedString alloc] initWithString:announcement
149  attributes:@{
150  UIAccessibilitySpeechAttributeQueueAnnouncement : @YES
151  }]);
152  }
153  }
154 
155  SemanticsObject* root = objects_[@(kRootNodeId)];
156 
157  bool routeChanged = false;
158  SemanticsObject* lastAdded = nil;
159 
160  if (root) {
161  if (!view_controller_.view.accessibilityElements) {
162  view_controller_.view.accessibilityElements =
163  @[ [root accessibilityContainer] ?: [NSNull null] ];
164  }
165  NSMutableArray<SemanticsObject*>* newRoutes = [[NSMutableArray alloc] init];
166  [root collectRoutes:newRoutes];
167  // Finds the last route that is not in the previous routes.
168  for (SemanticsObject* route in newRoutes) {
169  if (std::find(previous_routes_.begin(), previous_routes_.end(), [route uid]) ==
170  previous_routes_.end()) {
171  lastAdded = route;
172  }
173  }
174  // If all the routes are in the previous route, get the last route.
175  if (lastAdded == nil && [newRoutes count] > 0) {
176  int index = [newRoutes count] - 1;
177  lastAdded = [newRoutes objectAtIndex:index];
178  }
179  // There are two cases if lastAdded != nil
180  // 1. lastAdded is not in previous routes. In this case,
181  // [lastAdded uid] != previous_route_id_
182  // 2. All new routes are in previous routes and
183  // lastAdded = newRoutes.last.
184  // In the first case, we need to announce new route. In the second case,
185  // we need to announce if one list is shorter than the other.
186  if (lastAdded != nil &&
187  ([lastAdded uid] != previous_route_id_ || [newRoutes count] != previous_routes_.size())) {
188  previous_route_id_ = [lastAdded uid];
189  routeChanged = true;
190  }
191  previous_routes_.clear();
192  for (SemanticsObject* route in newRoutes) {
193  previous_routes_.push_back([route uid]);
194  }
195  } else {
196  view_controller_.viewIfLoaded.accessibilityElements = nil;
197  }
198 
199  NSMutableArray<NSNumber*>* doomed_uids = [NSMutableArray arrayWithArray:objects_.allKeys];
200  if (root) {
201  VisitObjectsRecursivelyAndRemove(root, doomed_uids);
202  }
203  [objects_ removeObjectsForKeys:doomed_uids];
204 
205  for (SemanticsObject* object in objects_.allValues) {
206  [object accessibilityBridgeDidFinishUpdate];
207  }
208 
209  if (!ios_delegate_->IsFlutterViewControllerPresentingModalViewController(view_controller_)) {
210  layoutChanged = layoutChanged || [doomed_uids count] > 0;
211 
212  if (routeChanged) {
213  NSString* routeName = [lastAdded routeName];
214  ios_delegate_->PostAccessibilityNotification(UIAccessibilityScreenChangedNotification,
215  routeName);
216  }
217 
218  if (layoutChanged) {
219  SemanticsObject* next = FindNextFocusableIfNecessary();
220  SemanticsObject* lastFocused = [objects_ objectForKey:@(last_focused_semantics_object_id_)];
221  // Only specify the focus item if the new focus is different, avoiding double focuses on the
222  // same item. See: https://github.com/flutter/flutter/issues/104176. If there is a route
223  // change, we always refocus.
224  ios_delegate_->PostAccessibilityNotification(
225  UIAccessibilityLayoutChangedNotification,
226  (routeChanged || next != lastFocused) ? next.nativeAccessibility : NULL);
227  } else if (scrollOccured) {
228  // TODO(chunhtai): figure out what string to use for notification. At this
229  // point, it is guarantee the previous focused object is still in the tree
230  // so that we don't need to worry about focus lost. (e.g. "Screen 0 of 3")
231  ios_delegate_->PostAccessibilityNotification(
232  UIAccessibilityPageScrolledNotification,
233  FindNextFocusableIfNecessary().nativeAccessibility);
234  }
235  }
236 }
237 
238 void AccessibilityBridge::DispatchSemanticsAction(int32_t uid, flutter::SemanticsAction action) {
239  platform_view_->DispatchSemanticsAction(uid, action, {});
240 }
241 
243  flutter::SemanticsAction action,
244  fml::MallocMapping args) {
245  platform_view_->DispatchSemanticsAction(uid, action, std::move(args));
246 }
247 
248 static void ReplaceSemanticsObject(SemanticsObject* oldObject,
249  SemanticsObject* newObject,
250  NSMutableDictionary<NSNumber*, SemanticsObject*>* objects) {
251  // `newObject` should represent the same id as `oldObject`.
252  FML_DCHECK(oldObject.node.id == newObject.uid);
253  NSNumber* nodeId = @(oldObject.node.id);
254  NSUInteger positionInChildlist = [oldObject.parent.children indexOfObject:oldObject];
255  oldObject.children = @[];
256  [oldObject.parent replaceChildAtIndex:positionInChildlist withChild:newObject];
257  [objects removeObjectForKey:nodeId];
258  objects[nodeId] = newObject;
259 }
260 
261 static SemanticsObject* CreateObject(const flutter::SemanticsNode& node,
262  const fml::WeakPtr<AccessibilityBridge>& weak_ptr) {
263  if (node.HasFlag(flutter::SemanticsFlags::kIsTextField) &&
264  !node.HasFlag(flutter::SemanticsFlags::kIsReadOnly)) {
265  // Text fields are backed by objects that implement UITextInput.
266  return [[TextInputSemanticsObject alloc] initWithBridge:weak_ptr uid:node.id];
267  } else if (!node.HasFlag(flutter::SemanticsFlags::kIsInMutuallyExclusiveGroup) &&
268  (node.HasFlag(flutter::SemanticsFlags::kHasToggledState) ||
269  node.HasFlag(flutter::SemanticsFlags::kHasCheckedState))) {
270  return [[FlutterSwitchSemanticsObject alloc] initWithBridge:weak_ptr uid:node.id];
271  } else if (node.HasFlag(flutter::SemanticsFlags::kHasImplicitScrolling)) {
272  return [[FlutterScrollableSemanticsObject alloc] initWithBridge:weak_ptr uid:node.id];
273  } else if (node.IsPlatformViewNode()) {
274  FlutterPlatformViewsController* platformViewsController =
275  weak_ptr->GetPlatformViewsController();
276  FlutterTouchInterceptingView* touchInterceptingView =
277  [platformViewsController flutterTouchInterceptingViewForId:node.platformViewId];
278  return [[FlutterPlatformViewSemanticsContainer alloc] initWithBridge:weak_ptr
279  uid:node.id
280  platformView:touchInterceptingView];
281  } else {
282  return [[FlutterSemanticsObject alloc] initWithBridge:weak_ptr uid:node.id];
283  }
284 }
285 
286 static bool DidFlagChange(const flutter::SemanticsNode& oldNode,
287  const flutter::SemanticsNode& newNode,
288  SemanticsFlags flag) {
289  return oldNode.HasFlag(flag) != newNode.HasFlag(flag);
290 }
291 
292 SemanticsObject* AccessibilityBridge::GetOrCreateObject(int32_t uid,
293  flutter::SemanticsNodeUpdates& updates) {
294  SemanticsObject* object = objects_[@(uid)];
295  if (!object) {
296  object = CreateObject(updates[uid], GetWeakPtr());
297  objects_[@(uid)] = object;
298  } else {
299  // Existing node case
300  auto nodeEntry = updates.find(object.node.id);
301  if (nodeEntry != updates.end()) {
302  // There's an update for this node
303  flutter::SemanticsNode node = nodeEntry->second;
304  if (DidFlagChange(object.node, node, flutter::SemanticsFlags::kIsTextField) ||
305  DidFlagChange(object.node, node, flutter::SemanticsFlags::kIsReadOnly) ||
306  DidFlagChange(object.node, node, flutter::SemanticsFlags::kHasCheckedState) ||
307  DidFlagChange(object.node, node, flutter::SemanticsFlags::kHasToggledState) ||
308  DidFlagChange(object.node, node, flutter::SemanticsFlags::kHasImplicitScrolling)) {
309  // The node changed its type. In this case, we cannot reuse the existing
310  // SemanticsObject implementation. Instead, we replace it with a new
311  // instance.
312  SemanticsObject* newSemanticsObject = CreateObject(node, GetWeakPtr());
313  ReplaceSemanticsObject(object, newSemanticsObject, objects_);
314  object = newSemanticsObject;
315  }
316  }
317  }
318  return object;
319 }
320 
321 void AccessibilityBridge::VisitObjectsRecursivelyAndRemove(SemanticsObject* object,
322  NSMutableArray<NSNumber*>* doomed_uids) {
323  [doomed_uids removeObject:@(object.uid)];
324  for (SemanticsObject* child in [object children])
325  VisitObjectsRecursivelyAndRemove(child, doomed_uids);
326 }
327 
328 SemanticsObject* AccessibilityBridge::FindNextFocusableIfNecessary() {
329  // This property will be -1 if the focus is outside of the flutter
330  // application. In this case, we should not refocus anything.
331  if (last_focused_semantics_object_id_ == kSemanticObjectIdInvalid) {
332  return nil;
333  }
334 
335  // Tries to refocus the previous focused semantics object to avoid random jumps.
336  return FindFirstFocusable(objects_[@(last_focused_semantics_object_id_)]);
337 }
338 
339 SemanticsObject* AccessibilityBridge::FindFirstFocusable(SemanticsObject* parent) {
340  SemanticsObject* currentObject = parent ?: objects_[@(kRootNodeId)];
341  if (!currentObject) {
342  return nil;
343  }
344  if (currentObject.isAccessibilityElement) {
345  return currentObject;
346  }
347 
348  for (SemanticsObject* child in [currentObject children]) {
349  SemanticsObject* candidate = FindFirstFocusable(child);
350  if (candidate) {
351  return candidate;
352  }
353  }
354  return nil;
355 }
356 
357 void AccessibilityBridge::HandleEvent(NSDictionary<NSString*, id>* annotatedEvent) {
358  NSString* type = annotatedEvent[@"type"];
359  if ([type isEqualToString:@"announce"]) {
360  NSString* message = annotatedEvent[@"data"][@"message"];
361  ios_delegate_->PostAccessibilityNotification(UIAccessibilityAnnouncementNotification, message);
362  }
363  if ([type isEqualToString:@"focus"]) {
364  SemanticsObject* node = objects_[annotatedEvent[@"nodeId"]];
365  ios_delegate_->PostAccessibilityNotification(UIAccessibilityLayoutChangedNotification, node);
366  }
367 }
368 
369 fml::WeakPtr<AccessibilityBridge> AccessibilityBridge::GetWeakPtr() {
370  return weak_factory_.GetWeakPtr();
371 }
372 
374  [objects_ removeAllObjects];
375  previous_route_id_ = 0;
376  previous_routes_.clear();
377 }
378 
379 } // namespace flutter
FlutterBasicMessageChannel
Definition: FlutterChannels.h:37
SemanticsObject::parent
SemanticsObject * parent
Definition: SemanticsObject.h:42
FlutterViewController
Definition: FlutterViewController.h:57
FlutterCustomAccessibilityAction
Definition: SemanticsObject.h:135
flutter::AccessibilityBridge::AccessibilityObjectDidLoseFocus
void AccessibilityObjectDidLoseFocus(int32_t id) override
FlutterEngine_Internal.h
flutter::AccessibilityBridge::textInputView
UIView< UITextInput > * textInputView() override
flutter::AccessibilityBridge::clearState
void clearState()
TextInputSemanticsObject.h
FlutterViewController::engine
FlutterEngine * engine
Definition: FlutterViewController.h:236
platform_view
std::unique_ptr< flutter::PlatformViewIOS > platform_view
Definition: FlutterEnginePlatformViewTest.mm:66
FlutterCustomAccessibilityAction::uid
int32_t uid
Definition: SemanticsObject.h:140
flutter::PlatformViewIOS::GetOwnerViewController
FlutterViewController * GetOwnerViewController() const __attribute__((cf_audited_transfer))
Definition: platform_view_ios.mm:78
FlutterStandardMessageCodec
Definition: FlutterCodecs.h:209
FlutterSemanticsObject
Definition: SemanticsObject.h:155
flutter::AccessibilityBridge::AccessibilityBridge
AccessibilityBridge()
Creates a new instance of a accessibility bridge.
Definition: accessibility_bridge.cc:23
flutter
Definition: accessibility_bridge.h:27
accessibility_bridge.h
SemanticsObject::node
flutter::SemanticsNode node
Definition: SemanticsObject.h:57
FlutterSwitchSemanticsObject
Definition: SemanticsObject.h:183
kRootNodeId
constexpr int32_t kRootNodeId
Definition: SemanticsObject.h:16
flutter::AccessibilityBridge::AccessibilityObjectDidBecomeFocused
void AccessibilityObjectDidBecomeFocused(int32_t id) override
FlutterReply
NS_ASSUME_NONNULL_BEGIN typedef void(^ FlutterReply)(id _Nullable reply)
SemanticsObject::children
NSArray< SemanticsObject * > * children
Definition: SemanticsObject.h:68
textInputPlugin
FlutterTextInputPlugin * textInputPlugin
Definition: FlutterTextInputPluginTest.mm:90
FlutterViewController_Internal.h
FlutterPlatformViewsController
Definition: FlutterPlatformViewsController.h:31
SemanticsObject::nativeAccessibility
id nativeAccessibility
Definition: SemanticsObject.h:83
TextInputSemanticsObject
Definition: TextInputSemanticsObject.h:20
FlutterPlatformViewSemanticsContainer
Definition: SemanticsObject.h:169
FlutterTouchInterceptingView
Definition: FlutterPlatformViews.mm:521
SemanticsObject::uid
int32_t uid
Definition: SemanticsObject.h:36
flutter::AccessibilityBridge::DispatchSemanticsAction
void DispatchSemanticsAction(int32_t id, flutter::SemanticsAction action) override
platform_view_ios.h
flutter::AccessibilityBridge::UpdateSemantics
void UpdateSemantics(flutter::SemanticsNodeUpdates nodes, const flutter::CustomAccessibilityActionUpdates &actions)
FLUTTER_ASSERT_ARC
Definition: FlutterChannelKeyResponder.mm:13
flutter::AccessibilityBridge::GetWeakPtr
fml::WeakPtr< AccessibilityBridge > GetWeakPtr()
flutter::AccessibilityBridge::HandleEvent
void HandleEvent(NSDictionary< NSString *, id > *annotatedEvent)
FlutterScrollableSemanticsObject
Definition: SemanticsObject.h:189
SemanticsObject
Definition: SemanticsObject.h:31
flutter::AccessibilityBridge::~AccessibilityBridge
~AccessibilityBridge()
Definition: accessibility_bridge.cc:34