Flutter iOS Embedder
FlutterMetalLayer.mm
Go to the documentation of this file.
1 // Copyright 2013 The Flutter Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file.
4 
6 
7 #include <CoreMedia/CoreMedia.h>
8 #include <IOSurface/IOSurfaceObjC.h>
9 #include <Metal/Metal.h>
10 #include <UIKit/UIKit.h>
11 
12 #import "flutter/shell/platform/darwin/common/InternalFlutterSwiftCommon/InternalFlutterSwiftCommon.h"
14 #import "flutter/shell/platform/darwin/ios/InternalFlutterSwift/InternalFlutterSwift.h"
15 
17 
18 @class FlutterTexture;
19 @class FlutterDrawable;
20 
21 @interface FlutterMetalLayer () {
22  id<MTLDevice> _preferredDevice;
23  CGSize _drawableSize;
24 
25  NSUInteger _nextDrawableId;
26 
27  // Access to these variables must be synchronized.
28  NSMutableSet<FlutterTexture*>* _availableTextures;
29  NSUInteger _totalTextures;
31 }
32 
33 - (void)presentTexture:(FlutterTexture*)texture;
34 - (void)returnTexture:(FlutterTexture*)texture;
35 
36 @end
37 
38 @interface FlutterTexture : NSObject
39 
40 @property(readonly, nonatomic) id<MTLTexture> texture;
41 @property(readonly, nonatomic) IOSurface* surface;
42 @property(readwrite, nonatomic) CFTimeInterval presentedTime;
43 @property(readwrite, atomic) BOOL waitingForCompletion;
44 
45 @end
46 
47 @implementation FlutterTexture
48 
49 - (instancetype)initWithTexture:(id<MTLTexture>)texture surface:(IOSurface*)surface {
50  if (self = [super init]) {
51  _texture = texture;
52  _surface = surface;
53  }
54  return self;
55 }
56 
57 @end
58 
59 @interface FlutterDrawable : NSObject <FlutterMetalDrawable> {
62  NSUInteger _drawableId;
63  BOOL _presented;
64 }
65 
66 - (instancetype)initWithTexture:(FlutterTexture*)texture
67  layer:(FlutterMetalLayer*)layer
68  drawableId:(NSUInteger)drawableId;
69 
70 @end
71 
72 @implementation FlutterDrawable
73 
74 - (instancetype)initWithTexture:(FlutterTexture*)texture
75  layer:(FlutterMetalLayer*)layer
76  drawableId:(NSUInteger)drawableId {
77  if (self = [super init]) {
78  _texture = texture;
79  _layer = layer;
80  _drawableId = drawableId;
81  }
82  return self;
83 }
84 
85 - (id<MTLTexture>)texture {
86  return self->_texture.texture;
87 }
88 
89 #pragma clang diagnostic push
90 #pragma clang diagnostic ignored "-Wunguarded-availability-new"
91 - (CAMetalLayer*)layer {
92  return (id)self->_layer;
93 }
94 #pragma clang diagnostic pop
95 
96 - (NSUInteger)drawableID {
97  return self->_drawableId;
98 }
99 
100 - (CFTimeInterval)presentedTime {
101  return 0;
102 }
103 
104 - (void)present {
105  [_layer presentTexture:self->_texture];
106  self->_presented = YES;
107 }
108 
109 - (void)dealloc {
110  if (!_presented) {
111  [_layer returnTexture:self->_texture];
112  }
113 }
114 
115 - (void)addPresentedHandler:(nonnull MTLDrawablePresentedHandler)block {
116  [FlutterLogger logWarning:@"FlutterMetalLayer drawable does not implement addPresentedHandler:"];
117 }
118 
119 - (void)presentAtTime:(CFTimeInterval)presentationTime {
120  [FlutterLogger logWarning:@"FlutterMetalLayer drawable does not implement presentAtTime:"];
121 }
122 
123 - (void)presentAfterMinimumDuration:(CFTimeInterval)duration {
124  [FlutterLogger
125  logWarning:@"FlutterMetalLayer drawable does not implement presentAfterMinimumDuration:"];
126 }
127 
128 - (void)flutterPrepareForPresent:(nonnull id<MTLCommandBuffer>)commandBuffer {
130  texture.waitingForCompletion = YES;
131  [commandBuffer addCompletedHandler:^(id<MTLCommandBuffer> buffer) {
132  texture.waitingForCompletion = NO;
133  }];
134 }
135 
136 @end
137 
138 @implementation FlutterMetalLayer
139 
140 - (instancetype)init {
141  if (self = [super init]) {
142  _preferredDevice = MTLCreateSystemDefaultDevice();
143  self.device = self.preferredDevice;
144  self.pixelFormat = MTLPixelFormatBGRA8Unorm;
145  _availableTextures = [[NSMutableSet alloc] init];
146 
147  [[NSNotificationCenter defaultCenter] addObserver:self
148  selector:@selector(didEnterBackground:)
149  name:UIApplicationDidEnterBackgroundNotification
150  object:nil];
151  }
152  return self;
153 }
154 
155 - (void)dealloc {
156  [[NSNotificationCenter defaultCenter] removeObserver:self];
157 }
158 
159 - (BOOL)isKindOfClass:(Class)aClass {
160 #pragma clang diagnostic push
161 #pragma clang diagnostic ignored "-Wunguarded-availability-new"
162  // Pretend that we're a CAMetalLayer so that the rest of Flutter plays along
163  if ([aClass isEqual:[CAMetalLayer class]]) {
164  return YES;
165  }
166 #pragma clang diagnostic pop
167  return [super isKindOfClass:aClass];
168 }
169 
170 - (void)setDrawableSize:(CGSize)drawableSize {
171  @synchronized(self) {
172  [_availableTextures removeAllObjects];
173  _front = nil;
174  _totalTextures = 0;
175  _drawableSize = drawableSize;
176  }
177 }
178 
179 - (void)didEnterBackground:(id)notification {
180  @synchronized(self) {
181  [_availableTextures removeAllObjects];
182  _totalTextures = _front != nil ? 1 : 0;
183  }
184 }
185 
186 - (CGSize)drawableSize {
187  @synchronized(self) {
188  return _drawableSize;
189  }
190 }
191 
192 - (IOSurface*)createIOSurface {
193  unsigned pixelFormat;
194  unsigned bytesPerElement;
195  if (self.pixelFormat == MTLPixelFormatRGBA16Float) {
196  pixelFormat = kCVPixelFormatType_64RGBAHalf;
197  bytesPerElement = 8;
198  } else if (self.pixelFormat == MTLPixelFormatBGRA8Unorm) {
199  pixelFormat = kCVPixelFormatType_32BGRA;
200  bytesPerElement = 4;
201  } else if (self.pixelFormat == MTLPixelFormatBGRA10_XR) {
202  pixelFormat = kCVPixelFormatType_40ARGBLEWideGamut;
203  bytesPerElement = 8;
204  } else {
205  NSString* errorMessage =
206  [NSString stringWithFormat:@"Unsupported pixel format: %lu", self.pixelFormat];
207  [FlutterLogger logError:errorMessage];
208  return nil;
209  }
210  size_t bytesPerRow =
211  IOSurfaceAlignProperty(kIOSurfaceBytesPerRow, _drawableSize.width * bytesPerElement);
212  size_t totalBytes =
213  IOSurfaceAlignProperty(kIOSurfaceAllocSize, _drawableSize.height * bytesPerRow);
214  NSDictionary* options = @{
215  (id)kIOSurfaceWidth : @(_drawableSize.width),
216  (id)kIOSurfaceHeight : @(_drawableSize.height),
217  (id)kIOSurfacePixelFormat : @(pixelFormat),
218  (id)kIOSurfaceBytesPerElement : @(bytesPerElement),
219  (id)kIOSurfaceBytesPerRow : @(bytesPerRow),
220  (id)kIOSurfaceAllocSize : @(totalBytes),
221  };
222 
223  IOSurfaceRef res = IOSurfaceCreate((CFDictionaryRef)options);
224  if (res == nil) {
225  NSString* errorMessage = [NSString
226  stringWithFormat:@"Failed to create IOSurface with options %@", options.debugDescription];
227  [FlutterLogger logError:errorMessage];
228  return nil;
229  }
230 
231  if (self.colorspace != nil) {
232  CFStringRef name = CGColorSpaceGetName(self.colorspace);
233  IOSurfaceSetValue(res, kIOSurfaceColorSpace, name);
234  } else {
235  IOSurfaceSetValue(res, kIOSurfaceColorSpace, kCGColorSpaceSRGB);
236  }
237  return (__bridge_transfer IOSurface*)res;
238 }
239 
240 - (FlutterTexture*)nextTexture {
241  CFTimeInterval start = CACurrentMediaTime();
242  while (true) {
243  FlutterTexture* texture = [self tryNextTexture];
244  if (texture != nil) {
245  return texture;
246  }
247  CFTimeInterval elapsed = CACurrentMediaTime() - start;
248  if (elapsed > 1.0) {
249  NSLog(@"Waited %f seconds for a drawable, giving up.", elapsed);
250  return nil;
251  }
252  }
253 }
254 
255 - (FlutterTexture*)tryNextTexture {
256  @synchronized(self) {
257  if (_front != nil && _front.waitingForCompletion) {
258  return nil;
259  }
260  if (_totalTextures < 3) {
261  ++_totalTextures;
262  IOSurface* surface = [self createIOSurface];
263  if (surface == nil) {
264  return nil;
265  }
266  MTLTextureDescriptor* textureDescriptor =
267  [MTLTextureDescriptor texture2DDescriptorWithPixelFormat:_pixelFormat
268  width:_drawableSize.width
269  height:_drawableSize.height
270  mipmapped:NO];
271 
272  if (_framebufferOnly) {
273  textureDescriptor.usage = MTLTextureUsageRenderTarget;
274  } else {
275  textureDescriptor.usage =
276  MTLTextureUsageRenderTarget | MTLTextureUsageShaderRead | MTLTextureUsageShaderWrite;
277  }
278  id<MTLTexture> texture = [self.device newTextureWithDescriptor:textureDescriptor
279  iosurface:(__bridge IOSurfaceRef)surface
280  plane:0];
281  FlutterTexture* flutterTexture = [[FlutterTexture alloc] initWithTexture:texture
282  surface:surface];
283  return flutterTexture;
284  } else {
285  // Prefer surface that is not in use and has been presented the longest
286  // time ago.
287  // When isInUse is false, the surface is definitely not used by the compositor.
288  // When isInUse is true, the surface may be used by the compositor.
289  // When both surfaces are in use, the one presented earlier will be returned.
290  // The assumption here is that the compositor is already aware of the
291  // newer texture and is unlikely to read from the older one, even though it
292  // has not decreased the use count yet (there seems to be certain latency).
293  FlutterTexture* res = nil;
294  for (FlutterTexture* texture in _availableTextures) {
295  if (res == nil) {
296  res = texture;
297  } else if (res.surface.isInUse && !texture.surface.isInUse) {
298  // prefer texture that is not in use.
299  res = texture;
300  } else if (res.surface.isInUse == texture.surface.isInUse &&
301  texture.presentedTime < res.presentedTime) {
302  // prefer texture with older presented time.
303  res = texture;
304  }
305  }
306  if (res != nil) {
307  [_availableTextures removeObject:res];
308  }
309  return res;
310  }
311  }
312 }
313 
314 - (id<CAMetalDrawable>)nextDrawable {
315  FlutterTexture* texture = [self nextTexture];
316  if (texture == nil) {
317  return nil;
318  }
319  FlutterDrawable* drawable = [[FlutterDrawable alloc] initWithTexture:texture
320  layer:self
321  drawableId:_nextDrawableId++];
322  return drawable;
323 }
324 
325 - (void)presentOnMainThread:(FlutterTexture*)texture {
326  if (texture.texture.width != _drawableSize.width ||
327  texture.texture.height != _drawableSize.height) {
328  // This texture was created with an old size, but the view has since been
329  // resized. Do not present this stale frame to avoid distortion. The texture
330  // will be correctly recycled on the next frame.
331  return;
332  }
333 
334  // This is needed otherwise frame gets skipped on touch begin / end. Go figure.
335  // Might also be placebo
336  [self setNeedsDisplay];
337 
338  [CATransaction begin];
339  [CATransaction setDisableActions:YES];
340  self.contents = texture.surface;
341  [CATransaction commit];
342 }
343 
344 - (void)presentTexture:(FlutterTexture*)texture {
345  @synchronized(self) {
346  if (texture.texture.width != _drawableSize.width ||
347  texture.texture.height != _drawableSize.height) {
348  return;
349  }
350  if (_front != nil) {
351  [_availableTextures addObject:_front];
352  }
353  _front = texture;
354  texture.presentedTime = CACurrentMediaTime();
355  if ([NSThread isMainThread]) {
356  [self presentOnMainThread:texture];
357  } else {
358  // Core animation layers can only be updated on main thread.
359  dispatch_async(dispatch_get_main_queue(), ^{
360  [self presentOnMainThread:texture];
361  });
362  }
363  }
364 }
365 
366 - (void)returnTexture:(FlutterTexture*)texture {
367  if (texture == nil) {
368  return;
369  }
370  @synchronized(self) {
371  if (texture.texture.width == _drawableSize.width &&
372  texture.texture.height == _drawableSize.height) {
373  [_availableTextures addObject:texture];
374  }
375  }
376 }
377 
378 + (BOOL)enabled {
379  static BOOL enabled = YES;
380  static BOOL didCheckInfoPlist = NO;
381  if (!didCheckInfoPlist) {
382  didCheckInfoPlist = YES;
383  NSNumber* use_flutter_metal_layer =
384  [[NSBundle mainBundle] objectForInfoDictionaryKey:@"FLTUseFlutterMetalLayer"];
385  if (use_flutter_metal_layer != nil && ![use_flutter_metal_layer boolValue]) {
386  enabled = NO;
387  }
388  }
389  return enabled;
390 }
391 
392 @end
id< MTLDevice > _preferredDevice
NSMutableSet< FlutterTexture * > * _availableTextures
FlutterTexture * _front
FlutterTexture * _texture
NSUInteger _drawableId
__weak FlutterMetalLayer * _layer
CGColorSpaceRef colorspace
nullable id< CAMetalDrawable > nextDrawable()
MTLPixelFormat pixelFormat
IOSurface * surface
id< MTLTexture > texture
CFTimeInterval presentedTime