Page 1
© 2016 Apple Inc. All rights reserved. Redistribution or public display not permitted without written permission from Apple.
…And getting the most out of Apple Pencil
App Frameworks #WWDC16
Session 220
Leveraging Touch Input on iOS
Dominik Wagner UIKit Engineer
Page 2
New and Recent Hardware
Page 3
iPhone 6s and iPhone 6s Plus3D Touch
A Peek at 3D Touch Presidio Thursday 4:00PM
Page 4
iPad Air 2 and iPad ProFaster Touch Scanning
Page 5
iPad ProApple Pencil
Precise location240 Hz scan rateTilt, orientation, and forcePalm rejection
Page 6
Apple TVSiri Remote
UIFocusEngineGame ControllerIndirect touches
Page 7
Apple TVSiri Remote
UIFocusEngineGame ControllerIndirect touches
Apple TV Tech Talks
Controlling Game Input for Apple TV Mission Wednesday 5:00PM
Page 8
Building a Drawing App
Page 9
AgendaBuilding a Drawing App
New APIStep-by-stepSample code available
Page 10
Say Hello to SpeedSketch
One sheet of paper you can draw onFull support for Apple Pencil and 3D Touch
Page 11
Model and captureBuilding a Drawing App
Page 12
Model and captureBuilding a Drawing App
Series of strokesUITouch in event callbacksCopy the relevant data
Page 13
Model and captureBuilding a Drawing App
struct StrokeSample {
let location: CGPoint
}StrokeSample
Page 14
class Stroke {
var samples: [StrokeSample] = []
func add(sample: StrokeSample)
}
Model and captureBuilding a Drawing App
Stroke
StrokeSample
StrokeSample
StrokeSample
Page 15
class Stroke {
var samples: [StrokeSample] = []
func add(sample: StrokeSample)
}
var state: StrokeState = .active
enum StrokeState {
case active
case done
case cancelled
}
Model and captureBuilding a Drawing App
Stroke
StrokeSample
StrokeSample
StrokeSample
Page 16
class StrokeCollection {
var strokes: [Stroke] = []
func add(doneStroke: stroke)
}
Model and captureBuilding a Drawing App
StrokeCollection
Stroke
StrokeSample
StrokeSample
StrokeSample
Stroke
Stroke
Page 17
class StrokeCollection {
var strokes: [Stroke] = []
func add(doneStroke: stroke)
}
var activeStroke: Stroke?
Model and captureBuilding a Drawing App
StrokeCollection
Stroke
StrokeSample
StrokeSample
StrokeSample
Stroke
Stroke
Page 18
Model and captureBuilding a Drawing App
Where to capture?• UIGestureRecognizer• UIView• Up the responder chain
Page 19
Custom UIGestureRecognizerTargeting the main view controllerView controller facilitates update
CaptureBuilding a Drawing App
Stroke View
Canvas View Controller
Stroke Gesture Recognizer
Page 20
import UIKit.UIGestureRecognizerSubclass
class StrokeGestureRecognizer: UIGestureRecognizer {
}
Page 21
import UIKit.UIGestureRecognizerSubclass
class StrokeGestureRecognizer: UIGestureRecognizer {
var stroke = Stroke()
}
Page 22
import UIKit.UIGestureRecognizerSubclass
class StrokeGestureRecognizer: UIGestureRecognizer {
var stroke = Stroke()
override func touchesBegan(_ touches: Set<UITouch>, with event: UIEvent?) {
}
}
Page 23
import UIKit.UIGestureRecognizerSubclass
class StrokeGestureRecognizer: UIGestureRecognizer {
var stroke = Stroke()
func appendTouches(_ touches: Set<UITouch>, event: UIEvent?) -> Bool …
override func touchesBegan(_ touches: Set<UITouch>, with event: UIEvent?) {
}
}
Page 24
import UIKit.UIGestureRecognizerSubclass
class StrokeGestureRecognizer: UIGestureRecognizer {
var stroke = Stroke()
func appendTouches(_ touches: Set<UITouch>, event: UIEvent?) -> Bool …
override func touchesBegan(_ touches: Set<UITouch>, with event: UIEvent?) {
}
}
if appendTouches(touches, event:event) { state = .began }
Page 25
import UIKit.UIGestureRecognizerSubclass
class StrokeGestureRecognizer: UIGestureRecognizer {
var stroke = Stroke()
func appendTouches(_ touches: Set<UITouch>, event: UIEvent?) -> Bool …
override func touchesBegan(_ touches: Set<UITouch>, with event: UIEvent?) { if appendTouches(touches, event:event) { state = .began } } override func touchesMoved(_ touches: Set<UITouch>, with event: UIEvent?) { if appendTouches(touches, event:event) { state = .changed } }
}
Page 26
import UIKit.UIGestureRecognizerSubclass
class StrokeGestureRecognizer: UIGestureRecognizer {
var stroke = Stroke()
func appendTouches(_ touches: Set<UITouch>, event: UIEvent?) -> Bool …
override func touchesBegan(_ touches: Set<UITouch>, with event: UIEvent?) { if appendTouches(touches, event:event) { state = .began } } override func touchesMoved(_ touches: Set<UITouch>, with event: UIEvent?) { if appendTouches(touches, event:event) { state = .changed } } override func touchesEnded(_ touches: Set<UITouch>, with event: UIEvent?) … override func touchesCancelled(_ touches: Set<UITouch>, with event: UIEvent?) …
}
Page 27
import UIKit.UIGestureRecognizerSubclass
class StrokeGestureRecognizer: UIGestureRecognizer {
var stroke = Stroke()
func appendTouches(_ touches: Set<UITouch>, event: UIEvent?) -> Bool …
override func touchesBegan(_ touches: Set<UITouch>, with event: UIEvent?) { if appendTouches(touches, event:event) { state = .began } } override func touchesMoved(_ touches: Set<UITouch>, with event: UIEvent?) { if appendTouches(touches, event:event) { state = .changed } } override func touchesEnded(_ touches: Set<UITouch>, with event: UIEvent?) … override func touchesCancelled(_ touches: Set<UITouch>, with event: UIEvent?) …
override func reset() { stroke = Stroke() super.reset() } }
Page 28
import UIKit.UIGestureRecognizerSubclass
class StrokeGestureRecognizer: UIGestureRecognizer {
var stroke = Stroke()
func appendTouches(_ touches: Set<UITouch>, event: UIEvent?) -> Bool …
override func touchesBegan(_ touches: Set<UITouch>, with event: UIEvent?) { if appendTouches(touches, event:event) { state = .began } } override func touchesMoved(_ touches: Set<UITouch>, with event: UIEvent?) { if appendTouches(touches, event:event) { state = .changed } } override func touchesEnded(_ touches: Set<UITouch>, with event: UIEvent?) … override func touchesCancelled(_ touches: Set<UITouch>, with event: UIEvent?) …
override func reset() { stroke = Stroke() super.reset() } }
Page 29
class CanvasViewController: UIViewController {
override func viewDidLoad() { super.viewDidLoad()
}
}
Page 30
class CanvasViewController: UIViewController {
override func viewDidLoad() { super.viewDidLoad() let strokeRecognizer = StrokeGestureRecognizer( target: self,
action: #selector(strokeUpdated(_:)) )
view.addGestureRecognizer(strokeRecognizer) }
}
Page 31
class CanvasViewController: UIViewController {
override func viewDidLoad() { super.viewDidLoad() let strokeRecognizer = StrokeGestureRecognizer( target: self,
action: #selector(strokeUpdated(_:)) )
view.addGestureRecognizer(strokeRecognizer) }
func strokeUpdated(_ strokeGesture: StrokeGestureRecognizer) { view.strokeToDraw = strokeGesture.stroke }
}
Page 32
class CanvasViewController: UIViewController {
override func viewDidLoad() { super.viewDidLoad() let strokeRecognizer = StrokeGestureRecognizer( target: self,
action: #selector(strokeUpdated(_:)) )
view.addGestureRecognizer(strokeRecognizer) }
func strokeUpdated(_ strokeGesture: StrokeGestureRecognizer) { view.strokeToDraw = strokeGesture.stroke }
}
Page 33
Let’s have a look
Page 36
Long, choppy lines
Excessive gap
Pencil
Page 37
Analysis of First Attempt
Missed events• Drawing engine• Did not use the new iOS 9.0 API
Page 38
Anatomy of a Stroke
Page 39
Anatomy of a Stroke
Page 40
Anatomy of a Stroke
Began
Ended
Page 41
Anatomy of a Stroke
Began
Ended
Page 42
Anatomy of a Stroke
Began
Ended
Moved
Moved
MovedMoved
Moved
Page 43
Began
Ended
Moved
Moved
MovedMoved
Moved
Anatomy of a Stroke
Page 44
Began
Ended
Moved
Moved
MovedMoved
Moved
Anatomy of a Stroke
Coalesced
Page 45
Began
Ended
Moved
Moved
MovedMoved
Moved
Anatomy of a Stroke
Coalesced
Page 46
Began
Ended
Moved
Moved
MovedMoved
Moved
Anatomy of a Stroke
Coalesced
Page 47
Coalesced Touches
class UIEvent {
public func coalescedTouches(for touch: UITouch) -> [UITouch]?
}
Page 48
Coalesced Touches
class StrokeGestureRecognizer: UIGestureRecognizer {
override func touchesMoved(_ touches: Set<UITouch>, with event: UIEvent?) {
if let touch = touches.first {
appendTouch(touch)
}
}
}
Page 49
Coalesced Touches
class StrokeGestureRecognizer: UIGestureRecognizer {
override func touchesMoved(_ touches: Set<UITouch>, with event: UIEvent?) {
if let touch = touches.first {
for coalescedTouch in event.coalescedTouches(for: touch) {
appendTouch(touch)
}
}
}
}
Page 52
PencilCoalesced touches (gray)
Live touch (black)Excessive gap
Too many coalesced touches
Page 53
Analysis of Second Attempt
Speed is still lackingUIKit helped us by coalescing
Page 54
Do Not Draw on Every Touch Event
Display refresh rate is 60 HzIncoming event frequency can reach 240 Hz and moreDo not try to draw faster than screen can refresh
Page 55
UIView
• Use setNeedsDisplay()
• Implement draw(_ rect: CGRect)
GLKView, MTLView
• Set the enableSetNeedsDisplay property to true
• If required, draw at a steady rate using a CADisplayLink object
When to Render?
Page 56
class StrokeCGView: UIView {
var strokeToDraw: Stroke? {
didSet {
drawImageAndUpdate()
}
}
}
Page 57
class StrokeCGView: UIView {
var strokeToDraw: Stroke? {
didSet {
setNeedsDisplay()
}
}
}
Page 58
Activate drawsAsynchronously on the layer
Mark only the changed areas• setNeedsDisplayIn(rect: changedRect)
Even Better
class StrokeCGView: UIView {
override init(frame: CGRect) {
super.init(frame: frame)
layer.drawsAsynchronously = true
}
}
Page 60
Steady amount of coalesced touches
Still some lag
Page 61
Improve Perceived Latency
Use predicted touches
class UIEvent {
public func predictedTouches(for touch: UITouch) -> [UITouch]?
}
Page 62
Predicted Touches
Add predicted touches to your data structure temporarilyChoose their appearance, depending on your app• To appear like actual touches• To appear as tentative
Page 63
Predicted Touches
class StrokeGestureRecognizer: UIGestureRecognizer {
override func touchesMoved(_ touches: Set<UITouch>, with event: UIEvent?) {
if let touch = touches.first {
for coalescedTouch in event.coalescedTouches(for:touch) {
appendTouch(touch)
}
}
}
}
Page 64
Predicted Touches
class StrokeGestureRecognizer: UIGestureRecognizer {
override func touchesMoved(_ touches: Set<UITouch>, with event: UIEvent?) {
if let touch = touches.first {
for coalescedTouch in event.coalescedTouches(for:touch) {
appendTouch(touch)
}
for predictedTouch in event.predictedTouches(for:touch) {
appendTouchTemporarily(touch)
}
}
}
}
Page 65
Predicted Touches
class StrokeGestureRecognizer: UIGestureRecognizer {
override func touchesMoved(_ touches: Set<UITouch>, with event: UIEvent?) {
if let touch = touches.first {
clearTemporaryTouches()
for coalescedTouch in event.coalescedTouches(for:touch) {
appendTouch(touch)
}
for predictedTouch in event.predictedTouches(for:touch) {
appendTouchTemporarily(touch)
}
}
}
}
Page 67
Predicted touches
Page 68
What Have We Seen so Far
Collect input data using a UIGestureRecognizerAccess coalesced touchesMake rendering fast and efficientUse predicted touches
Page 70
Touch typesApple Pencil
public var type: UITouchType { get }
Page 71
Touch typesApple Pencil
public var type: UITouchType { get }
enum UITouchType : Int { case direct case indirect case stylus }
Page 72
Higher precisionApple Pencil
func preciseLocation(in view: UIView?) -> CGPoint
func precisePreviousLocation(in view: UIView?) -> CGPoint
Page 73
public var force: CGFloat { get }
public var maximumPossibleForce: CGFloat { get }
Device Range
iPhone 6s (Plus) 0.0 - maximumPossibleForce
Apple Pencil on iPad Pro 0.0 - maximumPossibleForce
Touch on iPad Pro and all previous devices 0.0
ForceApple Pencil and 3D Touch
Page 74
ForceApple Pencil and 3D Touch
func touchesBegan(_ touches: Set<UITouch>, with event: UIEvent)
func touchesMoved(_ touches: Set<UITouch>, with event: UIEvent)
func touchesEnded(_ touches: Set<UITouch>, with event: UIEvent)
func touchesCancelled(_ touches: Set<UITouch>, with event: UIEvent)
Page 75
Tap recognitionApple Pencil and 3D Touch
func touchesMoved(_ touches: Set<UITouch>, with event: UIEvent) {
cancelTap()
}
Use UITapGestureRecognizer
Page 76
Add to the modelForce
struct StrokeSample { let location: CGPoint
}
Page 77
Add to the modelForce
struct StrokeSample { let location: CGPoint
}
var force: CGFloat?
Page 78
Width basedon force
Page 80
Altitude
TiltApple Pencil
Page 81
TiltApple Pencil
var altitudeAngle: CGFloat { get }
Page 82
OrientationApple Pencil
Page 83
OrientationApple Pencil
Page 84
OrientationApple Pencil
Page 85
OrientationApple Pencil
Page 86
OrientationApple Pencil
Azimuth
Page 87
AzimuthApple Pencil
func azimuthAngle(in view: UIView?) -> CGFloat
func azimuthUnitVector(in view: UIView?) -> CGVector
Page 88
ForceApple Pencil
Along axis
Page 89
ForceApple Pencil
Perpendicular todevice surface
Page 90
var perpendicularForce: CGFloat {
get {
return force / sin(altitudeAngle)
}
}
ForceApple Pencil
Page 91
var perpendicularForce: CGFloat {
get {
return force / sin(altitudeAngle)
}
}
ForceApple Pencil
min( , maximumPossibleForce)
Page 92
ForceApple Pencil
Page 93
Estimated propertiesApple Pencil
var estimatedProperties: UITouchProperties { get }
Page 94
Estimated propertiesApple Pencil
var estimatedProperties: UITouchProperties { get }
struct UITouchProperties : OptionSet {
}
Page 95
Estimated propertiesApple Pencil
var estimatedProperties: UITouchProperties { get }
struct UITouchProperties : OptionSet {
static var force: UITouchProperties { get }
}
Page 96
Estimated propertiesApple Pencil
var estimatedProperties: UITouchProperties { get }
struct UITouchProperties : OptionSet {
static var force: UITouchProperties { get }
static var azimuth: UITouchProperties { get }
static var altitude: UITouchProperties { get }
}
Page 97
Estimated propertiesApple Pencil
var estimatedProperties: UITouchProperties { get }
struct UITouchProperties : OptionSet {
static var force: UITouchProperties { get }
static var azimuth: UITouchProperties { get }
static var altitude: UITouchProperties { get }
static var location: UITouchProperties { get } }
Page 98
Estimated propertiesApple Pencil
var estimatedProperties: UITouchProperties { get }
struct UITouchProperties : OptionSet {
static var force: UITouchProperties { get }
static var azimuth: UITouchProperties { get }
static var altitude: UITouchProperties { get }
static var location: UITouchProperties { get }
var estimatedPropertiesExpectingUpdates: UITouchProperties { get }
}
Page 99
Estimated properties with updatesApple Pencil
func touchesBegan(_ touches: Set<UITouch>, with event: UIEvent)
func touchesMoved(_ touches: Set<UITouch>, with event: UIEvent)
func touchesEnded(_ touches: Set<UITouch>, with event: UIEvent)
func touchesCancelled(_ touches: Set<UITouch>, with event: UIEvent)
Page 100
Estimated properties with updatesApple Pencil
func touchesBegan(_ touches: Set<UITouch>, with event: UIEvent)
func touchesMoved(_ touches: Set<UITouch>, with event: UIEvent)
func touchesEnded(_ touches: Set<UITouch>, with event: UIEvent)
func touchesCancelled(_ touches: Set<UITouch>, with event: UIEvent)
func touchesEstimatedPropertiesUpdated(_ touches: Set<UITouch>)
Page 101
Check estimatedPropertiesExpectingUpdates
Use estimationUpdateIndex as key to index your sample
Look up the estimationUpdateIndex intouchesEstimatedPropertiesUpdated(_:)
Some updates will arrive after touchesEnded:
Estimated properties with updatesApple Pencil
Page 102
Estimated properties with updatesApple Pencil
override func touchesEstimatedPropertiesUpdated(_ touches: Set<UITouch>) { for touch in touches {
}
}
Page 103
Estimated properties with updatesApple Pencil
override func touchesEstimatedPropertiesUpdated(_ touches: Set<UITouch>) { for touch in touches {
let estimationIndex = touch.estimationUpdateIndex!
}
}
Page 104
Estimated properties with updatesApple Pencil
override func touchesEstimatedPropertiesUpdated(_ touches: Set<UITouch>) { for touch in touches {
let estimationIndex = touch.estimationUpdateIndex!
let (sample, sampleIndex) = samplesExpectingUpdates[estimationIndex]
}
}
Page 105
Estimated properties with updatesApple Pencil
override func touchesEstimatedPropertiesUpdated(_ touches: Set<UITouch>) { for touch in touches {
let estimationIndex = touch.estimationUpdateIndex!
let (sample, sampleIndex) = samplesExpectingUpdates[estimationIndex]
let updatedSample = updatedSample(with: touch)
}
}
Page 106
Estimated properties with updatesApple Pencil
override func touchesEstimatedPropertiesUpdated(_ touches: Set<UITouch>) { for touch in touches {
let estimationIndex = touch.estimationUpdateIndex!
let (sample, sampleIndex) = samplesExpectingUpdates[estimationIndex]
let updatedSample = updatedSample(with: touch)
stroke.update(sample: updatedSample, at: sampleIndex)
}
}
Page 107
Estimated properties with updatesApple Pencil
override func touchesEstimatedPropertiesUpdated(_ touches: Set<UITouch>) { for touch in touches {
let estimationIndex = touch.estimationUpdateIndex!
let (sample, sampleIndex) = samplesExpectingUpdates[estimationIndex]
let updatedSample = updatedSample(with: touch)
stroke.update(sample: updatedSample, at: sampleIndex)
if touch.estimatedPropertiesExpectingUpdates == [] {
samplesExpectingUpdates.removeValue(forKey: sampleIndex)
}
}
}
Page 110
CanvasFinishing Touches
84
Page 111
CanvasFinishing Touches
84
Page 112
CanvasFinishing Touches
84
Page 113
Adjusting gesturesFinishing Touches
Scroll view UIPanGestureRecognizer vs. StrokeGestureRecognizerDisable scrolling with Apple Pencil
class UIGestureRecognizer { public var allowedTouchTypes: [NSNumber]}
let pan = scrollView.panGestureRecognizer
pan.allowedTouchTypes = [UITouchType.direct.rawValue as NSNumber]
strokeRecognizer.allowedTouchTypes = [UITouchType.stylus.rawValue as NSNumber]
Page 114
Adjusting gesturesFinishing Touches
class UIGestureRecognizer {
public var requiresExclusiveTouchType: Bool
}
Page 115
Summary
New properties of UITouchCoalesced and predicted touchesProperty estimationAdjusting gestures
Page 118
More Information
https://developer.apple.com/wwdc16/220
Page 119
Related Sessions
Controlling Game Input for Apple TV Mission Wednesday 5:00PM
A Peek at 3D Touch Presidio Thursday 4:00PM
Advanced Touch Input on iOS WWDC 2015
Page 120
Labs
UIKit and UIKit Animations Lab Frameworks Lab C Thursday 1:00PM
Cocoa Touch and 3D Touch Lab Frameworks Lab C Friday 10:30AM