diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f3ecb08..15818de8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,28 @@ +# 2.25.2 (July 5 2023) + +- [Fix]: Fix crash on iOS when publishing a screen-sharing stream. + +# 2.25.1 (June 27 2023) + +- [Fix]: Fix camera lifecycle on Android. - issue #645 + +# 2.25.0 (May 17 2023) + +- [Update]: Update OpenTok Android SDK and OpenTok iOS SDK to version 2.25.1. + + Note that with this version, we are pinning the major and minor release versions + (2.25) to match the corresponding versions in the OpenTok Android and iOS SDKs. + + For iOS, note that this version supports iOS 13+, removes support for FAT binaries + and drops 32-bit support. The OpenTok iOS SDK is now available as the OTXCFramework + Pod file. (The OpenTok pod file was for FAT binaries.) + + See the release notes for the OpenTok [ioS SDK](https://tokbox.com/developer/sdks/ios/release-notes.html) + and the [Android SDK](https://tokbox.com/developer/sdks/android/release-notes.html). + +- [Fix]: Fixes an issue in which applications could not connect to a session when + the `proxyUrl` option for OTSession was set. - issue #645 + # 0.21.4 (April 12 2023) - [Update]: Revert OpenTok iOS SDK back 2.23.1. There are issues with diff --git a/README.md b/README.md index 322cf32b..696e33ad 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ If you've installed this package before, you may need to edit your `Podfile` and target '' do # Pods for - pod 'OpenTok', '2.23.1' + pod 'OTXCFramework', '2.25.1' end ``` diff --git a/android/build.gradle b/android/build.gradle index 313ea666..58266028 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -27,5 +27,5 @@ android { dependencies { implementation fileTree(dir: "libs", include: ["*.jar"]) implementation "com.facebook.react:react-native:${_reactNativeVersion}" // From node_modules - implementation 'com.opentok.android:opentok-android-sdk:2.24.2' + implementation 'com.opentok.android:opentok-android-sdk:2.25.1' } diff --git a/android/src/main/java/com/opentokreactnative/OTSessionManager.java b/android/src/main/java/com/opentokreactnative/OTSessionManager.java index dc3a921f..a647937b 100644 --- a/android/src/main/java/com/opentokreactnative/OTSessionManager.java +++ b/android/src/main/java/com/opentokreactnative/OTSessionManager.java @@ -4,14 +4,15 @@ * Created by manik on 1/29/18. */ +import android.os.Build; import android.util.Log; import android.widget.FrameLayout; import android.view.View; import androidx.annotation.Nullable; - import com.facebook.react.bridge.Arguments; import com.facebook.react.bridge.Callback; +import com.facebook.react.bridge.LifecycleEventListener; import com.facebook.react.bridge.ReactApplicationContext; import com.facebook.react.bridge.ReactContext; import com.facebook.react.bridge.ReactContextBaseJavaModule; @@ -56,7 +57,9 @@ public class OTSessionManager extends ReactContextBaseJavaModule SubscriberKit.AudioStatsListener, SubscriberKit.VideoStatsListener, SubscriberKit.VideoListener, - SubscriberKit.StreamListener{ + SubscriberKit.StreamListener, + LifecycleEventListener + { private ConcurrentHashMap connectionStatusMap = new ConcurrentHashMap<>(); private ArrayList jsEvents = new ArrayList(); @@ -72,6 +75,7 @@ public OTSessionManager(ReactApplicationContext reactContext) { super(reactContext); sharedState = OTRN.getSharedState(); + reactContext.addLifecycleEventListener(this); } @ReactMethod @@ -981,5 +985,32 @@ public void onStreamVideoTypeChanged(Session session, Stream stream, Stream.Stre sendEventMap(this.getReactApplicationContext(), session.getSessionId() + ":" + sessionPreface + "onStreamPropertyChanged", eventData); printLogs("onStreamVideoTypeChanged"); } + @Override + public void onHostResume() { + ConcurrentHashMap mPublishers = sharedState.getPublishers(); + + for (String key: mPublishers.keySet()) { + Publisher publisher = mPublishers.get(key); + + if (publisher != null) { + publisher.onResume(); + } + } + } + + @Override + public void onHostPause() { + ConcurrentHashMap mPublishers = sharedState.getPublishers(); + + for (String key: mPublishers.keySet()) { + Publisher publisher = mPublishers.get(key); + + if (publisher != null) { + publisher.onPause(); + } + } + } + @Override + public void onHostDestroy() {} } diff --git a/docs/OTSession.md b/docs/OTSession.md index a4a5881f..2d6011d5 100644 --- a/docs/OTSession.md +++ b/docs/OTSession.md @@ -150,9 +150,8 @@ The default value is false. **isCamera2Capable** (Boolean) -- Deprecated and ignored. Android only. -**proxyUrl** - -The proxy URL. This is an [add-on feature](https://www.vonage.com/communications-apis/video/pricing//plans) +**proxyUrl** (String) -- The proxy URL to use for the session. +This is an [add-on feature](https://www.vonage.com/communications-apis/video/pricing//plans) feature. See the [OpenTok IP Proxy](https://tokbox.com/developer/guides/ip-proxy/) developer guide. **useTextureViews** (Boolean) -- Set to `true` to use texture views. The default is `false`. Android only. diff --git a/ios/OpenTokReactNative/OTScreenCapture.h b/ios/OpenTokReactNative/OTScreenCapture.h index 0cc28476..665e6612 100644 --- a/ios/OpenTokReactNative/OTScreenCapture.h +++ b/ios/OpenTokReactNative/OTScreenCapture.h @@ -10,13 +10,6 @@ @protocol OTVideoCapture; -// defines for image scaling -// From https://bugs.chromium.org/p/webrtc/issues/detail?id=4643#c7 : -// Don't send any image larger than 1280px on either edge. Additionally, don't -// send any image with dimensions %16 != 0 -#define MAX_EDGE_SIZE_LIMIT 1280.0f -#define EDGE_DIMENSION_COMMON_FACTOR 16.0f - /** * Periodically sends video frames to an OpenTok Publisher by rendering the * CALayer for a UIView. @@ -30,9 +23,5 @@ */ - (instancetype)initWithView:(UIView*)view; -// private: declared here for testing scaling & padding function -+ (void)dimensionsForInputSize:(CGSize)input - containerSize:(CGSize*)destContainerSize - drawRect:(CGRect*)destDrawRect; @end diff --git a/ios/OpenTokReactNative/OTScreenCapture.m b/ios/OpenTokReactNative/OTScreenCapture.m index a8e804e0..1e363a76 100644 --- a/ios/OpenTokReactNative/OTScreenCapture.m +++ b/ios/OpenTokReactNative/OTScreenCapture.m @@ -7,21 +7,26 @@ #include #include +#import #import "OTScreenCapture.h" @implementation OTScreenCapture { - CMTime _minFrameDuration; dispatch_queue_t _queue; - dispatch_source_t _timer; CVPixelBufferRef _pixelBuffer; BOOL _capturing; OTVideoFrame* _videoFrame; UIView* _view; + CGFloat _screenScale; + CGContextRef _bitmapContext; + + CADisplayLink *_displayLink; + dispatch_semaphore_t _capturingSemaphore; } @synthesize videoCaptureConsumer; +@synthesize videoContentHint; #pragma mark - Class Lifecycle. @@ -30,16 +35,13 @@ - (instancetype)initWithView:(UIView *)view self = [super init]; if (self) { _view = view; - // Recommend sending 5 frames per second: Allows for higher image - // quality per frame - _minFrameDuration = CMTimeMake(1, 5); _queue = dispatch_queue_create("SCREEN_CAPTURE", NULL); - - OTVideoFormat *format = [[OTVideoFormat alloc] init]; - [format setPixelFormat:OTPixelFormatARGB]; - - _videoFrame = [[OTVideoFrame alloc] initWithFormat:format]; - + _screenScale = [[UIScreen mainScreen] scale]; + _displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(captureView)]; + _displayLink.preferredFramesPerSecond = 30.0; + _capturingSemaphore = dispatch_semaphore_create(1); + [self createPixelBuffer]; + [self createCGContextFromPixelBuffer]; } return self; } @@ -47,100 +49,111 @@ - (instancetype)initWithView:(UIView *)view - (void)dealloc { [self stopCapture]; - CVPixelBufferRelease(_pixelBuffer); + if(_bitmapContext) + CGContextRelease(_bitmapContext); + if(_pixelBuffer) + CVPixelBufferRelease(_pixelBuffer); } #pragma mark - Private Methods -/** - * Make sure receiving video frame container is setup for this image. - */ -- (void)checkImageSize:(CGImageRef)image { - CGFloat width = CGImageGetWidth(image); - CGFloat height = CGImageGetHeight(image); - - if (_videoFrame.format.imageHeight == height && - _videoFrame.format.imageWidth == width) - { - // don't rock the boat. if nothing has changed, don't update anything. +- (CMTime)getTimeStamp { + static mach_timebase_info_data_t time_info; + uint64_t time_stamp = 0; + if (time_info.denom == 0) { + (void) mach_timebase_info(&time_info); + } + time_stamp = mach_absolute_time(); + time_stamp *= time_info.numer; + time_stamp /= time_info.denom; + CMTime time = CMTimeMake(time_stamp, 1000); + return time; +} + +-(void)captureView +{ + if (!(_capturing && self.videoCaptureConsumer)) { return; } - - [_videoFrame.format.bytesPerRow removeAllObjects]; - [_videoFrame.format.bytesPerRow addObject:@(width * 4)]; - [_videoFrame.format setImageHeight:height]; - [_videoFrame.format setImageWidth:width]; - - CGSize frameSize = CGSizeMake(width, height); - NSDictionary *options = [NSDictionary dictionaryWithObjectsAndKeys: - @NO, - kCVPixelBufferCGImageCompatibilityKey, - @NO, - kCVPixelBufferCGBitmapContextCompatibilityKey, - nil]; - - if (NULL != _pixelBuffer) { - CVPixelBufferRelease(_pixelBuffer); + + // Wait until consumeImageBuffer is done. + if (dispatch_semaphore_wait(_capturingSemaphore, DISPATCH_TIME_NOW) != 0) { + return; } - CVReturn status = CVPixelBufferCreate(kCFAllocatorDefault, - frameSize.width, - frameSize.height, - kCVPixelFormatType_32ARGB, - (__bridge CFDictionaryRef)(options), - &_pixelBuffer); - - NSParameterAssert(status == kCVReturnSuccess && _pixelBuffer != NULL); + dispatch_async(dispatch_get_main_queue(), ^{ + [self.view.layer renderInContext:self->_bitmapContext]; + // Don't block the UI thread + dispatch_async(self->_queue, ^{ + CMTime time = [self getTimeStamp]; + [self.videoCaptureConsumer consumeImageBuffer:self->_pixelBuffer + orientation:OTVideoOrientationUp + timestamp:time + metadata:nil]; + // Signal for more frames + dispatch_semaphore_signal(self->_capturingSemaphore); + }); + }); +} +- (void)createCGContextFromPixelBuffer { + CGColorSpaceRef rgbColorSpace = CGColorSpaceCreateDeviceRGB(); + CVPixelBufferLockBaseAddress(_pixelBuffer, 0); + + _bitmapContext = CGBitmapContextCreate(CVPixelBufferGetBaseAddress(_pixelBuffer), + CVPixelBufferGetWidth(_pixelBuffer), + CVPixelBufferGetHeight(_pixelBuffer), + 8, CVPixelBufferGetBytesPerRow(_pixelBuffer), rgbColorSpace, + kCGBitmapByteOrder32Little | kCGImageAlphaPremultipliedFirst + ); + CGContextTranslateCTM(_bitmapContext, 0.0f, self.view.frame.size.height); + CGContextScaleCTM(_bitmapContext, self.view.layer.contentsScale, -self.view.layer.contentsScale); + CVPixelBufferUnlockBaseAddress(_pixelBuffer, 0); + CFRelease(rgbColorSpace); +} + +- (void)createPixelBuffer { + + CFDictionaryRef ioSurfaceProps = CFDictionaryCreate( kCFAllocatorDefault, NULL, NULL, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks ); + + NSDictionary *bufferAttributes = @{(id)kCVPixelBufferPixelFormatTypeKey : @(kCVPixelFormatType_32BGRA), + (id)kCVPixelBufferCGBitmapContextCompatibilityKey : @YES, + (id)kCVPixelBufferWidthKey : @(self.view.frame.size.width * _screenScale), + (id)kCVPixelBufferHeightKey : @(self.view.frame.size.height * _screenScale), + (id)kCVPixelBufferBytesPerRowAlignmentKey : + @(self.view.frame.size.width * _screenScale * 4), + (id)kCVPixelBufferIOSurfacePropertiesKey : (__bridge id)ioSurfaceProps + }; + CVPixelBufferCreate(kCFAllocatorDefault, + self.view.frame.size.width, + self.view.frame.size.height, + kCVPixelFormatType_32ARGB, + (__bridge CFDictionaryRef)(bufferAttributes), + &_pixelBuffer); + CFRelease(ioSurfaceProps); } #pragma mark - Capture lifecycle -/** - * Allocate capture resources; in this case we're just setting up a timer and - * block to execute periodically to send video frames. - */ - (void)initCapture { - __unsafe_unretained OTScreenCapture* _self = self; - _timer = dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, 0, 0, _queue); - - dispatch_source_set_timer(_timer, dispatch_walltime(NULL, 0), - 100ull * NSEC_PER_MSEC, 100ull * NSEC_PER_MSEC); - dispatch_source_set_event_handler(_timer, ^{ - @autoreleasepool { - __block UIImage* screen = [_self screenshot]; - CGImageRef paddedScreen = [self resizeAndPadImage:screen]; - [_self consumeFrame:paddedScreen]; - } - }); } - (void)releaseCapture { - _timer = nil; + [_displayLink invalidate]; } - (int32_t)startCapture { _capturing = YES; - - if (_timer) { - dispatch_resume(_timer); - } - + [_displayLink addToRunLoop:[NSRunLoop mainRunLoop] forMode:NSRunLoopCommonModes]; return 0; } - (int32_t)stopCapture { _capturing = NO; - - dispatch_sync(_queue, ^{ - if (self->_timer) { - dispatch_source_cancel(self->_timer); - } - }); - + [_displayLink invalidate]; return 0; } @@ -149,205 +162,10 @@ - (BOOL)isCaptureStarted return _capturing; } -#pragma mark - Screen capture implementation - -- (CVPixelBufferRef)pixelBufferFromCGImage:(CGImageRef)image -{ - CGFloat width = CGImageGetWidth(image); - CGFloat height = CGImageGetHeight(image); - CGSize frameSize = CGSizeMake(width, height); - CVPixelBufferLockBaseAddress(_pixelBuffer, 0); - void *pxdata = CVPixelBufferGetBaseAddress(_pixelBuffer); - - CGColorSpaceRef rgbColorSpace = CGColorSpaceCreateDeviceRGB(); - CGContextRef context = - CGBitmapContextCreate(pxdata, - frameSize.width, - frameSize.height, - 8, - CVPixelBufferGetBytesPerRow(_pixelBuffer), - rgbColorSpace, - kCGImageAlphaPremultipliedFirst | - kCGBitmapByteOrder32Little); - - - CGContextDrawImage(context, CGRectMake(0, 0, width, height), image); - CGColorSpaceRelease(rgbColorSpace); - CGContextRelease(context); - - CVPixelBufferUnlockBaseAddress(_pixelBuffer, 0); - - return _pixelBuffer; -} - - (int32_t)captureSettings:(OTVideoFormat*)videoFormat { videoFormat.pixelFormat = OTPixelFormatARGB; return 0; } -+ (void)dimensionsForInputSize:(CGSize)input - containerSize:(CGSize*)destContainerSize - drawRect:(CGRect*)destDrawRect -{ - CGFloat sourceWidth = input.width; - CGFloat sourceHeight = input.height; - double sourceAspectRatio = sourceWidth / sourceHeight; - - CGFloat destContainerWidth = sourceWidth; - CGFloat destContainerHeight = sourceHeight; - CGFloat destImageWidth = sourceWidth; - CGFloat destImageHeight = sourceHeight; - - // if image is wider than tall and width breaks edge size limit - if (MAX_EDGE_SIZE_LIMIT < sourceWidth && sourceAspectRatio >= 1.0) { - destContainerWidth = MAX_EDGE_SIZE_LIMIT; - destContainerHeight = destContainerWidth / sourceAspectRatio; - if (0 != fmod(destContainerHeight, EDGE_DIMENSION_COMMON_FACTOR)) { - // add padding to make height % 16 == 0 - destContainerHeight += - (EDGE_DIMENSION_COMMON_FACTOR - fmod(destContainerHeight, - EDGE_DIMENSION_COMMON_FACTOR)); - } - destImageWidth = destContainerWidth; - destImageHeight = destContainerWidth / sourceAspectRatio; - } - - // if image is taller than wide and height breaks edge size limit - if (MAX_EDGE_SIZE_LIMIT < destContainerHeight && sourceAspectRatio <= 1.0) { - destContainerHeight = MAX_EDGE_SIZE_LIMIT; - destContainerWidth = destContainerHeight * sourceAspectRatio; - if (0 != fmod(destContainerWidth, EDGE_DIMENSION_COMMON_FACTOR)) { - // add padding to make width % 16 == 0 - destContainerWidth += - (EDGE_DIMENSION_COMMON_FACTOR - fmod(destContainerWidth, - EDGE_DIMENSION_COMMON_FACTOR)); - } - destImageHeight = destContainerHeight; - destImageWidth = destContainerHeight * sourceAspectRatio; - } - - // ensure the dimensions of the resulting container are safe - if (fmod(destContainerWidth, EDGE_DIMENSION_COMMON_FACTOR) != 0) { - double remainder = fmod(destContainerWidth, - EDGE_DIMENSION_COMMON_FACTOR); - // increase the edge size only if doing so does not break the edge limit - if (destContainerWidth + (EDGE_DIMENSION_COMMON_FACTOR - remainder) > - MAX_EDGE_SIZE_LIMIT) - { - destContainerWidth -= remainder; - } else { - destContainerWidth += EDGE_DIMENSION_COMMON_FACTOR - remainder; - } - } - // ensure the dimensions of the resulting container are safe - if (fmod(destContainerHeight, EDGE_DIMENSION_COMMON_FACTOR) != 0) { - double remainder = fmod(destContainerHeight, - EDGE_DIMENSION_COMMON_FACTOR); - // increase the edge size only if doing so does not break the edge limit - if (destContainerHeight + (EDGE_DIMENSION_COMMON_FACTOR - remainder) > - MAX_EDGE_SIZE_LIMIT) - { - destContainerHeight -= remainder; - } else { - destContainerHeight += EDGE_DIMENSION_COMMON_FACTOR - remainder; - } - } - - destContainerSize->width = destContainerWidth; - destContainerSize->height = destContainerHeight; - - // scale and recenter source image to fit in destination container - if (sourceAspectRatio > 1.0) { - destDrawRect->origin.x = 0; - destDrawRect->origin.y = - (destContainerHeight - destImageHeight) / 2; - destDrawRect->size.width = destContainerWidth; - destDrawRect->size.height = - destContainerWidth / sourceAspectRatio; - } else { - destDrawRect->origin.x = - (destContainerWidth - destImageWidth) / 2; - destDrawRect->origin.y = 0; - destDrawRect->size.height = destContainerHeight; - destDrawRect->size.width = - destContainerHeight * sourceAspectRatio; - } - -} - -- (CGImageRef)resizeAndPadImage:(UIImage*)sourceUIImage { - CGImageRef sourceCGImage = [sourceUIImage CGImage]; - CGFloat sourceWidth = CGImageGetWidth(sourceCGImage); - CGFloat sourceHeight = CGImageGetHeight(sourceCGImage); - CGSize sourceSize = CGSizeMake(sourceWidth, sourceHeight); - CGSize destContainerSize = CGSizeZero; - CGRect destRectForSourceImage = CGRectZero; - - [OTScreenCapture dimensionsForInputSize:sourceSize - containerSize:&destContainerSize - drawRect:&destRectForSourceImage]; - - UIGraphicsBeginImageContextWithOptions(destContainerSize, NO, 1.0); - CGContextRef context = UIGraphicsGetCurrentContext(); - - // flip source image to match destination coordinate system - CGContextScaleCTM(context, 1.0, -1.0); - CGContextTranslateCTM(context, 0, -destRectForSourceImage.size.height); - CGContextDrawImage(context, destRectForSourceImage, sourceCGImage); - - // Clean up and get the new image. - UIImage *newImage = UIGraphicsGetImageFromCurrentImageContext(); - UIGraphicsEndImageContext(); - - return [newImage CGImage]; -} - -- (UIImage *)screenshot -{ - UIGraphicsBeginImageContextWithOptions(self.view.bounds.size, NO, 0.0); - [self.view drawViewHierarchyInRect:self.view.bounds afterScreenUpdates:NO]; - UIImage *image = UIGraphicsGetImageFromCurrentImageContext(); - UIGraphicsEndImageContext(); - return image; -} - -- (void) consumeFrame:(CGImageRef)frame { - - [self checkImageSize:frame]; - - static mach_timebase_info_data_t time_info; - uint64_t time_stamp = 0; - - if (!(_capturing && self.videoCaptureConsumer)) { - return; - } - - if (time_info.denom == 0) { - (void) mach_timebase_info(&time_info); - } - - time_stamp = mach_absolute_time(); - time_stamp *= time_info.numer; - time_stamp /= time_info.denom; - - CMTime time = CMTimeMake(time_stamp, 1000); - CVImageBufferRef ref = [self pixelBufferFromCGImage:frame]; - - CVPixelBufferLockBaseAddress(ref, 0); - - _videoFrame.timestamp = time; - _videoFrame.format.estimatedFramesPerSecond = - _minFrameDuration.timescale / _minFrameDuration.value; - _videoFrame.format.estimatedCaptureDelay = 100; - _videoFrame.orientation = OTVideoOrientationUp; - - [_videoFrame clearPlanes]; - [_videoFrame.planes addPointer:CVPixelBufferGetBaseAddress(ref)]; - [self.videoCaptureConsumer consumeFrame:_videoFrame]; - - CVPixelBufferUnlockBaseAddress(ref, 0); -} - - @end diff --git a/opentok-react-native.podspec b/opentok-react-native.podspec index 0e65b99d..7eb7cb13 100644 --- a/opentok-react-native.podspec +++ b/opentok-react-native.podspec @@ -10,12 +10,12 @@ Pod::Spec.new do |s| s.authors = package['author'] s.homepage = package['homepage'] - s.platform = :ios, "12.0" + s.platform = :ios, "13.0" s.swift_version = "4.2" s.source = { :git => "https://github.com/opentok/opentok-react-native.git", :tag => "v#{s.version}" } s.source_files = "ios/**/*.{h,m,swift}" s.dependency 'React' - s.dependency 'OTXCFramework','2.23.1' + s.dependency 'OTXCFramework','2.25.1' end diff --git a/package-lock.json b/package-lock.json index 0ddaccb9..befd45ad 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "opentok-react-native", - "version": "0.21.4", + "version": "2.25.2", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "opentok-react-native", - "version": "0.21.4", + "version": "2.25.2", "license": "MIT", "dependencies": { "axios": "^0.21.1", diff --git a/package.json b/package.json index 26ccc162..1e5bb467 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "opentok-react-native", - "version": "0.21.4", + "version": "2.25.2", "description": "React Native components for OpenTok iOS and Android SDKs", "main": "src/index.js", "homepage": "https://www.tokbox.com",