Added VisionTestApp for testing vision ops in iOS (#3629) (#3806)

Co-authored-by: Yuchen Huang <hyc@fb.com>

.gitignore

@@ -28,3 +28,6 @@ gen.yml
 *.orig
 *-checkpoint.ipynb
 *.venv
+
+## Xcode User settings
+xcuserdata/

ios/VisionTestApp/VisionTestApp.xcodeproj/project.xcworkspace/contents.xcworkspacedata

+<?xml version="1.0" encoding="UTF-8"?>
+<Workspace
+   version = "1.0">
+   <FileRef
+      location = "self:">
+   </FileRef>
+</Workspace>

ios/VisionTestApp/VisionTestApp.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist

+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>IDEDidComputeMac32BitWarning</key>
+	<true/>
+</dict>
+</plist>

ios/VisionTestApp/VisionTestApp/AppDelegate.h

+#import <UIKit/UIKit.h>
+
+@interface AppDelegate : UIResponder <UIApplicationDelegate>
+
+@property(strong, nonatomic) UIWindow *window;
+
+@end

ios/VisionTestApp/VisionTestApp/AppDelegate.m

+
+#import "AppDelegate.h"
+
+@interface AppDelegate ()
+
+@end
+
+@implementation AppDelegate
+
+
+- (BOOL)application:(UIApplication *)application didFinishLaunchingWithOptions:(NSDictionary *)launchOptions {
+    // Override point for customization after application launch.
+    return YES;
+}
+
+
+- (void)applicationWillResignActive:(UIApplication *)application {
+    // Sent when the application is about to move from active to inactive state. This can occur for certain types of temporary interruptions (such as an incoming phone call or SMS message) or when the user quits the application and it begins the transition to the background state.
+    // Use this method to pause ongoing tasks, disable timers, and invalidate graphics rendering callbacks. Games should use this method to pause the game.
+}
+
+
+- (void)applicationDidEnterBackground:(UIApplication *)application {
+    // Use this method to release shared resources, save user data, invalidate timers, and store enough application state information to restore your application to its current state in case it is terminated later.
+    // If your application supports background execution, this method is called instead of applicationWillTerminate: when the user quits.
+}
+
+
+- (void)applicationWillEnterForeground:(UIApplication *)application {
+    // Called as part of the transition from the background to the active state; here you can undo many of the changes made on entering the background.
+}
+
+
+- (void)applicationDidBecomeActive:(UIApplication *)application {
+    // Restart any tasks that were paused (or not yet started) while the application was inactive. If the application was previously in the background, optionally refresh the user interface.
+}
+
+
+- (void)applicationWillTerminate:(UIApplication *)application {
+    // Called when the application is about to terminate. Save data if appropriate. See also applicationDidEnterBackground:.
+}
+
+
+@end

ios/VisionTestApp/VisionTestApp/Assets.xcassets/AccentColor.colorset/Contents.json

+{
+  "colors" : [
+    {
+      "idiom" : "universal"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}

ios/VisionTestApp/VisionTestApp/Assets.xcassets/AppIcon.appiconset/Contents.json

+{
+  "images" : [
+    {
+      "idiom" : "iphone",
+      "scale" : "2x",
+      "size" : "20x20"
+    },
+    {
+      "idiom" : "iphone",
+      "scale" : "3x",
+      "size" : "20x20"
+    },
+    {
+      "idiom" : "iphone",
+      "scale" : "2x",
+      "size" : "29x29"
+    },
+    {
+      "idiom" : "iphone",
+      "scale" : "3x",
+      "size" : "29x29"
+    },
+    {
+      "idiom" : "iphone",
+      "scale" : "2x",
+      "size" : "40x40"
+    },
+    {
+      "idiom" : "iphone",
+      "scale" : "3x",
+      "size" : "40x40"
+    },
+    {
+      "idiom" : "iphone",
+      "scale" : "2x",
+      "size" : "60x60"
+    },
+    {
+      "idiom" : "iphone",
+      "scale" : "3x",
+      "size" : "60x60"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "1x",
+      "size" : "20x20"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "2x",
+      "size" : "20x20"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "1x",
+      "size" : "29x29"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "2x",
+      "size" : "29x29"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "1x",
+      "size" : "40x40"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "2x",
+      "size" : "40x40"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "1x",
+      "size" : "76x76"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "2x",
+      "size" : "76x76"
+    },
+    {
+      "idiom" : "ipad",
+      "scale" : "2x",
+      "size" : "83.5x83.5"
+    },
+    {
+      "idiom" : "ios-marketing",
+      "scale" : "1x",
+      "size" : "1024x1024"
+    }
+  ],
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}

ios/VisionTestApp/VisionTestApp/Assets.xcassets/Contents.json

+{
+  "info" : {
+    "author" : "xcode",
+    "version" : 1
+  }
+}

ios/VisionTestApp/VisionTestApp/Base.lproj/LaunchScreen.storyboard

+<?xml version="1.0" encoding="UTF-8"?>
+<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="17701" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" launchScreen="YES" useTraitCollections="YES" useSafeAreas="YES" colorMatched="YES" initialViewController="01J-lp-oVM">
+    <device id="retina6_1" orientation="portrait" appearance="light"/>
+    <dependencies>
+        <deployment identifier="iOS"/>
+        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="17703"/>
+        <capability name="Safe area layout guides" minToolsVersion="9.0"/>
+        <capability name="System colors in document resources" minToolsVersion="11.0"/>
+        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
+    </dependencies>
+    <scenes>
+        <!--View Controller-->
+        <scene sceneID="EHf-IW-A2E">
+            <objects>
+                <viewController id="01J-lp-oVM" sceneMemberID="viewController">
+                    <view key="view" contentMode="scaleToFill" id="Ze5-6b-2t3">
+                        <rect key="frame" x="0.0" y="0.0" width="414" height="896"/>
+                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                        <viewLayoutGuide key="safeArea" id="6Tk-OE-BBY"/>
+                        <color key="backgroundColor" systemColor="systemBackgroundColor"/>
+                    </view>
+                </viewController>
+                <placeholder placeholderIdentifier="IBFirstResponder" id="iYj-Kq-Ea1" userLabel="First Responder" sceneMemberID="firstResponder"/>
+            </objects>
+            <point key="canvasLocation" x="53" y="375"/>
+        </scene>
+    </scenes>
+    <resources>
+        <systemColor name="systemBackgroundColor">
+            <color white="1" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
+        </systemColor>
+    </resources>
+</document>

ios/VisionTestApp/VisionTestApp/Base.lproj/Main.storyboard

+<?xml version="1.0" encoding="UTF-8"?>
+<document type="com.apple.InterfaceBuilder3.CocoaTouch.Storyboard.XIB" version="3.0" toolsVersion="17701" targetRuntime="iOS.CocoaTouch" propertyAccessControl="none" useAutolayout="YES" useTraitCollections="YES" useSafeAreas="YES" colorMatched="YES" initialViewController="C8u-kp-ZGJ">
+    <device id="retina6_1" orientation="portrait" appearance="light"/>
+    <dependencies>
+        <deployment identifier="iOS"/>
+        <plugIn identifier="com.apple.InterfaceBuilder.IBCocoaTouchPlugin" version="17703"/>
+        <capability name="Safe area layout guides" minToolsVersion="9.0"/>
+        <capability name="System colors in document resources" minToolsVersion="11.0"/>
+        <capability name="documents saved in the Xcode 8 format" minToolsVersion="8.0"/>
+    </dependencies>
+    <scenes>
+        <!--Test Vision Ops-->
+        <scene sceneID="tne-QT-ifu">
+            <objects>
+                <viewController id="BYZ-38-t0r" customClass="ViewController" sceneMemberID="viewController">
+                    <view key="view" contentMode="scaleToFill" id="8bC-Xf-vdC">
+                        <rect key="frame" x="0.0" y="0.0" width="414" height="896"/>
+                        <autoresizingMask key="autoresizingMask" widthSizable="YES" heightSizable="YES"/>
+                        <subviews>
+                            <textView clipsSubviews="YES" multipleTouchEnabled="YES" contentMode="scaleToFill" textAlignment="natural" translatesAutoresizingMaskIntoConstraints="NO" id="gaT-0L-TqB">
+                                <rect key="frame" x="20" y="88" width="374" height="774"/>
+                                <color key="backgroundColor" systemColor="systemBackgroundColor"/>
+                                <color key="textColor" systemColor="labelColor"/>
+                                <fontDescription key="fontDescription" type="system" pointSize="14"/>
+                                <textInputTraits key="textInputTraits" autocapitalizationType="sentences"/>
+                            </textView>
+                        </subviews>
+                        <viewLayoutGuide key="safeArea" id="6Tk-OE-BBY"/>
+                        <color key="backgroundColor" systemColor="systemBackgroundColor"/>
+                        <constraints>
+                            <constraint firstItem="gaT-0L-TqB" firstAttribute="leading" secondItem="6Tk-OE-BBY" secondAttribute="leading" constant="20" id="3b1-x4-VD4"/>
+                            <constraint firstItem="gaT-0L-TqB" firstAttribute="bottom" secondItem="6Tk-OE-BBY" secondAttribute="bottom" id="URh-hA-LJV"/>
+                            <constraint firstItem="6Tk-OE-BBY" firstAttribute="trailing" secondItem="gaT-0L-TqB" secondAttribute="trailing" constant="20" id="avy-0f-meB"/>
+                            <constraint firstItem="gaT-0L-TqB" firstAttribute="top" secondItem="6Tk-OE-BBY" secondAttribute="top" id="d1Q-UA-AUb"/>
+                        </constraints>
+                    </view>
+                    <navigationItem key="navigationItem" title="Test Vision Ops" id="QV8-E1-9z9">
+                        <barButtonItem key="rightBarButtonItem" title="Redo" id="ZQ5-yr-k4l">
+                            <connections>
+                                <action selector="rerun:" destination="BYZ-38-t0r" id="F5t-Nr-XmE"/>
+                            </connections>
+                        </barButtonItem>
+                    </navigationItem>
+                    <connections>
+                        <outlet property="textView" destination="gaT-0L-TqB" id="lMF-Rf-ics"/>
+                    </connections>
+                </viewController>
+                <placeholder placeholderIdentifier="IBFirstResponder" id="dkx-z0-nzr" sceneMemberID="firstResponder"/>
+            </objects>
+            <point key="canvasLocation" x="1047.8260869565217" y="137.94642857142856"/>
+        </scene>
+        <!--Navigation Controller-->
+        <scene sceneID="LxF-da-Ea2">
+            <objects>
+                <navigationController automaticallyAdjustsScrollViewInsets="NO" id="C8u-kp-ZGJ" sceneMemberID="viewController">
+                    <toolbarItems/>
+                    <navigationBar key="navigationBar" contentMode="scaleToFill" id="SkD-La-Hwl">
+                        <rect key="frame" x="0.0" y="44" width="414" height="44"/>
+                        <autoresizingMask key="autoresizingMask"/>
+                    </navigationBar>
+                    <nil name="viewControllers"/>
+                    <connections>
+                        <segue destination="BYZ-38-t0r" kind="relationship" relationship="rootViewController" id="s4b-GK-ujM"/>
+                    </connections>
+                </navigationController>
+                <placeholder placeholderIdentifier="IBFirstResponder" id="tCY-46-VMM" userLabel="First Responder" customClass="UIResponder" sceneMemberID="firstResponder"/>
+            </objects>
+            <point key="canvasLocation" x="137.68115942028987" y="137.94642857142856"/>
+        </scene>
+    </scenes>
+    <resources>
+        <systemColor name="labelColor">
+            <color white="0.0" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
+        </systemColor>
+        <systemColor name="systemBackgroundColor">
+            <color white="1" alpha="1" colorSpace="custom" customColorSpace="genericGamma22GrayColorSpace"/>
+        </systemColor>
+    </resources>
+</document>

ios/VisionTestApp/VisionTestApp/Info.plist

+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>CFBundleDevelopmentRegion</key>
+	<string>$(DEVELOPMENT_LANGUAGE)</string>
+	<key>CFBundleExecutable</key>
+	<string>$(EXECUTABLE_NAME)</string>
+	<key>CFBundleIdentifier</key>
+	<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
+	<key>CFBundleInfoDictionaryVersion</key>
+	<string>6.0</string>
+	<key>CFBundleName</key>
+	<string>$(PRODUCT_NAME)</string>
+	<key>CFBundlePackageType</key>
+	<string>$(PRODUCT_BUNDLE_PACKAGE_TYPE)</string>
+	<key>CFBundleShortVersionString</key>
+	<string>1.0</string>
+	<key>CFBundleVersion</key>
+	<string>1</string>
+	<key>LSRequiresIPhoneOS</key>
+	<true/>
+	<key>UIApplicationSupportsIndirectInputEvents</key>
+	<true/>
+	<key>UILaunchStoryboardName</key>
+	<string>LaunchScreen</string>
+	<key>UIMainStoryboardFile</key>
+	<string>Main</string>
+	<key>UIRequiredDeviceCapabilities</key>
+	<array>
+		<string>armv7</string>
+	</array>
+	<key>UISupportedInterfaceOrientations</key>
+	<array>
+		<string>UIInterfaceOrientationPortrait</string>
+	</array>
+	<key>UISupportedInterfaceOrientations~ipad</key>
+	<array>
+		<string>UIInterfaceOrientationPortrait</string>
+		<string>UIInterfaceOrientationPortraitUpsideDown</string>
+		<string>UIInterfaceOrientationLandscapeLeft</string>
+		<string>UIInterfaceOrientationLandscapeRight</string>
+	</array>
+</dict>
+</plist>

ios/VisionTestApp/VisionTestApp/ModelRunner.h

+
+#import <Foundation/Foundation.h>
+
+NS_ASSUME_NONNULL_BEGIN
+
+@interface ModelRunner : NSObject
+
++ (NSString* )run;
++ (BOOL)setUp;
+
+@end
+
+NS_ASSUME_NONNULL_END

ios/VisionTestApp/VisionTestApp/ModelRunner.mm

+
+#import "ModelRunner.h"
+#include <string>
+#include <vector>
+#include "ATen/ATen.h"
+#include "caffe2/core/timer.h"
+#include "caffe2/utils/string_utils.h"
+#include "torch/csrc/autograd/grad_mode.h"
+#include "torch/csrc/jit/serialization/import.h"
+#include "torch/script.h"
+
+static NSString *model_name = @"frcnn_mnetv3";
+static NSString *model_suffix = @"pt";
+static NSString *model_path = nil;
+static int warmup = 5;
+static int iter = 20;
+
+@implementation ModelRunner
+
++ (NSString *)run {
+  std::vector<std::string> logs;
+#define UI_LOG(fmt, ...)                                          \
+  {                                                               \
+    NSString* log = [NSString stringWithFormat:fmt, __VA_ARGS__]; \
+    NSLog(@"%@", log);                                            \
+    logs.push_back(log.UTF8String);                               \
+  }
+  
+  auto module = torch::jit::load(std::string(model_path.UTF8String));
+  module.eval();
+  
+  std::vector<c10::IValue> inputs;
+  auto img_tensor = torch::ones({3, 224, 224}, at::ScalarType::Float);
+  inputs.push_back(c10::List<at::Tensor>(img_tensor));
+  torch::autograd::AutoGradMode guard(false);
+  at::AutoNonVariableTypeMode nonVarTypeModeGuard(true);
+  
+  UI_LOG(@"Running warmup runs...", nil);
+  for (int i = 0; i < warmup; ++i) {
+    module.forward(inputs);
+  }
+  UI_LOG(@"Warmup runs finished.\nMain runs...", nil);
+  caffe2::Timer timer;
+  auto millis = timer.MilliSeconds();
+  for (int i = 0; i < iter; ++i) {
+    module.forward(inputs);
+  }
+  millis = timer.MilliSeconds();
+  UI_LOG(@"Main run finished. \nMilliseconds per iter: %.3f", millis / iter, nil);
+  UI_LOG(@"Iters per second: : %.3f", 1000.0 * iter / millis, nil);
+  UI_LOG(@"Done.", nil);
+  
+  std::cout << module.forward(inputs) << std::endl;
+  
+  NSString* log_text = @"";
+  for (auto& msg : logs) {
+    log_text = [log_text stringByAppendingString:[NSString stringWithUTF8String:msg.c_str()]];
+    log_text = [log_text stringByAppendingString:@"\n"];
+  }
+  return log_text;
+}
+
++ (BOOL)setUp {
+  model_path = [[NSBundle mainBundle] pathForResource:model_name ofType:model_suffix];
+  if (![[NSFileManager defaultManager] fileExistsAtPath:model_path]) {
+    NSLog(@"Invalid model path!");
+    model_path = nil;
+    return NO;
+  }
+  return YES;
+}
+
+@end

ios/VisionTestApp/VisionTestApp/ViewController.h

+
+#import <UIKit/UIKit.h>
+
+@interface ViewController : UIViewController
+
+
+@end
+

ios/VisionTestApp/VisionTestApp/ViewController.mm

+
+#import "ViewController.h"
+#include <torch/script.h>
+#import "ModelRunner.h"
+
+@interface ViewController ()
+@property (weak, nonatomic) IBOutlet UITextView *textView;
+@end
+
+static NSString const *config_error_msg = @"Wrong model configurations... Please fix and click \"Redo\"";
+
+@implementation ViewController
+
+- (void)viewDidLoad {
+  [super viewDidLoad];
+  if ([ModelRunner setUp]) {
+    [self testModel];
+  } else {
+    self.textView.text = [config_error_msg copy];
+  }
+}
+
+
+- (IBAction)rerun:(id)sender {
+  self.textView.text = @"";
+  if (![ModelRunner setUp]) {
+    self.textView.text = [config_error_msg copy];
+    return;
+  }
+  dispatch_async(dispatch_get_main_queue(), ^{
+    [self testModel];
+  });
+}
+
+- (void)testModel {
+  dispatch_async(dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_DEFAULT, 0), ^{
+    NSString *text = [ModelRunner run];
+    dispatch_async(dispatch_get_main_queue(), ^{
+      self.textView.text = [self.textView.text stringByAppendingString:text];
+    });
+  });
+}
+
+@end

ios/VisionTestApp/VisionTestApp/main.m

+//
+//  main.m
+//  VisionTestApp
+//
+//  Created by Yuchen Huang on 3/31/21.
+//
+
+#import <UIKit/UIKit.h>
+#import "AppDelegate.h"
+
+int main(int argc, char * argv[]) {
+  NSString * appDelegateClassName;
+  @autoreleasepool {
+      // Setup code that might create autoreleased objects goes here.
+      appDelegateClassName = NSStringFromClass([AppDelegate class]);
+  }
+  return UIApplicationMain(argc, argv, nil, appDelegateClassName);
+}

ios/VisionTestApp/clean.sh

+#!/bin/bash
+set -ex -o pipefail
+
+TEST_APP_PATH=$(dirname $(realpath $0))
+cd ${TEST_APP_PATH}
+
+rm -rf ./install
+rm ./VisionTestApp/*.pt

ios/VisionTestApp/make_assets.py

+import torch
+import torchvision
+from torch.utils.mobile_optimizer import optimize_for_mobile
+
+print(torch.__version__)
+
+model = torchvision.models.detection.fasterrcnn_mobilenet_v3_large_320_fpn(
+    pretrained=True,
+    box_score_thresh=0.7,
+    rpn_post_nms_top_n_test=100,
+    rpn_score_thresh=0.4,
+    rpn_pre_nms_top_n_test=150)
+
+model.eval()
+script_model = torch.jit.script(model)
+opt_script_model = optimize_for_mobile(script_model)
+opt_script_model.save("VisionTestApp/frcnn_mnetv3.pt")

ios/VisionTestApp/setup.sh

+#!/bin/bash
+set -ex -o pipefail
+
+echo ""
+echo "DIR: $(pwd)"
+
+TEST_APP_PATH=$(dirname $(realpath $0))
+cd ${TEST_APP_PATH}
+
+PYTORCH_IOS_NIGHTLY_NAME=libtorch_ios_nightly_build.zip
+VISION_IOS_NIGHTLY_NAME=libtorchvision_ops_ios_nightly_build.zip
+
+echo "Downloading torch libs and vision libs..."
+wget https://ossci-ios-build.s3.amazonaws.com/${PYTORCH_IOS_NIGHTLY_NAME}
+wget https://ossci-ios-build.s3.amazonaws.com/${VISION_IOS_NIGHTLY_NAME}
+
+mkdir -p ./library/torch
+mkdir -p ./library/vision
+
+echo "Unziping torch libs and vision libs..."
+unzip -d ./library/torch ./${PYTORCH_IOS_NIGHTLY_NAME}
+unzip -d ./library/vision ./${VISION_IOS_NIGHTLY_NAME}
+
+cp ./library/vision/install/lib/*.a ./library/torch/install/lib
+cp -r ./library/torch/install .
+
+rm -rf ./library
+rm -rf ./*.zip
+
+echo "Generating the vision model..."
+python ./make_assets.py
+
+echo "Finished project setups."