diff --git a/benchmark/CMakeLists.txt b/benchmark/CMakeLists.txt index a619bfbd3..ab21fe330 100644 --- a/benchmark/CMakeLists.txt +++ b/benchmark/CMakeLists.txt @@ -26,6 +26,18 @@ target_include_directories( include(${PROJECT_SOURCE_DIR}/vendor/benchmark.cmake) +if(CMAKE_SYSTEM_NAME STREQUAL iOS) + set_target_properties(mbgl-vendor-benchmark PROPERTIES XCODE_ATTRIBUTE_IPHONEOS_DEPLOYMENT_TARGET "${IOS_DEPLOYMENT_TARGET}") + set_target_properties(mbgl-vendor-benchmark PROPERTIES XCODE_ATTRIBUTE_ENABLE_BITCODE "YES") + set_target_properties(mbgl-vendor-benchmark PROPERTIES XCODE_ATTRIBUTE_BITCODE_GENERATION_MODE bitcode) + set_target_properties(mbgl-vendor-benchmark PROPERTIES XCODE_ATTRIBUTE_ONLY_ACTIVE_ARCH $<$:YES>) + + set_target_properties(mbgl-benchmark PROPERTIES XCODE_ATTRIBUTE_IPHONEOS_DEPLOYMENT_TARGET "${IOS_DEPLOYMENT_TARGET}") + set_target_properties(mbgl-benchmark PROPERTIES XCODE_ATTRIBUTE_ENABLE_BITCODE "YES") + set_target_properties(mbgl-benchmark PROPERTIES XCODE_ATTRIBUTE_BITCODE_GENERATION_MODE bitcode) + set_target_properties(mbgl-benchmark PROPERTIES XCODE_ATTRIBUTE_ONLY_ACTIVE_ARCH $<$:YES>) +endif() + # Needed for testing private classes get_target_property(MBGL_CORE_PRIVATE_LIBRARIES mbgl-core LINK_LIBRARIES) diff --git a/benchmark/ios/BenchmarkApp_iphoneos13.2-arm64e-debug.xctestrun b/benchmark/ios/BenchmarkApp_iphoneos13.2-arm64e-debug.xctestrun new file mode 100644 index 000000000..4270610f0 --- /dev/null +++ b/benchmark/ios/BenchmarkApp_iphoneos13.2-arm64e-debug.xctestrun @@ -0,0 +1,68 @@ + + + + + BenchmarkAppTests + + BlueprintName + BenchmarkAppTests + BundleIdentifiersForCrashReportEmphasis + + com.mapbox.BenchmarkApp + com.mapbox.BenchmarkAppTests + + CommandLineArguments + + DependentProductPaths + + __TESTROOT__/Debug-iphoneos/BenchmarkApp.app + __TESTROOT__/Debug-iphoneos/BenchmarkApp.app/PlugIns/BenchmarkAppTests.xctest + + EnvironmentVariables + + OS_ACTIVITY_DT_MODE + YES + SQLITE_ENABLE_THREAD_ASSERTIONS + 1 + + IsAppHostedTestBundle + + ProductModuleName + BenchmarkAppTests + RunOrder + 0 + SystemAttachmentLifetime + deleteOnSuccess + TestBundlePath + __TESTHOST__/PlugIns/BenchmarkAppTests.xctest + TestHostBundleIdentifier + com.mapbox.BenchmarkApp + TestHostPath + __TESTROOT__/Debug-iphoneos/BenchmarkApp.app + TestLanguage + + TestRegion + + TestingEnvironmentVariables + + DYLD_FALLBACK_FRAMEWORK_PATH + + DYLD_INSERT_LIBRARIES + __PLATFORMS__/iPhoneOS.platform/Developer/usr/lib/libXCTestBundleInject.dylib + XCInjectBundleInto + unused + + ToolchainsSettingValue + + UITargetAppCommandLineArguments + + UserAttachmentLifetime + deleteOnSuccess + + __xctestrun_metadata__ + + FormatVersion + 1 + + + diff --git a/benchmark/ios/BenchmarkApp_iphoneos13.2-arm64e-release.xctestrun b/benchmark/ios/BenchmarkApp_iphoneos13.2-arm64e-release.xctestrun new file mode 100644 index 000000000..6c6d73f3b --- /dev/null +++ b/benchmark/ios/BenchmarkApp_iphoneos13.2-arm64e-release.xctestrun @@ -0,0 +1,68 @@ + + + + + BenchmarkAppTests + + BlueprintName + BenchmarkAppTests + BundleIdentifiersForCrashReportEmphasis + + com.mapbox.BenchmarkApp + com.mapbox.BenchmarkAppTests + + CommandLineArguments + + DependentProductPaths + + __TESTROOT__/Release-iphoneos/BenchmarkApp.app + __TESTROOT__/Release-iphoneos/BenchmarkApp.app/PlugIns/BenchmarkAppTests.xctest + + EnvironmentVariables + + OS_ACTIVITY_DT_MODE + YES + SQLITE_ENABLE_THREAD_ASSERTIONS + 1 + + IsAppHostedTestBundle + + ProductModuleName + BenchmarkAppTests + RunOrder + 0 + SystemAttachmentLifetime + deleteOnSuccess + TestBundlePath + __TESTHOST__/PlugIns/BenchmarkAppTests.xctest + TestHostBundleIdentifier + com.mapbox.BenchmarkApp + TestHostPath + __TESTROOT__/Release-iphoneos/BenchmarkApp.app + TestLanguage + + TestRegion + + TestingEnvironmentVariables + + DYLD_FALLBACK_FRAMEWORK_PATH + + DYLD_INSERT_LIBRARIES + __PLATFORMS__/iPhoneOS.platform/Developer/usr/lib/libXCTestBundleInject.dylib + XCInjectBundleInto + unused + + ToolchainsSettingValue + + UITargetAppCommandLineArguments + + UserAttachmentLifetime + deleteOnSuccess + + __xctestrun_metadata__ + + FormatVersion + 1 + + + diff --git a/benchmark/ios/Info.plist b/benchmark/ios/Info.plist new file mode 100644 index 000000000..0c6fd6144 --- /dev/null +++ b/benchmark/ios/Info.plist @@ -0,0 +1,62 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleDisplayName + ${PRODUCT_NAME} + CFBundleExecutable + BenchmarkApp + CFBundleIdentifier + com.mapbox.BenchmarkApp + CFBundleGetInfoString + com.mapbox.BenchmarkApp + CFBundleInfoDictionaryVersion + 1.0 + CFBundleLongVersionString + 1.0 + CFBundleName + com.mapbox.BenchmarkApp + CFBundlePackageType + APPL + CFBundleShortVersionString + 1.0 + CFBundleSignature + MBGL + CFBundleVersion + 7877 + CSResourcesFileMapped + + NSHumanReadableCopyright + © 2014–2020 Mapbox + LSMinimumSystemVersion + ${MACOSX_DEPLOYMENT_TARGET} + LSRequiresIPhoneOS + + UILaunchStoryboardName + LaunchScreen + UIMainStoryboardFile + Main + UIBackgroundModes + + fetch + remote-notification + + UIRequiredDeviceCapabilities + + armv7 + + UISupportedInterfaceOrientations + + UIInterfaceOrientationPortrait + UIInterfaceOrientationLandscapeLeft + UIInterfaceOrientationLandscapeRight + + NSAppTransportSecurity + + NSAllowsArbitraryLoads + + + + \ No newline at end of file diff --git a/benchmark/ios/README.md b/benchmark/ios/README.md new file mode 100644 index 000000000..5ee2a9aa3 --- /dev/null +++ b/benchmark/ios/README.md @@ -0,0 +1,7 @@ +# iOS BenchmarkRunner App + +This is a blank single-view-controller iOS app, linked against to a C++ static libraries, plus a a simple unit xctest. We use CMake to create an Xcode-friendly out-of-source build system. In another word, the build system is maintained by `CMakeLists` file,instead of a configured `.xcodeproj` file. + +This CMake project can build the executable BenchmarkApp which is linked to the static C++ library 'mbgl-benchmark'. CMakeLists.txt files are the only build configuration kept in source control. This is in contrast to committing the `.xcodeproj` directory which includes the backing XML, which is nonsensically hard to edit by hand. + +The test instantiates ObjC object of class `IosTestRunner` from the app, the class will instantiates a C++ object from the linked library `mbgl-benchmark` and calls running benchmark function on it. It subsequently deletes the C++ object pointer. In the end, the test will check the existence of test report, which is a json file that attach to xctest result. diff --git a/benchmark/ios/codesigning/BenchmarkApp.app.xcent.template b/benchmark/ios/codesigning/BenchmarkApp.app.xcent.template new file mode 100644 index 000000000..669e8cdc5 --- /dev/null +++ b/benchmark/ios/codesigning/BenchmarkApp.app.xcent.template @@ -0,0 +1,16 @@ + + + + + application-identifier + $TEAM_ID.com.mapbox.BenchmarkApp + com.apple.developer.team-identifier + $TEAM_ID + get-task-allow + + keychain-access-groups + + $TEAM_ID.com.mapbox.BenchmarkApp + + + \ No newline at end of file diff --git a/benchmark/ios/codesigning/BenchmarkAppTests.xctest.xcent.template b/benchmark/ios/codesigning/BenchmarkAppTests.xctest.xcent.template new file mode 100644 index 000000000..4070d8939 --- /dev/null +++ b/benchmark/ios/codesigning/BenchmarkAppTests.xctest.xcent.template @@ -0,0 +1,16 @@ + + + + + application-identifier + $TEAM_ID.com.mapbox.BenchmarkAppTests + com.apple.developer.team-identifier + $TEAM_ID + get-task-allow + + keychain-access-groups + + $TEAM_ID.com.mapbox.BenchmarkAppTests + + + \ No newline at end of file diff --git a/benchmark/ios/codesigning/XCTAutomationSupport.framework.xcent.template b/benchmark/ios/codesigning/XCTAutomationSupport.framework.xcent.template new file mode 100644 index 000000000..4070d8939 --- /dev/null +++ b/benchmark/ios/codesigning/XCTAutomationSupport.framework.xcent.template @@ -0,0 +1,16 @@ + + + + + application-identifier + $TEAM_ID.com.mapbox.BenchmarkAppTests + com.apple.developer.team-identifier + $TEAM_ID + get-task-allow + + keychain-access-groups + + $TEAM_ID.com.mapbox.BenchmarkAppTests + + + \ No newline at end of file diff --git a/benchmark/ios/codesigning/XCTest.framework.xcent.template b/benchmark/ios/codesigning/XCTest.framework.xcent.template new file mode 100644 index 000000000..4070d8939 --- /dev/null +++ b/benchmark/ios/codesigning/XCTest.framework.xcent.template @@ -0,0 +1,16 @@ + + + + + application-identifier + $TEAM_ID.com.mapbox.BenchmarkAppTests + com.apple.developer.team-identifier + $TEAM_ID + get-task-allow + + keychain-access-groups + + $TEAM_ID.com.mapbox.BenchmarkAppTests + + + \ No newline at end of file diff --git a/benchmark/ios/codesigning/codesign_all.sh b/benchmark/ios/codesigning/codesign_all.sh new file mode 100755 index 000000000..70f5aee74 --- /dev/null +++ b/benchmark/ios/codesigning/codesign_all.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +codesign --force --sign $CODESIGNIDENTITY --entitlements ../codesigning/BenchmarkApp.app.xcent --timestamp=none ./Release-iphoneos/BenchmarkApp.app +codesign --force --sign $CODESIGNIDENTITY --deep --preserve-metadata=identifier,entitlements,flags --timestamp=none ./Release-iphoneos/BenchmarkApp.app/Frameworks/libXCTestSwiftSupport.dylib +codesign --force --sign $CODESIGNIDENTITY --deep --preserve-metadata=identifier,entitlements,flags --timestamp=none ./Release-iphoneos/BenchmarkApp.app/Frameworks/libXCTestBundleInject.dylib +codesign --force --sign $CODESIGNIDENTITY --deep --preserve-metadata=identifier,entitlements,flags --timestamp=none ./Release-iphoneos/BenchmarkApp.app/Frameworks/XCTAutomationSupport.framework +codesign --force --sign $CODESIGNIDENTITY --deep --preserve-metadata=identifier,entitlements,flags --timestamp=none ./Release-iphoneos/BenchmarkApp.app/Frameworks/XCTest.framework +codesign --force --sign $CODESIGNIDENTITY --deep --entitlements ../codesigning/BenchmarkAppTests.xctest.xcent --timestamp=none ./Release-iphoneos/BenchmarkApp.app/PlugIns/BenchmarkAppTests.xctest diff --git a/benchmark/ios/codesigning/generate-entitlements.swift b/benchmark/ios/codesigning/generate-entitlements.swift new file mode 100755 index 000000000..83834483b --- /dev/null +++ b/benchmark/ios/codesigning/generate-entitlements.swift @@ -0,0 +1,40 @@ +#!/usr/bin/swift + +/// Generates Entitlement files .xcent for codesigning, containing the correct team id +import Swift +import Foundation + +let fileManager = FileManager.default + +let contentsOfCurrentWorkingDirectory = try FileManager.default.contentsOfDirectory(at: URL(fileURLWithPath: "./codesigning/"), includingPropertiesForKeys: nil, options: []) + +guard let rawTeamIdEnv = getenv("IOS_DEVELOPMENT_TEAM") else { + print("Please add a IOS_DEVELOPMENT_TEAM environment variable.") + exit(1) +} + +let teamId2 = String(utf8String: rawTeamIdEnv) + +for file in contentsOfCurrentWorkingDirectory { + if (file.pathExtension == "template") { + var content = "" + do { + content = try String(contentsOf: file, encoding: .utf8) + } + catch { + print("Error reading xcent file: \(error).") + } + + content = content.replacingOccurrences(of: "$TEAM_ID", with: teamId2!) + + var targetFile = file + targetFile.deletePathExtension() + do { + try content.write(to: targetFile, atomically: false, encoding: .utf8) + print("Entitlement " + targetFile.absoluteString + " generated.") + } + catch { + print("Error writing xcent file: \(error).") + } + } +} \ No newline at end of file diff --git a/benchmark/ios/codesigning/libXCTestSwiftSupport.dylib.xcent.template b/benchmark/ios/codesigning/libXCTestSwiftSupport.dylib.xcent.template new file mode 100644 index 000000000..4070d8939 --- /dev/null +++ b/benchmark/ios/codesigning/libXCTestSwiftSupport.dylib.xcent.template @@ -0,0 +1,16 @@ + + + + + application-identifier + $TEAM_ID.com.mapbox.BenchmarkAppTests + com.apple.developer.team-identifier + $TEAM_ID + get-task-allow + + keychain-access-groups + + $TEAM_ID.com.mapbox.BenchmarkAppTests + + + \ No newline at end of file diff --git a/benchmark/ios/fastlane/Appfile b/benchmark/ios/fastlane/Appfile new file mode 100644 index 000000000..af5ebdffa --- /dev/null +++ b/benchmark/ios/fastlane/Appfile @@ -0,0 +1,3 @@ +app_identifier("com.mapbox.BenchmarkApp") # The bundle identifier of your app +apple_id ENV['MAPBOX_APPLE_ID'] +team_id ENV['MAPBOX_APPLE_TEAM_ID'] # Developer Portal Team ID \ No newline at end of file diff --git a/benchmark/ios/fastlane/Matchfile b/benchmark/ios/fastlane/Matchfile new file mode 100644 index 000000000..f8fe80463 --- /dev/null +++ b/benchmark/ios/fastlane/Matchfile @@ -0,0 +1,5 @@ +git_url("git@github.com:mapbox/apple-certificates.git") +type("development") # The default type, can be: appstore, adhoc, enterprise or development +app_identifier(["com.mapbox.BenchmarkApp", "com.mapbox.BenchmarkAppTests"]) +username ENV['MAPBOX_APPLE_ID'] +keychain_name("fastlane_keychain") \ No newline at end of file diff --git a/benchmark/ios/iosTestRunner.h b/benchmark/ios/iosTestRunner.h new file mode 100644 index 000000000..d8cd7991b --- /dev/null +++ b/benchmark/ios/iosTestRunner.h @@ -0,0 +1,12 @@ +#import + +NSString *dataDir = @"benchmark-data"; +NSString *resultFilePath = @"benchmark/results.json"; + +__attribute__((visibility ("default"))) +@interface IosTestRunner : NSObject + +- (NSString*) getResultPath; +- (BOOL) getTestStatus; + +@end diff --git a/benchmark/ios/ios_test_runner.cpp b/benchmark/ios/ios_test_runner.cpp new file mode 100644 index 000000000..25bd3add5 --- /dev/null +++ b/benchmark/ios/ios_test_runner.cpp @@ -0,0 +1,34 @@ +#include + +#include + +#include + +#include +#include + +#define EXPORT __attribute__((visibility("default"))) + +EXPORT +bool TestRunner::startTest(const std::string& basePath) { + std::vector arguments = {"mbgl-benchmark-runner", + "--benchmark_repetitions=3", + "--benchmark_format=json", + "--benchmark_out=" + basePath + "/benchmark/results.json"}; + std::vector argv; + for (const auto& arg : arguments) { + argv.push_back(const_cast(arg.data())); + } + argv.push_back(nullptr); + + if (chdir(basePath.c_str())) { + mbgl::Log::Error(mbgl::Event::General, "Failed to change the directory to " + basePath); + return false; + } + + mbgl::Log::Info(mbgl::Event::General, "Start BenchmarkRunner"); + int status = mbgl::runBenchmark(static_cast(argv.size()), argv.data()); + mbgl::Log::Info(mbgl::Event::General, "BenchmarkRunner finished with status: '%d'", status); + + return status == 0; +} diff --git a/benchmark/ios/tests/Info.plist b/benchmark/ios/tests/Info.plist new file mode 100644 index 000000000..83e7a9a08 --- /dev/null +++ b/benchmark/ios/tests/Info.plist @@ -0,0 +1,22 @@ + + + + + CFBundleDevelopmentRegion + en + CFBundleExecutable + BenchmarkAppTests + CFBundleIdentifier + com.mapbox.BenchmarkAppTests + CFBundleInfoDictionaryVersion + 1.0 + CFBundleName + BenchmarkAppTests + CFBundlePackageType + BNDL + CFBundleShortVersionString + 1.0 + CFBundleVersion + 1 + + diff --git a/platform/ios/ios.cmake b/platform/ios/ios.cmake index f9f6f540c..eb1d339c9 100644 --- a/platform/ios/ios.cmake +++ b/platform/ios/ios.cmake @@ -265,4 +265,76 @@ if(MBGL_IOS_UNIT_TEST) set_target_properties(UnitTestsAppTests PROPERTIES XCODE_ATTRIBUTE_CODE_SIGNING_REQUIRED "NO") endif() +if(MBGL_IOS_BENCHMARK) + execute_process(COMMAND ditto ${PROJECT_SOURCE_DIR}/benchmark/fixtures ${CMAKE_CURRENT_BINARY_DIR}/benchmark-data/benchmark/fixtures) + execute_process( + COMMAND + ditto ${PROJECT_SOURCE_DIR}/test/fixtures/api/assets/streets + ${CMAKE_CURRENT_BINARY_DIR}/benchmark-data/test/fixtures/api/assets/streets + ) + + set(RESOURCES ${PROJECT_SOURCE_DIR}/platform/ios/test/common/Main.storyboard + ${PROJECT_SOURCE_DIR}/platform/ios/test/common/LaunchScreen.storyboard ${CMAKE_CURRENT_BINARY_DIR}/benchmark-data) + + add_executable( + BenchmarkApp + ${PROJECT_SOURCE_DIR}/platform/ios/test/common/ios_test_runner.hpp + ${PROJECT_SOURCE_DIR}/platform/ios/test/common/AppDelegate.h + ${PROJECT_SOURCE_DIR}/platform/ios/test/common/AppDelegate.m + ${PROJECT_SOURCE_DIR}/platform/ios/test/common/iosTestRunner.mm + ${PROJECT_SOURCE_DIR}/platform/ios/test/common/main.m + ${PROJECT_SOURCE_DIR}/benchmark/ios/ios_test_runner.cpp + ${PROJECT_SOURCE_DIR}/benchmark/ios/iosTestRunner.h + ${RESOURCES} + ) + initialize_ios_target(BenchmarkApp) + + target_include_directories( + BenchmarkApp + PUBLIC + ${PROJECT_SOURCE_DIR}/include + ${PROJECT_SOURCE_DIR}/platform/ios/test/common + ${PROJECT_SOURCE_DIR}/benchmark/include + ${PROJECT_SOURCE_DIR}/benchmark/ios + ) + + target_link_libraries( + BenchmarkApp + PRIVATE + "-framework CoreGraphics" + "-framework CoreLocation" + "-framework Foundation" + "-framework OpenGLES" + "-framework QuartzCore" + "-framework UIKit" + mbgl-compiler-options + -Wl,-force_load + mbgl-benchmark + ) + + set_target_properties(BenchmarkApp PROPERTIES MACOSX_BUNDLE TRUE) + set_target_properties(BenchmarkApp PROPERTIES MACOSX_BUNDLE_IDENTIFIER com.mapbox.BenchmarkApp) + set_target_properties(BenchmarkApp PROPERTIES MACOSX_BUNDLE_INFO_PLIST ${PROJECT_SOURCE_DIR}/benchmark/ios/Info.plist) + set_target_properties(BenchmarkApp PROPERTIES RESOURCE "${RESOURCES}") + set_target_properties(BenchmarkApp PROPERTIES XCODE_ATTRIBUTE_CODE_SIGN_IDENTITY "") + set_target_properties(BenchmarkApp PROPERTIES XCODE_ATTRIBUTE_CODE_SIGNING_REQUIRED "NO") + + find_package(XCTest REQUIRED) + + xctest_add_bundle(BenchmarkAppTests BenchmarkApp ${PROJECT_SOURCE_DIR}/platform/ios/test/common/Tests.m) + + target_include_directories( + BenchmarkAppTests + PUBLIC ${PROJECT_SOURCE_DIR}/benchmark/ios + ) + + xctest_add_test(XCTest.BenchmarkApp BenchmarkAppTests) + + set_target_properties(BenchmarkAppTests PROPERTIES XCODE_ATTRIBUTE_IPHONEOS_DEPLOYMENT_TARGET "${IOS_DEPLOYMENT_TARGET}") + set_target_properties(BenchmarkAppTests PROPERTIES XCODE_ATTRIBUTE_ONLY_ACTIVE_ARCH $<$:YES>) + set_target_properties(BenchmarkAppTests PROPERTIES MACOSX_BUNDLE_INFO_PLIST ${PROJECT_SOURCE_DIR}/benchmark/ios/tests/Info.plist) + set_target_properties(BenchmarkAppTests PROPERTIES XCODE_ATTRIBUTE_CODE_SIGN_IDENTITY "") + set_target_properties(BenchmarkAppTests PROPERTIES XCODE_ATTRIBUTE_CODE_SIGNING_REQUIRED "NO") +endif() + unset(IOS_DEPLOYMENT_TARGET CACHE)