[test-runner] Enable rebaseline for next-ios-render-test-runner (#16147)

.gitmodules

@@ -70,3 +70,6 @@
 [submodule "vendor/googletest"]
 	path = vendor/googletest
 	url = https://github.com/google/googletest.git
+[submodule "vendor/zip-archive"]
+	path = vendor/zip-archive
+	url = https://github.com/ZipArchive/ZipArchive.git

circle.yml

@@ -30,6 +30,7 @@ workflows:
             - linux-clang8-release
             - linux-gcc8-release
             - macos-xcode11-release
+            - ios-render-test-runner
       - build-template:
           name: android-armeabi-v7a-release
           executor_name: ubuntu-disco
@@ -389,6 +390,12 @@ jobs:
               xcparse attachments *.xcresult && ls
               cd ../.. && mkdir -p /tmp/tests/ios-render-test
               if ls render-test-app/TestLogs/*.html 1> /dev/null 2>&1; then cp render-test-app/TestLogs/*.html /tmp/tests/ios-render-test; fi
+              if [[ -n $(ls render-test-app/TestLogs/*.zip 2>/dev/null) ]]; then
+                cp render-test-app/TestLogs/*.zip metrics/next-ios-render-test-runner/
+                cd metrics/next-ios-render-test-runner/
+                unzip *.zip
+                rm -rf *.zip
+              fi
             fi
       - save
   android-benchmark-runner:

next/platform/ios/ios.cmake

@@ -95,6 +95,10 @@ target_link_libraries(
 
 if(MBGL_IOS_RENDER_TEST)
     set(CMAKE_OSX_ARCHITECTURES "armv7;i386;x86_64;arm64")
+
+    include(${PROJECT_SOURCE_DIR}/vendor/zip-archive.cmake)
+    initialize_ios_target(mbgl-vendor-zip-archive)
+
     set(PREPARE_CMD "${MBGL_ROOT}/render-test/ios/setup_test_data.sh")
     message("COMMAND: ${PREPARE_CMD}")
     execute_process(COMMAND ${PREPARE_CMD} RESULT_VARIABLE CMD_ERROR)
@@ -160,6 +164,7 @@ if(MBGL_IOS_RENDER_TEST)
             "-framework QuartzCore"
             "-framework UIKit"
             mbgl-render-test
+            mbgl-vendor-zip-archive
     )
 
     find_package(XCTest REQUIRED)

render-test/ios/README.md

@@ -0,0 +1,7 @@
+# iOS RenderTestRunner App
+
+This is a blank single-view-controller iOS app, linked against to a C++ static libraries, plus a a simple unit xctest. We use CMake to create an Xcode-friendly out-of-source build system. In another word, the build system is maintained by `CMakeLists` file,instead of a configured `.xcodeproj` file.
+
+This CMake project can build the executable RenderTestApp which is linked to the static C++ library 'mbgl-render-test'. CMakeLists.txt files are the only build configuration kept in source control. This is in contrast to committing the `.xcodeproj` directory which includes the backing XML, which is nonsensically hard to edit by hand.
+
+The test instantiates ObjC object of class `IosTestRunner` from the app, the class will instantiates a C++ object from the linked library `mbgl-render-test` and calls running render test function on it. It subsequently deletes the C++ object pointer. In the end, the test will check the existence of test reports, which are html files that attach to xctest result.

render-test/ios/iosTestRunner.h

@@ -5,6 +5,7 @@ __attribute__((visibility ("default")))
 
 - (NSString*) getStyleResultPath;
 - (NSString*) getMetricResultPath;
+- (NSString*) getMetricPath;
 - (BOOL) getTestStatus;
 
 @end

render-test/ios/iosTestRunner.mm

@@ -2,6 +2,8 @@
 
 #include "ios_test_runner.hpp"
 
+#import "SSZipArchive.h"
+
 #include <string>
 
 @interface IosTestRunner ()
@@ -90,6 +92,45 @@
                 NSLog(@"Metric test result file '%@' doese not exit ", self.metricResultPath);
                 self.testStatus = NO;
             }
+
+            NSString *rebaselinePath = [path stringByAppendingPathComponent:@"/baselines"];
+            BOOL needArchiving = NO;
+            BOOL isDir = NO;
+            NSArray *subpaths = [[NSArray alloc] init];
+            if ([fileManager fileExistsAtPath:rebaselinePath isDirectory: &isDir] && isDir){
+                subpaths = [fileManager subpathsAtPath:rebaselinePath];
+                for(NSString *path in subpaths)
+                {
+                    NSString *longPath = [rebaselinePath stringByAppendingPathComponent:path];
+                    if([fileManager fileExistsAtPath:longPath isDirectory:&isDir] && !isDir)
+                    {
+                        needArchiving = YES;
+                        break;
+                    }
+                }
+            }
+            else {
+               NSLog(@"Metric path '%@' doese not exit ", rebaselinePath);
+            }
+
+            if (needArchiving) {
+                NSString *archivePath = [path stringByAppendingString:@"/metrics.zip"];
+                BOOL success = [SSZipArchive createZipFileAtPath:archivePath
+                withContentsOfDirectory:rebaselinePath
+                    keepParentDirectory:NO
+                       compressionLevel:-1
+                               password:nil
+                                    AES:YES
+                        progressHandler:nil];
+
+                if (success) {
+                    NSLog(@"Successfully archive all of the metrics into metrics.zip");
+                    self.metricPath =  archivePath;
+                }
+                else {
+                    NSLog(@"Failed to archive rebaselined metrics into metrics.zip");
+                }
+            }
         }
 
         delete self.runner;
@@ -99,14 +140,18 @@
 }
 
 - (NSString*) getStyleResultPath {
-   return self.styleResultPath;
+    return self.styleResultPath;
 }
 
 - (NSString*) getMetricResultPath {
-   return self.metricResultPath;
+    return self.metricResultPath;
+}
+
+- (NSString*) getMetricPath {
+    return self.metricPath;
 }
 
 - (BOOL) getTestStatus {
-   return self.testStatus;
+    return self.testStatus;
 }
 @end

render-test/ios/tests/Tests.m

@@ -31,7 +31,7 @@
     XCTAssert(attachment1URL, @"Failed to attach test result '%@'", styleResult);
     attachment1URL.lifetime = XCTAttachmentLifetimeKeepAlways;
     [self addAttachment:attachment1URL];
-    
+
     fileFound = [fileManager fileExistsAtPath: metricResult];
     XCTAssert(fileFound, @"Test result html '%@' doese not exit", metricResult);
     NSURL *metricURL = [NSURL fileURLWithPath:metricResult];
@@ -40,6 +40,18 @@
     attachment2URL.lifetime = XCTAttachmentLifetimeKeepAlways;
     [self addAttachment:attachment2URL];
 
+    NSString* metrics = [runner getMetricPath];
+    if (metrics && [fileManager fileExistsAtPath: metrics]) {
+        NSURL *metricsURL = [NSURL fileURLWithPath:metrics];
+        XCTAttachment *attachmentMetricsURL = [XCTAttachment attachmentWithContentsOfFileAtURL: metricsURL];
+        XCTAssert(attachmentMetricsURL, @"Failed to attach test rebaselined metrics '%@'", metrics);
+        attachmentMetricsURL.lifetime = XCTAttachmentLifetimeKeepAlways;
+        [self addAttachment:attachmentMetricsURL];
+    }
+    else {
+        NSLog(@"No rebaselined metrics are found");
+    }
+
     BOOL success = [runner getTestStatus];
     XCTAssert(success, @"IOSTestRunner reports error because some of the tests are not passed, please check the test report");
 }

render-test/manifest_parser.cpp

@@ -280,6 +280,8 @@ mbgl::optional<Manifest> ManifestParser::parseManifest(const std::string& manife
             std::vector<mbgl::filesystem::path> expectedMetricPaths{expectedMetricPath};
 #if defined(__ANDROID__)
             expectedMetricPaths.emplace_back("/sdcard/baselines/");
+#elif defined(__APPLE__)
+            expectedMetricPaths.emplace_back(manifest.manifestPath + "/baselines/");
 #endif
             testPaths.emplace_back(testPath,
                                    getTestExpectations(defaultExpectationPath, testId, expectationPaths),

vendor/zip-archive.cmake

@@ -0,0 +1,61 @@
+if(TARGET mbgl-vendor-zip-archive)
+    return()
+endif()
+
+add_library(
+    mbgl-vendor-zip-archive STATIC
+    ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/SSZipArchive.h
+    ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/SSZipArchive.m
+    ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/SSZipCommon.h
+    ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/ZipArchive.h
+    ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz.h
+    ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_compat.c
+    ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_compat.h
+    ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_crypt.c
+    ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_crypt.h
+    ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_crypt_apple.c
+    ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_os.c
+    ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_os.h
+    ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_os_posix.c
+    ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_strm.c
+    ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_strm.h
+    ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_strm_buf.c
+    ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_strm_buf.h
+    ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_strm_mem.c
+    ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_strm_mem.h
+    ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_strm_os.h
+    ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_strm_os_posix.c
+    ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_strm_pkcrypt.c
+    ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_strm_pkcrypt.h
+    ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_strm_split.c
+    ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_strm_split.h
+    ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_strm_wzaes.c
+    ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_strm_wzaes.h
+    ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_strm_zlib.c
+    ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_strm_zlib.h
+    ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_zip.c
+    ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_zip.h
+    ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_zip_rw.c
+    ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive/minizip/mz_zip_rw.h
+)
+
+target_link_libraries(
+    mbgl-vendor-zip-archive
+    PRIVATE z
+)
+
+target_include_directories(
+    mbgl-vendor-zip-archive SYSTEM
+    INTERFACE ${CMAKE_CURRENT_LIST_DIR}/zip-archive/SSZipArchive
+)
+
+target_compile_definitions(
+    mbgl-vendor-zip-archive
+    PRIVATE
+        HAVE_INTTYPES_H
+        HAVE_PKCRYPT
+        HAVE_STDINT_H
+        HAVE_WZAES
+        HAVE_ZLIB
+)
+set_property(TARGET mbgl-vendor-zip-archive PROPERTY FOLDER RenderTestApp)