[test-runner] Enable rebaseline for next-ios-render-test-runner (#16147)

render-test/ios/README.md

@@ -0,0 +1,7 @@
+# iOS RenderTestRunner App
+
+This is a blank single-view-controller iOS app, linked against to a C++ static libraries, plus a a simple unit xctest. We use CMake to create an Xcode-friendly out-of-source build system. In another word, the build system is maintained by `CMakeLists` file,instead of a configured `.xcodeproj` file.
+
+This CMake project can build the executable RenderTestApp which is linked to the static C++ library 'mbgl-render-test'. CMakeLists.txt files are the only build configuration kept in source control. This is in contrast to committing the `.xcodeproj` directory which includes the backing XML, which is nonsensically hard to edit by hand.
+
+The test instantiates ObjC object of class `IosTestRunner` from the app, the class will instantiates a C++ object from the linked library `mbgl-render-test` and calls running render test function on it. It subsequently deletes the C++ object pointer. In the end, the test will check the existence of test reports, which are html files that attach to xctest result.

render-test/ios/iosTestRunner.h

@@ -5,6 +5,7 @@ __attribute__((visibility ("default")))
 
 - (NSString*) getStyleResultPath;
 - (NSString*) getMetricResultPath;
+- (NSString*) getMetricPath;
 - (BOOL) getTestStatus;
 
 @end

render-test/ios/iosTestRunner.mm

@@ -2,6 +2,8 @@
 
 #include "ios_test_runner.hpp"
 
+#import "SSZipArchive.h"
+
 #include <string>
 
 @interface IosTestRunner ()
@@ -90,6 +92,45 @@
                 NSLog(@"Metric test result file '%@' doese not exit ", self.metricResultPath);
                 self.testStatus = NO;
             }
+
+            NSString *rebaselinePath = [path stringByAppendingPathComponent:@"/baselines"];
+            BOOL needArchiving = NO;
+            BOOL isDir = NO;
+            NSArray *subpaths = [[NSArray alloc] init];
+            if ([fileManager fileExistsAtPath:rebaselinePath isDirectory: &isDir] && isDir){
+                subpaths = [fileManager subpathsAtPath:rebaselinePath];
+                for(NSString *path in subpaths)
+                {
+                    NSString *longPath = [rebaselinePath stringByAppendingPathComponent:path];
+                    if([fileManager fileExistsAtPath:longPath isDirectory:&isDir] && !isDir)
+                    {
+                        needArchiving = YES;
+                        break;
+                    }
+                }
+            }
+            else {
+               NSLog(@"Metric path '%@' doese not exit ", rebaselinePath);
+            }
+
+            if (needArchiving) {
+                NSString *archivePath = [path stringByAppendingString:@"/metrics.zip"];
+                BOOL success = [SSZipArchive createZipFileAtPath:archivePath
+                withContentsOfDirectory:rebaselinePath
+                    keepParentDirectory:NO
+                       compressionLevel:-1
+                               password:nil
+                                    AES:YES
+                        progressHandler:nil];
+
+                if (success) {
+                    NSLog(@"Successfully archive all of the metrics into metrics.zip");
+                    self.metricPath =  archivePath;
+                }
+                else {
+                    NSLog(@"Failed to archive rebaselined metrics into metrics.zip");
+                }
+            }
         }
 
         delete self.runner;
@@ -99,14 +140,18 @@
 }
 
 - (NSString*) getStyleResultPath {
-   return self.styleResultPath;
+    return self.styleResultPath;
 }
 
 - (NSString*) getMetricResultPath {
-   return self.metricResultPath;
+    return self.metricResultPath;
+}
+
+- (NSString*) getMetricPath {
+    return self.metricPath;
 }
 
 - (BOOL) getTestStatus {
-   return self.testStatus;
+    return self.testStatus;
 }
 @end

render-test/ios/tests/Tests.m

@@ -31,7 +31,7 @@
     XCTAssert(attachment1URL, @"Failed to attach test result '%@'", styleResult);
     attachment1URL.lifetime = XCTAttachmentLifetimeKeepAlways;
     [self addAttachment:attachment1URL];
-    
+
     fileFound = [fileManager fileExistsAtPath: metricResult];
     XCTAssert(fileFound, @"Test result html '%@' doese not exit", metricResult);
     NSURL *metricURL = [NSURL fileURLWithPath:metricResult];
@@ -40,6 +40,18 @@
     attachment2URL.lifetime = XCTAttachmentLifetimeKeepAlways;
     [self addAttachment:attachment2URL];
 
+    NSString* metrics = [runner getMetricPath];
+    if (metrics && [fileManager fileExistsAtPath: metrics]) {
+        NSURL *metricsURL = [NSURL fileURLWithPath:metrics];
+        XCTAttachment *attachmentMetricsURL = [XCTAttachment attachmentWithContentsOfFileAtURL: metricsURL];
+        XCTAssert(attachmentMetricsURL, @"Failed to attach test rebaselined metrics '%@'", metrics);
+        attachmentMetricsURL.lifetime = XCTAttachmentLifetimeKeepAlways;
+        [self addAttachment:attachmentMetricsURL];
+    }
+    else {
+        NSLog(@"No rebaselined metrics are found");
+    }
+
     BOOL success = [runner getTestStatus];
     XCTAssert(success, @"IOSTestRunner reports error because some of the tests are not passed, please check the test report");
 }

render-test/manifest_parser.cpp

@@ -280,6 +280,8 @@ mbgl::optional<Manifest> ManifestParser::parseManifest(const std::string& manife
             std::vector<mbgl::filesystem::path> expectedMetricPaths{expectedMetricPath};
 #if defined(__ANDROID__)
             expectedMetricPaths.emplace_back("/sdcard/baselines/");
+#elif defined(__APPLE__)
+            expectedMetricPaths.emplace_back(manifest.manifestPath + "/baselines/");
 #endif
             testPaths.emplace_back(testPath,
                                    getTestExpectations(defaultExpectationPath, testId, expectationPaths),