Storage hardening; Add error handling capabilities.

Segment.xcodeproj/project.pbxproj

@@ -34,6 +34,7 @@
 		465879BA2686560C00180335 /* watchOSDelegation.swift in Sources */ = {isa = PBXBuildFile; fileRef = 465879B82686560C00180335 /* watchOSDelegation.swift */; };
 		465879BB2686560C00180335 /* watchOSLifecycleMonitor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 465879B92686560C00180335 /* watchOSLifecycleMonitor.swift */; };
 		4663C729267A799100ADDD1A /* QueueTimer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 4663C728267A799100ADDD1A /* QueueTimer.swift */; };
+		466EC2CE28FB7D5D001B384E /* OutputFileStream.swift in Sources */ = {isa = PBXBuildFile; fileRef = 466EC2CD28FB7D5D001B384E /* OutputFileStream.swift */; };
 		46A018C225E5857D00F9CCD8 /* Context.swift in Sources */ = {isa = PBXBuildFile; fileRef = 46A018C125E5857D00F9CCD8 /* Context.swift */; };
 		46A018D425E6C9C200F9CCD8 /* LinuxUtils.swift in Sources */ = {isa = PBXBuildFile; fileRef = 46A018D325E6C9C200F9CCD8 /* LinuxUtils.swift */; };
 		46A018DA25E97FDF00F9CCD8 /* AppleUtils.swift in Sources */ = {isa = PBXBuildFile; fileRef = 46A018D925E97FDF00F9CCD8 /* AppleUtils.swift */; };
@@ -55,15 +56,13 @@
 		96259F8626CF1D45008AE301 /* ConsoleTarget.swift in Sources */ = {isa = PBXBuildFile; fileRef = 96259F8526CF1D45008AE301 /* ConsoleTarget.swift */; };
 		96469AF82706225900AC5772 /* SystemTarget.swift in Sources */ = {isa = PBXBuildFile; fileRef = 96469AF72706225900AC5772 /* SystemTarget.swift */; };
 		966945D7259BDCDD00271339 /* HTTPClient.swift in Sources */ = {isa = PBXBuildFile; fileRef = 967C40ED259A7311008EB0B6 /* HTTPClient.swift */; };
-		967C40DA258D472C008EB0B6 /* SegmentLog_Tests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 967C40D9258D472C008EB0B6 /* SegmentLog_Tests.swift */; };
 		967C40E3258D4DAF008EB0B6 /* Metrics_Tests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 967C40E2258D4DAF008EB0B6 /* Metrics_Tests.swift */; };
 		9692724E25A4E5B7009B5298 /* Startup.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9692724D25A4E5B7009B5298 /* Startup.swift */; };
 		9692726825A583A6009B5298 /* SegmentDestination.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9692726725A583A6009B5298 /* SegmentDestination.swift */; };
 		96A9624E2810C6B80011DE54 /* macOSLifecycleEvents.swift in Sources */ = {isa = PBXBuildFile; fileRef = 96A9624D2810C6B80011DE54 /* macOSLifecycleEvents.swift */; };
 		96A9668927BC137F00078F8B /* iOSLifecycle_Tests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 96A9668827BC137F00078F8B /* iOSLifecycle_Tests.swift */; };
 		96C33A9C25880A5E00F3D538 /* SegmentLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = 96C33A9B25880A5E00F3D538 /* SegmentLog.swift */; };
 		96C33AB1258961F500F3D538 /* Settings.swift in Sources */ = {isa = PBXBuildFile; fileRef = 96C33AB0258961F500F3D538 /* Settings.swift */; };
-		96C95B16271DE22700C3EB9A /* LogTarget_Tests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 96C95B15271DE22600C3EB9A /* LogTarget_Tests.swift */; };
 		96DBF37B26F39B5500724B0B /* Timeline_Tests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 96DBF37A26F39B5500724B0B /* Timeline_Tests.swift */; };
 		A31A16262576B6F200C9CDDF /* Timeline.swift in Sources */ = {isa = PBXBuildFile; fileRef = A31A16252576B6F200C9CDDF /* Timeline.swift */; };
 		A31A162F2576B73F00C9CDDF /* State.swift in Sources */ = {isa = PBXBuildFile; fileRef = A31A162E2576B73F00C9CDDF /* State.swift */; };
@@ -125,6 +124,7 @@
 		465879B82686560C00180335 /* watchOSDelegation.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = watchOSDelegation.swift; sourceTree = "<group>"; };
 		465879B92686560C00180335 /* watchOSLifecycleMonitor.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = watchOSLifecycleMonitor.swift; sourceTree = "<group>"; };
 		4663C728267A799100ADDD1A /* QueueTimer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = QueueTimer.swift; sourceTree = "<group>"; };
+		466EC2CD28FB7D5D001B384E /* OutputFileStream.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = OutputFileStream.swift; sourceTree = "<group>"; };
 		46A018C125E5857D00F9CCD8 /* Context.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Context.swift; sourceTree = "<group>"; };
 		46A018D325E6C9C200F9CCD8 /* LinuxUtils.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LinuxUtils.swift; sourceTree = "<group>"; };
 		46A018D925E97FDF00F9CCD8 /* AppleUtils.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppleUtils.swift; sourceTree = "<group>"; };
@@ -356,6 +356,7 @@
 				A3AEE1492580208E002386EB /* iso8601.swift */,
 				46FE4CDF25A53FAD003A7362 /* Storage.swift */,
 				4621080B2605332D00EBC4A8 /* KeyPath.swift */,
+				466EC2CD28FB7D5D001B384E /* OutputFileStream.swift */,
 				46022770261F7A4800A9E913 /* Atomic.swift */,
 				46E382E62654429A00BA2502 /* Utils.swift */,
 				4663C728267A799100ADDD1A /* QueueTimer.swift */,
@@ -507,7 +508,7 @@
 			attributes = {
 				LastSwiftMigration = 9999;
 				LastSwiftUpdateCheck = 1220;
-				LastUpgradeCheck = 1310;
+				LastUpgradeCheck = 1340;
 			};
 			buildConfigurationList = OBJ_2 /* Build configuration list for PBXProject "Segment" */;
 			compatibilityVersion = "Xcode 3.2";
@@ -570,6 +571,7 @@
 				9692726825A583A6009B5298 /* SegmentDestination.swift in Sources */,
 				4602276C261E7BF900A9E913 /* iOSDelegation.swift in Sources */,
 				46A018D425E6C9C200F9CCD8 /* LinuxUtils.swift in Sources */,
+				466EC2CE28FB7D5D001B384E /* OutputFileStream.swift in Sources */,
 				96C33A9C25880A5E00F3D538 /* SegmentLog.swift in Sources */,
 				46FE4C9725A3F35E003A7362 /* macOSLifecycleMonitor.swift in Sources */,
 				9620862C2575C0C800314F8D /* Events.swift in Sources */,
@@ -596,11 +598,9 @@
 				46210811260538BE00EBC4A8 /* KeyPath_Tests.swift in Sources */,
 				967C40E3258D4DAF008EB0B6 /* Metrics_Tests.swift in Sources */,
 				96A9668927BC137F00078F8B /* iOSLifecycle_Tests.swift in Sources */,
-				96C95B16271DE22700C3EB9A /* LogTarget_Tests.swift in Sources */,
 				A31A16512576C47400C9CDDF /* JSON_Tests.swift in Sources */,
 				46FE4D1D25A7A850003A7362 /* Storage_Tests.swift in Sources */,
 				46FE4CFB25A6C671003A7362 /* TestUtilities.swift in Sources */,
-				967C40DA258D472C008EB0B6 /* SegmentLog_Tests.swift in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};
@@ -792,7 +792,7 @@
 				SWIFT_ACTIVE_COMPILATION_CONDITIONS = "$(inherited) SWIFT_PACKAGE DEBUG";
 				SWIFT_OPTIMIZATION_LEVEL = "-Onone";
 				SWIFT_VERSION = 5.0;
-				TVOS_DEPLOYMENT_TARGET = 11.0;
+				TVOS_DEPLOYMENT_TARGET = 12.0;
 				USE_HEADERMAP = NO;
 				WATCHOS_DEPLOYMENT_TARGET = 7.1;
 			};
@@ -865,7 +865,7 @@
 				SWIFT_COMPILATION_MODE = wholemodule;
 				SWIFT_OPTIMIZATION_LEVEL = "-O";
 				SWIFT_VERSION = 5.0;
-				TVOS_DEPLOYMENT_TARGET = 11.0;
+				TVOS_DEPLOYMENT_TARGET = 12.0;
 				USE_HEADERMAP = NO;
 				WATCHOS_DEPLOYMENT_TARGET = 7.1;
 			};

Segment.xcodeproj/xcshareddata/xcschemes/Segment-Package.xcscheme

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <Scheme
-   LastUpgradeVersion = "1310"
+   LastUpgradeVersion = "1340"
    version = "1.3">
    <BuildAction
       parallelizeBuildables = "YES"

Sources/Segment/Analytics.swift

@@ -31,7 +31,7 @@ public class Analytics {
         self.configuration = configuration
 
         store = Store()
-        storage = Storage(store: self.store, writeKey: configuration.values.writeKey)
+        storage = Storage(store: self.store, writeKey: configuration.values.writeKey, monitor: configuration.values.storageMonitor)
         timeline = Timeline()
 
         // provide our default state

Sources/Segment/Configuration.swift

@@ -30,6 +30,7 @@ public class Configuration {
         var autoAddSegmentDestination: Bool = true
         var apiHost: String = HTTPClient.getDefaultAPIHost()
         var cdnHost: String = HTTPClient.getDefaultCDNHost()
+        var storageMonitor: ((Error) -> Void)?
     }
     internal var values: Values
 
@@ -95,5 +96,11 @@ public extension Configuration {
         values.cdnHost = value
         return self
     }
+
+    @discardableResult
+    func storageMonitor(_ value: @escaping (Error) -> Void) -> Configuration {
+        values.storageMonitor = value
+        return self
+    }
 }
 

Sources/Segment/Plugins/SegmentDestination.swift

@@ -39,7 +39,7 @@ public class SegmentDestination: DestinationPlugin {
         var cleanup: CleanupClosure? = nil
     }
 
-    private var httpClient: HTTPClient?
+    internal var httpClient: HTTPClient?
     private var uploads = [UploadTaskInfo]()
     private let uploadsQueue = DispatchQueue(label: "uploadsQueue.segment.com")
     private var storage: Storage?

Sources/Segment/Utilities/HTTPClient.swift

@@ -20,7 +20,7 @@ public class HTTPClient {
     private static let defaultAPIHost = "api.segment.io/v1"
     private static let defaultCDNHost = "cdn-settings.segment.com/v1"
 
-    private var session: URLSession
+    internal var session: URLSession
     private var apiHost: String
     private var apiKey: String
     private var cdnHost: String

Sources/Segment/Utilities/OutputFileStream.swift

@@ -0,0 +1,96 @@
+//
+//  OutputFileStream.swift
+//  
+//
+//  Created by Brandon Sneed on 10/15/22.
+//
+
+import Foundation
+
+/** C style output filestream ----------------- */
+
+#if os(Linux)
+import Glibc
+#else
+import Darwin.C
+#endif
+
+struct OutputFileStream {
+    enum OutputStreamError: Error {
+        case invalidPath(String)
+        case unableToOpen(String)
+        case unableToWrite
+    }
+
+    internal var filePointer: UnsafeMutablePointer<FILE>? = nil
+
+    init(fileURL: URL) throws {
+        let path = fileURL.path
+        guard path.isEmpty == false else { throw OutputStreamError.invalidPath(path) }
+
+        path.withCString { file in
+            filePointer = fopen(file, "w")
+        }
+        guard filePointer != nil else { throw OutputStreamError.unableToOpen(path) }
+    }
+
+    func write(_ data: Data) throws {
+        guard let string = String(data: data, encoding: .utf8) else { return }
+        try write(string)
+    }
+
+    func write(_ string: String) throws {
+        guard string.isEmpty == false else { return }
+        _ = try string.utf8.withContiguousStorageIfAvailable { str in
+            if let baseAddr = str.baseAddress {
+                fwrite(baseAddr, 1, str.count, filePointer)
+            } else {
+                throw OutputStreamError.unableToWrite
+            }
+            if ferror(filePointer) != 0 {
+                throw OutputStreamError.unableToWrite
+            }
+        }
+    }
+
+    func close() {
+        fclose(filePointer)
+    }
+}
+
+
+/** FileHandle version for comparison ------------------------------------ */
+/*
+struct OutputFileStream {
+    enum OutputStreamError: Error {
+        case unableToWrite
+        case unableToCreate(String)
+    }
+
+    internal var fileHandle: FileHandle
+
+    init(fileURL: URL) throws {
+        let created = FileManager.default.createFile(atPath: fileURL.path, contents: nil)
+        if !created { throw OutputStreamError.unableToCreate(fileURL.path) }
+        fileHandle = try FileHandle(forWritingTo: fileURL)
+    }
+
+    func write(_ data: Data) throws {
+        fileHandle.seekToEndOfFile()
+        if #available(macOS 10.15.4, iOS 13.4, *) {
+            try fileHandle.write(contentsOf: data)
+        } else {
+            fileHandle.write(data)
+        }
+    }
+
+    func write(_ string: String) throws {
+        guard let data = string.data(using: .utf8) else { throw OutputStreamError.unableToWrite }
+        try write(data)
+    }
+
+    func close() {
+        try? fileHandle.close()
+    }
+}
+*/

Sources/Segment/Utilities/Storage.swift

@@ -16,13 +16,18 @@ internal class Storage: Subscriber {
 
     // This queue synchronizes reads/writes.
     // Do NOT use it outside of: write, read, reset, remove.
-    let syncQueue = DispatchQueue(label: "storage.segment.com")
+    let syncQueue = DispatchQueue(label: "sync.segment.com")
 
-    private var fileHandle: FileHandle? = nil
+    private var outputStream: OutputFileStream? = nil
 
-    init(store: Store, writeKey: String) {
+    internal var onFinish: ((URL) -> Void)? = nil
+
+    internal let storageMonitor: ((Error) -> Void)?
+
+    init(store: Store, writeKey: String, monitor: ((Error) -> Void)?) {
         self.store = store
         self.writeKey = writeKey
+        self.storageMonitor = monitor
         self.userDefaults = UserDefaults(suiteName: "com.segment.storage.\(writeKey)")
         store.subscribe(self, handler: userInfoUpdate)
         store.subscribe(self, handler: systemUpdate)
@@ -242,8 +247,14 @@ extension Storage {
         if fm.fileExists(atPath: storeFile.path) == false {
             start(file: storeFile)
             newFile = true
-        } else if fileHandle == nil {
-            fileHandle = try? FileHandle(forWritingTo: file)
+        } else if outputStream == nil {
+            // this can happen if an instance was terminated before finishing a file.
+            do {
+                outputStream = try OutputFileStream(fileURL: storeFile)
+            } catch {
+                storageMonitor?(error)
+                Analytics.segmentLog(message: "Storage: Unable to open \(file), Error: \(error)", kind: .error)
+            }
         }
 
         // Verify file size isn't too large
@@ -258,38 +269,34 @@ extension Storage {
         }
 
         let jsonString = event.toString()
-        if let jsonData = jsonString.data(using: .utf8) {
-            fileHandle?.seekToEndOfFile()
-            // prepare for the next entry
-            if newFile == false {
-                fileHandle?.write(",".data(using: .utf8)!)
+        do {
+            if outputStream == nil {
+                Analytics.segmentLog(message: "Storage: Output stream is nil for \(storeFile)", kind: .error)
             }
-            // write the data
-            fileHandle?.write(jsonData)
-            if #available(tvOS 13, *) {
-                try? fileHandle?.synchronize()
+            if newFile == false {
+                // prepare for the next entry
+                try outputStream?.write(",")
             }
-        } else {
-            assert(false, "Storage: Unable to convert event to json!")
+            try outputStream?.write(jsonString)
+        } catch {
+            storageMonitor?(error)
+            Analytics.segmentLog(message: "Storage: Unable to write to \(storeFile), Error: \(error)", kind: .error)
         }
     }
 
     private func start(file: URL) {
         let contents = "{ \"batch\": ["
         do {
-            FileManager.default.createFile(atPath: file.path, contents: contents.data(using: .utf8))
-            fileHandle = try FileHandle(forWritingTo: file)
+            outputStream = try OutputFileStream(fileURL: file)
+            try outputStream?.write(contents)
         } catch {
-            assert(false, "Storage: failed to write \(file), error: \(error)")
+            storageMonitor?(error)
+            Analytics.segmentLog(message: "Storage: Unable to write to \(file), Error: \(error)", kind: .error)
         }
     }
 
     private func finish(file: URL) {
-        if self.fileHandle == nil {
-            self.fileHandle = try? FileHandle(forWritingTo: file)
-        }
-
-        guard let fileHandle = self.fileHandle else {
+        guard let outputStream = self.outputStream else {
             // we haven't actually started a file yet and being told to flush
             // so ignore it and get out.
             return
@@ -299,26 +306,25 @@ extension Storage {
 
         // write it to the existing file
         let fileEnding = "],\"sentAt\":\"\(sentAt)\",\"writeKey\":\"\(writeKey)\"}"
-        let endData = fileEnding.data(using: .utf8)
-        if let endData = endData {
-            fileHandle.seekToEndOfFile()
-            fileHandle.write(endData)
-            if #available(tvOS 13, *) {
-                try? fileHandle.synchronize()
-            }
-            fileHandle.closeFile()
-            self.fileHandle = nil
-        } else {
-            // something is wrong with this file :S
-            Analytics.segmentLog(message: "Event storage \(file) has some kind of problem.", kind: .error)
+        do {
+            try outputStream.write(fileEnding)
+        } catch {
+            storageMonitor?(error)
+            Analytics.segmentLog(message: "Storage: Unable to write to \(file), Error: \(error)", kind: .error)
         }
+        outputStream.close()
+        self.outputStream = nil
+        print("stream closed for \(file)")
 
         let tempFile = file.appendingPathExtension(Storage.tempExtension)
         do {
             try FileManager.default.moveItem(at: file, to: tempFile)
         } catch {
             Analytics.segmentLog(message: "Unable to rename to temp: \(file), Error: \(error)", kind: .error)
         }
+
+        // necessary for testing, do not use.
+        onFinish?(tempFile)
 
         let currentFile: Int = (userDefaults?.integer(forKey: Constants.events.rawValue) ?? 0) + 1
         userDefaults?.set(currentFile, forKey: Constants.events.rawValue)