From 11fb81373444809ccb1ce8586204855cbacbe5c5 Mon Sep 17 00:00:00 2001 From: Laurent Date: Fri, 9 Feb 2024 14:53:51 +0100 Subject: [PATCH] migration tests --- LeStorage.xcodeproj/project.pbxproj | 12 +++ LeStorage/ApiCall.swift | 4 +- LeStorage/Services.swift | 6 +- LeStorage/Storable.swift | 4 + LeStorage/Store.swift | 53 ++++++++++-- LeStorage/StoredCollection.swift | 97 +++++++++++++--------- LeStorage/Utils/Collection+Extension.swift | 6 +- LeStorage/Wip/Migration.swift | 85 +++++++++++++++++++ 8 files changed, 215 insertions(+), 52 deletions(-) create mode 100644 LeStorage/Wip/Migration.swift diff --git a/LeStorage.xcodeproj/project.pbxproj b/LeStorage.xcodeproj/project.pbxproj index 043fec0..fa490de 100644 --- a/LeStorage.xcodeproj/project.pbxproj +++ b/LeStorage.xcodeproj/project.pbxproj @@ -22,6 +22,7 @@ C4A47D6B2B71244100ADC637 /* Collection+Extension.swift in Sources */ = {isa = PBXBuildFile; fileRef = C4A47D6A2B71244100ADC637 /* Collection+Extension.swift */; }; C4A47D6D2B71364600ADC637 /* ModelObject.swift in Sources */ = {isa = PBXBuildFile; fileRef = C4A47D6C2B71364600ADC637 /* ModelObject.swift */; }; C4A47D6F2B7154F600ADC637 /* README.md in Resources */ = {isa = PBXBuildFile; fileRef = C4A47D6E2B7154F600ADC637 /* README.md */; }; + C4A47D812B7665AD00ADC637 /* Migration.swift in Sources */ = {isa = PBXBuildFile; fileRef = C4A47D802B76658F00ADC637 /* Migration.swift */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ @@ -51,6 +52,7 @@ C4A47D6A2B71244100ADC637 /* Collection+Extension.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Collection+Extension.swift"; sourceTree = ""; }; C4A47D6C2B71364600ADC637 /* ModelObject.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ModelObject.swift; sourceTree = ""; }; C4A47D6E2B7154F600ADC637 /* README.md */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = net.daringfireball.markdown; path = README.md; sourceTree = SOURCE_ROOT; }; + C4A47D802B76658F00ADC637 /* Migration.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Migration.swift; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -102,6 +104,7 @@ C425D4572B6D2519002A7B48 /* Store.swift */, C4A47D642B6E92FE00ADC637 /* Storable.swift */, C4A47D4E2B6D280200ADC637 /* StoredCollection.swift */, + C4A47D822B7665BC00ADC637 /* Wip */, C4A47D582B6D352900ADC637 /* Utils */, ); path = LeStorage; @@ -126,6 +129,14 @@ path = Utils; sourceTree = ""; }; + C4A47D822B7665BC00ADC637 /* Wip */ = { + isa = PBXGroup; + children = ( + C4A47D802B76658F00ADC637 /* Migration.swift */, + ); + path = Wip; + sourceTree = ""; + }; /* End PBXGroup section */ /* Begin PBXHeadersBuildPhase section */ @@ -244,6 +255,7 @@ C4A47D652B6E92FE00ADC637 /* Storable.swift in Sources */, C4A47D6D2B71364600ADC637 /* ModelObject.swift in Sources */, C4A47D4F2B6D280200ADC637 /* StoredCollection.swift in Sources */, + C4A47D812B7665AD00ADC637 /* Migration.swift in Sources */, C4A47D672B6FF83A00ADC637 /* ApiCall.swift in Sources */, C425D4582B6D2519002A7B48 /* Store.swift in Sources */, C4A47D6B2B71244100ADC637 /* Collection+Extension.swift in Sources */, diff --git a/LeStorage/ApiCall.swift b/LeStorage/ApiCall.swift index ab0214c..1853a2b 100644 --- a/LeStorage/ApiCall.swift +++ b/LeStorage/ApiCall.swift @@ -28,7 +28,7 @@ class ApiCall : ModelObject, Storable, SomeCall { var dataId: String /// The content of the call - var body: Data + var body: String /// The number of times the call has been executed var attemptsCount: Int = 0 @@ -36,7 +36,7 @@ class ApiCall : ModelObject, Storable, SomeCall { /// The date of the last execution var lastAttemptDate: Date = Date() - init(url: String, method: String, dataId: String, body: Data) { + init(url: String, method: String, dataId: String, body: String) { self.url = url self.method = method self.dataId = dataId diff --git a/LeStorage/Services.swift b/LeStorage/Services.swift index 3e2ffb9..c6e95fa 100644 --- a/LeStorage/Services.swift +++ b/LeStorage/Services.swift @@ -109,9 +109,9 @@ class Services { } fileprivate func _createCall(method: Method, instance: T) throws -> ApiCall { - let data = try instance.jsonData() + let jsonString = try instance.jsonString() let url = self._baseURL + T.resourceName() + "/" - return ApiCall(url: url, method: method.rawValue, dataId: String(instance.id), body: data) + return ApiCall(url: url, method: method.rawValue, dataId: String(instance.id), body: jsonString) } func runApiCall(_ apiCall: ApiCall) async throws -> T { @@ -130,7 +130,7 @@ class Services { } var request = URLRequest(url: url) request.httpMethod = apiCall.method - request.httpBody = apiCall.body + request.httpBody = apiCall.body.data(using: .utf8) request.setValue("application/json", forHTTPHeaderField: "Content-Type") return request } diff --git a/LeStorage/Storable.swift b/LeStorage/Storable.swift index e5d439d..e1352b6 100644 --- a/LeStorage/Storable.swift +++ b/LeStorage/Storable.swift @@ -18,4 +18,8 @@ extension Storable { return Store.main.findById(id) } + static func fileName() -> String { + return self.resourceName() + ".json" + } + } diff --git a/LeStorage/Store.swift b/LeStorage/Store.swift index 436fcf0..639d5a2 100644 --- a/LeStorage/Store.swift +++ b/LeStorage/Store.swift @@ -35,21 +35,22 @@ public class Store { fileprivate var _collections: [String : any SomeCollection] = [:] fileprivate var _apiCallsCollections: [String : any SomeCollection] = [:] -// fileprivate var _apiCallTimers: [String: Timer] = [:] + fileprivate var _migrations: [SomeMigration] = [] + + fileprivate lazy var _migrationCollection: StoredCollection = { StoredCollection(synchronized: false, store: Store.main, asynchronousIO: false) + }() public init() { } public func registerCollection(synchronized: Bool) -> StoredCollection { // register collection - let collection = StoredCollection(synchronized: synchronized, store: self, loadCompletion: { _ in - - }) + let collection = StoredCollection(synchronized: synchronized, store: Store.main, loadCompletion: nil) self._collections[T.resourceName()] = collection if synchronized { // register additional collection for api calls - let apiCallCollection = StoredCollection>(synchronized: false, store: self, loadCompletion: { apiCallCollection in - self._reloadTimers(collection: apiCallCollection) + let apiCallCollection = StoredCollection>(synchronized: false, store: Store.main, loadCompletion: { apiCallCollection in + self._rescheduleCalls(collection: apiCallCollection) }) self._apiCallsCollections[T.resourceName()] = apiCallCollection } @@ -88,9 +89,47 @@ public class Store { try self.collection().deleteDependencies(items) } + // MARK: - Migration + + public func addMigration(_ migration: SomeMigration) { + self._migrations.append(migration) + } + + func performMigrationIfNecessary(_ collection: StoredCollection) async throws { + + // Check for migrations + let migrations = self._migrations.filter { $0.resourceName == T.resourceName() } + if migrations.isEmpty { return } + + // Check for applied migrations + var version: Int = -1 + var performedMigration: MigrationHistory? = self._migrationCollection.first(where: { $0.resourceName == T.resourceName() }) + if let performedMigration { + version = Int(performedMigration.version) + } + + // Apply necessary migrations + let applicableMigrations = migrations.filter { $0.version > version } + .sorted(keyPath: \.version) + for migration in applicableMigrations { + + Logger.log("Start migration for \(migration.resourceName), version: \(migration.version)") + + try migration.migrate(synchronized: collection.synchronized) + + if let performedMigration { + performedMigration.version = migration.version + } else { + performedMigration = MigrationHistory(version: migration.version, resourceName: migration.resourceName) + } + + self._migrationCollection.addOrUpdate(instance: performedMigration!) + } + } + // MARK: - Api call rescheduling - fileprivate func _reloadTimers(collection: StoredCollection>) { + fileprivate func _rescheduleCalls(collection: StoredCollection>) { for apiCall in collection { self.startCallsRescheduling(apiCall: apiCall) } diff --git a/LeStorage/StoredCollection.swift b/LeStorage/StoredCollection.swift index 1c885c6..30aca4f 100644 --- a/LeStorage/StoredCollection.swift +++ b/LeStorage/StoredCollection.swift @@ -32,13 +32,9 @@ public class StoredCollection : RandomAccessCollection, SomeCollec /// The reference to the Store fileprivate var _store: Store - fileprivate var loadCompletion: (StoredCollection) -> () - - /// Returns the default filename for the collection - fileprivate var _fileName: String { - return T.resourceName() + ".json" - } - + /// Notifies the closure when the loading is done + fileprivate var loadCompletion: ((StoredCollection) -> ())? = nil + /// Indicates whether the collection has changed, thus requiring a write operation fileprivate var _hasChanged: Bool = false { didSet { @@ -52,47 +48,54 @@ public class StoredCollection : RandomAccessCollection, SomeCollec } } - init(synchronized: Bool, store: Store, loadCompletion: @escaping (StoredCollection) -> ()) { + fileprivate var asynchronousIO: Bool = true + + init(synchronized: Bool, store: Store, asynchronousIO: Bool = true, loadCompletion: ((StoredCollection) -> ())? = nil) { self.synchronized = synchronized + self.asynchronousIO = asynchronousIO self._store = store self.loadCompletion = loadCompletion self._load() } + + // MARK: - Loading - /// Launches a load operation if the file exists + /// Migrates if necessary and asynchronously decodes the json file fileprivate func _load() { do { - let url = try FileUtils.directoryURLForFileName(self._fileName) + let url = try FileUtils.directoryURLForFileName(T.fileName()) if FileManager.default.fileExists(atPath: url.path()) { - self._loadAsync() - } else { - try? self.loadDataFromServer() + + if self.asynchronousIO { + Task(priority: .high) { + try await Store.main.performMigrationIfNecessary(self) + try self._decodeJSONFile() + } + } else { + try self._decodeJSONFile() + } + } +// else { +// try? self.loadDataFromServer() +// } } catch { Logger.log(error) } } - - /// Loads asynchronously into memory the objects contained inside the collection file - fileprivate func _loadAsync() { - DispatchQueue(label: "lestorage.queue.read", qos: .background).async { - do { - let jsonString = try FileUtils.readDocumentFile(fileName: self._fileName) - if let decoded: [T] = try jsonString.decodeArray() { - DispatchQueue.main.sync { - Logger.log("loaded \(self._fileName) with \(decoded.count) items") - self.items = decoded - self.loadCompletion(self) - - NotificationCenter.default.post(name: NSNotification.Name.CollectionDidLoad, object: self) - } - } - } catch { - Logger.error(error) // TODO how to notify the main project + + fileprivate func _decodeJSONFile() throws { + let jsonString = try FileUtils.readDocumentFile(fileName: T.fileName()) + if let decoded: [T] = try jsonString.decodeArray() { + DispatchQueue.main.sync { + Logger.log("loaded \(T.fileName()) with \(decoded.count) items") + self.items = decoded + self.loadCompletion?(self) + + NotificationCenter.default.post(name: NSNotification.Name.CollectionDidLoad, object: self) } } - } public func loadDataFromServer() throws { @@ -108,6 +111,8 @@ public class StoredCollection : RandomAccessCollection, SomeCollec } } + // MARK: - Basic operations + /// Adds or updates the provided instance inside the collection /// Adds it if its id is not found, and otherwise updates it public func addOrUpdate(instance: T) { @@ -136,6 +141,16 @@ public class StoredCollection : RandomAccessCollection, SomeCollec self._sendDeletionIfNecessary(instance) } + public func batchInsert(_ sequence: any Sequence) { + defer { + self._hasChanged = true + } + self.items.append(contentsOf: sequence) + for instance in sequence { + self._sendUpdateIfNecessary(instance) + } + } + /// Returns the instance corresponding to the provided [id] public func findById(_ id: String) -> T? { return self.items.first(where: { $0.id == id }) @@ -169,18 +184,22 @@ public class StoredCollection : RandomAccessCollection, SomeCollec /// Schedules a write operation fileprivate func _scheduleWrite() { - self._write() + if self.asynchronousIO { + DispatchQueue(label: "lestorage.queue.write", qos: .utility).async { + self._write() + } + } else { + self._write() + } } /// Writes all the items as a json array inside a file fileprivate func _write() { - DispatchQueue(label: "lestorage.queue.write", qos: .background).async { - do { - let jsonString = try self.items.jsonString() - let _ = try FileUtils.writeToDocumentDirectory(content: jsonString, fileName: self._fileName) - } catch { - Logger.error(error) // TODO how to notify the main project - } + do { + let jsonString: String = try self.items.jsonString() + let _ = try FileUtils.writeToDocumentDirectory(content: jsonString, fileName: T.fileName()) + } catch { + Logger.error(error) // TODO how to notify the main project } } diff --git a/LeStorage/Utils/Collection+Extension.swift b/LeStorage/Utils/Collection+Extension.swift index ba65dc7..74e883d 100644 --- a/LeStorage/Utils/Collection+Extension.swift +++ b/LeStorage/Utils/Collection+Extension.swift @@ -11,7 +11,11 @@ extension Array { func sorted(keyPath: KeyPath, ascending: Bool = true) -> [Element] { return self.sorted { e1, e2 in - return e1[keyPath: keyPath] > e2[keyPath: keyPath] + if ascending { + return e1[keyPath: keyPath] < e2[keyPath: keyPath] + } else { + return e1[keyPath: keyPath] > e2[keyPath: keyPath] + } } } diff --git a/LeStorage/Wip/Migration.swift b/LeStorage/Wip/Migration.swift new file mode 100644 index 0000000..67bcc5c --- /dev/null +++ b/LeStorage/Wip/Migration.swift @@ -0,0 +1,85 @@ +// +// Migration.swift +// LeStorage +// +// Created by Laurent Morvillier on 07/02/2024. +// + +import Foundation + +public protocol MigrationSource : Storable { + associatedtype Destination : Storable + func migrate() -> Destination +} + +public protocol SomeMigration { + func migrate(synchronized: Bool) throws + var version: UInt { get } + var resourceName: String { get } +} + +public class Migration : SomeMigration where S.Destination == D { + + public var version: UInt + + public var resourceName: String { + return S.resourceName() + } + + public init(version: UInt) { + self.version = version + } + + public func migrate(synchronized: Bool) throws { + try self._migrateMainCollection() + if synchronized { + try self._migrateApiCallsCollection() + } + } + + fileprivate func _migrateMainCollection() throws { + let jsonString = try FileUtils.readDocumentFile(fileName: S.fileName()) + if let decoded: [S] = try jsonString.decodeArray() { + + let migratedObjects: [D] = decoded.map { $0.migrate() } + + let jsonString: String = try migratedObjects.jsonString() + let _ = try FileUtils.writeToDocumentDirectory(content: jsonString, fileName: D.fileName()) + } + } + + fileprivate func _migrateApiCallsCollection() throws { + let jsonString = try FileUtils.readDocumentFile(fileName: ApiCall.fileName()) + if let apiCalls: [ApiCall] = try jsonString.decodeArray() { + + let migratedCalls = try apiCalls.map { apiCall in + if let source: S = try apiCall.body.decode() { + let migrated: D = source.migrate() + apiCall.body = try migrated.jsonString() + } + return apiCall + } + + let jsonString = try migratedCalls.jsonString() + let _ = try FileUtils.writeToDocumentDirectory(content: jsonString, fileName: ApiCall.fileName()) + + Logger.log("Ended _migrateApiCallsCollection: \(jsonString)") + } + } + +} + +class MigrationHistory: ModelObject, Storable { + + public static func resourceName() -> String { "migration_history" } + + public var id: String = Store.randomId() + public var version: UInt + public var resourceName: String + + init(version: UInt, resourceName: String) { + self.version = version + self.resourceName = resourceName + } + +}