Add .notedb database file format (#16)

### Added

- Commonplace Book now supports a new file format: .notedb, which is a single sqlite file. It is compatible with iCloud Document Storage though.
- Database storage is the default for new documents.
- Database storage has full-text search!

Author: bdewey

.swiftlint.yml

@@ -19,6 +19,8 @@ opt_in_rules:
 identifier_name:
   excluded:
     - i
+    - id
+    - db
 
 line_length: 100
 

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## Unreleased
+
+### Added
+
+- Commonplace Book now supports a new file format: .notedb, which is a single sqlite file. It is compatible with iCloud Document Storage though.
+- Database storage is the default for new documents.
+- Database storage has full-text search!
+
 ## [0.14.0] - 2020-01-02
 
 ### Changed

CommonplaceBookApp.xcodeproj/project.pbxproj

@@ -49,14 +49,14 @@ final class AppDelegate: UIResponder, UIApplicationDelegate {
 
     let window = UIWindow(frame: UIScreen.main.bounds)
 
-    let browser = UIDocumentBrowserViewController(forOpeningFilesWithContentTypes: ["org.brians-brain.notebundle"])
+    let browser = UIDocumentBrowserViewController(forOpeningFilesWithContentTypes: ["org.brians-brain.notebundle", "org.brians-brain.notedb"])
     browser.delegate = self
 
     window.rootViewController = browser
     window.makeKeyAndVisible()
     self.window = window
 
-    if let openedDocumentBookmarkData = openedDocumentBookmark {
+    if !isUITesting, let openedDocumentBookmarkData = openedDocumentBookmark {
       DDLogInfo("Bookmark data exists for an open document")
       var isStale: Bool = false
       do {
@@ -78,7 +78,7 @@ final class AppDelegate: UIResponder, UIApplicationDelegate {
     makeMetadataProvider(completion: { metadataProviderResult in
       switch metadataProviderResult {
       case .success(let metadataProvider):
-        self.openDocument(at: metadataProvider.container.appendingPathComponent("archive.notebundle"), from: viewController, animated: false)
+        self.openDocument(at: metadataProvider.container.appendingPathComponent("archive.notedb"), from: viewController, animated: false)
       case .failure(let error):
         let messageText = "Error opening Notebook: \(error.localizedDescription)"
         let alertController = UIAlertController(title: "Error", message: messageText, preferredStyle: .alert)
@@ -143,8 +143,12 @@ final class AppDelegate: UIResponder, UIApplicationDelegate {
     }
   }
 
+  private lazy var isUITesting: Bool = {
+    CommandLine.arguments.contains("--uitesting")
+  }()
+
   private func makeMetadataProvider(completion: @escaping (Result<FileMetadataProvider, Swift.Error>) -> Void) {
-    if CommandLine.arguments.contains("--uitesting") {
+    if isUITesting {
       let container = FileManager.default.temporaryDirectory.appendingPathComponent("uitesting")
       completion(makeDirectoryProvider(at: container, deleteExistingContents: true))
       return
@@ -202,10 +206,18 @@ extension AppDelegate: UIDocumentBrowserViewControllerDelegate {
   /// - parameter controller: The view controller from which to present the DocumentListViewController
   private func openDocument(at url: URL, from controller: UIDocumentBrowserViewController, animated: Bool) {
     DDLogInfo("Opening document at \(url)")
-    let noteArchiveDocument = NoteDocumentStorage(
-      fileURL: url,
-      parsingRules: ParsingRules.commonplace
-    )
+    let noteArchiveDocument: NoteStorage
+    if url.pathExtension == "notebundle" {
+      noteArchiveDocument = NoteDocumentStorage(
+        fileURL: url,
+        parsingRules: ParsingRules.commonplace
+      )
+    } else if url.pathExtension == "notedb" {
+      noteArchiveDocument = NoteSqliteStorage(fileURL: url, parsingRules: ParsingRules.commonplace)
+    } else {
+      assertionFailure("Unknown note format: \(url.pathExtension)")
+      return
+    }
     DDLogInfo("Using document at \(noteArchiveDocument.fileURL)")
     let documentListViewController = DocumentListViewController(notebook: noteArchiveDocument)
     documentListViewController.didTapFilesAction = { [weak self] in
@@ -229,11 +241,11 @@ extension AppDelegate: UIDocumentBrowserViewControllerDelegate {
       noteIdentifierCopy.flatMap { documentListViewController.showPage(with: $0) }
       let properties: [String: String] = [
         "Success": success.description,
-        "documentState": String(describing: noteArchiveDocument.documentState),
-        "previousError": noteArchiveDocument.previousError?.localizedDescription ?? "nil",
+//        "documentState": String(describing: noteArchiveDocument.documentState),
+//        "previousError": noteArchiveDocument.previousError?.localizedDescription ?? "nil",
       ]
       DDLogInfo("In open completion handler. \(properties)")
-      if success {
+      if success, !self.isUITesting {
         self.openedDocumentBookmark = try? url.bookmarkData()
       }
     })
@@ -248,23 +260,17 @@ extension AppDelegate: UIDocumentBrowserViewControllerDelegate {
 
   func documentBrowser(_ controller: UIDocumentBrowserViewController, didRequestDocumentCreationWithHandler importHandler: @escaping (URL?, UIDocumentBrowserViewController.ImportMode) -> Void) {
     let directoryURL = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).last!
-    let url = directoryURL.appendingPathComponent(UUID().uuidString).appendingPathExtension("notebundle")
-    let document = NoteDocumentStorage(fileURL: url, parsingRules: ParsingRules.commonplace)
-    document.save(to: url, for: .forCreating) { saveSuccess in
-      guard saveSuccess else {
-        DDLogError("Could not save document to \(url): \(document.previousError?.localizedDescription ?? "nil")")
-        importHandler(nil, .none)
-        return
-      }
-      document.close { closeSuccess in
-        guard closeSuccess else {
-          DDLogError("Could not close document at \(url): \(document.previousError?.localizedDescription ?? "nil")")
-          importHandler(nil, .none)
-          return
-        }
-        importHandler(url, .copy)
-      }
+    let url = directoryURL.appendingPathComponent("commonplace").appendingPathExtension("notedb")
+    do {
+      let document = NoteSqliteStorage(fileURL: url, parsingRules: ParsingRules.commonplace)
+      try document.open()
+      try document.flush()
+    } catch {
+      DDLogError("Error creating new document: \(error)")
+      importHandler(nil, .none)
+      return
     }
+    importHandler(url, .copy)
   }
 
   func documentBrowser(_ controller: UIDocumentBrowserViewController, didImportDocumentAt sourceURL: URL, toDestinationURL destinationURL: URL) {

CommonplaceBookApp.xcworkspace/xcshareddata/swiftpm/Package.resolved

@@ -7,7 +7,7 @@ import UIKit
 /// Uniquely identifies a challenge.
 public struct ChallengeIdentifier: Hashable {
   /// The SHA1 digest of the template that created this challenge.
-  public let templateDigest: String?
+  public var templateDigest: String?
 
   /// The index of this challenge in the template's challenges array.
   public let index: Int

CommonplaceBookApp/AppDelegate.swift

@@ -40,11 +40,7 @@ open class ChallengeTemplate: Codable {
   /// the template before creating any challenges from it.
   ///
   /// Can only be set from nil to non-nil once; immutable once set.
-  public var templateIdentifier: String? {
-    willSet {
-      precondition(templateIdentifier == nil)
-    }
-  }
+  public var templateIdentifier: String?
 
   /// Subclasses should override and return their particular type.
   /// This is a computed, rather than a stored, property so it does not get serialized.

CommonplaceBookApp/Challenge.swift

@@ -46,7 +46,6 @@ final class DocumentListViewController: UIViewController {
   public let notebook: NoteStorage
   public var didTapFilesAction: (() -> Void)?
   private var dataSource: DocumentTableController?
-  private var currentSpotlightQuery: CSSearchQuery?
   private var notebookSubscription: AnyCancellable?
 
   private lazy var documentBrowserButton: UIBarButtonItem = {
@@ -277,16 +276,11 @@ extension DocumentListViewController: UISearchResultsUpdating, UISearchBarDelega
     guard searchController.isActive else {
       dataSource?.hashtags = []
       dataSource?.filteredPageIdentifiers = nil
-      currentSpotlightQuery = nil
       updateStudySession()
       return
     }
     let pattern = searchController.searchBar.text ?? ""
-    var queryString = """
-    contentDescription == "*\(pattern)*"dc
-    """
     if let selectedHashtag = searchController.searchBar.searchTextField.tokens.first?.representedObject as? String {
-      queryString.append(" && keywords == \"\(selectedHashtag)\"dc")
       dataSource?.hashtags = []
       dataSource?.filteredHashtag = selectedHashtag
     } else {
@@ -295,23 +289,13 @@ extension DocumentListViewController: UISearchResultsUpdating, UISearchBarDelega
         .filter { $0.fuzzyMatch(pattern: pattern) }
       dataSource?.filteredHashtag = nil
     }
-    DDLogInfo("Issuing query: \(queryString)")
-    currentSpotlightQuery?.cancel()
-    let query = CSSearchQuery(queryString: queryString, attributes: nil)
-    var allIdentifiers: [Note.Identifier] = []
-    query.foundItemsHandler = { items in
-      allIdentifiers.append(contentsOf: items.map { Note.Identifier(rawValue: $0.uniqueIdentifier) })
-    }
-    query.completionHandler = { _ in
-      DDLogInfo("Found identifiers: \(allIdentifiers)")
-      DispatchQueue.main.async {
-        if searchController.isActive, self.currentSpotlightQuery == query {
-          self.dataSource?.filteredPageIdentifiers = Set(allIdentifiers)
-        }
-      }
+    DDLogInfo("Issuing query: \(pattern)")
+    do {
+      let allIdentifiers = try notebook.search(for: pattern)
+      dataSource?.filteredPageIdentifiers = Set(allIdentifiers)
+    } catch {
+      DDLogError("Error issuing full text query: \(error)")
     }
-    query.start()
-    currentSpotlightQuery = query
   }
 
   func searchBarTextDidEndEditing(_ searchBar: UISearchBar) {
@@ -329,8 +313,12 @@ extension DocumentListViewController: StudyViewControllerDelegate {
     _ studyViewController: StudyViewController,
     didFinishSession session: StudySession
   ) {
-    notebook.updateStudySessionResults(session, on: Date())
-    updateStudySession()
+    do {
+      try notebook.updateStudySessionResults(session, on: Date())
+      updateStudySession()
+    } catch {
+      DDLogError("Unexpected error recording study session results: \(error)")
+    }
   }
 
   func studyViewControllerDidCancel(_ studyViewController: StudyViewController) {

CommonplaceBookApp/ChallengeTemplate.swift

@@ -341,7 +341,7 @@ private extension DocumentTableController {
     notebook.studySession(filter: nil, date: Date()) { studySession in
       self.cardsPerDocument = studySession
         .reduce(into: [Note.Identifier: Int]()) { cardsPerDocument, card in
-          cardsPerDocument[card.properties.documentName] = cardsPerDocument[card.properties.documentName, default: 0] + 1
+          cardsPerDocument[card.noteIdentifier] = cardsPerDocument[card.noteIdentifier, default: 0] + 1
         }
       DDLogInfo(
         "studySession.count = \(studySession.count). " +

CommonplaceBookApp/DocumentListViewController.swift

@@ -75,7 +75,7 @@ struct EditVocabularyView: View {
   private var selectedImage: SwiftUI.Image? {
     if
       let key = vocabularyTemplate.imageAsset,
-      let data = notebook.data(for: key),
+      let data = try? notebook.data(for: key),
       let uiImage = UIImage(data: data) {
       return Image(uiImage: uiImage)
     } else {
@@ -87,9 +87,14 @@ struct EditVocabularyView: View {
   // TODO: Only store asset data when we commit this association?
   private func onSelectedImage(encodedImage: EncodedImage) {
     DDLogInfo("Selected image: \(encodedImage)")
-    let key = notebook.storeAssetData(encodedImage.data, typeHint: encodedImage.encoding)
-    DDLogInfo("Saved image data as asset \(key)")
-    vocabularyTemplate.imageAsset = key
+    let key = encodedImage.data.sha1Digest() + "." + encodedImage.encoding
+    do {
+      let actualKey = try notebook.storeAssetData(encodedImage.data, key: key)
+      vocabularyTemplate.imageAsset = actualKey
+      DDLogInfo("Saved image data as asset \(actualKey)")
+    } catch {
+      DDLogError("Unexpected error saving image: \(error)")
+    }
   }
 }