# #Fin Rememberer
- [Medium Link](https://medium.com/@tonysu1204/fin-rememberer-dfb02cec9a3e)
- [GitHub Link](https://github.com/Ateto1204/Rememberer-iOS.git)
## 使用後台 API
### URLSession version
- URLSession
- JSONDecoder
- JSONEncoder
:::spoiler Code
```swift=
import Foundation
import Alamofire
import Combine
class OpenAIService {
let endpointURL = "https://api.openai.com/v1/chat/completions"
func sendMessage(messages: [Message], completion: @escaping (OpenAIChatResponse?) -> Void) {
let openAIMessages = messages.map({ OpenAIChatMessage(role: $0.role, content: $0.content) })
let body = OpenAIChatBody(model: "gpt-3.5-turbo-16k-0613", messages: openAIMessages)
guard let url = URL(string: endpointURL) else {
completion(nil)
return
}
var request = URLRequest(url: url)
request.httpMethod = "POST"
request.setValue("Bearer \(Constants.openAIAPIKey)", forHTTPHeaderField: "Authorization")
request.setValue("application/json", forHTTPHeaderField: "Content-Type")
do {
let requestBody = try JSONEncoder().encode(body)
request.httpBody = requestBody
} catch {
completion(nil)
return
}
URLSession.shared.dataTask(with: request) { data, response, error in
guard let data = data, error == nil else {
completion(nil)
return
}
do {
let openAIResponse = try JSONDecoder().decode(OpenAIChatResponse.self, from: data)
completion(openAIResponse)
} catch {
completion(nil)
}
}.resume()
}
}
struct OpenAIChatBody: Encodable {
let model: String
let messages: [OpenAIChatMessage]
}
struct OpenAIChatMessage: Codable {
let role: SenderRole
let content: String
}
enum SenderRole: String, Codable {
case system
case user
case assistant
}
struct OpenAIChatResponse: Decodable {
let choices: [OpenAIChatChoice]
}
struct OpenAIChatChoice: Decodable {
let message: OpenAIChatMessage
}
```
:::
### Alamofire version
- import [Alamofire Package](https://github.com/Alamofire/Alamofire.git)
- async/await request
:::spoiler Code
```swift=
import Foundation
import Alamofire
import Combine
class OpenAIService {
let endpointURL = "https://api.openai.com/v1/chat/completions"
func sendMessage(messages: [Message]) async -> OpenAIChatResponse? {
let openAIMessages = messages.map({OpenAIChatMessage(role: $0.role, content: $0.content)})
let body = OpenAIChatBody(model: "gpt-3.5-turbo-16k", messages: openAIMessages)
let headers: HTTPHeaders = [
"Authorization": "Bearer \(Constants.openAIAPIKey)"
]
return try? await AF.request(endpointURL, method: .post, parameters: body, encoder: .json, headers: headers).serializingDecodable(OpenAIChatResponse.self).value
}
}
struct OpenAIChatBody: Encodable {
let model: String
let messages: [OpenAIChatMessage]
}
struct OpenAIChatMessage: Codable {
let role: SenderRole
let content: String
}
enum SenderRole: String, Codable {
case system
case user
case assistant
}
struct OpenAIChatResponse: Decodable {
let choices: [OpenAIChatChoice]
}
struct OpenAIChatChoice: Decodable {
let message: OpenAIChatMessage
}
```
:::
## 定義 @Observable 的 class 串接網路 API 抓資料
:::spoiler Code
```swift=
extension ChatView {
class ViewModel: ObservableObject {
@Published var messages: [Message] = []
@Published var currentInput: String = ""
private let openAIService = OpenAIService()
init(initString: String) {
self.currentInput = initString
}
func updateCurrentInput(input: String) {
self.currentInput = input
}
func sendMessage() {
let newMessage = Message(id: UUID(), role: .user, content: currentInput, createAt: Date())
messages.append(newMessage)
currentInput = ""
Task {
let response = await openAIService.sendMessage(messages: messages)
guard let receiveOpenAIMessage = response?.choices.first?.message else {
print("Had no received message")
sendMessage()
return
}
let receiveMessage = Message(id: UUID(), role: receiveOpenAIMessage.role, content: receiveOpenAIMessage.content, createAt: Date())
await MainActor.run {
messages.append(receiveMessage)
}
}
}
}
}
```
:::
## 顯示資料下載中
- import [Lottie Package](https://github.com/Airbnb/Lottie-ios.git)
- display ContentUnavailableView if error
:::spoiler Code
```swift=
if viewModel.hasResponse {
if !viewModel.response.isEmpty {
VStack {
questionView(content: viewModel.response, retry: 0)
Spacer()
Spacer()
}
}
} else if viewModel.requestCrash {
ContentUnavailableView("Generating Fail", systemImage: "exclamationmark.triangle.fill")
} else {
LottieView(loopMode: .loop, source: "Loading")
.scaleEffect(0.5)
}
```
:::
### 沒有網路時使用 ContentUnavailableView 顯示錯誤
:::spoiler Code
```swift=
if networkManager.isNetworkAvailable {
// Statements
} else {
VStack {
ContentUnavailableView("No Internet Connect", systemImage: "wifi.slash")
Spacer()
}
}
```
:::
## 使用 TipKit 顯示 App 的操作說明 / 功能介紹
:::spoiler Code
```swift=
private let tip = QuestionDetailTip()
Button {
showExplanation = true
} label: {
HStack {
Spacer()
VStack {
Text("Question")
.font(.title3)
.foregroundColor(.secondary)
.padding(.bottom, 8)
Text(question)
.fontWeight(.semibold)
.multilineTextAlignment(.center)
.foregroundColor(Color.primary)
.font(.title3)
.transition(.scale)
.lineSpacing(1.5)
.modifier(ShakeEffect(animatableData: CGFloat(animateShake)))
.padding(.leading)
}
.padding(30)
Spacer()
}
.frame(height: 250)
.background(Color(uiColor: .secondarySystemBackground))
.cornerRadius(10)
.padding()
.popoverTip(tip, arrowEdge: .top) // Using TipKit
}
.sheet(isPresented: $showExplanation, content: {
VStack {
Text("Expanation")
.font(.title3)
.foregroundColor(.secondary)
.padding()
Text(ans)
.padding(.leading, 12)
.padding(.trailing, 12)
Text(explanation)
.padding()
Button {
viewModel.sendMessage()
} label: {
Text("Next")
}
}
})
struct QuestionDetailTip: Tip {
var title: Text {
Text("Press the question")
}
var message: Text? {
Text("Press the question the see more detail.")
}
var image: Image? {
Image(systemName: "quote.bubble")
}
}
```
:::
## 使用 SPM 加入第三方套件
- [Alamofire](https://github.com/Alamofire/Alamofire.git)
- [Lottie](https://github.com/Airbnb/Lottie-ios.git)
- [SwipeCellKit](https://github.com/SwipeCellKit/SwipeCellKit.git)
## 沒教過的功能技術
### 外掛 HUD View
:::spoiler Code 1
```swift=
struct HUD<Content: View>: View {
@Environment(\.colorScheme) private var colorScheme
@ViewBuilder let content: Content
var body: some View {
content
.padding(.horizontal, 12)
.padding(16)
.background(
Capsule()
.foregroundColor(colorScheme == .dark ? Color(UIColor.secondarySystemBackground) : Color(UIColor.systemBackground))
.shadow(color: Color(.black).opacity(0.15), radius: 10, x: 0, y: 4)
)
.padding(20)
}
}
```
:::
:::spoiler Code 2
```swift=
if showingHUD {
HUD {
if(self.currentAnswerIsCorrect) {
HStack(spacing: 25) {
HStack {
Image(systemName: "checkmark.circle.fill")
.foregroundColor(.green)
Text("That's correct")
.padding(.leading, 5)
.foregroundColor(Color.primary)
}
}
} else {
HStack {
Image(systemName: "xmark.circle.fill")
.foregroundColor(.red)
Text("That's wrong, try again")
.padding(.leading, 5)
}
}
}
.zIndex(1)
.transition(AnyTransition.move(edge: .bottom).combined(with: .opacity))
.padding(.bottom)
}
```
:::
### 利用正弦函數實作波動 SHM 特效動畫 (左右晃動/震動)
:::spoiler Code 1
```swift=
struct ShakeEffect: GeometryEffect {
var animatableData: CGFloat
func effectValue(size: CGSize) -> ProjectionTransform {
ProjectionTransform(CGAffineTransform(translationX: 10 * sin(animatableData * .pi * CGFloat(3)), y: 0))
}
}
```
:::
:::spoiler Code 2
```swift=
Text(question)
.fontWeight(.semibold)
.multilineTextAlignment(.center)
.foregroundColor(Color.primary)
.font(.title3)
.transition(.scale)
.lineSpacing(1.5)
.modifier(ShakeEffect(animatableData: CGFloat(animateShake))) // switch on if user press wrong choice
.padding(.leading)
```
:::
### 開啟相簿並將選擇結果自動進行文字辨識
- 透過 UIViewController 實作 ImagePicker
- UIImagePickerControllerDelegate Protocol
- 圖片選擇結果遞交給 TextRecognizer 進行文字辨識
:::spoiler Code
```swift=
import SwiftUI
import Foundation
import UIKit
struct ImagePicker: UIViewControllerRepresentable {
@Binding var selectedImage: UIImage?
@Binding var isPickerShowing: Bool
private let completionHandler: ([String]?) -> Void
init(selectedImage: Binding<UIImage?>, isPickerShowing: Binding<Bool>, completion: @escaping ([String]?) -> Void) {
_selectedImage = selectedImage
_isPickerShowing = isPickerShowing
self.completionHandler = completion
}
func makeUIViewController(context: Context) -> some UIViewController {
let imagePicker = UIImagePickerController()
imagePicker.sourceType = .photoLibrary
imagePicker.delegate = context.coordinator
return imagePicker
}
func updateUIViewController(_ uiViewController: UIViewControllerType, context: Context) {
}
func makeCoordinator() -> Coordinator {
return Coordinator(self, completion: completionHandler)
}
class Coordinator: NSObject, UIImagePickerControllerDelegate, UINavigationControllerDelegate {
private let completionHandler: ([String]?) -> Void
var parent: ImagePicker
init(_ picker: ImagePicker, completion: @escaping ([String]?) -> Void) {
self.parent = picker
self.completionHandler = completion
}
func imagePickerController(_ picker: UIImagePickerController, didFinishPickingMediaWithInfo info: [UIImagePickerController.InfoKey : Any]) {
print("image selected")
if let image = info[UIImagePickerController.InfoKey.originalImage] as? UIImage {
DispatchQueue.main.async { [self] in
self.parent.selectedImage = image
let recognizer = TextRecognizer(photoScan: image)
recognizer.recognizeText(withCompletionHandler: completionHandler)
}
}
parent.isPickerShowing = false
}
func imagePickerControllerDidCancel(_ picker: UIImagePickerController) {
print("cancelled")
parent.isPickerShowing = false
}
}
}
```
:::
### 開啟相機並預設自動捕捉文字辨識結果
- 透過 UIViewController 實作 CameraView
- VNDocumentCameraViewControllerDelegate Protocol
- 可更改為不要自動捕捉
:::spoiler Code
```swift=
import VisionKit
import SwiftUI
struct ScannerView: UIViewControllerRepresentable {
// 基於 UIViewControllerRepresentable Protocol,
// 在 ScannerView 被呼叫時 makeCoordinator() 會自動被呼叫
func makeCoordinator() -> Coordinator {
return Coordinator(completion: completionHandler)
}
final class Coordinator: NSObject, VNDocumentCameraViewControllerDelegate {
private let completionHandler: ([String]?) -> Void
init(completion: @escaping ([String]?) -> Void) {
self.completionHandler = completion
}
func documentCameraViewController(_ controller: VNDocumentCameraViewController, didFinishWith scan: VNDocumentCameraScan) {
let recognizer = TextRecognizer(cameraScan: scan)
recognizer.recognizeText(withCompletionHandler: completionHandler)
}
func documentCameraViewController(_ controller: VNDocumentCameraViewController, didFailWithError error: Error) {
completionHandler(nil)
}
func documentCameraViewControllerDidCancel(_ controller: VNDocumentCameraViewController) {
completionHandler(nil)
}
}
func makeUIViewController(context: Context) -> VNDocumentCameraViewController {
let viewController = VNDocumentCameraViewController()
viewController.delegate = context.coordinator
return viewController
}
func updateUIViewController(_ uiViewController: VNDocumentCameraViewController, context: Context) {
}
typealias UIViewControllerType = VNDocumentCameraViewController
private let completionHandler: ([String]?) -> Void
init(completion: @escaping ([String]?) -> Void) {
self.completionHandler = completion
}
}
```
:::
### 基於 OCR 光學文字辨識實作 TextRecognizer
- VNDocumentCameraScan, VNRecognizedTextObservation, VNImageRequestHandler, UIImage, CGImage 各種資料型別轉換
:::spoiler Code
```swift=
import Foundation
import Vision
import VisionKit
import SwiftUI
final class TextRecognizer {
let cameraScan: VNDocumentCameraScan?
let photoScan: UIImage?
init(cameraScan: VNDocumentCameraScan) {
self.cameraScan = cameraScan
self.photoScan = nil
}
init(photoScan: UIImage) {
self.cameraScan = nil
self.photoScan = photoScan
}
private let queue = DispatchQueue(label: "scan-codes", qos: .default, attributes: [], autoreleaseFrequency: .workItem)
func recognizeText(withCompletionHandler completionHandler: @escaping ([String]) -> Void) {
queue.async {
let images: [CGImage?]
if let cameraScan = self.cameraScan {
images = (0..<cameraScan.pageCount).compactMap {
cameraScan.imageOfPage(at: $0).cgImage
}
} else if let photoScan = self.photoScan {
// Assuming photoScan is a single page UIImage
images = [photoScan.cgImage]
} else {
images = []
}
let imagesAndRequests = images.map { (image: $0, request: VNRecognizeTextRequest()) }
let textPerPage = imagesAndRequests.map { image, request -> String in
guard let image = image else { return "" }
let handler = VNImageRequestHandler(cgImage: image, options: [:])
do {
try handler.perform([request])
guard let observations = request.results as? [VNRecognizedTextObservation] else { return "" }
return observations.compactMap { $0.topCandidates(1).first?.string }.joined(separator: "\n")
} catch {
print(error)
return ""
}
}
DispatchQueue.main.async {
completionHandler(textPerPage)
}
}
}
}
```
:::
### 基於 UITableView 實作 SwipeCellView
- 基於 UITableView 實現 element 的滑動刪除
- 基於 UIViewController 實現 UITableView
- 基於 ViewControllerRepresentable 實現 UIViewController
:::spoiler Code
```swift=
struct ViewControllerRepresentable: UIViewControllerRepresentable {
private var controller = ViewController()
func makeUIViewController(context: Context) -> ViewController {
return controller
}
func updateUIViewController(_ uiViewController: ViewController, context: Context) {
}
func addData() {
controller.addResource()
}
func refresh() {
controller.loadData()
}
}
class ViewController: UIViewController, ObservableObject {
@Published var resources: [Resource] = [Resource.demoResource]
private let tableView = UITableView()
override func viewDidLoad() {
super.viewDidLoad()
// Setting tableView
tableView.dataSource = self
tableView.delegate = self
tableView.register(UITableViewCell.self, forCellReuseIdentifier: "cell")
// Adding tableView to AppView
view.addSubview(tableView)
// Setting tableView with Auto Layout
tableView.translatesAutoresizingMaskIntoConstraints = false
NSLayoutConstraint.activate([
tableView.topAnchor.constraint(equalTo: view.safeAreaLayoutGuide.topAnchor),
tableView.leadingAnchor.constraint(equalTo: view.safeAreaLayoutGuide.leadingAnchor),
tableView.trailingAnchor.constraint(equalTo: view.safeAreaLayoutGuide.trailingAnchor),
tableView.bottomAnchor.constraint(equalTo: view.safeAreaLayoutGuide.bottomAnchor),
])
}
func loadData() {
tableView.reloadData()
}
func addResource() {
resources.append(Resource(title: "New resource"))
tableView.reloadData()
}
}
extension ViewController: UITableViewDataSource, UITableViewDelegate {
func tableView(_ tableView: UITableView, numberOfRowsInSection section: Int) -> Int {
resources.count
}
func tableView(_ tableView: UITableView, cellForRowAt indexPath: IndexPath) -> UITableViewCell {
let cell = UITableViewCell()
if !resources[indexPath.row].tags.isEmpty {
cell.textLabel?.text = resources[indexPath.row].tags[0] + " | "
} else {
cell.textLabel?.text = ". default | "
}
cell.textLabel?.text?.append(resources[indexPath.row].title)
return cell
}
func tableView(_ tableView: UITableView, didSelectRowAt indexPath: IndexPath) {
let destinationView = ResourceContentView(resource: resources[indexPath.row])
let hostingController = UIHostingController(rootView: destinationView)
navigationController?.pushViewController(hostingController, animated: true)
tableView.deselectRow(at: indexPath, animated: true)
}
func tableView(_ tableView: UITableView, trailingSwipeActionsConfigurationForRowAt indexPath: IndexPath) -> UISwipeActionsConfiguration? {
let deleteAction = UIContextualAction(style: .destructive, title: nil) {_, _, completion in
self.resources.remove(at: indexPath.row)
tableView.deleteRows(at: [indexPath], with: .automatic)
completion(true)
}
deleteAction.image = UIImage(systemName: "trash")
deleteAction.backgroundColor = .systemRed
let config = UISwipeActionsConfiguration(actions: [deleteAction])
self.loadData()
return config
}
}
```
:::
## 特製程式碼講解
### 將 GPT 回傳文字結果進行嚴格字串處理
- 基於 prompt engineering 定義嚴格回傳格式
- 使用 String.component(separatedBy: String) 進行字串切割
- 使用多層 guard 對切割結果進行檢查
- 當切割結果不理想時重新進行切割
- 當多次重新切割之結果不理想時重新發送 API request
- 當多次發送 API request 失敗時顯示 ContentUnavailableView
:::spoiler Code
```swift=
func questionView(content: String, retry: Int) -> some View {
let components: [String] = content.components(separatedBy: "Component: ").filter { !$0.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty }
guard components.count >= 4 else {
print("Retry: \(components.count)")
if retry > 8 {
print("Resend")
return Text("")
.onAppear(perform: {
viewModel.sendMessage()
})
}
else { return questionView(content: content, retry: retry + 1)}
}
let question: String = components[0]
var choices: [String] = components[1].components(separatedBy: .newlines)
guard choices.count >= 4 else {
if retry > 8 {
print("Resend")
return Text("")
.onAppear(perform: {
viewModel.sendMessage()
})
}
else { return questionView(content: content, retry: retry + 1) }
}
let ans: String = components[2]
let explanation: String = components[3]
return VStack(alignment: .leading, spacing: 5) {
Button {
showExplanation = true
} label: {
HStack {
Spacer()
VStack {
Text("Question")
.font(.title3)
.foregroundColor(.secondary)
.padding(.bottom, 8)
Text(question)
.fontWeight(.semibold)
.multilineTextAlignment(.center)
.foregroundColor(Color.primary)
.font(.title3)
.transition(.scale)
.lineSpacing(1.5)
.modifier(ShakeEffect(animatableData: CGFloat(animateShake)))
.padding(.leading)
}
.padding(30)
Spacer()
}
.frame(height: 250)
.background(Color(uiColor: .secondarySystemBackground))
.cornerRadius(10)
.padding()
.popoverTip(tip, arrowEdge: .top)
}
.sheet(isPresented: $showExplanation, content: {
VStack {
Text("Expanation")
.font(.title3)
.foregroundColor(.secondary)
.padding()
Text(ans)
.padding(.leading, 12)
.padding(.trailing, 12)
Text(explanation)
.padding()
Button {
viewModel.sendMessage()
} label: {
Text("Next")
}
}
})
ForEach(choices.indices) { idx in
if idx < choices.count && !choices[idx].isEmpty {
Button {
if choices[idx].first == ans.first {
self.currentAnswerIsCorrect = true
}
withAnimation(Animation.timingCurve(0.47, 1.62, 0.45, 0.99, duration: 0.4)) {
showingHUD.toggle()
isAnimating = true
}
if self.currentAnswerIsCorrect {
DispatchQueue.main.asyncAfter(deadline: .now() + (1.7)) {
viewModel.sendMessage()
withAnimation() {
showingHUD = false
isAnimating = false
self.currentAnswerIsCorrect = false
}
}
} else {
withAnimation(.default) {
animateShake += 1
}
DispatchQueue.main.asyncAfter(deadline: .now() + (2.5)) {
withAnimation() {
showingHUD = false
isAnimating = false
self.currentAnswerIsCorrect = false
}
}
}
} label: {
HStack {
Text(choices[idx])
.font(.callout)
.foregroundColor(.accentColor)
.padding(EdgeInsets())
Spacer()
}
.padding(12)
.background(Color.accentColor.opacity(0.13))
.cornerRadius(12)
.padding(EdgeInsets(top: 0, leading: 8, bottom: 0, trailing: 8))
}
.padding(3)
}
}
Spacer()
}
.disabled(isAnimating)
}
```
:::
## 寒假計畫
- 加入 ShareLink 提供分享答題結果服務
- 加入 searchable 提供基於 tag 搜尋 Resource 服務
- 加入使用者登入服務
- import [SQLite](https://github.com/stephencelis/SQLite.swift) package 儲存使用者資料
## 未來展望
### 影像視覺分析
- 晃動校正
- 雜訊過濾
### NLP
- 錯字校正
- 雜訊過濾
- 題庫生成
<!-- {%hackmd /@Ateto/Style %} -->