From 288a9bf2160c91b47171579a16032271275a89c6 Mon Sep 17 00:00:00 2001
From: Ufuk Kaan Tugay <ufuk.tugay@ruhr-uni-bochum.de>
Date: Wed, 15 Sep 2021 11:17:04 +0000
Subject: [PATCH] Replace ViewController.swift
---
.../ViewController.swift | 304 +++++++++++-------
1 file changed, 189 insertions(+), 115 deletions(-)
diff --git a/Objekterkennung in AR mit ARKit und CoreML/ViewController.swift b/Objekterkennung in AR mit ARKit und CoreML/ViewController.swift
index 5c69123..4964128 100644
--- a/Objekterkennung in AR mit ARKit und CoreML/ViewController.swift
+++ b/Objekterkennung in AR mit ARKit und CoreML/ViewController.swift
@@ -12,15 +12,61 @@ import Vision
class ViewController: UIViewController, ARSCNViewDelegate {
- @IBOutlet var sceneView: ARSCNView!
+ @IBOutlet var sceneView: ARSCNView!
+
var detectionMode = "click"
- var node_array: [SCNNode] = []
var scene_bounds: (CGFloat,CGFloat) = (0,0)
var object_detector = ObjectDetection()
var ar_functions = View()
-
let label = UILabel(frame: CGRect(x: 5, y: 20, width: 400, height: 30))
+ var start_position_array: [simd_float4x4] = []
+ var classification_object = ""
+ var bounding_box_object = SCNNode()
+ var text_object = SCNNode()
+ var text_node: SCNNode {
+ return text_object
+ }
+ var bounding_box_node: SCNNode {
+ return bounding_box_object
+ }
+
+ var test_node: SCNNode{
+ switch classification_object {
+ case "90T_Tyre":
+ return ar_functions.getTire90T()
+ case "beam_0412-140":
+ return ar_functions.getBeam0412_140()
+ case "beam_0412-92":
+ return ar_functions.getBeam0412_092()
+ case "beam_0824-112":
+ return ar_functions.getBeam0824_112()
+ case "beam_0824-16":
+ return ar_functions.getBeam0824_016()
+ case "beam_0824-80":
+ return ar_functions.getBeam0824_080()
+ case "beam_0824-96":
+ return ar_functions.getBeam0824_096()
+ case "bracket_3x3":
+ return ar_functions.getBracket3x3()
+ case "plate_3x6":
+ return ar_functions.getPlate3x6()
+ case "unnamed_nut":
+ return ar_functions.getBrassStud()
+ case "unnamed_angle":
+ return ar_functions.getPlate45()
+ default:
+ let no_model_node = SCNNode()
+ no_model_node.name = "no_model"
+ return no_model_node
+
+ }
+
+ }
+
+ var start_position: simd_float4x4{
+ return sceneView.session.currentFrame!.camera.transform
+ }
func createLabel(){
if detectionMode == "loop"{
@@ -33,26 +79,7 @@ class ViewController: UIViewController, ARSCNViewDelegate {
self.view.addSubview(label)
}
-
- func createButtons(){
- let mode_button = UIButton()
- mode_button.frame = CGRect(x:20, y: 700, width: 200, height: 100)
- mode_button.backgroundColor = .white
- mode_button.setTitle("Change Detection Mode", for: .normal)
- mode_button.setTitleColor( .black, for: .normal)
- mode_button.addTarget(self, action: #selector(changeDetectionMode), for: .touchUpInside)
-
-
- let detect_button = UIButton()
- detect_button.frame = CGRect(x:240, y: 700, width: 200, height: 100)
- detect_button.backgroundColor = .white
- detect_button.setTitle("Perform Detection", for: .normal)
- detect_button.setTitleColor( .black, for: .normal)
- detect_button.addTarget(self, action: #selector(detect), for: .touchUpInside)
-
- self.view.addSubview(mode_button)
- self.view.addSubview(detect_button)
- }
+
@objc func changeDetectionMode(){
if detectionMode == "loop" {
@@ -73,13 +100,13 @@ class ViewController: UIViewController, ARSCNViewDelegate {
}
-
+
override func viewDidLoad() {
super.viewDidLoad()
let configuration = ARWorldTrackingConfiguration()
- configuration.planeDetection = .horizontal
+ configuration.planeDetection = [.horizontal, .vertical]
sceneView.scene = SCNScene()
sceneView.session.run(configuration)
@@ -90,122 +117,141 @@ class ViewController: UIViewController, ARSCNViewDelegate {
createButtons()
createLabel()
}
-
- override func viewWillAppear(_ animated: Bool) {
- super.viewWillAppear(animated)
- }
-
- override func viewWillDisappear(_ animated: Bool) {
- super.viewWillDisappear(animated)
- }
-
+
+ func renderer(_ renderer: SCNSceneRenderer, didAdd node: SCNNode, for anchor: ARAnchor) {
+ if anchor.name == "model" {
+ node.addChildNode(test_node)
+
+ }
+ else if anchor.name == "bb"{
+ bounding_box_node.position = node.position
+ node.addChildNode(bounding_box_node)
+ }
+ else if anchor.name == "text"{
+ text_node.position = node.position
+ node.addChildNode(text_node)
+
+ }
+ }
func performDetection() {
+
guard let frame = sceneView.session.currentFrame else {
return
}
-
+
if frame.anchors.isEmpty {
print("anchors are empty")
return
- }
- let start_time = Date()
+ }
+
+ sceneView.scene = SCNScene()
+
object_detector.detectObjects(on: sceneView.session.currentFrame!.capturedImage)
- let end_timee = Date()
- print(end_timee.timeIntervalSince(start_time))
- if let new_detection = self.object_detector.detection_object_array.first {
- if new_detection.confidence > 0.7 {
- sceneView.scene = SCNScene()
- }
-
- }
for observations in self.object_detector.detection_object_array{
-
- if observations.confidence > 0.7 {
+ if observations.confidence > 0.5 {
for observations in self.object_detector.detection_object_array{
- //print(observations)
- //print(observations.boundingBox.midX)
- //print(observations.boundingBox.midY)
DispatchQueue.main.async {
self.scene_bounds = self.sceneBounds()
}
+ classification_object = observations.classification
+
+ if test_node.name == "no_model" {
+ continue
+ }
+ var translation = matrix_identity_float4x4
+ translation.columns.3.z = test_node.position.z
+ translation.columns.3.x = test_node.position.x
+ translation.columns.3.y = test_node.position.y
+ let transform = simd_mul(start_position_array[0], translation)
+ let model_anchor = ARAnchor(name: "model", transform: transform)
+
+ sceneView.session.add(anchor: model_anchor)
- print(observations.classification)
let detectionLocation = VNImageRectForNormalizedRect(
observations.boundingBox,
Int(self.scene_bounds.0),
Int(self.scene_bounds.1))
-
- //self.objectDetector.detection_object_array.removeAll()
- drawModel(observations.classification)
- //let bb2d_view = ar_functions.drawboundingbox2d(detectionLocation: detectionLocation)
- //bb2d_view.removeFromSuperview()
- //self.view.addSubview(bb2d_view)
-
- ar_functions.drawBoundingBox3D(detectionLocation: detectionLocation, vision_bb: observations.boundingBox, sceneView: sceneView)
- sceneView.scene.rootNode.addChildNode(ar_functions.draw_text_node(detectionLocation: detectionLocation, classification: observations.classification, sceneView: sceneView ))
- let end_time = Date()
- print(end_time.timeIntervalSince(start_time))
-
- }
-
-// sceneView.scene.rootNode.enumerateChildNodes { (node, stop) in
-// node.removeFromParentNode() }
+ let bb_node = ar_functions.drawBoundingBox3D(detectionLocation: detectionLocation, sceneView: sceneView)
+ bounding_box_object = bb_node
+ let anchor_bb = ARAnchor(name: "bb", transform: bb_node.simdTransform)
+ sceneView.session.add(anchor: anchor_bb)
+
+ let drawn_text_node = ar_functions.draw_text_node(detectionLocation: detectionLocation, classification: observations.classification, sceneView: sceneView)
+ text_object = drawn_text_node
+ let anchor_text = ARAnchor(name: "text", transform: drawn_text_node.simdTransform)
+ sceneView.session.add(anchor: anchor_text)
+ }
}
}
self.object_detector.detection_object_array.removeAll()
-
}
-// func removeNodes(){
-// self.sceneView.scene.rootNode.enumerateChildNodes { (existingNode, _) in
-// node_array.append(existingNode)
-// existingNode.removeFromParentNode()
-// node_array.removeFirst()
-// }
-// }
- func drawModel(_ object_type: String){
- switch object_type {
- case ("plate_3x6"):
- sceneView.scene.rootNode.addChildNode(ar_functions.getPlate3x6())
-
- case ("90T_Tyre"):
- sceneView.scene.rootNode.addChildNode(ar_functions.getTire90T())
-
- case ("beam_0412-140"):
- sceneView.scene.rootNode.addChildNode(ar_functions.getBeam0412_140())
- case ("beam_0412-92"):
- sceneView.scene.rootNode.addChildNode(ar_functions.getBeam0412_092())
-
- case ("beam_0824-112"):
- sceneView.scene.rootNode.addChildNode(ar_functions.getBeam0824_096())
-
- case ("beam_0824-16"):
- sceneView.scene.rootNode.addChildNode(ar_functions.getBeam0824_016())
- case ("beam_0824-80"):
- sceneView.scene.rootNode.addChildNode(ar_functions.getBeam0824_080())
- case ("beam_0824-96"):
- sceneView.scene.rootNode.addChildNode(ar_functions.getBeam0824_096())
- case ("bracket_3x3"):
- sceneView.scene.rootNode.addChildNode(ar_functions.getBracket3x3())
- case ("unnamed_angle"):
- sceneView.scene.rootNode.addChildNode(ar_functions.getPlate45())
- case ("unnamed_nut"):
- sceneView.scene.rootNode.addChildNode(ar_functions.getBrassStud())
- default:
- print("Kein Modell für diese Detection vorhanden")
- }
+// func drawModel(_ object_type: String){
+// switch object_type {
+// case ("plate_3x6"):
+// sceneView.scene.rootNode.addChildNode(ar_functions.getPlate3x6())
+// // case ("90T_Tyre"):
+// //sceneView.scene.rootNode.addChildNode(ar_functions.getTire90T())
+//
+// case ("beam_0412-140"):
+// sceneView.scene.rootNode.addChildNode(ar_functions.getBeam0412_140())
+//
+// case ("beam_0412-92"):
+// sceneView.scene.rootNode.addChildNode(ar_functions.getBeam0412_092())
+//
+// case ("beam_0824-112"):
+// sceneView.scene.rootNode.addChildNode(ar_functions.getBeam0824_096())
+//
+// case ("beam_0824-16"):
+// sceneView.scene.rootNode.addChildNode(ar_functions.getBeam0824_016())
+// case ("beam_0824-80"):
+// sceneView.scene.rootNode.addChildNode(ar_functions.getBeam0824_080())
+// case ("beam_0824-96"):
+// sceneView.scene.rootNode.addChildNode(ar_functions.getBeam0824_096())
+// case ("bracket_3x3"):
+// sceneView.scene.rootNode.addChildNode(ar_functions.getBracket3x3())
+// case ("unnamed_angle"):
+// sceneView.scene.rootNode.addChildNode(ar_functions.getPlate45())
+// case ("unnamed_nut"):
+// sceneView.scene.rootNode.addChildNode(ar_functions.getBrassStud())
+// default:
+// print("Kein Modell für diese Detection vorhanden")
+// }
+// }
+
+ func createButtons(){
+ let mode_button = UIButton()
+ mode_button.frame = CGRect(x:20, y: 680, width: 200, height: 100)
+ mode_button.backgroundColor = .white
+ mode_button.setTitle("Change Detection Mode", for: .normal)
+ mode_button.setTitleColor( .black, for: .normal)
+ mode_button.addTarget(self, action: #selector(changeDetectionMode), for: .touchUpInside)
-
- }
-
-
+ let detect_button = UIButton()
+ detect_button.frame = CGRect(x:240, y: 680, width: 200, height: 100)
+ detect_button.backgroundColor = .white
+ detect_button.setTitle("Perform Detection", for: .normal)
+ detect_button.setTitleColor( .black, for: .normal)
+ detect_button.addTarget(self, action: #selector(detect), for: .touchUpInside)
+
+ let position_button = UIButton()
+ position_button.frame = CGRect(x: 460,y: 680, width: 200, height: 100)
+ position_button.backgroundColor = .white
+ position_button.setTitle("Select Position", for: .normal)
+ position_button.setTitleColor(.black, for: .normal)
+ position_button.addTarget(self, action: #selector(generateStartPosition), for: .touchUpInside)
+ position_button.tag = 192
+ self.view.addSubview(mode_button)
+ self.view.addSubview(detect_button)
+ self.view.addSubview(position_button)
+ }
func sceneBounds() -> (CGFloat, CGFloat) {
return (self.sceneView.bounds.width,self.sceneView.bounds.height)
@@ -215,20 +261,48 @@ class ViewController: UIViewController, ARSCNViewDelegate {
if detectionMode == "loop" {
let dispatch_queue = DispatchQueue.main
scene_bounds = sceneBounds()
- dispatch_queue.asyncAfter(deadline: .now() + 1, execute: DispatchWorkItem{self.performDetection();self.detectionLoop()})
+ dispatch_queue.asyncAfter(deadline: .now() + 0.2 , execute: DispatchWorkItem{self.performDetection();self.detectionLoop()})
}
-
else if detectionMode == "click" {
performDetection()
}
-
}
+
+ @objc func generateStartPosition(){
+ while true{
+ if let start_pos = sceneView.session.currentFrame {
+ start_position_array.append(start_pos.camera.transform)
+ print("start position bestimmt")
+ self.view.subviews.forEach ({
+ if $0 is UIButton {
+ if $0.tag == 192{
+ $0.removeFromSuperview()
+ }
+ }
+ })
-
+
+ break
+ }
+ }
+ }
+
func session(_ session: ARSession) {
-
detectionLoop()
}
+
+ override func viewDidAppear(_ animated: Bool) {
+ super.viewDidAppear(animated)
+ }
+
+ override func viewWillAppear(_ animated: Bool) {
+ super.viewWillAppear(animated)
+ }
+
+ override func viewWillDisappear(_ animated: Bool) {
+ super.viewWillDisappear(animated)
+ }
+
}
--
GitLab