added detection based positioning and node constraints for text label

Objekterkennung in AR mit ARKit und CoreML/View.swift

@@ -37,18 +37,19 @@ class View {
     }
     
     // this function is inspired by a solution provided in  https://stackoverflow.com/questions/50731775/arkit-how-to-contain-scntext-within-another-scnnode-speech-bubble
-    func draw_text_node(detectionLocation bound_box2d: CGRect, classification class_name: String) -> SCNNode{
-        let string = class_name
+    func draw_text_node(detectionLocation bound_box2d: CGRect, classification class_name: String, sceneView: ARSCNView) -> SCNNode{
 
-        let text = SCNText(string: string, extrusionDepth: 0.1)
+        let text = SCNText(string: class_name, extrusionDepth: 0.1)
         
         let textNode = SCNNode(geometry: text)
+        
         let fontScale: Float = 0.001
         textNode.scale = SCNVector3(fontScale, fontScale, fontScale)
         let (min, max) = (text.boundingBox.min, text.boundingBox.max)
         let dx = min.x + 0.5 * (max.x - min.x)
         let dy = min.y + 0.5 * (max.y - min.y)
         let dz = min.z + 0.5 * (max.z - min.z)
+        
         textNode.pivot = SCNMatrix4MakeTranslation(dx, dy, dz)
         textNode.geometry?.firstMaterial?.diffuse.contents = UIColor.black
         let width = (max.x - min.x) * fontScale
@@ -61,6 +62,16 @@ class View {
         textNode.eulerAngles = planeNode.eulerAngles
         planeNode.addChildNode(textNode)
         
+        let hit_test = sceneView.hitTest(CGPoint(x: bound_box2d.midX, y: bound_box2d.midY) , types: [.featurePoint])
+
+        if let nearest_point = hit_test.first {
+            let transform_point = nearest_point.worldTransform
+            let point_coordinates_3d = SCNVector3Make(transform_point.columns.3.x,transform_point.columns.3.y, transform_point.columns.3.z*2)
+            planeNode.position = point_coordinates_3d
+        }
+        let constraints = SCNBillboardConstraint()
+        constraints.freeAxes = SCNBillboardAxis.Y
+        planeNode.constraints = [constraints]
         return planeNode
     }
     

Objekterkennung in AR mit ARKit und CoreML/ViewController.swift

@@ -14,9 +14,60 @@ class ViewController: UIViewController, ARSCNViewDelegate {
     
     @IBOutlet var sceneView: ARSCNView!
     
-    private var detectionOverlay: CALayer! = nil
-    let uiview = UIView()
+    let label = UILabel(frame: CGRect(x: 5, y: 20, width: 400, height: 30))
+    
+    func createLabel(){
+        if detectionMode == "loop"{
+            label.text = "Detection Mode: Loop"
+        }
+        else if detectionMode == "click"{
+            label.text = "Detection Mode: On Button Press"
+        }
+        label.textAlignment = .left
+        self.view.addSubview(label)
 
+    }
+
+    func createButtons(){
+        let mode_button = UIButton()
+        mode_button.frame = CGRect(x:20, y: 700, width: 200, height: 100)
+        mode_button.backgroundColor = .white
+        mode_button.setTitle("Change Detection Mode", for: .normal)
+        mode_button.setTitleColor( .black, for: .normal)
+        mode_button.addTarget(self, action: #selector(changeDetectionMode), for: .touchUpInside)
+        
+        
+        let detect_button = UIButton()
+        detect_button.frame = CGRect(x:240, y: 700, width: 200, height: 100)
+        detect_button.backgroundColor = .white
+        detect_button.setTitle("Perform Detection", for: .normal)
+        detect_button.setTitleColor( .black, for: .normal)
+        detect_button.addTarget(self, action: #selector(detect), for: .touchUpInside)
+        
+        self.view.addSubview(mode_button)
+        self.view.addSubview(detect_button)
+    }
+    
+    @objc func changeDetectionMode(){
+        if detectionMode == "loop" {
+            detectionMode = "click"
+            createLabel()
+            print("Detection mode switched to click")
+            
+        }
+        else if detectionMode == "click" {
+            detectionMode = "loop"
+            print("Detection mode switched to loop")
+            createLabel()
+            detectionLoop()
+        }
+    }
+    @objc func detect() {
+        performDetection()
+        print("detection performed")
+    }
+    var detectionMode = "click"
+    let uiview = UIView()
     var node_array: [SCNNode] = []
     var scene_bounds: (CGFloat,CGFloat) = (0,0)
     var objectDetector = ObjectDetection()
@@ -33,7 +84,7 @@ class ViewController: UIViewController, ARSCNViewDelegate {
         self.view.addSubview(uiview)
     }
 
-    
+
     override func viewDidLoad() {
         super.viewDidLoad()
 
@@ -46,6 +97,8 @@ class ViewController: UIViewController, ARSCNViewDelegate {
         sceneView.showsStatistics = true
         sceneView.debugOptions = [.showFeaturePoints]
         sceneView.autoenablesDefaultLighting = true
+        createButtons()
+        createLabel()
     }
     
     override func viewWillAppear(_ animated: Bool) {
@@ -89,7 +142,7 @@ class ViewController: UIViewController, ARSCNViewDelegate {
                         self.scene_bounds = self.sceneBounds()
                     }
                     
-                    print(observations.boundingBox)
+                    print(observations.classification)
                     let detectionLocation = VNImageRectForNormalizedRect(
                         observations.boundingBox,
                         Int(self.scene_bounds.0),
@@ -97,11 +150,10 @@ class ViewController: UIViewController, ARSCNViewDelegate {
                     
                     
                     //self.objectDetector.detection_object_array.removeAll()
-                    print(detectionLocation)
                     drawModel(observations.classification)
-                    drawboundingbox2d(detectionLocation: detectionLocation)
+                    //drawboundingbox2d(detectionLocation: detectionLocation)
                     models3d.drawBoundingBox3D(detectionLocation: detectionLocation, vision_bb: observations.boundingBox, sceneView: sceneView)
-                    sceneView.scene.rootNode.addChildNode(models3d.draw_text_node(detectionLocation: detectionLocation, classification: observations.classification ))
+                    sceneView.scene.rootNode.addChildNode(models3d.draw_text_node(detectionLocation: detectionLocation, classification: observations.classification, sceneView: sceneView ))
                     let end_time = Date()
                     print(end_time.timeIntervalSince(start_time))
                 
@@ -127,19 +179,16 @@ class ViewController: UIViewController, ARSCNViewDelegate {
         switch  object_type {
         case ("plate_3x6"):
             sceneView.scene.rootNode.addChildNode(models3d.getPlate3x6Node())
-            removeNodes()
 
         case ("90T_Tyre"):
             sceneView.scene.rootNode.addChildNode(models3d.getTire90T())
-            removeNodes()
 
         case ("beam_0412-140"):
             sceneView.scene.rootNode.addChildNode(models3d.getBeam0412_140Node())
-            removeNodes()
+
         case ("beam_0412-92"):
             sceneView.scene.rootNode.addChildNode(models3d.getBeam0412_092Node())
-            removeNodes()
-//          TODO
+
 //        case ("beam_0824-112"):
 //            removeNodes()
 //            sceneView.scene.rootNode.addChildNode(models3d.getBeam0824_096())
@@ -150,16 +199,12 @@ class ViewController: UIViewController, ARSCNViewDelegate {
             sceneView.scene.rootNode.addChildNode(models3d.getBeam0824_080())
         case ("beam_0824-96"):
             sceneView.scene.rootNode.addChildNode(models3d.getBeam0824_096())
-            removeNodes()
         case ("bracket_3x3"):
             sceneView.scene.rootNode.addChildNode(models3d.getBracket3x3())
-            removeNodes()
         case ("unnamed_angle"):
             sceneView.scene.rootNode.addChildNode(models3d.getPlate3x6Node())
-            removeNodes()
         case ("unnamed_nut"):
             sceneView.scene.rootNode.addChildNode(models3d.getBrassStud())
-            removeNodes()
         default:
             print("Kein Modell für diese Detection vorhanden")
         }
@@ -175,10 +220,15 @@ class ViewController: UIViewController, ARSCNViewDelegate {
     }
     
     func detectionLoop() {
-        
+        if detectionMode == "loop" {
         let dispatch_queue = DispatchQueue.main
         scene_bounds = sceneBounds()
         dispatch_queue.asyncAfter(deadline: .now() + 1, execute: DispatchWorkItem{self.performDetection();self.detectionLoop()})
+        }
+        
+        else if detectionMode == "click" {
+            performDetection()
+        }
    
     }