Merge pull request #3 from neocortex-link/feature/request-unification

API Updates / Request Unification

Author: srcnalt

Editor/AudioReceiverEditor.cs renamed to Editor/NeocortexAudioReceiverEditor.cs

@@ -4,20 +4,26 @@
 namespace Neocortex.Editor
 {
     [CustomEditor(typeof(NeocortexAudioReceiver))]
-    public class AudioReceiverEditor : UnityEditor.Editor
+    public class NeocortexAudioReceiverEditor : UnityEditor.Editor
     {
-        private const string MIC_INDEX_KEY = "neocortex-mix-index";
+        private const string MIC_INDEX_KEY = "neocortex-mic-index";
 
         private string[] microphoneOptions;
         private int selectedMicrophoneIndex;
 
+        private SerializedProperty onAudioRecorded;
+        private SerializedProperty onRecordingFailed;
+        
         private void OnEnable()
         {
             microphoneOptions = Microphone.devices;
             selectedMicrophoneIndex = PlayerPrefs.GetInt(MIC_INDEX_KEY, 0);
 
             NeocortexAudioReceiver audioReceiver = (NeocortexAudioReceiver)target;
             audioReceiver.SelectedMicrophone = microphoneOptions[selectedMicrophoneIndex];
+            
+            onAudioRecorded = serializedObject.FindProperty("OnAudioRecorded");
+            onRecordingFailed = serializedObject.FindProperty("OnRecordingFailed");
         }
 
         public override void OnInspectorGUI()
@@ -36,6 +42,14 @@ public override void OnInspectorGUI()
             {
                 EditorGUILayout.LabelField("No microphones available.");
             }
+            
+            GUILayout.Space(8);
+            EditorGUILayout.PropertyField(onAudioRecorded);
+            serializedObject.ApplyModifiedProperties();
+            
+            GUILayout.Space(8);
+            EditorGUILayout.PropertyField(onRecordingFailed);
+            serializedObject.ApplyModifiedProperties();
         }
     }
 }

Editor/AudioReceiverEditor.cs.meta renamed to Editor/NeocortexAudioReceiverEditor.cs.meta

@@ -1,4 +1,3 @@
-using System;
 using UnityEditor;
 using UnityEngine;
 

Editor/UI/NeocortexChatPanelEditor.cs

@@ -44,38 +44,108 @@ The `Neocortex Smart Agent` component is the main component that allows you to i
   <img width="393" alt="neocortex_unity_smart_agent_component" src="https://github.com/user-attachments/assets/9613bb88-87a9-4ba5-b412-d404c0bf63e3">
 </p>
 
-**public async void Send(string message)**
-  - Sends a message to the Neocortex project
+**public async void TextToText(string message)**
+  - Send a text message to the Neocortex project, and expect a text response.
   - Parameters:
-    - `message`: The message to send
+    - `message`: The text message to send.
   - Example:
     ```csharp
     var smartAgent = GetComponent<NeocortexSmartAgent>();
-    smartAgent.Send("Hello, Neocortex!");
+    smartAgent.OnChatResponseReceived.AddListener((response) =>
+    {
+        Debug.Log($"Message: {response.message}");
+        Debug.Log($"Action: {response.action}");
+    });
+    smartAgent.TextToText("Hello, Neocortex!");
+    ```
+
+**public async void TextToAudio(string message)**
+  - Send a text message to the Neocortex project, and expect a audio response.
+  - Parameters:
+    - `message`: The text message to send.
+  - Example:
+    ```csharp
+    var audioSource = GetComponent<AudioSource>();
+    var smartAgent = GetComponent<NeocortexSmartAgent>();
+    smartAgent.OnChatResponseReceived.AddListener((response) =>
+    {
+        Debug.Log($"Message: {response.message}");
+        Debug.Log($"Action: {response.action}");
+    });
+    smartAgent.OnAudioResponseReceived.AddListener((audioClip) =>
+    {
+        audioSource.clip = audioClip;
+        audioSource.Play();
+    });
+    
+    smartAgent.TextToAudio("Hello, Neocortex!");
     ```
 
-**public async void Send(byte[] audio)**
-  - Sends an audio clip to the Neocortex project. This method is used with `NeocortexAudioReceiver` component to send audio data to the Neocortex project.
+**public async void AudioToText(AudioClip audio)**
+  - Sends an audio clip to the Neocortex project. This method is used with `NeocortexAudioReceiver` component to send audio data.
   - Parameters:
-    - `audio`: The audio data to send
+    - `audioClip`: The audio clip to send.
   - Example:
     ```csharp
     var smartAgent = GetComponent<NeocortexSmartAgent>();
+    smartAgent.OnTranscriptionReceived.AddListener((message) =>
+    {
+        Debug.Log($"You: {message}");
+    });
+
     var audioReceiver = GetComponent<NeocortexAudioReceiver>();
-    audioReceiver.OnAudioRecorded += (audioData) =>
+    audioReceiver.OnAudioRecorded.AddListener((audioClip) =>
     {
-        smartAgent.Send(audioData);
-    };
+        Debug.Log($"Audio Data Length: {audioClip.samples}");
+        smartAgent.AudioToText(audioClip);
+    });
 
-    // Start recording audio for 5 seconds
+    // Start recording audio for 3 seconds
     audioReceiver.StartMicrophone();
-    await Task.Delay(5000);
+    await Task.Delay(3000);
     audioReceiver.StopMicrophone();
     ```
-**public event Action<ChatResponse> OnChatResponseReceived**
-  - Event that is triggered when the Neocortex project responds to a message
+
+**public async void AudioToAudio(AudioClip audio)**
+  - Sends an audio clip to the Neocortex project and expects an audio response. This method is used with `NeocortexAudioReceiver` component to send audio data.
   - Parameters:
-    - `response`: The response from the Neocortex project
+    - `audioClip`: The audio clip to send.
+  - Example:
+    ```csharp
+    var audioSource = GetComponent<AudioSource>();
+    var smartAgent = GetComponent<NeocortexSmartAgent>();
+    smartAgent.OnAudioResponseReceived.AddListener((audioClip) =>
+    {
+        audioSource.clip = audioClip;
+        audioSource.Play();
+    });
+    smartAgent.OnTranscriptionReceived.AddListener((message) =>
+    {
+        Debug.Log($"You: {message}");
+    });
+    smartAgent.OnChatResponseReceived.AddListener((response) =>
+    {
+        Debug.Log($"Message: {response.message}");
+        Debug.Log($"Action: {response.action}");
+    });
+
+    var audioReceiver = GetComponent<NeocortexAudioReceiver>();
+    audioReceiver.OnAudioRecorded.AddListener((audioClip) =>
+    {
+        Debug.Log($"Audio Data Length: {audioClip.samples}");
+        smartAgent.AudioToAudio(audioClip);
+    });
+
+    // Start recording audio for 3 seconds
+    audioReceiver.StartMicrophone();
+    await Task.Delay(3000);
+    audioReceiver.StopMicrophone();
+    ```
+
+**public UnityEvent<ChatResponse> OnChatResponseReceived**
+  - Event that is triggered when the Neocortex project responds to a text message.
+  - Parameters:
+    - `response`: The response from the Neocortex project.
   - Example:
     ```csharp
     var smartAgent = GetComponent<NeocortexSmartAgent>();
@@ -86,10 +156,10 @@ The `Neocortex Smart Agent` component is the main component that allows you to i
     };
     ```
 
-**public event Action<string> OnTranscriptionReceived**
-  - Event that is triggered when the Neocortex project transcribes an audio message to text
+**public UnityEvent<string> OnTranscriptionReceived**
+  - Event that is triggered when the Neocortex project transcribes an audio message to text.
   - Parameters:
-    - `message`: The transcribed message
+    - `message`: The transcribed audio message.
   - Example:
     ```csharp
     var smartAgent = GetComponent<NeocortexSmartAgent>();
@@ -99,10 +169,10 @@ The `Neocortex Smart Agent` component is the main component that allows you to i
     };
     ```
 
-**public event Action<AudioClip> OnAudioResponseReceived**
-  - Event that is triggered when the Neocortex project responds with an audio message
+**public UnityEvent<AudioClip> OnAudioResponseReceived**
+  - Event that is triggered when the Neocortex project responds with an audio message.
   - Parameters:
-    - `audioClip`: The audio clip received from the Neocortex project
+    - `audioClip`: The audio clip received from the Neocortex project.
   - Example:
     ```csharp
     var audioSource = GetComponent<AudioSource>();
@@ -114,6 +184,19 @@ The `Neocortex Smart Agent` component is the main component that allows you to i
     };
     ```
 
+**public UnityEvent<string> OnRequestFailed**
+  - Event that is triggered when a request to the Neocortex project fails.
+  - Parameters:
+    - `error`: The error message.
+  - Example:
+    ```csharp
+    var smartAgent = GetComponent<NeocortexSmartAgent>();
+    smartAgent.OnRequestFailed += (error) =>
+    {
+        Debug.LogError(error);
+    };
+    ```
+
 ### NeocortexAudioReceiver component
 The `NeocortexAudioReceiver` component is used to record audio data from the microphone via loudness of the souned, so you can have a hands free chat with the smart agent. On this component you can:
 - pick the microphone device to use
@@ -140,17 +223,17 @@ The `NeocortexAudioReceiver` component is used to record audio data from the mic
   audioReceiver.StopMicrophone();
   ```
 
-**public event UnityAction<byte[]> OnAudioRecorded**
-  - Event that is triggered when audio data is recorded from the microphone
-  - Parameters:
-    - `audioData`: The recorded audio data
+**public UnityEvent<AudioClip> OnAudioRecorded OnAudioRecorded**
+  - Event that is triggered when audio data is recorded from the microphone.
+  - Returns:
+    - `audioClip`: The recorded audio clip.
   - Example:
   ```csharp
   var audioReceiver = GetComponent<NeocortexAudioReceiver>();
-  audioReceiver.OnAudioRecorded += (audioData) =>
+  audioReceiver.OnAudioRecorded.AddListener((audioClip) =>
   {
-      Debug.Log($"Audio Data Length: {audioData.Length}");
-  };
+      Debug.Log($"Audio Data Length: {audioClip.samples}");
+  });
   ```
 
 ## Sample Projects

README.md

@@ -462,7 +462,6 @@ GameObject:
   - component: {fileID: 2962727209791358683}
   - component: {fileID: 7710414645333600691}
   - component: {fileID: 973319749406882948}
-  - component: {fileID: 3425882831741277229}
   m_Layer: 5
   m_Name: Chat State
   m_TagString: Untagged
@@ -531,50 +530,6 @@ MonoBehaviour:
   m_FillOrigin: 0
   m_UseSpriteMesh: 0
   m_PixelsPerUnitMultiplier: 1
---- !u!114 &3425882831741277229
-MonoBehaviour:
-  m_ObjectHideFlags: 0
-  m_CorrespondingSourceObject: {fileID: 0}
-  m_PrefabInstance: {fileID: 0}
-  m_PrefabAsset: {fileID: 0}
-  m_GameObject: {fileID: 969423940934023843}
-  m_Enabled: 1
-  m_EditorHideFlags: 0
-  m_Script: {fileID: 11500000, guid: 4e29b1a8efbd4b44bb3f3716e73f07ff, type: 3}
-  m_Name: 
-  m_EditorClassIdentifier: 
-  m_Navigation:
-    m_Mode: 3
-    m_WrapAround: 0
-    m_SelectOnUp: {fileID: 0}
-    m_SelectOnDown: {fileID: 0}
-    m_SelectOnLeft: {fileID: 0}
-    m_SelectOnRight: {fileID: 0}
-  m_Transition: 1
-  m_Colors:
-    m_NormalColor: {r: 1, g: 1, b: 1, a: 1}
-    m_HighlightedColor: {r: 0.9607843, g: 0.9607843, b: 0.9607843, a: 1}
-    m_PressedColor: {r: 0.78431374, g: 0.78431374, b: 0.78431374, a: 1}
-    m_SelectedColor: {r: 0.9607843, g: 0.9607843, b: 0.9607843, a: 1}
-    m_DisabledColor: {r: 0.78431374, g: 0.78431374, b: 0.78431374, a: 0.5019608}
-    m_ColorMultiplier: 1
-    m_FadeDuration: 0.1
-  m_SpriteState:
-    m_HighlightedSprite: {fileID: 0}
-    m_PressedSprite: {fileID: 0}
-    m_SelectedSprite: {fileID: 0}
-    m_DisabledSprite: {fileID: 0}
-  m_AnimationTriggers:
-    m_NormalTrigger: Normal
-    m_HighlightedTrigger: Highlighted
-    m_PressedTrigger: Pressed
-    m_SelectedTrigger: Selected
-    m_DisabledTrigger: Disabled
-  m_Interactable: 1
-  m_TargetGraphic: {fileID: 973319749406882948}
-  m_OnClick:
-    m_PersistentCalls:
-      m_Calls: []
 --- !u!1 &2381234143618097104
 GameObject:
   m_ObjectHideFlags: 0

Resources/Prefabs/Audio Chat Input.prefab

@@ -0,0 +1,14 @@
+using System;
+using UnityEngine.Networking;
+
+namespace Neocortex.API
+{
+    [Serializable]
+    public class ApiPayload
+    {
+        public string url;
+        public string method = UnityWebRequest.kHttpVerbPOST;
+        public byte[] data;
+        public bool isAudio;
+    }
+}